diff --git a/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift b/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift index f63c9e2f9b..40ed686a5e 100644 --- a/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift +++ b/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift @@ -1,5 +1,5 @@ // Generated by Scripts/regenerate-codex-parser-hash.sh. Do not edit by hand. enum CodexParserHash { - static let value = "aa0b0865c496e548" + static let value = "dcab8c067490a89e" } diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageCache.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageCache.swift index 2bdffa03d7..8dcb034715 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageCache.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageCache.swift @@ -123,18 +123,8 @@ enum CostUsageCacheIO { cache.producerKey = producerKey ?? self.currentProducerKey(provider: provider) cache.timeZoneIdentifier = calendar.timeZone.identifier - let tmp = dir.appendingPathComponent(".tmp-\(UUID().uuidString).json", isDirectory: false) let data = (try? JSONEncoder().encode(cache)) ?? Data() - do { - try data.write(to: tmp, options: [.atomic]) - if FileManager.default.fileExists(atPath: url.path) { - _ = try FileManager.default.replaceItemAt(url, withItemAt: tmp) - } else { - try FileManager.default.moveItem(at: tmp, to: url) - } - } catch { - try? FileManager.default.removeItem(at: tmp) - } + try? data.write(to: url, options: [.atomic]) } static func currentProducerKey( @@ -173,6 +163,8 @@ struct CostUsageCache: Codable { var codexPreviousReport: CostUsageCodexPreviousReport? /// Persistent session-id discovery and generation-scoped negative lookups for fork parents. var codexSessionDiscovery: CostUsageCodexSessionDiscovery? + /// Resumable bounded discovery for recently modified rollouts in older date partitions. + var codexActiveLookbackState: CostUsageCodexActiveLookbackState? /// filePath -> file usage var files: [String: CostUsageFileUsage] = [:] @@ -184,6 +176,15 @@ struct CostUsageCache: Codable { var roots: [String: Int64]? } +struct CostUsageCodexActiveLookbackState: Codable { + var scanSinceKey: String + var rootPaths: [String] + var nextDayKeyByRoot: [String: String] = [:] + var completedRootPaths: [String] = [] + var pendingFilePaths: [String] = [] + var legacyRecursivePendingRootPaths: [String] = [] +} + struct CostUsageCodexSessionDiscovery: Codable { struct DirectoryStamp: Codable, Equatable { var mtimeUnixMs: Int64 diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift index cdcad4999d..f69842ca5a 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift @@ -1819,7 +1819,7 @@ enum CostUsageScanner { root: root, scanSinceKey: scanSinceKey, scanUntilKey: scanUntilKey, - calendar: calendar) + calendar: calendar).files let flat = self.listCodexSessionFilesFlat(root: root, scanSinceKey: scanSinceKey, scanUntilKey: scanUntilKey) let recursive = includeRecursive ? self.listCodexLegacySessionFilesRecursive(root: root) : [] var seen: Set = [] @@ -2078,33 +2078,32 @@ enum CostUsageScanner { indexedBytes: anchor.indexedBytes) == anchor } - private static func listCodexRecentlyModifiedFiles( + private static func listCodexRecentlyModifiedPartitionFiles( root: URL, scanSinceKey: String, - scanUntilKey: String, modifiedSince: Date, - calendar: Calendar = .current) -> [URL] + scanBudget: CodexScanBudget, + resumeDayKey: String?, + calendar: Calendar = .current) -> CodexDatePartitionListing { let lookbackSinceKey = self.dayKey( scanSinceKey, addingDays: -self.codexActiveSessionLookbackDays, calendar: calendar) ?? scanSinceKey + let lookbackUntilKey = self.dayKey(scanSinceKey, addingDays: -1, calendar: calendar) + ?? lookbackSinceKey let partitioned = self.listCodexSessionFilesByDatePartition( root: root, scanSinceKey: lookbackSinceKey, - scanUntilKey: scanUntilKey, - calendar: calendar) - let partitionedModified = self.filterRecentlyModified(files: partitioned, modifiedSince: modifiedSince) - - let legacyRecursive = self.listCodexRecentlyModifiedFilesRecursive(root: root, modifiedSince: modifiedSince) - var seen = Set(partitionedModified.map(\.path)) - var out = partitionedModified - for fileURL in legacyRecursive where !seen.contains(fileURL.path) { - seen.insert(fileURL.path) - out.append(fileURL) - } - return out + scanUntilKey: lookbackUntilKey, + calendar: calendar, + scanBudget: scanBudget, + resumeDayKey: resumeDayKey) + return CodexDatePartitionListing( + files: self.filterRecentlyModified(files: partitioned.files, modifiedSince: modifiedSince), + isComplete: partitioned.isComplete, + nextDayKey: partitioned.nextDayKey) } private static func filterRecentlyModified(files: [URL], modifiedSince: Date) -> [URL] { @@ -2178,9 +2177,9 @@ enum CostUsageScanner { } static func isWithinCodexRoots(fileURL: URL, roots: [URL]) -> Bool { - let filePath = fileURL.standardizedFileURL.path + let filePath = self.codexResolvedPath(fileURL) return roots.contains { root in - let rootPath = root.standardizedFileURL.path + let rootPath = self.codexResolvedPath(root) if filePath == rootPath { return true } @@ -2189,19 +2188,57 @@ enum CostUsageScanner { } } + private static func codexResolvedPath(_ url: URL) -> String { + let path = url.resolvingSymlinksInPath().standardizedFileURL.path + if path.hasPrefix("/private/var/") { + return String(path.dropFirst("/private".count)) + } + return path + } + + private struct CodexDatePartitionListing { + let files: [URL] + let isComplete: Bool + let nextDayKey: String? + } + private static func listCodexSessionFilesByDatePartition( root: URL, scanSinceKey: String, scanUntilKey: String, - calendar: Calendar = .current) -> [URL] + calendar: Calendar = .current, + scanBudget: CodexScanBudget? = nil, + resumeDayKey: String? = nil) -> CodexDatePartitionListing { - guard FileManager.default.fileExists(atPath: root.path) else { return [] } + guard FileManager.default.fileExists(atPath: root.path) else { + return CodexDatePartitionListing(files: [], isComplete: true, nextDayKey: nil) + } let calendar = CostUsageDayRange.localGregorianCalendar(matching: calendar) var out: [URL] = [] - var date = Self.parseDayKey(scanSinceKey, calendar: calendar) ?? Date() - let untilDate = Self.parseDayKey(scanUntilKey, calendar: calendar) ?? date + let sinceDate = Self.parseDayKey(scanSinceKey, calendar: calendar) ?? Date() + let untilDate = Self.parseDayKey(scanUntilKey, calendar: calendar) ?? sinceDate + let resumedDate = resumeDayKey.flatMap { Self.parseDayKey($0, calendar: calendar) } + var date = if let resumedDate, resumedDate >= sinceDate, resumedDate <= untilDate { + resumedDate + } else { + sinceDate + } while date <= untilDate { + let admittedWork: Int64 + if let scanBudget { + switch scanBudget.admit(workBytes: 1) { + case let .allow(allowance): admittedWork = allowance + case .deferBudget: + return CodexDatePartitionListing( + files: out, + isComplete: false, + nextDayKey: CostUsageDayRange.dayKey(from: date, calendar: calendar)) + } + } else { + admittedWork = 0 + } + let comps = calendar.dateComponents([.year, .month, .day], from: date) let y = String(format: "%04d", comps.year ?? 1970) let m = String(format: "%02d", comps.month ?? 1) @@ -2220,11 +2257,114 @@ enum CostUsageScanner { out.append(item) } } + scanBudget?.complete(admittedWorkBytes: admittedWork, actualWorkBytes: admittedWork) date = calendar.date(byAdding: .day, value: 1, to: date) ?? untilDate.addingTimeInterval(1) } - return out + return CodexDatePartitionListing(files: out, isComplete: true, nextDayKey: nil) + } + + private static func codexActiveLookbackState( + cache: CostUsageCache, + roots: [URL], + scanSinceKey: String, + includeLegacyRecursiveScan: Bool) -> CostUsageCodexActiveLookbackState + { + let rootPaths = roots.map(Self.codexResolvedPath).sorted() + if let cached = cache.codexActiveLookbackState, + cached.scanSinceKey == scanSinceKey, + cached.rootPaths == rootPaths + { + return cached + } + return CostUsageCodexActiveLookbackState( + scanSinceKey: scanSinceKey, + rootPaths: rootPaths, + legacyRecursivePendingRootPaths: includeLegacyRecursiveScan ? rootPaths : []) + } + + private static func advanceCodexActiveLookback( + root: URL, + range: CostUsageDayRange, + modifiedSince: Date, + scanBudget: CodexScanBudget, + state: inout CostUsageCodexActiveLookbackState) + { + let rootPath = Self.codexResolvedPath(root) + var completedRootPaths = Set(state.completedRootPaths) + var pendingFilePaths = Set(state.pendingFilePaths) + if !completedRootPaths.contains(rootPath) { + let listing = Self.listCodexRecentlyModifiedPartitionFiles( + root: root, + scanSinceKey: range.scanSinceKey, + modifiedSince: modifiedSince, + scanBudget: scanBudget, + resumeDayKey: state.nextDayKeyByRoot[rootPath], + calendar: range.calendar) + pendingFilePaths.formUnion(listing.files.map(Self.codexResolvedPath)) + if listing.isComplete { + completedRootPaths.insert(rootPath) + state.nextDayKeyByRoot.removeValue(forKey: rootPath) + } else if let nextDayKey = listing.nextDayKey { + state.nextDayKeyByRoot[rootPath] = nextDayKey + } + } + + var legacyPendingRoots = Set(state.legacyRecursivePendingRootPaths) + if completedRootPaths.contains(rootPath), legacyPendingRoots.remove(rootPath) != nil { + // This recursive walk belongs only to the cold-start cycle. Later warm cycles + // retain the bounded partition discovery above. + let legacy = Self.listCodexRecentlyModifiedFilesRecursive( + root: root, + modifiedSince: modifiedSince) + pendingFilePaths.formUnion(legacy.map(Self.codexResolvedPath)) + } + state.completedRootPaths = completedRootPaths.sorted() + state.pendingFilePaths = pendingFilePaths.sorted() + state.legacyRecursivePendingRootPaths = legacyPendingRoots.sorted() + } + + private static func appendPendingCodexActiveLookbackFiles( + state: inout CostUsageCodexActiveLookbackState, + roots: [URL], + seenPaths: inout Set, + files: inout [URL]) + { + state.pendingFilePaths = state.pendingFilePaths.filter { path in + FileManager.default.fileExists(atPath: path) + && Self.isWithinCodexRoots(fileURL: URL(fileURLWithPath: path), roots: roots) + } + var seenFileIDs = Set(files.compactMap { Self.codexFileMetadata(fileURL: $0).fileId }) + for path in state.pendingFilePaths where !seenPaths.contains(path) { + let fileID = Self.codexFileMetadata(fileURL: URL(fileURLWithPath: path)).fileId + if let fileID, !seenFileIDs.insert(fileID).inserted { + continue + } + seenPaths.insert(path) + files.append(URL(fileURLWithPath: path)) + } + } + + private static func finalizedCodexActiveLookbackState( + _ state: CostUsageCodexActiveLookbackState, + cache: CostUsageCache) -> CostUsageCodexActiveLookbackState? + { + var state = state + state.pendingFilePaths.removeAll { path in + guard FileManager.default.fileExists(atPath: path) else { return true } + let fileID = Self.codexFileMetadata(fileURL: URL(fileURLWithPath: path)).fileId + if let fileID { + return cache.files.values.contains { + $0.codexScanFileId == fileID && $0.codexScanComplete == true + } + } + return cache.files[path]?.codexScanComplete == true + } + let isComplete = Set(state.completedRootPaths) == Set(state.rootPaths) + && state.pendingFilePaths.isEmpty + && state.legacyRecursivePendingRootPaths.isEmpty + return isComplete ? nil : state } private static func listCodexSessionFilesFlat(root: URL, scanSinceKey: String, scanUntilKey: String) -> [URL] { @@ -4048,6 +4188,7 @@ enum CostUsageScanner { Self.dropCachedCodexFile(path: metadata.path, cached: cache.files[metadata.path], cache: &cache) return } + Self.reconcileCodexCachePathAliases(metadata: metadata, cache: &cache) let cached = cache.files[metadata.path] @@ -4358,6 +4499,15 @@ enum CostUsageScanner { let cachedUntilKey = cache.scanUntilKey let shouldRunColdCacheLookback = cache.files.isEmpty || plan.rootsChanged let coldCacheLookbackStart = Self.localStartOfDay(range.scanSinceKey, calendar: options.calendar) + let scanBudget = CodexScanBudget( + maxFileBytes: options.maxCodexSessionFileBytes, + maxBytesPerRefresh: options.maxCodexScanBytesPerRefresh, + maxDuration: options.maxCodexScanDurationPerRefresh) + var activeLookbackState = Self.codexActiveLookbackState( + cache: cache, + roots: plan.roots, + scanSinceKey: range.scanSinceKey, + includeLegacyRecursiveScan: shouldRunColdCacheLookback) var seenPaths: Set = [] var files: [URL] = [] for root in plan.roots { @@ -4372,22 +4522,31 @@ enum CostUsageScanner { files.append(fileURL) } - if shouldRunColdCacheLookback, let coldCacheLookbackStart { - let recentlyModifiedFiles = Self.listCodexRecentlyModifiedFiles( + // The lookback runs on every refresh, not just cold ones: a session + // resumed in an older date partition is appended to in place, so the + // in-window partition listing never sees it and `cachedCodexSessionFiles` + // cannot either until it has been scanned once. Without this, such a + // session's usage stays invisible until a forced rescan. + // + // Partition discovery and any discovered candidates persist across bounded + // passes. That prevents a small budget from restarting at the oldest day or + // rediscovering a file without ever leaving enough budget to parse it. + if let coldCacheLookbackStart { + Self.advanceCodexActiveLookback( root: root, - scanSinceKey: range.scanSinceKey, - scanUntilKey: range.scanUntilKey, + range: range, modifiedSince: coldCacheLookbackStart, - calendar: options.calendar) - for fileURL in recentlyModifiedFiles.sorted(by: { $0.path < $1.path }) - where !seenPaths.contains(fileURL.path) - { - seenPaths.insert(fileURL.path) - files.append(fileURL) - } + scanBudget: scanBudget, + state: &activeLookbackState) } } + Self.appendPendingCodexActiveLookbackFiles( + state: &activeLookbackState, + roots: plan.roots, + seenPaths: &seenPaths, + files: &files) + for fileURL in Self.cachedCodexSessionFiles( cache: cache, range: range, @@ -4404,10 +4563,6 @@ enum CostUsageScanner { } var filePathsInScan = Set(files.map(\.path)) - let scanBudget = CodexScanBudget( - maxFileBytes: options.maxCodexSessionFileBytes, - maxBytesPerRefresh: options.maxCodexScanBytesPerRefresh, - maxDuration: options.maxCodexScanDurationPerRefresh) let fileIndex = CodexSessionFileIndex( files: files, roots: plan.roots, @@ -4443,6 +4598,9 @@ enum CostUsageScanner { context: scanContext, cache: &cache, inheritedResolver: inheritedResolver)) + cache.codexActiveLookbackState = Self.finalizedCodexActiveLookbackState( + activeLookbackState, + cache: cache) if scanBudget.resumedPartialFileCount > 0 || scanBudget.deferredByBudgetFileCount > 0 || scanBudget.deferredByTimeBudgetFileCount > 0 @@ -4517,6 +4675,7 @@ enum CostUsageScanner { || cache.files.values.contains { $0.codexScanComplete == false } || cache.files.values.contains { $0.hasBufferedCodexForkRetryLines } || fileIndex.hasPendingDiscovery + || cache.codexActiveLookbackState != nil cache.codexScanCatchUpPending = catchUpPending cache.codexPreviousReport = catchUpPending ? previousReport : nil if plan.hasPriorityMetadata { @@ -4711,6 +4870,27 @@ enum CostUsageScanner { return lhs.path < rhs.path } } + + private static func reconcileCodexCachePathAliases( + metadata: CodexFileMetadata, + cache: inout CostUsageCache) + { + guard let fileID = metadata.fileId else { return } + var aliases = cache.files.compactMap { path, usage in + path != metadata.path && usage.codexScanFileId == fileID ? path : nil + }.sorted() + guard !aliases.isEmpty else { return } + + if cache.files[metadata.path] == nil, let migratedPath = aliases.first { + cache.files[metadata.path] = cache.files.removeValue(forKey: migratedPath) + aliases.removeFirst() + } + for alias in aliases { + guard let stale = cache.files[alias] else { continue } + Self.applyFileDays(cache: &cache, fileDays: stale.days, sign: -1) + cache.files.removeValue(forKey: alias) + } + } } // swiftlint:enable type_body_length diff --git a/Tests/CodexBarTests/CostUsagePerformanceGateTests.swift b/Tests/CodexBarTests/CostUsagePerformanceGateTests.swift index 504004e9dd..eeafac36f1 100644 --- a/Tests/CodexBarTests/CostUsagePerformanceGateTests.swift +++ b/Tests/CodexBarTests/CostUsagePerformanceGateTests.swift @@ -360,6 +360,7 @@ struct CostUsagePerformanceGateTests { maxCodexSessionFileBytes: slice, maxCodexScanBytesPerRefresh: slice) options.refreshMinIntervalSeconds = 0 + options.maxCodexScanBytesPerRefresh += Self.codexLookbackDiscoveryWork(options: options) _ = CostUsageScanner.loadDailyReport( provider: .codex, @@ -406,6 +407,7 @@ struct CostUsagePerformanceGateTests { maxCodexSessionFileBytes: slice, maxCodexScanBytesPerRefresh: slice) options.refreshMinIntervalSeconds = 0 + options.maxCodexScanBytesPerRefresh += Self.codexLookbackDiscoveryWork(options: options) _ = CostUsageScanner.loadDailyReport( provider: .codex, @@ -1253,7 +1255,7 @@ extension CostUsagePerformanceGateTests { let childSize = CostUsageScanner.codexFileMetadata(fileURL: childURL).size options.maxCodexSessionFileBytes = 64 * 1024 * 1024 - options.maxCodexScanBytesPerRefresh = childSize + options.maxCodexScanBytesPerRefresh = childSize + Self.codexLookbackDiscoveryWork(options: options) options.preferNewestCodexSessionsFirst = true _ = CostUsageScanner.loadDailyReport( provider: .codex, @@ -1401,6 +1403,13 @@ extension CostUsagePerformanceGateTests { throw CocoaError(.fileWriteUnknown) } } + + private static func codexLookbackDiscoveryWork(options: CostUsageScanner.Options) -> Int64 { + let existingRootCount = CostUsageScanner.codexSessionsRoots(options: options).count { + FileManager.default.fileExists(atPath: $0.path) + } + return Int64(CostUsageScanner.codexActiveSessionLookbackDays * existingRootCount) + } } private final class HeadParseCounter: @unchecked Sendable { diff --git a/Tests/CodexBarTests/CostUsageScannerBreakdownTests.swift b/Tests/CodexBarTests/CostUsageScannerBreakdownTests.swift index a8b6474035..0b62a2482d 100644 --- a/Tests/CodexBarTests/CostUsageScannerBreakdownTests.swift +++ b/Tests/CodexBarTests/CostUsageScannerBreakdownTests.swift @@ -1203,6 +1203,9 @@ struct CostUsageScannerBreakdownTests { now: day, options: options) #expect(wide.summary?.totalTokens == 30) + try FileManager.default.setAttributes( + [.modificationDate: olderDay], + ofItemAtPath: olderFile.path) var cache = CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot) cache.codexProjectMetadataVersion = nil @@ -6339,6 +6342,9 @@ struct CostUsageScannerBreakdownTests { ], ], ])) + try FileManager.default.setAttributes( + [.modificationDate: archivedDay], + ofItemAtPath: archivedURL.path) var options = CostUsageScanner.Options( codexSessionsRoot: env.codexSessionsRoot, diff --git a/TestsLinux/CodexWarmCacheResumeLinuxTests.swift b/TestsLinux/CodexWarmCacheResumeLinuxTests.swift new file mode 100644 index 0000000000..f33a0b9dca --- /dev/null +++ b/TestsLinux/CodexWarmCacheResumeLinuxTests.swift @@ -0,0 +1,283 @@ +import Foundation +import Testing +@testable import CodexBarCore + +/// A Codex session resumed in an older date partition must keep being counted. +/// +/// Candidate files come from three sources: the day directories inside the scan +/// window, the flat root, and paths already present in `cache.files`. The +/// mtime-based lookback that would otherwise catch a rollout living outside the +/// window is gated behind `cache.files.isEmpty || plan.rootsChanged`, i.e. cold +/// cache only. A rollout that was never scanned and sits in an out-of-window +/// partition therefore stays invisible on every warm refresh. +struct CodexWarmCacheResumeLinuxTests { + private struct Environment { + let root: URL + let cacheRoot: URL + let codexSessionsRoot: URL + + init() throws { + self.root = FileManager.default.temporaryDirectory + .appendingPathComponent("codexbar-warm-resume-\(UUID().uuidString)", isDirectory: true) + self.cacheRoot = self.root.appendingPathComponent("cache", isDirectory: true) + self.codexSessionsRoot = self.root.appendingPathComponent("sessions", isDirectory: true) + try FileManager.default.createDirectory(at: self.cacheRoot, withIntermediateDirectories: true) + try FileManager.default.createDirectory(at: self.codexSessionsRoot, withIntermediateDirectories: true) + } + + func cleanup() { + try? FileManager.default.removeItem(at: self.root) + } + + func localNoon(year: Int, month: Int, day: Int) throws -> Date { + var comps = DateComponents() + comps.calendar = Calendar.current + comps.timeZone = TimeZone.current + comps.year = year + comps.month = month + comps.day = day + comps.hour = 12 + guard let date = comps.date else { + throw NSError(domain: "CodexWarmCacheResumeLinuxTests", code: 1) + } + return date + } + + func isoString(for date: Date) -> String { + let fmt = ISO8601DateFormatter() + fmt.formatOptions = [.withInternetDateTime] + return fmt.string(from: date) + } + + /// Writes into the `YYYY/MM/DD` partition for `day`, like Codex does. + @discardableResult + func writeSession(day: Date, filename: String, contents: String) throws -> URL { + let comps = Calendar.current.dateComponents([.year, .month, .day], from: day) + let dir = self.codexSessionsRoot + .appendingPathComponent(String(format: "%04d", comps.year ?? 1970), isDirectory: true) + .appendingPathComponent(String(format: "%02d", comps.month ?? 1), isDirectory: true) + .appendingPathComponent(String(format: "%02d", comps.day ?? 1), isDirectory: true) + try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) + let fileURL = dir.appendingPathComponent(filename) + try contents.write(to: fileURL, atomically: true, encoding: .utf8) + return fileURL + } + + func jsonl(_ objects: [[String: Any]]) throws -> String { + try objects + .map { try String(decoding: JSONSerialization.data(withJSONObject: $0), as: UTF8.self) } + .joined(separator: "\n") + "\n" + } + } + + private static let model = "openai/gpt-5.2-codex" + + private static func turnContext(iso: String) -> [String: Any] { + ["type": "turn_context", "timestamp": iso, "payload": ["model": self.model]] + } + + private static func tokenCount(iso: String, input: Int, cached: Int, output: Int) -> [String: Any] { + [ + "type": "event_msg", + "timestamp": iso, + "payload": [ + "type": "token_count", + "info": [ + "total_token_usage": [ + "input_tokens": input, + "cached_input_tokens": cached, + "output_tokens": output, + "reasoning_output_tokens": 0, + ], + "model": self.model, + ], + ], + ] + } + + private static func totalTokens(_ report: CostUsageDailyReport) -> Int { + report.data.reduce(0) { $0 + ($1.totalTokens ?? 0) } + } + + private static func persistedCache(cacheRoot: URL) throws -> CostUsageCache { + let data = try Data(contentsOf: CostUsageCacheIO.cacheFileURL(provider: .codex, cacheRoot: cacheRoot)) + return try JSONDecoder().decode(CostUsageCache.self, from: data) + } + + @Test + func `a session resumed in an older partition keeps being counted on a warm cache`() throws { + let env = try Environment() + defer { env.cleanup() } + + // "Today" for the scan, and a session that started outside a short history + // window. The gap stays inside `codexActiveSessionLookbackDays`, which is the + // range the active-session lookback is designed to cover. + let today = try env.localNoon(year: 2026, month: 7, day: 20) + let oldDay = try env.localNoon(year: 2026, month: 7, day: 1) + let windowStart = try env.localNoon(year: 2026, month: 7, day: 15) + + // An in-window session, so the cache is warm (cache.files is not empty) and + // the cold-cache lookback does not run on later refreshes. + try env.writeSession( + day: today, + filename: "rollout-recent.jsonl", + contents: env.jsonl([ + Self.turnContext(iso: env.isoString(for: today)), + Self.tokenCount(iso: env.isoString(for: today), input: 100, cached: 0, output: 10), + ])) + + // The resumed session: its rollout lives in an old partition. + let oldFile = try env.writeSession( + day: oldDay, + filename: "rollout-resumed.jsonl", + contents: env.jsonl([ + Self.turnContext(iso: env.isoString(for: oldDay)), + Self.tokenCount(iso: env.isoString(for: oldDay), input: 50, cached: 0, output: 5), + ])) + + var options = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + claudeProjectsRoots: nil, + cacheRoot: env.cacheRoot) + options.refreshMinIntervalSeconds = 0 + + // Warm the cache over the short window. The old rollout is out of window and + // is not expected to appear here. + let warmup = CostUsageScanner.loadDailyReport( + provider: .codex, + since: windowStart, + until: today, + now: today, + options: options) + let warmupTotal = Self.totalTokens(warmup) + + // The user resumes that old session: Codex appends to the original file, in + // its original partition. The appended turn is dated today. + let resumeISO = env.isoString(for: today.addingTimeInterval(60)) + try (env.jsonl([ + Self.turnContext(iso: env.isoString(for: oldDay)), + Self.tokenCount(iso: env.isoString(for: oldDay), input: 50, cached: 0, output: 5), + Self.turnContext(iso: resumeISO), + Self.tokenCount(iso: resumeISO, input: 4000, cached: 0, output: 400), + ])).write(to: oldFile, atomically: true, encoding: .utf8) + + let afterResume = CostUsageScanner.loadDailyReport( + provider: .codex, + since: windowStart, + until: today, + now: today.addingTimeInterval(120), + options: options) + + // A forced rescan re-arms the cold path and sees everything, which is the + // reference value the warm refresh should match. + var forcedOptions = options + forcedOptions.forceRescan = true + let forced = CostUsageScanner.loadDailyReport( + provider: .codex, + since: windowStart, + until: today, + now: today.addingTimeInterval(180), + options: forcedOptions) + + let resumedTokens = Self.totalTokens(forced) - warmupTotal + #expect(resumedTokens > 0, "fixture sanity: the forced rescan must observe the resumed turn") + + // The warm refresh must not silently drop the resumed session's new tokens. + #expect( + Self.totalTokens(afterResume) == Self.totalTokens(forced), + """ + warm refresh reported \(Self.totalTokens(afterResume)) tokens but a forced rescan \ + reported \(Self.totalTokens(forced)); the resumed session in an older partition was \ + never re-scanned + """) + } + + @Test + func `older partition discovery shares the bounded scan budget`() throws { + let env = try Environment() + defer { env.cleanup() } + + let today = try env.localNoon(year: 2026, month: 7, day: 20) + let oldDay = try env.localNoon(year: 2026, month: 7, day: 1) + let windowStart = try env.localNoon(year: 2026, month: 7, day: 15) + try env.writeSession( + day: today, + filename: "rollout-recent.jsonl", + contents: env.jsonl([ + Self.turnContext(iso: env.isoString(for: today)), + Self.tokenCount(iso: env.isoString(for: today), input: 100, cached: 0, output: 10), + ])) + let oldFile = try env.writeSession( + day: oldDay, + filename: "rollout-resumed.jsonl", + contents: env.jsonl([ + Self.turnContext(iso: env.isoString(for: oldDay)), + Self.tokenCount(iso: env.isoString(for: oldDay), input: 50, cached: 0, output: 5), + ])) + + var options = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + claudeProjectsRoots: nil, + cacheRoot: env.cacheRoot) + options.refreshMinIntervalSeconds = 0 + let warmup = CostUsageScanner.loadDailyReport( + provider: .codex, + since: windowStart, + until: today, + now: today, + options: options) + + let resumeISO = env.isoString(for: today.addingTimeInterval(60)) + try env.jsonl([ + Self.turnContext(iso: env.isoString(for: oldDay)), + Self.tokenCount(iso: env.isoString(for: oldDay), input: 50, cached: 0, output: 5), + Self.turnContext(iso: resumeISO), + Self.tokenCount(iso: resumeISO, input: 4000, cached: 0, output: 400), + ]).write(to: oldFile, atomically: true, encoding: .utf8) + + // Ten units cannot reach the old partition in one pass. The persisted cursor must + // advance rather than restarting at the oldest lookback day on every refresh. + options.maxCodexScanBytesPerRefresh = 10 + let firstBudgeted = CostUsageScanner.loadDailyReport( + provider: .codex, + since: windowStart, + until: today, + now: today.addingTimeInterval(120), + options: options) + #expect(Self.totalTokens(firstBudgeted) == Self.totalTokens(warmup)) + let firstState = try #require( + Self.persistedCache(cacheRoot: env.cacheRoot).codexActiveLookbackState) + let sessionsRootPath = env.codexSessionsRoot.standardizedFileURL.path + let firstNextDay = try #require(firstState.nextDayKeyByRoot[sessionsRootPath]) + #expect(!firstState.pendingFilePaths.contains(oldFile.standardizedFileURL.path)) + + let secondBudgeted = CostUsageScanner.loadDailyReport( + provider: .codex, + since: windowStart, + until: today, + now: today.addingTimeInterval(180), + options: options) + #expect(Self.totalTokens(secondBudgeted) == Self.totalTokens(warmup)) + let secondState = try #require( + Self.persistedCache(cacheRoot: env.cacheRoot).codexActiveLookbackState) + #expect(secondState.nextDayKeyByRoot[sessionsRootPath].map { $0 > firstNextDay } == true) + #expect(secondState.pendingFilePaths.contains(oldFile.standardizedFileURL.path)) + + options.maxCodexScanBytesPerRefresh = 512 * 1024 * 1024 + let caughtUp = CostUsageScanner.loadDailyReport( + provider: .codex, + since: windowStart, + until: today, + now: today.addingTimeInterval(240), + options: options) + var forcedOptions = options + forcedOptions.forceRescan = true + let forced = CostUsageScanner.loadDailyReport( + provider: .codex, + since: windowStart, + until: today, + now: today.addingTimeInterval(300), + options: forcedOptions) + #expect(Self.totalTokens(caughtUp) == Self.totalTokens(forced)) + } +}