diff --git a/CHANGELOG.md b/CHANGELOG.md index db52336b934..6fb4f44f1e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## 0.48.2 — Unreleased ### Fixed +- Codex: restore JSON-cache retention semantics lost in the SQLite cutover — discovery pruning now reaches the scanner's round-tripped payload so deleted files stop resurfacing, the row budget never sacrifices in-window or recently active sessions, and fork-parent protection again drops stale lineage-only parents (refs #2760). - Codex: the SQLite cost store no longer deletes the whole database on transient failures — lock contention from a concurrent CLI/app writer, disk-full, or a constraint violation now preserve history and only genuine corruption or schema drift triggers a rebuild, which is now logged (refs #2760). - Menu: let long metric reset and pace details wrap to two lines instead of truncating, without clipping cached card heights (#2742). Thanks @Yuxin-Qiao! - Menu: let compact metric detail and reset rows wrap to a second line instead of truncating, so non-English locales keep the full pace and reset information (refs #2182). Thanks @Yuxin-Qiao! diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageStore+CodexCache.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageStore+CodexCache.swift index c81110c1a60..c4d83f4e2cf 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageStore+CodexCache.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageStore+CodexCache.swift @@ -18,7 +18,9 @@ extension CostUsageStore { _ cache: CostUsageCache, calendar: Calendar, requestedScanWindow: (sinceKey: String, untilKey: String), - reportWindow: (sinceKey: String, untilKey: String)? = nil) -> CostUsageStoreBudgetResult + reportWindow: (sinceKey: String, untilKey: String)? = nil, + rowBudget: Int = CostUsageStore.defaultRowBudget, + fileBudgetBytes: Int64 = CostUsageStore.defaultFileBudgetBytes) -> CostUsageStoreBudgetResult { let previous = self.readSnapshot() let canReuseStoredRows = previous.metadata.timeZoneIdentifier == calendar.timeZone.identifier @@ -42,8 +44,8 @@ extension CostUsageStore { _ = self.setDiscoveryState(Self.discoveryState(cache.codexSessionDiscovery)) _ = self.setLookbackState(Self.lookbackState(cache.codexActiveLookbackState)) let result = self.enforceBudgets( - maxRows: Self.defaultRowBudget, - maxFileBytes: Self.defaultFileBudgetBytes, + maxRows: rowBudget, + maxFileBytes: fileBudgetBytes, requestedSinceDay: requestedScanWindow.sinceKey, requestedUntilDay: requestedScanWindow.untilKey, calendar: calendar) diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageStore+Retention.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageStore+Retention.swift index 6e6bf7dd1a8..746a601f5b9 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageStore+Retention.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageStore+Retention.swift @@ -10,7 +10,11 @@ import CSQLite3 extension CostUsageStore { @discardableResult - func retainDayWindow(sinceDay: String, untilDay: String) -> CostUsageStoreRetentionResult { + func retainDayWindow( + sinceDay: String, + untilDay: String, + calendar: Calendar = .current) -> CostUsageStoreRetentionResult + { guard sinceDay <= untilDay else { return CostUsageStoreRetentionResult( deletedFiles: 0, @@ -24,7 +28,7 @@ extension CostUsageStore { deletedFileDayAggregates: 0, deletedDayAggregates: 0) return self.withDatabase(default: fallback) { database in - try Self.prune(database, sinceDay: sinceDay, untilDay: untilDay) + try Self.prune(database, sinceDay: sinceDay, untilDay: untilDay, calendar: calendar) } } @@ -42,7 +46,8 @@ extension CostUsageStore { private static func prune( _ database: OpaquePointer, sinceDay: String, - untilDay: String) throws -> CostUsageStoreRetentionResult + untilDay: String, + calendar: Calendar) throws -> CostUsageStoreRetentionResult { try self.inTransaction(database) { let beforeFiles = try self.scalarInt(database, "SELECT COUNT(*) FROM files") @@ -50,17 +55,27 @@ extension CostUsageStore { let beforeFileAggregates = try self.scalarInt(database, "SELECT COUNT(*) FROM file_day_aggregates") let beforeAggregates = try self.scalarInt(database, "SELECT COUNT(*) FROM day_aggregates") - let candidates = try self.retentionCandidates( - database, - sinceDay: sinceDay, - untilDay: untilDay) + let activeWindow = self.activeWindowMs(sinceDay: sinceDay, untilDay: untilDay, calendar: calendar) let deleteFile = try self.prepare(database, "DELETE FROM files WHERE path = ?") defer { sqlite3_finalize(deleteFile) } - for candidate in candidates { - sqlite3_reset(deleteFile) - sqlite3_clear_bindings(deleteFile) - self.bind(candidate.path, to: deleteFile, at: 1) - try self.stepDone(deleteFile, database: database) + var candidates: [RetentionCandidate] = [] + // Iterate to a fixpoint: deleting a stale child releases the fork protection on + // its stale parent, which must then be pruned in the same pass instead of + // lingering as an unreferenced out-of-window row. + while true { + let round = try self.retentionCandidates( + database, + sinceDay: sinceDay, + untilDay: untilDay, + activeWindow: activeWindow) + guard !round.isEmpty else { break } + for candidate in round { + sqlite3_reset(deleteFile) + sqlite3_clear_bindings(deleteFile) + self.bind(candidate.path, to: deleteFile, at: 1) + try self.stepDone(deleteFile, database: database) + } + candidates += round } let deleteFileAggregates = try self.prepare( @@ -105,13 +120,37 @@ extension CostUsageStore { var sessionID: String? } + /// Milliseconds spanned by the scan window's local days; a session file whose mtime + /// falls inside it is still active (it may hold unscanned in-window rows) even when its + /// scanned coverage is entirely out of window, so retention must not drop it. + static func activeWindowMs( + sinceDay: String?, + untilDay: String?, + calendar: Calendar) -> Range? + { + guard let sinceDay, let untilDay, + let since = CostUsageScanner.parseDayKey(sinceDay, calendar: calendar), + let until = CostUsageScanner.parseDayKey(untilDay, calendar: calendar) + else { return nil } + let scanCalendar = CostUsageScanner.CostUsageDayRange.localGregorianCalendar(matching: calendar) + let end = scanCalendar.date(byAdding: .day, value: 1, to: until) ?? until + let lower = Int64(since.timeIntervalSince1970 * 1000) + let upper = Int64(end.timeIntervalSince1970 * 1000) + guard lower < upper else { return nil } + return lower.. [RetentionCandidate] + untilDay: String, + activeWindow: Range?) throws -> [RetentionCandidate] { + // Fork parents stay protected only while a surviving child still needs the parent's + // baseline; lineage-only children (dependency key "not required") never resolve + // inherited totals, so they do not keep a stale parent alive. let statement = try self.prepare(database, """ - SELECT f.path, f.session_id + SELECT f.path, f.session_id, f.mtime_ms FROM files f WHERE f.scan_complete = 1 AND f.coverage_since_day IS NOT NULL @@ -122,6 +161,7 @@ extension CostUsageStore { f.session_id IS NULL OR NOT EXISTS ( SELECT 1 FROM fork_lineage l WHERE l.forked_from_id = f.session_id AND l.file_id != f.id + AND (l.dependency_key IS NULL OR l.dependency_key != ?) ) ) ORDER BY f.updated_at_ms, f.path @@ -129,11 +169,15 @@ extension CostUsageStore { defer { sqlite3_finalize(statement) } self.bind(sinceDay, to: statement, at: 1) self.bind(untilDay, to: statement, at: 2) + self.bind(CostUsageScanner.codexForkDependencyNotRequiredKey, to: statement, at: 3) var values: [RetentionCandidate] = [] var result = sqlite3_step(statement) while result == SQLITE_ROW { guard let path = self.columnText(statement, at: 0) else { throw StoreError.invalidData } - values.append(RetentionCandidate(path: path, sessionID: self.columnText(statement, at: 1))) + let mtime = sqlite3_column_int64(statement, 2) + if activeWindow?.contains(mtime) != true { + values.append(RetentionCandidate(path: path, sessionID: self.columnText(statement, at: 1))) + } result = sqlite3_step(statement) } guard result == SQLITE_DONE else { throw StoreError.sqlite(result) } @@ -158,7 +202,35 @@ extension CostUsageStore { state.filePathBySessionID = state.filePathBySessionID.filter { !sessionIDs.contains($0.key) && !paths.contains($0.value) } - state.nextFileIndex = min(state.nextFileIndex, state.filePaths.count) + // Cursors may point past the shortened arrays, and pruned coverage is no longer + // complete; reset both so the next discovery pass re-enqueues remaining files + // instead of trusting stale coverage. + state.nextFileIndex = 0 + state.nextDirectoryIndex = 0 + state.validationDirectoryIndex = 0 + state.isComplete = false + // The scanner round-trips discovery through the opaque payload, so the payload must + // be pruned in lockstep with the typed columns or the deleted files resurface on the + // next load. + if let payload = state.payload, + var discovery = try? JSONDecoder().decode(CostUsageCodexSessionDiscovery.self, from: payload) + { + discovery.filePaths.removeAll { paths.contains($0) } + discovery.fileStamps = discovery.fileStamps.filter { !paths.contains($0.key) } + discovery.filePathBySessionId = discovery.filePathBySessionId.filter { + !sessionIDs.contains($0.key) && !paths.contains($0.value) + } + discovery.missingSessionIds.removeAll { sessionIDs.contains($0) } + discovery.pendingSessionIds.removeAll { sessionIDs.contains($0) } + if let head = discovery.headScan, paths.contains(head.path) { + discovery.headScan = nil + } + discovery.nextFileIndex = 0 + discovery.nextDirectoryIndex = 0 + discovery.validationDirectoryIndex = 0 + discovery.isComplete = false + state.payload = (try? JSONEncoder().encode(discovery)) ?? state.payload + } try self.writeSingleton(state, database: database, table: "discovery_state") } } @@ -192,15 +264,21 @@ extension CostUsageStore { if initialRows > Int64(rowLimit) || initialBytes > byteLimit, let sinceDay, let untilDay, sinceDay <= untilDay { - _ = try Self.prune(database, sinceDay: sinceDay, untilDay: untilDay) + _ = try Self.prune(database, sinceDay: sinceDay, untilDay: untilDay, calendar: calendar) } var catchUpRequired = false + // The row budget mirrors the former entry budget: it only ever removes files the + // requested window does not read. In-window and recently active files are never + // dropped just for a row overage; the byte budget below is the sole authority + // that may sacrifice in-window data. while try Self.rowCount(database) > Int64(rowLimit) { guard try Self.deleteOldestRetainedFile( database, sinceDay: sinceDay, - calendar: calendar) else { break } + untilDay: untilDay, + calendar: calendar, + protectRequestedWindow: true) else { break } catchUpRequired = true } try Self.reclaimFreePages(database) @@ -210,7 +288,9 @@ extension CostUsageStore { guard try Self.deleteOldestRetainedFile( database, sinceDay: sinceDay, - calendar: calendar) + untilDay: untilDay, + calendar: calendar, + protectRequestedWindow: false) else { guard try Self.stripOldestRebuildableDetail(database) else { break } catchUpRequired = true @@ -246,7 +326,9 @@ extension CostUsageStore { private static func deleteOldestRetainedFile( _ database: OpaquePointer, sinceDay: String?, - calendar: Calendar) throws -> Bool + untilDay: String?, + calendar: Calendar, + protectRequestedWindow: Bool) throws -> Bool { let statement = try self.prepare(database, """ SELECT f.id, f.path, f.mtime_ms, f.coverage_since_day, f.coverage_until_day @@ -264,16 +346,24 @@ extension CostUsageStore { """) defer { sqlite3_finalize(statement) } self.bind(CostUsageScanner.codexForkDependencyNotRequiredKey, to: statement, at: 1) - let activeSinceMs = sinceDay.flatMap { CostUsageScanner.parseDayKey($0, calendar: calendar) } - .map { Int64($0.timeIntervalSince1970 * 1000) } + let activeWindow = self.activeWindowMs(sinceDay: sinceDay, untilDay: untilDay, calendar: calendar) var result = sqlite3_step(statement) while result == SQLITE_ROW { let mtime = sqlite3_column_int64(statement, 2) let coverageSince = self.columnText(statement, at: 3) let coverageUntil = self.columnText(statement, at: 4) - let zeroDayRecentlyActive = coverageSince == nil && coverageUntil == nil - && activeSinceMs.map { mtime >= $0 } == true - if !zeroDayRecentlyActive { + let recentlyActive = activeWindow?.contains(mtime) == true + let zeroDayRecentlyActive = coverageSince == nil && coverageUntil == nil && recentlyActive + let touchesWindow: Bool = if let sinceDay, let untilDay, + let coverageSince, let coverageUntil + { + coverageUntil >= sinceDay && coverageSince <= untilDay + } else { + false + } + let protected = zeroDayRecentlyActive + || (protectRequestedWindow && (touchesWindow || recentlyActive)) + if !protected { let fileID = sqlite3_column_int64(statement, 0) try self.execute(database, "DELETE FROM files WHERE id = \(fileID)") return sqlite3_changes(database) > 0 diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageStore.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageStore.swift index e003feb59f5..36181521699 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageStore.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageStore.swift @@ -114,7 +114,9 @@ extension CostUsageStore { _ cache: CostUsageCache, calendar: Calendar, requestedScanWindow: (sinceKey: String, untilKey: String), - reportWindow: (sinceKey: String, untilKey: String)? = nil) -> CostUsageStoreBudgetResult + reportWindow: (sinceKey: String, untilKey: String)? = nil, + rowBudget: Int = CostUsageStore.defaultRowBudget, + fileBudgetBytes: Int64 = CostUsageStore.defaultFileBudgetBytes) -> CostUsageStoreBudgetResult { Self.sharedExecutor.sync { self.assumeIsolated { store in @@ -122,7 +124,9 @@ extension CostUsageStore { cache, calendar: calendar, requestedScanWindow: requestedScanWindow, - reportWindow: reportWindow) + reportWindow: reportWindow, + rowBudget: rowBudget, + fileBudgetBytes: fileBudgetBytes) } } } diff --git a/Tests/CodexBarTests/CostUsageStoreTests.swift b/Tests/CodexBarTests/CostUsageStoreTests.swift index 976c764504c..4ea42d67e05 100644 --- a/Tests/CodexBarTests/CostUsageStoreTests.swift +++ b/Tests/CodexBarTests/CostUsageStoreTests.swift @@ -570,6 +570,220 @@ extension CostUsageStoreTests { #expect(discovery.pendingSessionIDs.isEmpty) #expect(discovery.filePathBySessionID.isEmpty) } + + @Test + func `retention prunes the discovery payload in sync with typed state`() async throws { + let fixture = try StoreFixture() + defer { fixture.remove() } + let store = CostUsageStore(cacheRoot: fixture.root) + var old = Self.file(path: "/rollouts/old.jsonl", day: "2026-07-01") + old.sessionID = "old-session" + var kept = Self.file(path: "/rollouts/kept.jsonl", day: "2026-08-02") + kept.sessionID = "kept-session" + #expect(await store.upsertFile(old)) + #expect(await store.upsertFile(kept)) + let discovery = CostUsageCodexSessionDiscovery( + roots: ["/rollouts"], + generation: nil, + directoryStamps: [:], + directoryPaths: [], + nextDirectoryIndex: 3, + filePaths: [old.path, kept.path], + nextFileIndex: 2, + fileStamps: [ + old.path: .init(mtimeUnixMs: 1, size: 100, fileId: nil), + kept.path: .init(mtimeUnixMs: 1, size: 100, fileId: nil), + ], + headScan: nil, + filePathBySessionId: [ + "old-session": old.path, + "kept-session": kept.path, + ], + missingSessionIds: ["old-session"], + pendingSessionIds: [], + validationDirectoryIndex: 1, + isComplete: true) + var state = Self.discoveryState(paths: [old.path, kept.path]) + state.payload = try JSONEncoder().encode(discovery) + #expect(await store.setDiscoveryState(state)) + + _ = await store.retainDayWindow(sinceDay: "2026-08-01", untilDay: "2026-08-03") + + // The scanner round-trips discovery through the opaque payload, so pruning must + // reach it or deleted files resurface on the next load. + let cache = store.syncLoadCodexCache(calendar: .current) + let pruned = try #require(cache.codexSessionDiscovery) + #expect(pruned.filePaths == [kept.path]) + #expect(pruned.fileStamps[old.path] == nil) + #expect(pruned.fileStamps[kept.path] != nil) + #expect(pruned.filePathBySessionId["old-session"] == nil) + #expect(pruned.filePathBySessionId["kept-session"] != nil) + #expect(pruned.missingSessionIds.isEmpty) + #expect(pruned.isComplete == false) + #expect(pruned.nextFileIndex == 0) + #expect(pruned.nextDirectoryIndex == 0) + } + + @Test + func `retention does not protect a parent referenced only by a lineage only child`() async throws { + let fixture = try StoreFixture() + defer { fixture.remove() } + let store = CostUsageStore(cacheRoot: fixture.root) + var parent = Self.file(path: "/rollouts/parent.jsonl", day: "2026-07-01") + parent.sessionID = "parent-session" + let child = Self.file(path: "/rollouts/child.jsonl", day: "2026-08-02") + var lineage = Self.lineage(path: child.path) + lineage.forkedFromID = "parent-session" + lineage.dependencyKey = CostUsageScanner.codexForkDependencyNotRequiredKey + #expect(await store.upsertFile(parent)) + #expect(await store.upsertFile(child)) + #expect(await store.upsertForkLineage(lineage)) + + _ = await store.retainDayWindow(sinceDay: "2026-08-01", untilDay: "2026-08-03") + + // Lineage-only children never resolve inherited totals, so they do not keep a + // stale out-of-window parent alive. + #expect(await store.fetchFile(path: parent.path) == nil) + #expect(await store.fetchFile(path: child.path) != nil) + } + + @Test + func `retention drops a stale parent referenced only by a stale child`() async throws { + let fixture = try StoreFixture() + defer { fixture.remove() } + let store = CostUsageStore(cacheRoot: fixture.root) + var parent = Self.file(path: "/rollouts/parent.jsonl", day: "2026-07-01", updatedAt: 1) + parent.sessionID = "parent-session" + var child = Self.file(path: "/rollouts/child.jsonl", day: "2026-07-02", updatedAt: 2) + child.sessionID = "child-session" + var lineage = Self.lineage(path: child.path) + lineage.forkedFromID = "parent-session" + #expect(await store.upsertFile(parent)) + #expect(await store.upsertFile(child)) + #expect(await store.upsertForkLineage(lineage)) + + let result = await store.retainDayWindow(sinceDay: "2026-08-01", untilDay: "2026-08-03") + + // One pass: deleting the stale child releases the parent's fork protection, and the + // parent must not linger as an unreferenced out-of-window row. + #expect(result.deletedFiles == 2) + #expect(await store.fetchFile(path: parent.path) == nil) + #expect(await store.fetchFile(path: child.path) == nil) + } + + @Test + func `retention preserves an out of window file modified inside the window`() async throws { + let fixture = try StoreFixture() + defer { fixture.remove() } + let store = CostUsageStore(cacheRoot: fixture.root) + let calendar = CostUsageScanner.CostUsageDayRange.localGregorianCalendar() + let activeDate = try #require(CostUsageScanner.parseDayKey("2026-08-02", calendar: calendar)) + var active = Self.file(path: "/rollouts/active.jsonl", day: "2026-07-01") + active.mtimeUnixMs = Int64(activeDate.addingTimeInterval(3600).timeIntervalSince1970 * 1000) + var stale = Self.file(path: "/rollouts/stale.jsonl", day: "2026-07-01") + stale.mtimeUnixMs = 1 + #expect(await store.upsertFile(active)) + #expect(await store.upsertFile(stale)) + + _ = await store.retainDayWindow(sinceDay: "2026-08-01", untilDay: "2026-08-03", calendar: calendar) + + // A recent mtime means the file may hold unscanned in-window rows; dropping it would + // force a rediscovery and full reparse on every refresh. + #expect(await store.fetchFile(path: active.path) != nil) + #expect(await store.fetchFile(path: stale.path) == nil) + } +} + +extension CostUsageStoreTests { + @Test + func `save preserves an existing complete previous report across repeated trims`() async throws { + let fixture = try StoreFixture() + defer { fixture.remove() } + let store = CostUsageStore(cacheRoot: fixture.root) + var cache = CostUsageCache() + cache.scanSinceKey = "2026-06-01" + cache.scanUntilKey = "2026-07-01" + var older = CostUsageFileUsage( + mtimeUnixMs: 1, + size: 100, + days: ["2026-06-05": ["gpt-5.5": [1, 0, 0]]]) + older.sessionId = "older-session" + var recent = CostUsageFileUsage( + mtimeUnixMs: 1, + size: 100, + days: ["2026-06-28": ["gpt-5.5": [1, 0, 0]]]) + recent.sessionId = "recent-session" + cache.files = [ + "/sessions/older.jsonl": older, + "/sessions/recent.jsonl": recent, + ] + cache.days = [ + "2026-06-05": ["gpt-5.5": [1, 0, 0]], + "2026-06-28": ["gpt-5.5": [1, 0, 0]], + ] + // Simulate an already pending catch-up pass with a complete previous report. + cache.codexScanCatchUpPending = true + cache.codexPreviousReport = CostUsageCodexPreviousReport( + report: CostUsageDailyReport(data: [ + CostUsageDailyReport.Entry( + date: "2026-06-05", + inputTokens: 1, + outputTokens: 0, + totalTokens: 1, + costUSD: nil, + modelsUsed: nil, + modelBreakdowns: nil), + ], summary: nil), + cache: cache) + + let result = store.syncSaveCodexCache( + cache, + calendar: .current, + requestedScanWindow: (sinceKey: "2026-06-01", untilKey: "2026-07-01"), + reportWindow: (sinceKey: "2026-06-01", untilKey: "2026-07-01"), + fileBudgetBytes: 1) + + #expect(result.catchUpRequired) + let metadata = await store.fetchMetadata() + let payload = try #require(metadata.previousReportPayload) + let preserved = try JSONDecoder().decode(CostUsageCodexPreviousReport.self, from: payload) + #expect(preserved.data.count == 1) + #expect(preserved.data.first?.date == "2026-06-05") + #expect(preserved.data.contains { $0.date == "2026-06-28" } == false) + } + + @Test + func `save catch up report honors a non-gregorian system calendar`() async throws { + let fixture = try StoreFixture() + defer { fixture.remove() } + let store = CostUsageStore(cacheRoot: fixture.root) + var calendar = Calendar(identifier: .buddhist) + calendar.timeZone = try #require(TimeZone(secondsFromGMT: 0)) + var cache = CostUsageCache() + cache.scanSinceKey = "2026-06-01" + cache.scanUntilKey = "2026-07-01" + var entry = CostUsageFileUsage( + mtimeUnixMs: 1, + size: 100, + days: ["2026-06-05": ["gpt-5.5": [1, 0, 0]]]) + entry.sessionId = "in-window-session" + cache.files = ["/sessions/in-window.jsonl": entry] + cache.days = ["2026-06-05": ["gpt-5.5": [1, 0, 0]]] + + let result = store.syncSaveCodexCache( + cache, + calendar: calendar, + requestedScanWindow: (sinceKey: "2026-06-01", untilKey: "2026-07-01"), + reportWindow: (sinceKey: "2026-06-01", untilKey: "2026-07-01"), + fileBudgetBytes: 1) + + #expect(result.catchUpRequired) + let metadata = await store.fetchMetadata() + let payload = try #require(metadata.previousReportPayload) + let previous = try JSONDecoder().decode(CostUsageCodexPreviousReport.self, from: payload) + #expect(previous.data.contains { $0.date == "2026-06-05" }) + #expect(previous.data.contains { $0.date == "1483-06-05" } == false) + } } extension CostUsageStoreTests { @@ -673,7 +887,7 @@ extension CostUsageStoreTests { } @Test - func `in window budget trim marks catch up and preserves previous report`() async throws { + func `in window byte budget trim marks catch up and preserves previous report`() async throws { let fixture = try StoreFixture() defer { fixture.remove() } let store = CostUsageStore(cacheRoot: fixture.root) @@ -682,12 +896,18 @@ extension CostUsageStoreTests { metadata.lastScanUnixMs = 1234 metadata.previousReportPayload = previous #expect(await store.setMetadata(metadata)) - #expect(await store.upsertFile(Self.file(path: "/rollouts/one.jsonl", day: "2026-08-01", updatedAt: 1))) - #expect(await store.upsertFile(Self.file(path: "/rollouts/two.jsonl", day: "2026-08-01", updatedAt: 2))) + for (index, path) in ["/rollouts/one.jsonl", "/rollouts/two.jsonl"].enumerated() { + #expect(await store.upsertFile(Self.file(path: path, day: "2026-08-01", updatedAt: Int64(index)))) + let row = CostUsageStoreUsageRow( + path: path, + rowIndex: 0, + payload: Data(repeating: UInt8(index), count: 256 * 1024)) + #expect(await store.replaceUsageRows(path: path, rows: [row])) + } let result = await store.enforceBudgets( - maxRows: 1, - maxFileBytes: .max, + maxRows: .max, + maxFileBytes: 1, requestedSinceDay: "2026-08-01", requestedUntilDay: "2026-08-02") let retained = await store.fetchMetadata() @@ -698,6 +918,106 @@ extension CostUsageStoreTests { #expect(retained.previousReportPayload == previous) } + @Test + func `row budget never deletes files touching the requested window`() async throws { + let fixture = try StoreFixture() + defer { fixture.remove() } + let store = CostUsageStore(cacheRoot: fixture.root) + #expect(await store.upsertFile(Self.file(path: "/rollouts/stale.jsonl", day: "2026-06-01", updatedAt: 0))) + #expect(await store.upsertFile(Self.file(path: "/rollouts/one.jsonl", day: "2026-08-01", updatedAt: 1))) + #expect(await store.upsertFile(Self.file(path: "/rollouts/two.jsonl", day: "2026-08-02", updatedAt: 2))) + + let result = await store.enforceBudgets( + maxRows: 1, + maxFileBytes: .max, + requestedSinceDay: "2026-08-01", + requestedUntilDay: "2026-08-02") + + // The out-of-window file is pruned, but in-window files stay even though the row + // count remains above the cap: the former entry budget never dropped in-window data. + #expect(result.rowCount == 2) + #expect(await store.fetchFile(path: "/rollouts/stale.jsonl") == nil) + #expect(await store.fetchFile(path: "/rollouts/one.jsonl") != nil) + #expect(await store.fetchFile(path: "/rollouts/two.jsonl") != nil) + #expect(await store.fetchMetadata().catchUpPending == false) + } + + @Test + func `byte budget strips detail from a protected file instead of stalling`() async throws { + let fixture = try StoreFixture() + defer { fixture.remove() } + let store = CostUsageStore(cacheRoot: fixture.root) + var file = Self.file(path: "/rollouts/resuming.jsonl", day: "2026-08-01") + file.scanState.isComplete = false + #expect(await store.upsertFile(file)) + #expect(await store.appendTokenSnapshots([Self.snapshot(path: file.path, eventIndex: 0)])) + #expect(await store.replaceUsageRows(path: file.path, rows: [CostUsageStoreUsageRow( + path: file.path, + rowIndex: 0, + payload: Data(repeating: 7, count: 256 * 1024))])) + #expect(await store.upsertAccumulator(Self.accumulator(path: file.path))) + + let result = await store.enforceBudgets( + maxRows: .max, + maxFileBytes: 1, + requestedSinceDay: "2026-08-01", + requestedUntilDay: "2026-08-02") + + let stripped = try #require(await store.fetchFile(path: file.path)) + #expect(result.catchUpRequired) + #expect(stripped.parsedBytes == 0) + #expect(stripped.scanState.isComplete == false) + #expect(stripped.scanState.resumePayload == nil) + #expect(await store.fetchTokenSnapshots(path: file.path).isEmpty) + #expect(await store.fetchUsageRows(path: file.path).isEmpty) + #expect(await store.fetchAccumulator(path: file.path) == nil) + #expect(await store.fetchMetadata().catchUpPending) + } + + @Test + func `byte budget compacts a fork parent required by a surviving child`() async throws { + let fixture = try StoreFixture() + defer { fixture.remove() } + let store = CostUsageStore(cacheRoot: fixture.root) + var parent = Self.file(path: "/rollouts/parent.jsonl", day: "2026-08-01", updatedAt: 1) + parent.sessionID = "parent-session" + #expect(await store.upsertFile(parent)) + #expect(await store.upsertForkLineage(CostUsageStoreForkLineage( + path: parent.path, + sessionID: "parent-session", + forkedFromID: nil, + forkTimestamp: nil, + dependencyKey: nil, + subagentState: nil, + accountingState: nil))) + #expect(await store.replaceUsageRows(path: parent.path, rows: [CostUsageStoreUsageRow( + path: parent.path, + rowIndex: 0, + payload: Data(repeating: 5, count: 256 * 1024))])) + var child = Self.file(path: "/rollouts/child.jsonl", day: "2026-08-01", updatedAt: 2) + child.sessionID = "child-session" + child.scanState.isComplete = false + var lineage = Self.lineage(path: child.path) + lineage.forkedFromID = "parent-session" + #expect(await store.upsertFile(child)) + #expect(await store.upsertForkLineage(lineage)) + + let result = await store.enforceBudgets( + maxRows: .max, + maxFileBytes: 1, + requestedSinceDay: "2026-08-01", + requestedUntilDay: "2026-08-02") + + // The parent cannot be deleted while the incomplete child still needs its baseline, + // so byte pressure compacts its rebuildable detail instead. + let compacted = try #require(await store.fetchFile(path: parent.path)) + #expect(result.catchUpRequired) + #expect(compacted.parsedBytes == 0) + #expect(compacted.scanState.isComplete == false) + #expect(await store.fetchUsageRows(path: parent.path).isEmpty) + #expect(await store.fetchFile(path: child.path) != nil) + } + @Test func `read only WAL reader keeps a consistent snapshot during write`() async throws { let fixture = try StoreFixture()