From e2ffbf5807ddaff9a6739c257fa6ba20b38a75c4 Mon Sep 17 00:00:00 2001 From: Chris Ayers Date: Sat, 25 Jul 2026 11:35:54 -0400 Subject: [PATCH 1/4] fix: bound Codex cost scans on giant session corpora Local token-cost history walks Codex rollout JSONL under ~/.codex/sessions. On machines with multi-GB rollouts this can peg a core for hours on the cost-usage-scan queue while quitting/relaunching only restarts the same backlog. Rate-limit/account usage probes are a separate path and are unaffected. - Cap per-file cold/full scan work (default 256 MiB) - Cap newly-read bytes per refresh (default 512 MiB) and defer the rest - Prefer newest session files first so recent usage lands before catch-up - Add performance gates for oversized skip, budget deferral, and fork work sizing --- CHANGELOG.md | 2 + .../Generated/CodexParserHash.generated.swift | 2 +- .../Vendored/CostUsage/CostUsageScanner.swift | 149 +++++++++++++++++- .../CostUsagePerformanceGateTests.swift | 89 +++++++++++ 4 files changed, 237 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f7f2272d59..f268af76ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### Fixed +- Cost usage: bound Codex local session scans with per-file and per-refresh byte budgets, and prefer newest rollouts first, so multi-GB session corpora cannot peg a CPU core for hours while rate-limit/account usage probing remains unchanged. + ## 0.45.2 — 2026-07-19 ### Fixed diff --git a/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift b/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift index d46bf4b5b2..691c45c03a 100644 --- a/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift +++ b/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift @@ -1,5 +1,5 @@ // Generated by Scripts/regenerate-codex-parser-hash.sh. Do not edit by hand. enum CodexParserHash { - static let value = "48ac20dad61e9a7f" + static let value = "f9a2af0883938ebf" } diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift index 7ff583f33d..a8fd2ddb85 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift @@ -33,6 +33,14 @@ enum CostUsageScanner { var claudeLogProviderFilter: ClaudeLogProviderFilter = .all /// Force a full rescan, ignoring per-file cache and incremental offsets. var forceRescan: Bool = false + /// Skip full/cold parses of individual Codex rollout files larger than this. + /// Fresh cached entries are still reused. Default 256 MiB. + var maxCodexSessionFileBytes: Int64 = 256 * 1024 * 1024 + /// Soft budget for newly-read Codex session bytes in one refresh. + /// Remaining dirty files are deferred to later refreshes. Default 512 MiB. + var maxCodexScanBytesPerRefresh: Int64 = 512 * 1024 * 1024 + /// Prefer newest session files first so recent usage lands before catch-up work. + var preferNewestCodexSessionsFirst: Bool = true init( codexSessionsRoot: URL? = nil, @@ -40,7 +48,10 @@ enum CostUsageScanner { cacheRoot: URL? = nil, codexTraceDatabaseURL: URL? = nil, claudeLogProviderFilter: ClaudeLogProviderFilter = .all, - forceRescan: Bool = false) + forceRescan: Bool = false, + maxCodexSessionFileBytes: Int64 = 256 * 1024 * 1024, + maxCodexScanBytesPerRefresh: Int64 = 512 * 1024 * 1024, + preferNewestCodexSessionsFirst: Bool = true) { self.codexSessionsRoot = codexSessionsRoot self.claudeProjectsRoots = claudeProjectsRoots @@ -48,6 +59,50 @@ enum CostUsageScanner { self.codexTraceDatabaseURL = codexTraceDatabaseURL self.claudeLogProviderFilter = claudeLogProviderFilter self.forceRescan = forceRescan + self.maxCodexSessionFileBytes = max(0, maxCodexSessionFileBytes) + self.maxCodexScanBytesPerRefresh = max(0, maxCodexScanBytesPerRefresh) + self.preferNewestCodexSessionsFirst = preferNewestCodexSessionsFirst + } + } + + /// Per-refresh work limiter for Codex cost scans. Prevents multi-GB rollout corpora from + /// monopolizing a core for hours while still allowing progressive catch-up. + final class CodexScanBudget: @unchecked Sendable { + let maxFileBytes: Int64 + let maxBytesPerRefresh: Int64 + private(set) var bytesConsumed: Int64 = 0 + private(set) var skippedOversizedFileCount = 0 + private(set) var deferredByBudgetFileCount = 0 + + init(maxFileBytes: Int64, maxBytesPerRefresh: Int64) { + self.maxFileBytes = max(0, maxFileBytes) + self.maxBytesPerRefresh = max(0, maxBytesPerRefresh) + } + + enum Admission { + case allow + case skipOversized + case deferBudget + } + + func admit(workBytes: Int64) -> Admission { + let work = max(0, workBytes) + if self.maxFileBytes > 0, work > self.maxFileBytes { + self.skippedOversizedFileCount += 1 + return .skipOversized + } + if self.maxBytesPerRefresh > 0, + self.bytesConsumed > 0, + self.bytesConsumed + work > self.maxBytesPerRefresh + { + self.deferredByBudgetFileCount += 1 + return .deferBudget + } + return .allow + } + + func consume(workBytes: Int64) { + self.bytesConsumed += max(0, workBytes) } } @@ -408,6 +463,7 @@ enum CostUsageScanner { let changedPriorityTurnIDs: Set let resources: CodexScanResources let checkCancellation: CancellationCheck? + let scanBudget: CodexScanBudget? } final class CodexCanonicalProjectPathResolver { @@ -2732,10 +2788,60 @@ enum CostUsageScanner { if try Self.keepCachedCodexFileIfFresh(input: input, context: context, cache: &cache, state: &state) { return } + + let pendingWorkBytes = Self.pendingCodexScanWorkBytes(metadata: metadata, cached: cached) + if let budget = context.scanBudget { + switch budget.admit(workBytes: pendingWorkBytes) { + case .allow: + break + case .skipOversized: + Self.log.warning( + "Skipping oversized Codex session during cost scan", + metadata: [ + "path": metadata.path, + "bytes": "\(metadata.size)", + "limit": "\(budget.maxFileBytes)", + ]) + // Preserve any prior contribution instead of thrashing on multi-GB rollouts. + // Fresh caches were already handled above; stale caches stay until a future + // refresh with room, or until the file shrinks under the limit. + return + case .deferBudget: + Self.log.debug( + "Deferring Codex session cost scan until a later refresh", + metadata: [ + "path": metadata.path, + "pendingBytes": "\(pendingWorkBytes)", + "consumed": "\(budget.bytesConsumed)", + "limit": "\(budget.maxBytesPerRefresh)", + ]) + // Preserve stale cache so later refreshes can resume catch-up. + return + } + } + if try Self.appendCodexFileIncrementIfPossible(input: input, context: context, cache: &cache, state: &state) { + context.scanBudget?.consume(workBytes: pendingWorkBytes) return } try Self.rescanCodexFile(input: input, context: context, cache: &cache, state: &state) + context.scanBudget?.consume(workBytes: pendingWorkBytes) + } + + static func pendingCodexScanWorkBytes(metadata: CodexFileMetadata, cached: CostUsageFileUsage?) -> Int64 { + guard let cached else { return max(0, metadata.size) } + if cached.mtimeUnixMs == metadata.mtimeUnixMs, cached.size == metadata.size { + return 0 + } + let startOffset = cached.parsedBytes ?? cached.size + if metadata.size > cached.size, + startOffset > 0, + startOffset <= metadata.size, + cached.forkedFromId == nil + { + return max(0, metadata.size - startOffset) + } + return max(0, metadata.size) } private static func makeCodexRefreshPlan( @@ -2893,6 +2999,10 @@ enum CostUsageScanner { files.append(fileURL) } + if options.preferNewestCodexSessionsFirst { + files = Self.sortedCodexSessionFilesNewestFirst(files) + } + let filePathsInScan = Set(files.map(\.path)) var scanState = CodexScanState() let fileIndex = CodexSessionFileIndex( @@ -2913,12 +3023,16 @@ enum CostUsageScanner { modelsDevCatalog: plan.modelsDevCatalog, modelsDevCacheRoot: options.cacheRoot, priorityTurns: plan.priorityTurns) + let scanBudget = CodexScanBudget( + maxFileBytes: options.maxCodexSessionFileBytes, + maxBytesPerRefresh: options.maxCodexScanBytesPerRefresh) let scanContext = Self.codexFileScanContext( range: range, options: options, plan: plan, resources: resources, - checkCancellation: checkCancellation) + checkCancellation: checkCancellation, + scanBudget: scanBudget) for fileURL in files { try Self.scanCodexFile( fileURL: fileURL, @@ -2926,6 +3040,17 @@ enum CostUsageScanner { cache: &cache, state: &scanState) } + if scanBudget.skippedOversizedFileCount > 0 || scanBudget.deferredByBudgetFileCount > 0 { + Self.log.info( + "Codex cost scan applied work limits", + metadata: [ + "skippedOversized": "\(scanBudget.skippedOversizedFileCount)", + "deferredByBudget": "\(scanBudget.deferredByBudgetFileCount)", + "bytesConsumed": "\(scanBudget.bytesConsumed)", + "maxFileBytes": "\(scanBudget.maxFileBytes)", + "maxBytesPerRefresh": "\(scanBudget.maxBytesPerRefresh)", + ]) + } try checkCancellation?() Self.pruneForceRescanFilesOutsideWindow( @@ -3004,7 +3129,8 @@ enum CostUsageScanner { options: Options, plan: CodexRefreshPlan, resources: CodexScanResources, - checkCancellation: CancellationCheck?) -> CodexFileScanContext + checkCancellation: CancellationCheck?, + scanBudget: CodexScanBudget? = nil) -> CodexFileScanContext { CodexFileScanContext( range: range, @@ -3015,7 +3141,22 @@ enum CostUsageScanner { requiresTurnIDCache: plan.needsTurnIDCacheMigration, changedPriorityTurnIDs: plan.changedPriorityTurnIDs, resources: resources, - checkCancellation: checkCancellation) + checkCancellation: checkCancellation, + scanBudget: scanBudget) + } + + static func sortedCodexSessionFilesNewestFirst(_ files: [URL]) -> [URL] { + files.sorted { lhs, rhs in + let left = Self.codexFileMetadata(fileURL: lhs) + let right = Self.codexFileMetadata(fileURL: rhs) + if left.mtimeUnixMs != right.mtimeUnixMs { + return left.mtimeUnixMs > right.mtimeUnixMs + } + if left.size != right.size { + return left.size < right.size + } + return lhs.path < rhs.path + } } } diff --git a/Tests/CodexBarTests/CostUsagePerformanceGateTests.swift b/Tests/CodexBarTests/CostUsagePerformanceGateTests.swift index b5295f1cd6..369f27554a 100644 --- a/Tests/CodexBarTests/CostUsagePerformanceGateTests.swift +++ b/Tests/CodexBarTests/CostUsagePerformanceGateTests.swift @@ -285,6 +285,95 @@ struct CostUsagePerformanceGateTests { #expect(catalogLoadCount == 1) } + @Test + func `oversized codex session files are skipped instead of fully rescanned`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + let day = try env.makeLocalNoon(year: 2026, month: 5, day: 10) + let small = try Self.writeSyntheticCodexCorpus(env: env, day: day, files: 1, turnsPerFile: 2) + let smallURL = try #require(small.first) + let giantURL = try env.writeCodexSessionFile( + day: day, + filename: "giant-session.jsonl", + contents: String(repeating: "x", count: 8_192) + "\n") + + var options = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + claudeProjectsRoots: nil, + cacheRoot: env.cacheRoot, + codexTraceDatabaseURL: env.root.appendingPathComponent("missing.sqlite"), + maxCodexSessionFileBytes: 1_024, + maxCodexScanBytesPerRefresh: 64 * 1024 * 1024) + options.refreshMinIntervalSeconds = 0 + + let report = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day, + options: options) + let cache = CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot) + + #expect(report.summary?.totalTokens != nil) + #expect(cache.files[smallURL.path] != nil) + #expect(cache.files[giantURL.path] == nil) + #expect(report.data.isEmpty == false) + } + + @Test + func `per refresh byte budget defers later dirty files`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + let day = try env.makeLocalNoon(year: 2026, month: 5, day: 10) + let urls = try Self.writeSyntheticCodexCorpus(env: env, day: day, files: 3, turnsPerFile: 3) + // Make deterministic order by newest-first: touch later files later. + let older = try #require(urls.first) + let newer = try #require(urls.last) + let olderDate = day.addingTimeInterval(-3_600) + let newerDate = day + try FileManager.default.setAttributes([.modificationDate: olderDate], ofItemAtPath: older.path) + try FileManager.default.setAttributes([.modificationDate: newerDate], ofItemAtPath: newer.path) + + let newestMeta = CostUsageScanner.codexFileMetadata(fileURL: newer) + var options = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + claudeProjectsRoots: nil, + cacheRoot: env.cacheRoot, + codexTraceDatabaseURL: env.root.appendingPathComponent("missing.sqlite"), + maxCodexSessionFileBytes: 64 * 1024 * 1024, + // Enough for the newest file only; remaining dirty files defer. + maxCodexScanBytesPerRefresh: max(1, newestMeta.size), + preferNewestCodexSessionsFirst: true) + options.refreshMinIntervalSeconds = 0 + + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day, + options: options) + let cache = CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot) + + #expect(cache.files[newer.path] != nil) + #expect(cache.files[older.path] == nil) + } + + @Test + func `pending work bytes treat fork files as full rescan work`() { + let metadata = CostUsageScanner.CodexFileMetadata( + path: "/tmp/forked.jsonl", + mtimeUnixMs: 2, + size: 1_000, + fileId: "1:2") + let cached = CostUsageFileUsage( + mtimeUnixMs: 1, + size: 400, + days: [:], + parsedBytes: 400, + forkedFromId: "parent-session") + #expect(CostUsageScanner.pendingCodexScanWorkBytes(metadata: metadata, cached: cached) == 1_000) + } + private static func writeSyntheticCodexCorpus( env: CostUsageTestEnvironment, day: Date, From 49d0966838363fe636011d708017ff583cdf6e5e Mon Sep 17 00:00:00 2001 From: Chris Ayers Date: Sat, 25 Jul 2026 11:50:25 -0400 Subject: [PATCH 2/4] fix: close cost-scan budget bypasses on forced rescans and fork parents Address Codex review P1s on #2452: - pendingCodexScanWorkBytes no longer reports zero for unchanged size/mtime entries after keepCached fails. Forced full rescans (window/pricing/priority invalidation) now charge the full file. - CodexInheritedTotalsResolver shares the scan budget and skips or defers oversized parent baseline snapshot reads instead of parsing multi-GB parents while processing small fork/subagent children. - Add gates for forced-rescan work sizing and oversized-parent skip. --- CHANGELOG.md | 1 + .../Generated/CodexParserHash.generated.swift | 2 +- .../Vendored/CostUsage/CostUsageScanner.swift | 63 +++++++++++++++--- .../CostUsagePerformanceGateTests.swift | 65 +++++++++++++++++++ 4 files changed, 122 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f268af76ce..2e6da42458 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Fixed - Cost usage: bound Codex local session scans with per-file and per-refresh byte budgets, and prefer newest rollouts first, so multi-GB session corpora cannot peg a CPU core for hours while rate-limit/account usage probing remains unchanged. +- Cost usage: charge forced full rescans of unchanged cached files against the scan budget, and apply the same oversized/budget limits to inherited parent baseline reads used by forks/subagents. ## 0.45.2 — 2026-07-19 diff --git a/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift b/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift index 691c45c03a..b197397cef 100644 --- a/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift +++ b/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift @@ -1,5 +1,5 @@ // Generated by Scripts/regenerate-codex-parser-hash.sh. Do not edit by hand. enum CodexParserHash { - static let value = "f9a2af0883938ebf" + static let value = "cc9fab41073492bc" } diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift index a8fd2ddb85..5e9e31bfd8 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift @@ -675,11 +675,17 @@ enum CostUsageScanner { private let fileIndex: CodexSessionFileIndex private let checkCancellation: CancellationCheck? + private let scanBudget: CodexScanBudget? private var snapshotResolutions: [String: SnapshotResolution] = [:] - init(fileIndex: CodexSessionFileIndex, checkCancellation: CancellationCheck?) { + init( + fileIndex: CodexSessionFileIndex, + checkCancellation: CancellationCheck?, + scanBudget: CodexScanBudget? = nil) + { self.fileIndex = fileIndex self.checkCancellation = checkCancellation + self.scanBudget = scanBudget } func inheritedTotals(for sessionId: String, atOrBefore cutoffTimestamp: String) throws -> CodexForkBaseline { @@ -749,6 +755,43 @@ enum CostUsageScanner { return resolution } + let parentMetadata = CostUsageScanner.codexFileMetadata(fileURL: fileURL) + if let budget = self.scanBudget { + switch budget.admit(workBytes: parentMetadata.size) { + case .allow: + break + case .skipOversized: + CostUsageScanner.log.warning( + "Skipping oversized Codex parent session during inherited baseline read", + metadata: [ + "sessionId": sessionId, + "path": fileURL.path, + "bytes": "\(parentMetadata.size)", + "limit": "\(budget.maxFileBytes)", + ]) + let resolution = SnapshotResolution( + dependencyKey: self.dependencyKey(for: sessionId, fileURL: fileURL), + snapshots: nil) + self.snapshotResolutions[sessionId] = resolution + return resolution + case .deferBudget: + CostUsageScanner.log.debug( + "Deferring Codex parent session baseline read until a later refresh", + metadata: [ + "sessionId": sessionId, + "path": fileURL.path, + "pendingBytes": "\(parentMetadata.size)", + "consumed": "\(budget.bytesConsumed)", + "limit": "\(budget.maxBytesPerRefresh)", + ]) + let resolution = SnapshotResolution( + dependencyKey: self.dependencyKey(for: sessionId, fileURL: fileURL), + snapshots: nil) + self.snapshotResolutions[sessionId] = resolution + return resolution + } + } + for _ in 0..<2 { let dependencyKeyBeforeParse = self.dependencyKey(for: sessionId, fileURL: fileURL) let parsed = try CostUsageScanner.parseCodexTokenSnapshots( @@ -765,6 +808,7 @@ enum CostUsageScanner { dependencyKey: dependencyKeyAfterParse, snapshots: nil) self.snapshotResolutions[sessionId] = resolution + self.scanBudget?.consume(workBytes: parentMetadata.size) return resolution } if parsedSessionId != sessionId { @@ -779,12 +823,14 @@ enum CostUsageScanner { dependencyKey: dependencyKeyAfterParse, snapshots: nil) self.snapshotResolutions[sessionId] = resolution + self.scanBudget?.consume(workBytes: parentMetadata.size) return resolution } let resolution = SnapshotResolution( dependencyKey: dependencyKeyAfterParse, snapshots: parsed.snapshots) self.snapshotResolutions[sessionId] = resolution + self.scanBudget?.consume(workBytes: parentMetadata.size) return resolution } @@ -2829,10 +2875,10 @@ enum CostUsageScanner { } static func pendingCodexScanWorkBytes(metadata: CodexFileMetadata, cached: CostUsageFileUsage?) -> Int64 { + // Called only after keepCachedCodexFileIfFresh failed. Even when size/mtime still match + // (forced full rescan, priority invalidation, fork-dependency drift, etc.), the scanner + // will read the whole file — never report zero pending work in that case. guard let cached else { return max(0, metadata.size) } - if cached.mtimeUnixMs == metadata.mtimeUnixMs, cached.size == metadata.size { - return 0 - } let startOffset = cached.parsedBytes ?? cached.size if metadata.size > cached.size, startOffset > 0, @@ -3013,9 +3059,13 @@ enum CostUsageScanner { roots: plan.roots, knownExistingPaths: filePathsInScan), checkCancellation: checkCancellation) + let scanBudget = CodexScanBudget( + maxFileBytes: options.maxCodexSessionFileBytes, + maxBytesPerRefresh: options.maxCodexScanBytesPerRefresh) let inheritedResolver = CodexInheritedTotalsResolver( fileIndex: fileIndex, - checkCancellation: checkCancellation) + checkCancellation: checkCancellation, + scanBudget: scanBudget) let resources = CodexScanResources( fileIndex: fileIndex, inheritedResolver: inheritedResolver, @@ -3023,9 +3073,6 @@ enum CostUsageScanner { modelsDevCatalog: plan.modelsDevCatalog, modelsDevCacheRoot: options.cacheRoot, priorityTurns: plan.priorityTurns) - let scanBudget = CodexScanBudget( - maxFileBytes: options.maxCodexSessionFileBytes, - maxBytesPerRefresh: options.maxCodexScanBytesPerRefresh) let scanContext = Self.codexFileScanContext( range: range, options: options, diff --git a/Tests/CodexBarTests/CostUsagePerformanceGateTests.swift b/Tests/CodexBarTests/CostUsagePerformanceGateTests.swift index 369f27554a..fb02b9f1e4 100644 --- a/Tests/CodexBarTests/CostUsagePerformanceGateTests.swift +++ b/Tests/CodexBarTests/CostUsagePerformanceGateTests.swift @@ -374,6 +374,71 @@ struct CostUsagePerformanceGateTests { #expect(CostUsageScanner.pendingCodexScanWorkBytes(metadata: metadata, cached: cached) == 1_000) } + @Test + func `pending work bytes charge full file for forced rescans of unchanged cache entries`() { + let metadata = CostUsageScanner.CodexFileMetadata( + path: "/tmp/unchanged.jsonl", + mtimeUnixMs: 42, + size: 2_000_000_000, + fileId: "9:9") + let cached = CostUsageFileUsage( + mtimeUnixMs: 42, + size: 2_000_000_000, + days: ["2026-05-10": ["gpt-5.2-codex": [100, 20, 10]]], + parsedBytes: 2_000_000_000, + sessionId: "session-unchanged") + // keepCached can still reject this (forceFullScan / priority / fork dependency). + // Budget must not report zero pending work or multi-GB forced rescans slip through. + #expect(CostUsageScanner.pendingCodexScanWorkBytes(metadata: metadata, cached: cached) == 2_000_000_000) + } + + @Test + func `oversized parent baseline reads are skipped for small fork children`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + let day = try env.makeLocalNoon(year: 2026, month: 5, day: 10) + let iso = env.isoString(for: day) + + // Parent is intentionally larger than the per-file budget. + let parentBody = ([ + #"{"type":"session_meta","timestamp":"\#(iso)","payload":{"session_id":"parent-giant"}}"#, + #"{"type":"turn_context","timestamp":"\#(iso)","payload":{"model":"openai/gpt-5.2-codex"}}"#, + #"{"type":"event_msg","timestamp":"\#(iso)","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":500,"cached_input_tokens":50,"output_tokens":25},"model":"openai/gpt-5.2-codex"}}}"#, + ] + Array(repeating: "x", count: 4_096)).joined(separator: "\n") + "\n" + _ = try env.writeCodexSessionFile(day: day, filename: "parent-giant.jsonl", contents: parentBody) + + let childBody = [ + #"{"type":"session_meta","timestamp":"\#(iso)","payload":{"session_id":"child-small","forked_from_id":"parent-giant"}}"#, + #"{"type":"turn_context","timestamp":"\#(iso)","payload":{"model":"openai/gpt-5.2-codex"}}"#, + #"{"type":"event_msg","timestamp":"\#(iso)","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":600,"cached_input_tokens":60,"output_tokens":30},"model":"openai/gpt-5.2-codex"}}}"#, + ].joined(separator: "\n") + "\n" + let childURL = try env.writeCodexSessionFile(day: day, filename: "child-small.jsonl", contents: childBody) + + var options = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + claudeProjectsRoots: nil, + cacheRoot: env.cacheRoot, + codexTraceDatabaseURL: env.root.appendingPathComponent("missing.sqlite"), + maxCodexSessionFileBytes: 1_024, + maxCodexScanBytesPerRefresh: 64 * 1024 * 1024) + options.refreshMinIntervalSeconds = 0 + + let started = Date() + let report = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day, + options: options) + let elapsed = Date().timeIntervalSince(started) + let cache = CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot) + + #expect(elapsed < 2.0) + #expect(cache.files[childURL.path] != nil) + // Child still contributes local tokens even if parent baseline is unresolved/skipped. + #expect((report.summary?.totalTokens ?? 0) > 0) + } + private static func writeSyntheticCodexCorpus( env: CostUsageTestEnvironment, day: Date, From 556e3c6c6282312d513b1424221e5ae672b87a0e Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 28 Jul 2026 17:59:05 -0700 Subject: [PATCH 3/4] fix: validate resumable scan targets --- .../Generated/CodexParserHash.generated.swift | 2 +- .../Vendored/CostUsage/CostUsageJsonl.swift | 4 +- .../CostUsageScanner+CacheHelpers.swift | 9 +++- .../Vendored/CostUsage/CostUsageScanner.swift | 15 +++--- .../CostUsagePerformanceGateTests.swift | 53 +++++++++++++++++++ 5 files changed, 73 insertions(+), 10 deletions(-) diff --git a/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift b/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift index 79ac4a9b79..c8d4bb0634 100644 --- a/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift +++ b/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift @@ -1,5 +1,5 @@ // Generated by Scripts/regenerate-codex-parser-hash.sh. Do not edit by hand. enum CodexParserHash { - static let value = "5d6191ead6bfe597" + static let value = "0f95246e89ad5bca" } diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift index afad1cb09a..4911826e5b 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift @@ -304,7 +304,7 @@ enum CostUsageJsonl { var committedOffset = lineStartOffset var jsonTailState = resumeState?.jsonTailState ?? JSONTailState() let fileSize = (try? FileManager.default.attributesOfItem(atPath: fileURL.path)[.size] as? NSNumber)? - .int64Value ?? 0 + .int64Value func appendSegment(_ bytes: UnsafePointer, count: Int) { guard count > 0 else { return } @@ -356,7 +356,7 @@ enum CostUsageJsonl { try checkCancellation?() let remaining = maxBytesToRead.map { max(0, $0 - bytesRead) } if remaining == 0 { - if startOffset + bytesRead >= fileSize, hasCompleteJSONTail() { + if let fileSize, startOffset + bytesRead >= fileSize, hasCompleteJSONTail() { flushLine() committedOffset = startOffset + bytesRead lineStartOffset = committedOffset diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner+CacheHelpers.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner+CacheHelpers.swift index 0568b4ba42..b000ded093 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner+CacheHelpers.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner+CacheHelpers.swift @@ -991,17 +991,24 @@ extension CostUsageScanner { } // Subagent shape depends on the complete lineage prefix. Appended metadata can change an // independent counter into a copied-prefix rollout, so a tail-only parse is not sound. + let startOffset = cached.parsedBytes ?? cached.size + let hasMatchingResumeOffset = cached.codexJSONLResumeState?.offset == nil + || cached.codexJSONLResumeState?.offset == startOffset let isResumablePartial = cached.codexScanComplete == false + && cached.codexScanFileId != nil && cached.codexScanFileId == input.metadata.fileId && cached.codexScanTargetSize == input.metadata.size && cached.mtimeUnixMs == input.metadata.mtimeUnixMs + && hasMatchingResumeOffset + if cached.codexScanComplete == false, !isResumablePartial { + return false + } if !isResumablePartial, try Self.codexFileIsSubagentThread( fileURL: input.fileURL, checkCancellation: context.checkCancellation) { return false } - let startOffset = cached.parsedBytes ?? cached.size let initialCountedTotals = cached.lastCountedTotals ?? cached.lastTotals let initialRawTotalsBaseline = cached.lastRawTotalsBaseline ?? cached.lastTotals let initialHasDivergentTotals = cached.hasDivergentTotals ?? (cached.lastTotals == nil) diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift index 3a6cbc115d..c305584d81 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift @@ -2902,12 +2902,15 @@ enum CostUsageScanner { // (forced full rescan, priority invalidation, fork-dependency drift, etc.), the scanner // will read the whole file — never report zero pending work in that case. guard let cached else { return max(0, metadata.size) } - if cached.codexScanComplete == false, - cached.codexScanFileId == metadata.fileId, - cached.codexScanTargetSize == metadata.size, - cached.mtimeUnixMs == metadata.mtimeUnixMs - { - return max(0, metadata.size - (cached.parsedBytes ?? 0)) + if cached.codexScanComplete == false { + if cached.codexScanFileId != nil, + cached.codexScanFileId == metadata.fileId, + cached.codexScanTargetSize == metadata.size, + cached.mtimeUnixMs == metadata.mtimeUnixMs + { + return max(0, metadata.size - (cached.parsedBytes ?? 0)) + } + return max(0, metadata.size) } let startOffset = cached.parsedBytes ?? cached.size if metadata.size > cached.size, diff --git a/Tests/CodexBarTests/CostUsagePerformanceGateTests.swift b/Tests/CodexBarTests/CostUsagePerformanceGateTests.swift index 696bd0abb1..c202d20157 100644 --- a/Tests/CodexBarTests/CostUsagePerformanceGateTests.swift +++ b/Tests/CodexBarTests/CostUsagePerformanceGateTests.swift @@ -389,6 +389,59 @@ struct CostUsagePerformanceGateTests { #expect(second.codexScanTargetSize == metadata.size) } + @Test + func `oversized codex progress restarts when the target size changes`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + let day = try env.makeLocalNoon(year: 2026, month: 5, day: 10) + let files = try Self.writeSyntheticCodexCorpus(env: env, day: day, files: 1, turnsPerFile: 8) + let fileURL = try #require(files.first) + let originalMetadata = CostUsageScanner.codexFileMetadata(fileURL: fileURL) + let slice = max(1, originalMetadata.size / 4) + var options = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + claudeProjectsRoots: nil, + cacheRoot: env.cacheRoot, + codexTraceDatabaseURL: env.root.appendingPathComponent("missing.sqlite"), + maxCodexSessionFileBytes: slice, + maxCodexScanBytesPerRefresh: slice) + options.refreshMinIntervalSeconds = 0 + + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day, + options: options) + let first = try #require(CostUsageCacheIO.load( + provider: .codex, + cacheRoot: env.cacheRoot).files.values.first) + #expect(first.parsedBytes == slice) + #expect(first.codexScanComplete == false) + + let original = try String(contentsOf: fileURL, encoding: .utf8) + try (original + String(repeating: " ", count: 512)).write(to: fileURL, atomically: false, encoding: .utf8) + try FileManager.default.setAttributes( + [.modificationDate: day.addingTimeInterval(60)], + ofItemAtPath: fileURL.path) + let changedMetadata = CostUsageScanner.codexFileMetadata(fileURL: fileURL) + #expect(changedMetadata.size != originalMetadata.size) + + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day, + options: options) + let restarted = try #require(CostUsageCacheIO.load( + provider: .codex, + cacheRoot: env.cacheRoot).files.values.first) + #expect(restarted.parsedBytes == slice) + #expect(restarted.codexScanTargetSize == changedMetadata.size) + #expect(restarted.codexScanFileId == changedMetadata.fileId) + #expect(restarted.codexScanComplete == false) + } + @Test func `single oversized jsonl record resumes without stalling`() throws { let env = try CostUsageTestEnvironment() From 44858e1fc53e08d90f401c63955da1a406fcc3eb Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 28 Jul 2026 22:28:24 -0700 Subject: [PATCH 4/4] fix: budget subagent reclassification rescans --- .../Generated/CodexParserHash.generated.swift | 2 +- .../Vendored/CostUsage/CostUsageScanner.swift | 21 +++++++++++++++++-- ...exSubagentAccountingIntegrationTests.swift | 7 +++++-- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift b/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift index c8d4bb0634..b3ebf1b2d6 100644 --- a/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift +++ b/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift @@ -1,5 +1,5 @@ // Generated by Scripts/regenerate-codex-parser-hash.sh. Do not edit by hand. enum CodexParserHash { - static let value = "0f95246e89ad5bca" + static let value = "d91f9d2087a99035" } diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift index c305584d81..7b239e6214 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift @@ -2888,13 +2888,30 @@ enum CostUsageScanner { context.scanBudget?.consume(workBytes: allowedWorkBytes) return } + let fullRescanWorkBytes = max(0, metadata.size) + let fullRescanAllowedBytes: Int64 + if fullRescanWorkBytes == pendingWorkBytes { + fullRescanAllowedBytes = allowedWorkBytes + } else if let budget = context.scanBudget { + switch budget.admit(workBytes: fullRescanWorkBytes) { + case let .allow(allowance): + fullRescanAllowedBytes = allowance + case .deferBudget: + // No work was consumed by the rejected incremental path, so this is only + // reachable when the refresh budget has no allowance for the full rescan. + return + } + } else { + fullRescanAllowedBytes = fullRescanWorkBytes + } + try Self.rescanCodexFile( input: input, context: context, cache: &cache, state: &state, - maxBytesToRead: allowedWorkBytes) - context.scanBudget?.consume(workBytes: allowedWorkBytes) + maxBytesToRead: fullRescanAllowedBytes) + context.scanBudget?.consume(workBytes: fullRescanAllowedBytes) } static func pendingCodexScanWorkBytes(metadata: CodexFileMetadata, cached: CostUsageFileUsage?) -> Int64 { diff --git a/Tests/CodexBarTests/CodexSubagentAccountingIntegrationTests.swift b/Tests/CodexBarTests/CodexSubagentAccountingIntegrationTests.swift index fc7f57eb46..91eeebb957 100644 --- a/Tests/CodexBarTests/CodexSubagentAccountingIntegrationTests.swift +++ b/Tests/CodexBarTests/CodexSubagentAccountingIntegrationTests.swift @@ -507,7 +507,7 @@ struct CodexSubagentAccountingIntegrationTests { } @Test - func `appended ancestor metadata reclassifies the complete subagent rollout`() throws { + func `bounded append fallback reclassifies the complete subagent rollout`() throws { let env = try CostUsageTestEnvironment() defer { env.cleanup() } @@ -535,7 +535,9 @@ struct CodexSubagentAccountingIntegrationTests { var options = CostUsageScanner.Options( codexSessionsRoot: env.codexSessionsRoot, claudeProjectsRoots: nil, - cacheRoot: env.cacheRoot) + cacheRoot: env.cacheRoot, + maxCodexSessionFileBytes: 4096, + maxCodexScanBytesPerRefresh: 4096) options.refreshMinIntervalSeconds = 0 let first = CostUsageScanner.loadDailyReport( provider: .codex, @@ -579,6 +581,7 @@ struct CodexSubagentAccountingIntegrationTests { #expect(usage.sessionId == "growing-child") #expect(usage.forkedFromId == "growing-parent") #expect(usage.forkBaselineDependencyKey == CostUsageScanner.codexForkDependencyNotRequiredKey) + #expect(usage.codexScanComplete == true) } private func turnContext(timestamp: String, model: String) -> [String: Any] {