diff --git a/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift b/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift index 97b204e636..22be3d7abc 100644 --- a/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift +++ b/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift @@ -1,5 +1,5 @@ // Generated by Scripts/regenerate-codex-parser-hash.sh. Do not edit by hand. enum CodexParserHash { - static let value = "7378e1f7e954ea1f" + static let value = "6f689d90f8eedcbd" } diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageCache.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageCache.swift index a4a6ec7c1b..d4f675bd2d 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageCache.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageCache.swift @@ -159,6 +159,13 @@ struct CostUsageFileUsage: Codable { var codexWorkspaceContentFingerprint: String? var codexRows: [CostUsageScanner.CodexUsageRow]? var claudeRows: [CostUsageScanner.ClaudeUsageRow]? + /// Identity and target size for an in-progress bounded Codex parse. + var codexScanFileId: String? + var codexScanTargetSize: Int64? + var codexScanComplete: Bool? + var codexJSONLResumeState: CostUsageJsonl.ResumeState? + /// Compact relevant events retained while a subagent rollout awaits full-shape classification. + var codexBufferedSubagentLines: [CostUsageScanner.CodexBufferedFastLine]? } struct CostUsageCodexSessionMetadata: Codable, Equatable { diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift index fd17555f81..4911826e5b 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift @@ -6,8 +6,23 @@ enum CostUsageJsonl { let wasTruncated: Bool } - private struct JSONTailState { - private enum ScalarState { + struct ResumeState: Codable { + let offset: Int64 + fileprivate let lineStartOffset: Int64 + fileprivate let prefix: Data + fileprivate let lineBytes: Int + fileprivate let truncated: Bool + fileprivate let jsonTailState: JSONTailState + } + + struct ScanProgress { + let committedOffset: Int64 + let readOffset: Int64 + let resumeState: ResumeState? + } + + fileprivate struct JSONTailState: Codable { + private enum ScalarState: Codable { case notScalar case trueLiteral(Int) case falseLiteral(Int) @@ -16,7 +31,7 @@ enum CostUsageJsonl { case invalid } - private enum NumberState { + private enum NumberState: Codable { private enum ByteKind { case zero case digit @@ -234,6 +249,7 @@ enum CostUsageJsonl { offset: offset, maxLineBytes: maxLineBytes, prefixBytes: prefixBytes, + maxBytesToRead: nil, checkCancellation: nil, onLine: onLine) } @@ -244,25 +260,51 @@ enum CostUsageJsonl { offset: Int64 = 0, maxLineBytes: Int, prefixBytes: Int, + maxBytesToRead: Int64? = nil, checkCancellation: (() throws -> Void)? = nil, onLine: (Line) -> Void) throws -> Int64 + { + try self.scanBounded( + fileURL: fileURL, + offset: offset, + maxLineBytes: maxLineBytes, + prefixBytes: prefixBytes, + maxBytesToRead: maxBytesToRead, + resumeState: nil, + checkCancellation: checkCancellation, + onLine: onLine).committedOffset + } + + // swiftlint:disable:next function_parameter_count + static func scanBounded( + fileURL: URL, + offset: Int64 = 0, + maxLineBytes: Int, + prefixBytes: Int, + maxBytesToRead: Int64?, + resumeState: ResumeState?, + checkCancellation: (() throws -> Void)? = nil, + onLine: (Line) -> Void) throws -> ScanProgress { let handle = try FileHandle(forReadingFrom: fileURL) defer { try? handle.close() } - let startOffset = max(0, offset) + let startOffset = resumeState?.offset ?? max(0, offset) if startOffset > 0 { try handle.seek(toOffset: UInt64(startOffset)) } - var current = Data() + var current = resumeState?.prefix ?? Data() current.reserveCapacity(4 * 1024) - var lineBytes = 0 - var truncated = false + var lineBytes = resumeState?.lineBytes ?? 0 + var truncated = resumeState?.truncated ?? false var bytesRead: Int64 = 0 - var committedOffset = startOffset - var jsonTailState = JSONTailState() + var lineStartOffset = resumeState?.lineStartOffset ?? startOffset + var committedOffset = lineStartOffset + var jsonTailState = resumeState?.jsonTailState ?? JSONTailState() + let fileSize = (try? FileManager.default.attributesOfItem(atPath: fileURL.path)[.size] as? NSNumber)? + .int64Value func appendSegment(_ bytes: UnsafePointer, count: Int) { guard count > 0 else { return } @@ -288,6 +330,17 @@ enum CostUsageJsonl { jsonTailState.reset() } + func currentResumeState() -> ResumeState? { + guard lineBytes > 0 else { return nil } + return ResumeState( + offset: startOffset + bytesRead, + lineStartOffset: lineStartOffset, + prefix: current, + lineBytes: lineBytes, + truncated: truncated, + jsonTailState: jsonTailState) + } + func hasCompleteJSONTail() -> Bool { guard jsonTailState.isStructurallyComplete else { return false } if truncated { @@ -301,12 +354,23 @@ enum CostUsageJsonl { while true { try checkCancellation?() + let remaining = maxBytesToRead.map { max(0, $0 - bytesRead) } + if remaining == 0 { + if let fileSize, startOffset + bytesRead >= fileSize, hasCompleteJSONTail() { + flushLine() + committedOffset = startOffset + bytesRead + lineStartOffset = committedOffset + } + break + } let reachedEOF = try autoreleasepool { - let chunk = try handle.read(upToCount: 256 * 1024) ?? Data() + let readCount = min(256 * 1024, Int(remaining ?? Int64(256 * 1024))) + let chunk = try handle.read(upToCount: readCount) ?? Data() if chunk.isEmpty { if hasCompleteJSONTail() { flushLine() committedOffset = startOffset + bytesRead + lineStartOffset = committedOffset } return true } @@ -323,6 +387,7 @@ enum CostUsageJsonl { appendSegment(base.advanced(by: segmentStart), count: index - segmentStart) flushLine() committedOffset = chunkStartOffset + Int64(index + 1) + lineStartOffset = committedOffset segmentStart = index + 1 } else { jsonTailState.append(base[index]) @@ -341,6 +406,9 @@ enum CostUsageJsonl { try checkCancellation?() } - return committedOffset + return ScanProgress( + committedOffset: committedOffset, + readOffset: startOffset + bytesRead, + resumeState: currentResumeState()) } } diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner+CacheHelpers.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner+CacheHelpers.swift index 377c48e18d..a720f3ecb4 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner+CacheHelpers.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner+CacheHelpers.swift @@ -302,7 +302,12 @@ extension CostUsageScanner { codexPriorityTokens: [String: [String: Int]]? = nil, codexTurnIDs: [String]? = nil, codexRows: [CodexUsageRow]? = nil, - claudeRows: [ClaudeUsageRow]? = nil) -> CostUsageFileUsage + claudeRows: [ClaudeUsageRow]? = nil, + codexScanFileId: String? = nil, + codexScanTargetSize: Int64? = nil, + codexScanComplete: Bool? = nil, + codexJSONLResumeState: CostUsageJsonl.ResumeState? = nil, + codexBufferedSubagentLines: [CodexBufferedFastLine]? = nil) -> CostUsageFileUsage { CostUsageFileUsage( mtimeUnixMs: mtimeUnixMs, @@ -333,7 +338,12 @@ extension CostUsageScanner { codexPriorityTokens: codexPriorityTokens, codexTurnIDs: codexTurnIDs, codexRows: codexRows, - claudeRows: claudeRows) + claudeRows: claudeRows, + codexScanFileId: codexScanFileId, + codexScanTargetSize: codexScanTargetSize, + codexScanComplete: codexScanComplete, + codexJSONLResumeState: codexJSONLResumeState, + codexBufferedSubagentLines: codexBufferedSubagentLines) } static func needsCodexCostCache(_ usage: CostUsageFileUsage) -> Bool { @@ -950,6 +960,7 @@ extension CostUsageScanner { let needsSessionId = cached.sessionId == nil guard cached.mtimeUnixMs == input.metadata.mtimeUnixMs, cached.size == input.metadata.size, + cached.codexScanComplete != false, !needsSessionId, !context.forceFullScan else { return false } @@ -1025,7 +1036,8 @@ extension CostUsageScanner { input: CodexFileScanInput, context: CodexFileScanContext, cache: inout CostUsageCache, - state: inout CodexScanState) throws -> Bool + state: inout CodexScanState, + maxBytesToRead: Int64? = nil) throws -> Bool { try context.checkCancellation?() guard let cached = input.cached, cached.sessionId != nil, !context.forceFullScan else { return false } @@ -1035,13 +1047,24 @@ extension CostUsageScanner { } // Subagent shape depends on the complete lineage prefix. Appended metadata can change an // independent counter into a copied-prefix rollout, so a tail-only parse is not sound. - if try Self.codexFileIsSubagentThread( + let startOffset = cached.parsedBytes ?? cached.size + let hasMatchingResumeOffset = cached.codexJSONLResumeState?.offset == nil + || cached.codexJSONLResumeState?.offset == startOffset + let isResumablePartial = cached.codexScanComplete == false + && cached.codexScanFileId != nil + && cached.codexScanFileId == input.metadata.fileId + && cached.codexScanTargetSize == input.metadata.size + && cached.mtimeUnixMs == input.metadata.mtimeUnixMs + && hasMatchingResumeOffset + if cached.codexScanComplete == false, !isResumablePartial { + return false + } + if !isResumablePartial, try Self.codexFileIsSubagentThread( fileURL: input.fileURL, checkCancellation: context.checkCancellation) { return false } - let startOffset = cached.parsedBytes ?? cached.size let initialCountedTotals = cached.lastCountedTotals ?? cached.lastTotals let initialRawTotalsBaseline = cached.lastRawTotalsBaseline ?? cached.lastTotals let initialHasDivergentTotals = cached.hasDivergentTotals ?? (cached.lastTotals == nil) @@ -1051,11 +1074,13 @@ extension CostUsageScanner { (cached.hasInterleavedTotals == true && cached.lastRawTotalsWatermark == nil) || (cached.lastRawTotalsWatermark != nil && cached.hasInterleavedTotals == nil) || (initialHasDivergentTotals && cached.lastRawTotalsWatermark == nil) - let canIncremental = input.metadata.size > cached.size && startOffset > 0 + let canIncremental = startOffset > 0 && startOffset <= input.metadata.size - && initialCountedTotals != nil - && cached.forkedFromId == nil - && !hasIncompleteInterleaveState + && (isResumablePartial + || (input.metadata.size > cached.size + && initialCountedTotals != nil + && cached.forkedFromId == nil + && !hasIncompleteInterleaveState)) guard canIncremental else { return false } let delta = try Self.parseCodexFileCancellable( @@ -1071,8 +1096,11 @@ extension CostUsageScanner { initialHasInterleavedTotals: cached.hasInterleavedTotals ?? false, initialCodexTurnID: cached.lastCodexTurnID, initialCodexUsageRowIndex: Self.nextCodexUsageRowIndex(cached.codexRows), + initialBufferedSubagentLines: cached.codexBufferedSubagentLines, + initialJSONLResumeState: cached.codexJSONLResumeState, + maxBytesToRead: maxBytesToRead, checkCancellation: context.checkCancellation) - if delta.forkedFromId != nil { + if delta.forkedFromId != nil, !isResumablePartial { return false } let migrated = Self.codexFileUsageWithCostCache(cached, context: context) @@ -1086,6 +1114,10 @@ extension CostUsageScanner { let codexSession = cachedSessionMetadata.merging(delta.codexSession) let sessionId = codexSession.sessionId ?? delta.sessionId ?? cached.sessionId let projectPath = delta.projectPath ?? cached.projectPath + let forkBaselineDependencyKey = Self.codexForkBaselineDependencyKey( + parentSessionId: delta.forkedFromId, + dependsOnParentTotals: delta.dependsOnParentTotals, + inheritedResolver: context.resources.inheritedResolver) let canonicalProjectPath = delta.projectPath.map { context.resources.projectPathResolver.canonicalProjectPath(for: $0) } ?? cached.canonicalProjectPath ?? context.resources.projectPathResolver.canonicalProjectPath(for: projectPath) @@ -1153,6 +1185,7 @@ extension CostUsageScanner { lastCodexTurnID: delta.lastCodexTurnID, sessionId: sessionId, forkedFromId: codexSession.forkedFromId ?? delta.forkedFromId ?? migratedCached.forkedFromId, + forkBaselineDependencyKey: forkBaselineDependencyKey ?? migratedCached.forkBaselineDependencyKey, projectPath: projectPath, canonicalProjectPath: canonicalProjectPath, codexSession: codexSession.isEmpty ? nil : codexSession, @@ -1181,7 +1214,12 @@ extension CostUsageScanner { Self.mergeCodexRows(retainedCachedRows, rows: uniqueRows, sessionId: sessionId) ?? [], priorityTurns: context.resources.priorityTurns, modelsDevCatalog: context.resources.modelsDevCatalog, - modelsDevCacheRoot: context.resources.modelsDevCacheRoot)) + modelsDevCacheRoot: context.resources.modelsDevCacheRoot), + codexScanFileId: input.metadata.fileId, + codexScanTargetSize: input.metadata.size, + codexScanComplete: delta.parsedBytes >= input.metadata.size && delta.jsonlResumeState == nil, + codexJSONLResumeState: delta.jsonlResumeState, + codexBufferedSubagentLines: delta.bufferedSubagentLines) .refreshingCodexWorkspaceUsageFingerprint() Self.rememberScannedCodexFile( input: input, @@ -1196,7 +1234,8 @@ extension CostUsageScanner { input: CodexFileScanInput, context: CodexFileScanContext, cache: inout CostUsageCache, - state: inout CodexScanState) throws + state: inout CodexScanState, + maxBytesToRead: Int64? = nil) throws { try context.checkCancellation?() if let cached = input.cached { @@ -1210,6 +1249,7 @@ extension CostUsageScanner { let parsed = try Self.parseCodexFileCancellable( fileURL: input.fileURL, range: context.range, + maxBytesToRead: maxBytesToRead, inheritedTotalsResolver: context.resources.inheritedResolver.inheritedTotals(for:atOrBefore:), checkCancellation: context.checkCancellation) let forkBaselineDependencyKey = Self.codexForkBaselineDependencyKey( @@ -1238,7 +1278,8 @@ extension CostUsageScanner { if let sessionId, state.contributingSessionIds.contains(sessionId), uniqueRows.isEmpty, - usageDays.isEmpty + usageDays.isEmpty, + parsed.bufferedSubagentLines == nil { cache.files.removeValue(forKey: input.metadata.path) return @@ -1320,7 +1361,12 @@ extension CostUsageScanner { Self.mergeCodexRows(migratedCached?.codexRows, rows: uniqueRows, sessionId: sessionId) ?? [], priorityTurns: context.resources.priorityTurns, modelsDevCatalog: context.resources.modelsDevCatalog, - modelsDevCacheRoot: context.resources.modelsDevCacheRoot)) + modelsDevCacheRoot: context.resources.modelsDevCacheRoot), + codexScanFileId: input.metadata.fileId, + codexScanTargetSize: input.metadata.size, + codexScanComplete: parsed.parsedBytes >= input.metadata.size && parsed.jsonlResumeState == nil, + codexJSONLResumeState: parsed.jsonlResumeState, + codexBufferedSubagentLines: parsed.bufferedSubagentLines) .refreshingCodexWorkspaceUsageFingerprint() Self.applyFileDays(cache: &cache, fileDays: cache.files[input.metadata.path]?.days ?? [:], sign: 1) Self.rememberScannedCodexFile( diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift index c229f5d505..7740992df1 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift @@ -33,6 +33,14 @@ enum CostUsageScanner { var claudeLogProviderFilter: ClaudeLogProviderFilter = .all /// Force a full rescan, ignoring per-file cache and incremental offsets. var forceRescan: Bool = false + /// Maximum bounded slice read from one Codex rollout per refresh. Larger files + /// resume from cached progress on later refreshes. Default 256 MiB. + var maxCodexSessionFileBytes: Int64 = 256 * 1024 * 1024 + /// Soft budget for newly-read Codex session bytes in one refresh. + /// Remaining dirty files are deferred to later refreshes. Default 512 MiB. + var maxCodexScanBytesPerRefresh: Int64 = 512 * 1024 * 1024 + /// Prefer newest session files first so recent usage lands before catch-up work. + var preferNewestCodexSessionsFirst: Bool = true init( codexSessionsRoot: URL? = nil, @@ -40,7 +48,10 @@ enum CostUsageScanner { cacheRoot: URL? = nil, codexTraceDatabaseURL: URL? = nil, claudeLogProviderFilter: ClaudeLogProviderFilter = .all, - forceRescan: Bool = false) + forceRescan: Bool = false, + maxCodexSessionFileBytes: Int64 = 256 * 1024 * 1024, + maxCodexScanBytesPerRefresh: Int64 = 512 * 1024 * 1024, + preferNewestCodexSessionsFirst: Bool = true) { self.codexSessionsRoot = codexSessionsRoot self.claudeProjectsRoots = claudeProjectsRoots @@ -48,6 +59,50 @@ enum CostUsageScanner { self.codexTraceDatabaseURL = codexTraceDatabaseURL self.claudeLogProviderFilter = claudeLogProviderFilter self.forceRescan = forceRescan + self.maxCodexSessionFileBytes = max(0, maxCodexSessionFileBytes) + self.maxCodexScanBytesPerRefresh = max(0, maxCodexScanBytesPerRefresh) + self.preferNewestCodexSessionsFirst = preferNewestCodexSessionsFirst + } + } + + /// Per-refresh work limiter for Codex cost scans. Prevents multi-GB rollout corpora from + /// monopolizing a core for hours while still allowing progressive catch-up. + final class CodexScanBudget: @unchecked Sendable { + let maxFileBytes: Int64 + let maxBytesPerRefresh: Int64 + private(set) var bytesConsumed: Int64 = 0 + private(set) var resumedPartialFileCount = 0 + private(set) var deferredByBudgetFileCount = 0 + + init(maxFileBytes: Int64, maxBytesPerRefresh: Int64) { + self.maxFileBytes = max(0, maxFileBytes) + self.maxBytesPerRefresh = max(0, maxBytesPerRefresh) + } + + enum Admission { + case allow(Int64) + case deferBudget + } + + func admit(workBytes: Int64) -> Admission { + let work = max(0, workBytes) + let refreshRemaining = self.maxBytesPerRefresh > 0 + ? max(0, self.maxBytesPerRefresh - self.bytesConsumed) + : Int64.max + if work > 0, refreshRemaining == 0 { + self.deferredByBudgetFileCount += 1 + return .deferBudget + } + let fileAllowance = self.maxFileBytes > 0 ? self.maxFileBytes : Int64.max + let allowance = min(work, fileAllowance, refreshRemaining) + if allowance < work { + self.resumedPartialFileCount += 1 + } + return .allow(allowance) + } + + func consume(workBytes: Int64) { + self.bytesConsumed += max(0, workBytes) } } @@ -69,6 +124,8 @@ enum CostUsageScanner { let projectPath: String? let codexSession: CostUsageCodexSessionMetadata let rows: [CodexUsageRow] + let jsonlResumeState: CostUsageJsonl.ResumeState? + let bufferedSubagentLines: [CodexBufferedFastLine]? } struct CodexUsageRow: Codable, Equatable { @@ -516,6 +573,7 @@ enum CostUsageScanner { let changedPriorityTurnIDs: Set let resources: CodexScanResources let checkCancellation: CancellationCheck? + let scanBudget: CodexScanBudget? } final class CodexCanonicalProjectPathResolver { @@ -727,11 +785,17 @@ enum CostUsageScanner { private let fileIndex: CodexSessionFileIndex private let checkCancellation: CancellationCheck? + private let scanBudget: CodexScanBudget? private var snapshotResolutions: [String: SnapshotResolution] = [:] - init(fileIndex: CodexSessionFileIndex, checkCancellation: CancellationCheck?) { + init( + fileIndex: CodexSessionFileIndex, + checkCancellation: CancellationCheck?, + scanBudget: CodexScanBudget? = nil) + { self.fileIndex = fileIndex self.checkCancellation = checkCancellation + self.scanBudget = scanBudget } func inheritedTotals(for sessionId: String, atOrBefore cutoffTimestamp: String) throws -> CodexForkBaseline { @@ -801,6 +865,43 @@ enum CostUsageScanner { return resolution } + let parentMetadata = CostUsageScanner.codexFileMetadata(fileURL: fileURL) + if let budget = self.scanBudget { + switch budget.admit(workBytes: parentMetadata.size) { + case let .allow(allowance) where allowance >= parentMetadata.size: + break + case .allow: + CostUsageScanner.log.warning( + "Deferring oversized Codex parent baseline read while its file scan resumes", + metadata: [ + "sessionId": sessionId, + "path": fileURL.path, + "bytes": "\(parentMetadata.size)", + "slice": "\(budget.maxFileBytes)", + ]) + let resolution = SnapshotResolution( + dependencyKey: self.dependencyKey(for: sessionId, fileURL: fileURL), + snapshots: nil) + self.snapshotResolutions[sessionId] = resolution + return resolution + case .deferBudget: + CostUsageScanner.log.debug( + "Deferring Codex parent session baseline read until a later refresh", + metadata: [ + "sessionId": sessionId, + "path": fileURL.path, + "pendingBytes": "\(parentMetadata.size)", + "consumed": "\(budget.bytesConsumed)", + "limit": "\(budget.maxBytesPerRefresh)", + ]) + let resolution = SnapshotResolution( + dependencyKey: self.dependencyKey(for: sessionId, fileURL: fileURL), + snapshots: nil) + self.snapshotResolutions[sessionId] = resolution + return resolution + } + } + for _ in 0..<2 { let dependencyKeyBeforeParse = self.dependencyKey(for: sessionId, fileURL: fileURL) let parsed = try CostUsageScanner.parseCodexTokenSnapshots( @@ -817,6 +918,7 @@ enum CostUsageScanner { dependencyKey: dependencyKeyAfterParse, snapshots: nil) self.snapshotResolutions[sessionId] = resolution + self.scanBudget?.consume(workBytes: parentMetadata.size) return resolution } if parsedSessionId != sessionId { @@ -831,12 +933,14 @@ enum CostUsageScanner { dependencyKey: dependencyKeyAfterParse, snapshots: nil) self.snapshotResolutions[sessionId] = resolution + self.scanBudget?.consume(workBytes: parentMetadata.size) return resolution } let resolution = SnapshotResolution( dependencyKey: dependencyKeyAfterParse, snapshots: parsed.snapshots) self.snapshotResolutions[sessionId] = resolution + self.scanBudget?.consume(workBytes: parentMetadata.size) return resolution } @@ -1399,7 +1503,7 @@ enum CostUsageScanner { return String(filename[matchRange]) } - private struct CodexSessionMetadata { + struct CodexSessionMetadata: Codable { let sessionId: String? let forkedFromId: String? let forkTimestamp: String? @@ -1407,14 +1511,14 @@ enum CostUsageScanner { let isSubagentThread: Bool } - private struct CodexTurnContextMetadata { + struct CodexTurnContextMetadata: Codable { let timestamp: String? let model: String? let cwd: String? let title: String? } - private struct CodexTokenCountRecord { + struct CodexTokenCountRecord: Codable { let timestamp: String let model: String? let turnID: String? @@ -1422,7 +1526,7 @@ enum CostUsageScanner { let total: CostUsageCodexTotals? } - private enum CodexFastLine { + enum CodexFastLine: Codable { case sessionMeta(CodexSessionMetadata) case turnContext(CodexTurnContextMetadata) case interAgentCommunication(triggerTurn: Bool) @@ -1439,7 +1543,7 @@ enum CostUsageScanner { } } - private struct CodexBufferedFastLine { + struct CodexBufferedFastLine: Codable { let lineIndex: Int let line: CodexFastLine } @@ -2189,7 +2293,9 @@ enum CostUsageScanner { title: nil, startedAtUnixMs: nil, latestActivityUnixMs: nil), - rows: []) + rows: [], + jsonlResumeState: nil, + bufferedSubagentLines: nil) } // swiftlint:disable:next cyclomatic_complexity function_body_length @@ -2206,6 +2312,9 @@ enum CostUsageScanner { initialHasInterleavedTotals: Bool = false, initialCodexTurnID: String? = nil, initialCodexUsageRowIndex: Int = 0, + initialBufferedSubagentLines: [CodexBufferedFastLine]? = nil, + initialJSONLResumeState: CostUsageJsonl.ResumeState? = nil, + maxBytesToRead: Int64? = nil, inheritedTotalsResolver: ((String, String) throws -> CodexForkBaseline)? = nil, checkCancellation: CancellationCheck? = nil) throws -> CodexParseResult { @@ -2617,12 +2726,17 @@ enum CostUsageScanner { let maxLineBytes = 256 * 1024 let prefixBytes = maxLineBytes - var pendingSubagentLines: [CodexBufferedFastLine]? + var pendingSubagentLines = initialBufferedSubagentLines - if startOffset == 0, - let metadata = try Self.parseCodexSessionMetadata( - fileURL: fileURL, - checkCancellation: checkCancellation) + if let initialBufferedSubagentLines, startOffset > 0 { + for buffered in initialBufferedSubagentLines { + guard case let .sessionMeta(metadata) = buffered.line else { continue } + try handleSessionMetadata(metadata) + } + } else if startOffset == 0, + let metadata = try Self.parseCodexSessionMetadata( + fileURL: fileURL, + checkCancellation: checkCancellation) { try handleSessionMetadata(metadata) if metadata.isSubagentThread { @@ -2641,13 +2755,17 @@ enum CostUsageScanner { } var parsedBytes: Int64 - var physicalLineIndex = 0 + let targetSize = Self.codexFileMetadata(fileURL: fileURL).size + var physicalLineIndex = (initialBufferedSubagentLines?.last?.lineIndex ?? -1) + 1 + var jsonlResumeState = initialJSONLResumeState do { - parsedBytes = try CostUsageJsonl.scan( + let scanProgress = try CostUsageJsonl.scanBounded( fileURL: fileURL, offset: startOffset, maxLineBytes: maxLineBytes, prefixBytes: prefixBytes, + maxBytesToRead: maxBytesToRead, + resumeState: initialJSONLResumeState, checkCancellation: checkCancellation, onLine: { line in let lineIndex = physicalLineIndex @@ -2838,11 +2956,13 @@ enum CostUsageScanner { } } }) + parsedBytes = scanProgress.readOffset + jsonlResumeState = scanProgress.resumeState if let deferredError { throw deferredError } - if let pendingSubagentLines { + if let pendingSubagentLines, parsedBytes >= targetSize, jsonlResumeState == nil { // Same-leaf metadata can fill lineage fields after the opening record. Collect it // before replay so copied-prefix totals never run once on the wrong baseline, and // so an owned-suffix filter cannot discard the only fork identifier. @@ -2957,6 +3077,7 @@ enum CostUsageScanner { "Codex cost usage failed while scanning session file", metadata: ["path": fileURL.path, "error": error.localizedDescription]) parsedBytes = startOffset + jsonlResumeState = initialJSONLResumeState } return CodexParseResult( @@ -2980,7 +3101,9 @@ enum CostUsageScanner { || (subagentCounterSemantics != .independent && !usesLocalSubagentBoundary)), projectPath: projectPath, codexSession: codexSession, - rows: rows) + rows: rows, + jsonlResumeState: jsonlResumeState, + bufferedSubagentLines: parsedBytes < targetSize || jsonlResumeState != nil ? pendingSubagentLines : nil) } private static func codexTurnID(from payload: [String: Any]) -> String? { @@ -3012,10 +3135,89 @@ enum CostUsageScanner { if try Self.keepCachedCodexFileIfFresh(input: input, context: context, cache: &cache, state: &state) { return } - if try Self.appendCodexFileIncrementIfPossible(input: input, context: context, cache: &cache, state: &state) { + + let pendingWorkBytes = Self.pendingCodexScanWorkBytes(metadata: metadata, cached: cached) + let allowedWorkBytes: Int64 + if let budget = context.scanBudget { + switch budget.admit(workBytes: pendingWorkBytes) { + case let .allow(allowance): + allowedWorkBytes = allowance + case .deferBudget: + Self.log.debug( + "Deferring Codex session cost scan until a later refresh", + metadata: [ + "path": metadata.path, + "pendingBytes": "\(pendingWorkBytes)", + "consumed": "\(budget.bytesConsumed)", + "limit": "\(budget.maxBytesPerRefresh)", + ]) + // Preserve stale cache so later refreshes can resume catch-up. + return + } + } else { + allowedWorkBytes = pendingWorkBytes + } + + if try Self.appendCodexFileIncrementIfPossible( + input: input, + context: context, + cache: &cache, + state: &state, + maxBytesToRead: allowedWorkBytes) + { + context.scanBudget?.consume(workBytes: allowedWorkBytes) return } - try Self.rescanCodexFile(input: input, context: context, cache: &cache, state: &state) + let fullRescanWorkBytes = max(0, metadata.size) + let fullRescanAllowedBytes: Int64 + if fullRescanWorkBytes == pendingWorkBytes { + fullRescanAllowedBytes = allowedWorkBytes + } else if let budget = context.scanBudget { + switch budget.admit(workBytes: fullRescanWorkBytes) { + case let .allow(allowance): + fullRescanAllowedBytes = allowance + case .deferBudget: + // No work was consumed by the rejected incremental path, so this is only + // reachable when the refresh budget has no allowance for the full rescan. + return + } + } else { + fullRescanAllowedBytes = fullRescanWorkBytes + } + + try Self.rescanCodexFile( + input: input, + context: context, + cache: &cache, + state: &state, + maxBytesToRead: fullRescanAllowedBytes) + context.scanBudget?.consume(workBytes: fullRescanAllowedBytes) + } + + static func pendingCodexScanWorkBytes(metadata: CodexFileMetadata, cached: CostUsageFileUsage?) -> Int64 { + // Called only after keepCachedCodexFileIfFresh failed. Even when size/mtime still match + // (forced full rescan, priority invalidation, fork-dependency drift, etc.), the scanner + // will read the whole file — never report zero pending work in that case. + guard let cached else { return max(0, metadata.size) } + if cached.codexScanComplete == false { + if cached.codexScanFileId != nil, + cached.codexScanFileId == metadata.fileId, + cached.codexScanTargetSize == metadata.size, + cached.mtimeUnixMs == metadata.mtimeUnixMs + { + return max(0, metadata.size - (cached.parsedBytes ?? 0)) + } + return max(0, metadata.size) + } + let startOffset = cached.parsedBytes ?? cached.size + if metadata.size > cached.size, + startOffset > 0, + startOffset <= metadata.size, + cached.forkedFromId == nil + { + return max(0, metadata.size - startOffset) + } + return max(0, metadata.size) } private static func makeCodexRefreshPlan( @@ -3113,6 +3315,7 @@ enum CostUsageScanner { shouldRefresh: shouldRefresh) } + // swiftlint:disable:next function_body_length private static func loadCodexDaily( range: CostUsageDayRange, now: Date, @@ -3173,6 +3376,10 @@ enum CostUsageScanner { files.append(fileURL) } + if options.preferNewestCodexSessionsFirst { + files = Self.sortedCodexSessionFilesNewestFirst(files) + } + let filePathsInScan = Set(files.map(\.path)) var scanState = CodexScanState() let fileIndex = CodexSessionFileIndex( @@ -3183,9 +3390,13 @@ enum CostUsageScanner { roots: plan.roots, knownExistingPaths: filePathsInScan), checkCancellation: checkCancellation) + let scanBudget = CodexScanBudget( + maxFileBytes: options.maxCodexSessionFileBytes, + maxBytesPerRefresh: options.maxCodexScanBytesPerRefresh) let inheritedResolver = CodexInheritedTotalsResolver( fileIndex: fileIndex, - checkCancellation: checkCancellation) + checkCancellation: checkCancellation, + scanBudget: scanBudget) let resources = CodexScanResources( fileIndex: fileIndex, inheritedResolver: inheritedResolver, @@ -3198,7 +3409,8 @@ enum CostUsageScanner { options: options, plan: plan, resources: resources, - checkCancellation: checkCancellation) + checkCancellation: checkCancellation, + scanBudget: scanBudget) for fileURL in files { try Self.scanCodexFile( fileURL: fileURL, @@ -3206,6 +3418,17 @@ enum CostUsageScanner { cache: &cache, state: &scanState) } + if scanBudget.resumedPartialFileCount > 0 || scanBudget.deferredByBudgetFileCount > 0 { + Self.log.info( + "Codex cost scan applied work limits", + metadata: [ + "partialFiles": "\(scanBudget.resumedPartialFileCount)", + "deferredByBudget": "\(scanBudget.deferredByBudgetFileCount)", + "bytesConsumed": "\(scanBudget.bytesConsumed)", + "maxFileBytes": "\(scanBudget.maxFileBytes)", + "maxBytesPerRefresh": "\(scanBudget.maxBytesPerRefresh)", + ]) + } try checkCancellation?() Self.pruneForceRescanFilesOutsideWindow( @@ -3284,7 +3507,8 @@ enum CostUsageScanner { options: Options, plan: CodexRefreshPlan, resources: CodexScanResources, - checkCancellation: CancellationCheck?) -> CodexFileScanContext + checkCancellation: CancellationCheck?, + scanBudget: CodexScanBudget? = nil) -> CodexFileScanContext { CodexFileScanContext( range: range, @@ -3295,7 +3519,22 @@ enum CostUsageScanner { requiresTurnIDCache: plan.needsTurnIDCacheMigration, changedPriorityTurnIDs: plan.changedPriorityTurnIDs, resources: resources, - checkCancellation: checkCancellation) + checkCancellation: checkCancellation, + scanBudget: scanBudget) + } + + static func sortedCodexSessionFilesNewestFirst(_ files: [URL]) -> [URL] { + files.sorted { lhs, rhs in + let left = Self.codexFileMetadata(fileURL: lhs) + let right = Self.codexFileMetadata(fileURL: rhs) + if left.mtimeUnixMs != right.mtimeUnixMs { + return left.mtimeUnixMs > right.mtimeUnixMs + } + if left.size != right.size { + return left.size < right.size + } + return lhs.path < rhs.path + } } } diff --git a/Tests/CodexBarTests/CodexSubagentAccountingIntegrationTests.swift b/Tests/CodexBarTests/CodexSubagentAccountingIntegrationTests.swift index fc7f57eb46..91eeebb957 100644 --- a/Tests/CodexBarTests/CodexSubagentAccountingIntegrationTests.swift +++ b/Tests/CodexBarTests/CodexSubagentAccountingIntegrationTests.swift @@ -507,7 +507,7 @@ struct CodexSubagentAccountingIntegrationTests { } @Test - func `appended ancestor metadata reclassifies the complete subagent rollout`() throws { + func `bounded append fallback reclassifies the complete subagent rollout`() throws { let env = try CostUsageTestEnvironment() defer { env.cleanup() } @@ -535,7 +535,9 @@ struct CodexSubagentAccountingIntegrationTests { var options = CostUsageScanner.Options( codexSessionsRoot: env.codexSessionsRoot, claudeProjectsRoots: nil, - cacheRoot: env.cacheRoot) + cacheRoot: env.cacheRoot, + maxCodexSessionFileBytes: 4096, + maxCodexScanBytesPerRefresh: 4096) options.refreshMinIntervalSeconds = 0 let first = CostUsageScanner.loadDailyReport( provider: .codex, @@ -579,6 +581,7 @@ struct CodexSubagentAccountingIntegrationTests { #expect(usage.sessionId == "growing-child") #expect(usage.forkedFromId == "growing-parent") #expect(usage.forkBaselineDependencyKey == CostUsageScanner.codexForkDependencyNotRequiredKey) + #expect(usage.codexScanComplete == true) } private func turnContext(timestamp: String, model: String) -> [String: Any] { diff --git a/Tests/CodexBarTests/CostUsagePerformanceGateTests.swift b/Tests/CodexBarTests/CostUsagePerformanceGateTests.swift index b5295f1cd6..c202d20157 100644 --- a/Tests/CodexBarTests/CostUsagePerformanceGateTests.swift +++ b/Tests/CodexBarTests/CostUsagePerformanceGateTests.swift @@ -285,6 +285,381 @@ struct CostUsagePerformanceGateTests { #expect(catalogLoadCount == 1) } + @Test + func `oversized codex session is fully accounted across bounded refreshes`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + let day = try env.makeLocalNoon(year: 2026, month: 5, day: 10) + let files = try Self.writeSyntheticCodexCorpus(env: env, day: day, files: 1, turnsPerFile: 8) + let oversizedURL = try #require(files.first) + let metadata = CostUsageScanner.codexFileMetadata(fileURL: oversizedURL) + + var baselineOptions = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + claudeProjectsRoots: nil, + cacheRoot: env.root.appendingPathComponent("baseline-cache"), + codexTraceDatabaseURL: env.root.appendingPathComponent("missing.sqlite"), + maxCodexSessionFileBytes: 0, + maxCodexScanBytesPerRefresh: 0) + baselineOptions.refreshMinIntervalSeconds = 0 + let baseline = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day, + options: baselineOptions) + + var options = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + claudeProjectsRoots: nil, + cacheRoot: env.cacheRoot, + codexTraceDatabaseURL: env.root.appendingPathComponent("missing.sqlite"), + maxCodexSessionFileBytes: max(1, metadata.size / 4), + maxCodexScanBytesPerRefresh: max(1, metadata.size / 4)) + options.refreshMinIntervalSeconds = 0 + + var offsets: [Int64] = [] + var report: CostUsageDailyReport? + for _ in 0..<12 { + report = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day, + options: options) + let cached = try #require(CostUsageCacheIO.load( + provider: .codex, + cacheRoot: env.cacheRoot).files.values.first) + offsets.append(cached.parsedBytes ?? 0) + if cached.codexScanComplete == true { + break + } + } + + #expect(offsets.count > 1) + #expect(zip(offsets, offsets.dropFirst()).allSatisfy { $0 <= $1 }) + #expect(offsets.last == metadata.size) + #expect(report?.summary?.totalTokens == baseline.summary?.totalTokens) + #expect(report?.data.map(\.totalTokens) == baseline.data.map(\.totalTokens)) + } + + @Test + func `oversized codex progress survives cache round trip`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + let day = try env.makeLocalNoon(year: 2026, month: 5, day: 10) + let files = try Self.writeSyntheticCodexCorpus(env: env, day: day, files: 1, turnsPerFile: 8) + let fileURL = try #require(files.first) + let metadata = CostUsageScanner.codexFileMetadata(fileURL: fileURL) + let slice = max(1, metadata.size / 4) + var options = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + claudeProjectsRoots: nil, + cacheRoot: env.cacheRoot, + codexTraceDatabaseURL: env.root.appendingPathComponent("missing.sqlite"), + maxCodexSessionFileBytes: slice, + maxCodexScanBytesPerRefresh: slice) + options.refreshMinIntervalSeconds = 0 + + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day, + options: options) + let cacheData = try Data(contentsOf: CostUsageCacheIO.cacheFileURL(provider: .codex, cacheRoot: env.cacheRoot)) + let roundTripped = try JSONDecoder().decode(CostUsageCache.self, from: cacheData) + let first = try #require(roundTripped.files.values.first) + let firstOffset = try #require(first.parsedBytes) + #expect(first.codexScanFileId == metadata.fileId) + #expect(first.codexScanTargetSize == metadata.size) + #expect(first.codexScanComplete == false) + + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day, + options: options) + let second = try #require(CostUsageCacheIO.load( + provider: .codex, + cacheRoot: env.cacheRoot).files.values.first) + #expect((second.parsedBytes ?? 0) > firstOffset) + #expect(second.codexScanFileId == metadata.fileId) + #expect(second.codexScanTargetSize == metadata.size) + } + + @Test + func `oversized codex progress restarts when the target size changes`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + let day = try env.makeLocalNoon(year: 2026, month: 5, day: 10) + let files = try Self.writeSyntheticCodexCorpus(env: env, day: day, files: 1, turnsPerFile: 8) + let fileURL = try #require(files.first) + let originalMetadata = CostUsageScanner.codexFileMetadata(fileURL: fileURL) + let slice = max(1, originalMetadata.size / 4) + var options = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + claudeProjectsRoots: nil, + cacheRoot: env.cacheRoot, + codexTraceDatabaseURL: env.root.appendingPathComponent("missing.sqlite"), + maxCodexSessionFileBytes: slice, + maxCodexScanBytesPerRefresh: slice) + options.refreshMinIntervalSeconds = 0 + + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day, + options: options) + let first = try #require(CostUsageCacheIO.load( + provider: .codex, + cacheRoot: env.cacheRoot).files.values.first) + #expect(first.parsedBytes == slice) + #expect(first.codexScanComplete == false) + + let original = try String(contentsOf: fileURL, encoding: .utf8) + try (original + String(repeating: " ", count: 512)).write(to: fileURL, atomically: false, encoding: .utf8) + try FileManager.default.setAttributes( + [.modificationDate: day.addingTimeInterval(60)], + ofItemAtPath: fileURL.path) + let changedMetadata = CostUsageScanner.codexFileMetadata(fileURL: fileURL) + #expect(changedMetadata.size != originalMetadata.size) + + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day, + options: options) + let restarted = try #require(CostUsageCacheIO.load( + provider: .codex, + cacheRoot: env.cacheRoot).files.values.first) + #expect(restarted.parsedBytes == slice) + #expect(restarted.codexScanTargetSize == changedMetadata.size) + #expect(restarted.codexScanFileId == changedMetadata.fileId) + #expect(restarted.codexScanComplete == false) + } + + @Test + func `single oversized jsonl record resumes without stalling`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + let day = try env.makeLocalNoon(year: 2026, month: 5, day: 10) + let iso = env.isoString(for: day) + let model = "openai/gpt-5.2-codex" + let contents = [ + #"{"type":"session_meta","timestamp":"\#(iso)","payload":{"session_id":"long-record"}}"#, + #"{"type":"turn_context","timestamp":"\#(iso)","payload":{"model":"\#(model)"}}"#, + #"{"type":"event_msg","timestamp":"\#(iso)","payload":{"type":"token_count","padding":""# + + String(repeating: "x", count: 4096) + + + #"","info":{"total_token_usage":{"input_tokens":500,"cached_input_tokens":50,"# + + #""output_tokens":25},"model":"\#(model)"}}}"#, + ].joined(separator: "\n") + "\n" + _ = try env.writeCodexSessionFile(day: day, filename: "long-record.jsonl", contents: contents) + + var baselineOptions = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + claudeProjectsRoots: nil, + cacheRoot: env.root.appendingPathComponent("baseline-cache"), + codexTraceDatabaseURL: env.root.appendingPathComponent("missing.sqlite"), + maxCodexSessionFileBytes: 0, + maxCodexScanBytesPerRefresh: 0) + baselineOptions.refreshMinIntervalSeconds = 0 + let baseline = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day, + options: baselineOptions) + + var options = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + claudeProjectsRoots: nil, + cacheRoot: env.cacheRoot, + codexTraceDatabaseURL: env.root.appendingPathComponent("missing.sqlite"), + maxCodexSessionFileBytes: 256, + maxCodexScanBytesPerRefresh: 256) + options.refreshMinIntervalSeconds = 0 + + var offsets: [Int64] = [] + var sawPartialRecord = false + var report: CostUsageDailyReport? + for _ in 0..<24 { + report = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day, + options: options) + let cached = try #require(CostUsageCacheIO.load( + provider: .codex, + cacheRoot: env.cacheRoot).files.values.first) + offsets.append(cached.parsedBytes ?? 0) + sawPartialRecord = sawPartialRecord || cached.codexJSONLResumeState != nil + if cached.codexScanComplete == true { + break + } + } + + #expect(sawPartialRecord) + #expect(zip(offsets, offsets.dropFirst()).allSatisfy { $0 < $1 }) + #expect(report?.summary?.totalTokens == baseline.summary?.totalTokens) + } + + @Test + func `codex scan budget never admits more than its remaining allowance`() { + let budget = CostUsageScanner.CodexScanBudget(maxFileBytes: 100, maxBytesPerRefresh: 150) + guard case let .allow(first) = budget.admit(workBytes: 1000) else { + Issue.record("expected first bounded admission") + return + } + #expect(first == 100) + budget.consume(workBytes: first) + + guard case let .allow(second) = budget.admit(workBytes: 1000) else { + Issue.record("expected remaining-budget admission") + return + } + #expect(second == 50) + budget.consume(workBytes: second) + guard case .deferBudget = budget.admit(workBytes: 1) else { + Issue.record("expected exhausted budget to defer") + return + } + #expect(budget.bytesConsumed == 150) + } + + @Test + func `per refresh byte budget defers later dirty files`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + let day = try env.makeLocalNoon(year: 2026, month: 5, day: 10) + let urls = try Self.writeSyntheticCodexCorpus(env: env, day: day, files: 3, turnsPerFile: 3) + // Make deterministic order by newest-first: touch later files later. + let older = try #require(urls.first) + let middle = try #require(urls.dropFirst().first) + let newer = try #require(urls.last) + let olderDate = day.addingTimeInterval(-3600) + let middleDate = day.addingTimeInterval(-1800) + let newerDate = day + try FileManager.default.setAttributes([.modificationDate: olderDate], ofItemAtPath: older.path) + try FileManager.default.setAttributes([.modificationDate: middleDate], ofItemAtPath: middle.path) + try FileManager.default.setAttributes([.modificationDate: newerDate], ofItemAtPath: newer.path) + + let newestMeta = CostUsageScanner.codexFileMetadata(fileURL: newer) + var options = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + claudeProjectsRoots: nil, + cacheRoot: env.cacheRoot, + codexTraceDatabaseURL: env.root.appendingPathComponent("missing.sqlite"), + maxCodexSessionFileBytes: 64 * 1024 * 1024, + // Enough for the newest file only; remaining dirty files defer. + maxCodexScanBytesPerRefresh: max(1, newestMeta.size), + preferNewestCodexSessionsFirst: true) + options.refreshMinIntervalSeconds = 0 + + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day, + options: options) + let cache = CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot) + let cachedNames = Set(cache.files.keys.map { URL(fileURLWithPath: $0).lastPathComponent }) + + #expect(cachedNames.contains(newer.lastPathComponent)) + #expect(!cachedNames.contains(older.lastPathComponent)) + } + + @Test + func `pending work bytes treat fork files as full rescan work`() { + let metadata = CostUsageScanner.CodexFileMetadata( + path: "/tmp/forked.jsonl", + mtimeUnixMs: 2, + size: 1000, + fileId: "1:2") + let cached = CostUsageFileUsage( + mtimeUnixMs: 1, + size: 400, + days: [:], + parsedBytes: 400, + forkedFromId: "parent-session") + #expect(CostUsageScanner.pendingCodexScanWorkBytes(metadata: metadata, cached: cached) == 1000) + } + + @Test + func `pending work bytes charge full file for forced rescans of unchanged cache entries`() { + let metadata = CostUsageScanner.CodexFileMetadata( + path: "/tmp/unchanged.jsonl", + mtimeUnixMs: 42, + size: 2_000_000_000, + fileId: "9:9") + let cached = CostUsageFileUsage( + mtimeUnixMs: 42, + size: 2_000_000_000, + days: ["2026-05-10": ["gpt-5.2-codex": [100, 20, 10]]], + parsedBytes: 2_000_000_000, + sessionId: "session-unchanged") + // keepCached can still reject this (forceFullScan / priority / fork dependency). + // Budget must not report zero pending work or multi-GB forced rescans slip through. + #expect(CostUsageScanner.pendingCodexScanWorkBytes(metadata: metadata, cached: cached) == 2_000_000_000) + } + + @Test + func `oversized parent baseline reads defer for small fork children`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + let day = try env.makeLocalNoon(year: 2026, month: 5, day: 10) + let iso = env.isoString(for: day) + + // Parent is intentionally larger than the per-file budget. + let parentBody = ([ + #"{"type":"session_meta","timestamp":"\#(iso)","payload":{"session_id":"parent-giant"}}"#, + #"{"type":"turn_context","timestamp":"\#(iso)","payload":{"model":"openai/gpt-5.2-codex"}}"#, + #"{"type":"event_msg","timestamp":"\#(iso)","payload":{"type":"token_count","info":"# + + #"{"total_token_usage":{"input_tokens":500,"cached_input_tokens":50,"output_tokens":25},"# + + #""model":"openai/gpt-5.2-codex"}}}"#, + ] + Array(repeating: "x", count: 4096)).joined(separator: "\n") + "\n" + _ = try env.writeCodexSessionFile(day: day, filename: "parent-giant.jsonl", contents: parentBody) + + let childBody = [ + #"{"type":"session_meta","timestamp":"\#(iso)","payload":{"session_id":"child-small","# + + #""forked_from_id":"parent-giant"}}"#, + #"{"type":"turn_context","timestamp":"\#(iso)","payload":{"model":"openai/gpt-5.2-codex"}}"#, + #"{"type":"event_msg","timestamp":"\#(iso)","payload":{"type":"token_count","info":"# + + #"{"total_token_usage":{"input_tokens":600,"cached_input_tokens":60,"output_tokens":30},"# + + #""model":"openai/gpt-5.2-codex"}}}"#, + ].joined(separator: "\n") + "\n" + let childURL = try env.writeCodexSessionFile(day: day, filename: "child-small.jsonl", contents: childBody) + + var options = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + claudeProjectsRoots: nil, + cacheRoot: env.cacheRoot, + codexTraceDatabaseURL: env.root.appendingPathComponent("missing.sqlite"), + maxCodexSessionFileBytes: 1024, + maxCodexScanBytesPerRefresh: 64 * 1024 * 1024) + options.refreshMinIntervalSeconds = 0 + + let started = Date() + let report = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day, + options: options) + let elapsed = Date().timeIntervalSince(started) + let cache = CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot) + + #expect(elapsed < 2.0) + #expect(cache.files.keys.contains { URL(fileURLWithPath: $0).lastPathComponent == childURL.lastPathComponent }) + // Child still contributes local tokens while the parent baseline is unresolved. + #expect((report.summary?.totalTokens ?? 0) > 0) + } + private static func writeSyntheticCodexCorpus( env: CostUsageTestEnvironment, day: Date,