diff --git a/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift b/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift index e16a88f5cb..3802864e25 100644 --- a/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift +++ b/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift @@ -1,5 +1,5 @@ // Generated by Scripts/regenerate-codex-parser-hash.sh. Do not edit by hand. enum CodexParserHash { - static let value = "9b26fd821cf090dc" + static let value = "577571954a2d36bc" } diff --git a/Sources/CodexBarCore/PiSessionCostCache.swift b/Sources/CodexBarCore/PiSessionCostCache.swift index 355be9c18b..7f690c1025 100644 --- a/Sources/CodexBarCore/PiSessionCostCache.swift +++ b/Sources/CodexBarCore/PiSessionCostCache.swift @@ -2,7 +2,7 @@ import Foundation enum PiSessionCostCacheIO { /// Artifact schema version. Pricing changes are tracked separately by `pricingKey`. - private static let artifactVersion = 5 + private static let artifactVersion = 6 private static func defaultCacheRoot() -> URL { let root = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first! @@ -56,7 +56,7 @@ struct PiSessionCostCache: Codable { var daysByProvider: [String: [String: [String: PiPackedUsage]]] = [:] var files: [String: PiSessionFileUsage] = [:] - init(version: Int = 5) { + init(version: Int = 6) { self.version = version } } diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageCache.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageCache.swift index c1575c0ec2..7323a07c43 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageCache.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageCache.swift @@ -11,9 +11,9 @@ enum CostUsageCacheIO { private static func artifactVersion(for provider: UsageProvider) -> Int { switch provider { case .codex: - 9 + 10 case .claude, .vertexai: - 4 + 5 default: 1 } diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift index 35f58b446a..fd17555f81 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift @@ -6,6 +6,220 @@ enum CostUsageJsonl { let wasTruncated: Bool } + private struct JSONTailState { + private enum ScalarState { + case notScalar + case trueLiteral(Int) + case falseLiteral(Int) + case nullLiteral(Int) + case number(NumberState) + case invalid + } + + private enum NumberState { + private enum ByteKind { + case zero + case digit + case decimalPoint + case exponentMarker + case sign + case whitespace + case other + + init(_ byte: UInt8) { + switch byte { + case 0x30: self = .zero + case 0x31...0x39: self = .digit + case 0x2E: self = .decimalPoint + case 0x65, 0x45: self = .exponentMarker + case 0x2B, 0x2D: self = .sign + case 0x20, 0x09, 0x0A, 0x0D: self = .whitespace + default: self = .other + } + } + } + + case sign + case zero + case integer + case decimalPoint + case fraction + case exponentMarker + case exponentSign + case exponentDigits + case finished + case invalid + + var canCommitAtEOF: Bool { + switch self { + case .finished, .invalid: + true + case .sign, .zero, .integer, .decimalPoint, .fraction, + .exponentMarker, .exponentSign, .exponentDigits: + false + } + } + + func appending(_ byte: UInt8) -> Self { + switch (self, ByteKind(byte)) { + case (.invalid, _): .invalid + case (.finished, .whitespace): .finished + case (.sign, .zero): .zero + case (.sign, .digit): .integer + case (.zero, .decimalPoint): .decimalPoint + case (.zero, .exponentMarker): .exponentMarker + case (.integer, .zero), (.integer, .digit): .integer + case (.integer, .decimalPoint): .decimalPoint + case (.integer, .exponentMarker): .exponentMarker + case (.decimalPoint, .zero), (.decimalPoint, .digit): .fraction + case (.fraction, .zero), (.fraction, .digit): .fraction + case (.fraction, .exponentMarker): .exponentMarker + case (.exponentMarker, .sign): .exponentSign + case (.exponentMarker, .zero), (.exponentMarker, .digit): .exponentDigits + case (.exponentSign, .zero), (.exponentSign, .digit): .exponentDigits + case (.exponentDigits, .zero), (.exponentDigits, .digit): .exponentDigits + case (.zero, .whitespace), + (.integer, .whitespace), + (.fraction, .whitespace), + (.exponentDigits, .whitespace): .finished + default: .invalid + } + } + } + + private static let trueLiteral = Array("true".utf8) + private static let falseLiteral = Array("false".utf8) + private static let nullLiteral = Array("null".utf8) + + private var containerDepth = 0 + private var insideString = false + private var escaping = false + private var sawNonWhitespace = false + private var scalarState = ScalarState.notScalar + + mutating func reset() { + self = Self() + } + + var isStructurallyComplete: Bool { + guard self.sawNonWhitespace else { return false } + switch self.scalarState { + case .notScalar: + return !self.insideString && self.containerDepth == 0 + case let .trueLiteral(matched): + return matched == Self.trueLiteral.count + case let .falseLiteral(matched): + return matched == Self.falseLiteral.count + case let .nullLiteral(matched): + return matched == Self.nullLiteral.count + case let .number(state): + return state.canCommitAtEOF + case .invalid: + return true + } + } + + mutating func append(_ byte: UInt8) { + if !self.sawNonWhitespace { + self.start(byte) + return + } + + guard !self.appendScalar(byte) else { return } + self.appendContainer(byte) + } + + private mutating func start(_ byte: UInt8) { + guard !Self.isWhitespace(byte) else { return } + self.sawNonWhitespace = true + switch byte { + case 0x22: + self.insideString = true + case 0x7B, 0x5B: + self.containerDepth = 1 + case 0x74: + self.scalarState = .trueLiteral(1) + case 0x66: + self.scalarState = .falseLiteral(1) + case 0x6E: + self.scalarState = .nullLiteral(1) + case 0x2D: + self.scalarState = .number(.sign) + case 0x30: + self.scalarState = .number(.zero) + case 0x31...0x39: + self.scalarState = .number(.integer) + default: + self.scalarState = .invalid + } + } + + private mutating func appendScalar(_ byte: UInt8) -> Bool { + switch self.scalarState { + case let .trueLiteral(matched): + self.scalarState = self.advanceLiteral(byte, expected: Self.trueLiteral, matched: matched) + .map(ScalarState.trueLiteral) ?? .invalid + return true + case let .falseLiteral(matched): + self.scalarState = self.advanceLiteral(byte, expected: Self.falseLiteral, matched: matched) + .map(ScalarState.falseLiteral) ?? .invalid + return true + case let .nullLiteral(matched): + self.scalarState = self.advanceLiteral(byte, expected: Self.nullLiteral, matched: matched) + .map(ScalarState.nullLiteral) ?? .invalid + return true + case let .number(state): + self.scalarState = .number(state.appending(byte)) + return true + case .invalid: + return true + case .notScalar: + return false + } + } + + private mutating func appendContainer(_ byte: UInt8) { + if self.insideString { + if self.escaping { + self.escaping = false + } else if byte == 0x5C { + self.escaping = true + } else if byte == 0x22 { + self.insideString = false + } + return + } + + switch byte { + case 0x20, 0x09, 0x0D: + return + case 0x22: + self.insideString = true + case 0x7B, 0x5B: + self.containerDepth += 1 + case 0x7D, 0x5D: + self.containerDepth = max(0, self.containerDepth - 1) + default: + break + } + } + + private func advanceLiteral( + _ byte: UInt8, + expected: [UInt8], + matched: Int) -> Int? + { + if matched < expected.count { + return byte == expected[matched] ? matched + 1 : nil + } + return Self.isWhitespace(byte) ? matched : nil + } + + private static func isWhitespace(_ byte: UInt8) -> Bool { + byte == 0x20 || byte == 0x09 || byte == 0x0A || byte == 0x0D + } + } + @discardableResult static func scan( fileURL: URL, @@ -47,6 +261,8 @@ enum CostUsageJsonl { var lineBytes = 0 var truncated = false var bytesRead: Int64 = 0 + var committedOffset = startOffset + var jsonTailState = JSONTailState() func appendSegment(_ bytes: UnsafePointer, count: Int) { guard count > 0 else { return } @@ -69,6 +285,18 @@ enum CostUsageJsonl { current.removeAll(keepingCapacity: true) lineBytes = 0 truncated = false + jsonTailState.reset() + } + + func hasCompleteJSONTail() -> Bool { + guard jsonTailState.isStructurallyComplete else { return false } + if truncated { + // The full record is intentionally not retained. Its incremental state is enough + // to keep incomplete containers, strings, literals, and numbers retriable. + return true + } + guard lineBytes == current.count else { return false } + return (try? JSONSerialization.jsonObject(with: current, options: [.fragmentsAllowed])) != nil } while true { @@ -76,12 +304,16 @@ enum CostUsageJsonl { let reachedEOF = try autoreleasepool { let chunk = try handle.read(upToCount: 256 * 1024) ?? Data() if chunk.isEmpty { - flushLine() + if hasCompleteJSONTail() { + flushLine() + committedOffset = startOffset + bytesRead + } return true } try checkCancellation?() bytesRead += Int64(chunk.count) + let chunkStartOffset = startOffset + bytesRead - Int64(chunk.count) chunk.withUnsafeBytes { rawBuffer in guard let base = rawBuffer.bindMemory(to: UInt8.self).baseAddress else { return } var segmentStart = 0 @@ -90,7 +322,10 @@ enum CostUsageJsonl { if base[index] == 0x0A { appendSegment(base.advanced(by: segmentStart), count: index - segmentStart) flushLine() + committedOffset = chunkStartOffset + Int64(index + 1) segmentStart = index + 1 + } else { + jsonTailState.append(base[index]) } index += 1 } @@ -100,10 +335,12 @@ enum CostUsageJsonl { } return false } - if reachedEOF { break } + if reachedEOF { + break + } try checkCancellation?() } - return startOffset + bytesRead + return committedOffset } } diff --git a/Tests/CodexBarTests/CostUsageCacheTests.swift b/Tests/CodexBarTests/CostUsageCacheTests.swift index 9cc6946f8d..54dd8e9df1 100644 --- a/Tests/CodexBarTests/CostUsageCacheTests.swift +++ b/Tests/CodexBarTests/CostUsageCacheTests.swift @@ -11,9 +11,75 @@ struct CostUsageCacheTests { let claudeURL = CostUsageCacheIO.cacheFileURL(provider: .claude, cacheRoot: root) let vertexURL = CostUsageCacheIO.cacheFileURL(provider: .vertexai, cacheRoot: root) - #expect(codexURL.lastPathComponent == "codex-v9.json") - #expect(claudeURL.lastPathComponent == "claude-v4.json") - #expect(vertexURL.lastPathComponent == "vertexai-v4.json") + #expect(codexURL.lastPathComponent == "codex-v10.json") + #expect(claudeURL.lastPathComponent == "claude-v5.json") + #expect(vertexURL.lastPathComponent == "vertexai-v5.json") + } + + @Test + func `cost cache ignores predecessor artifact with persisted offset`() throws { + let root = try self.makeTemporaryCacheRoot() + defer { try? FileManager.default.removeItem(at: root) } + + let legacyURL = root + .appendingPathComponent("cost-usage", isDirectory: true) + .appendingPathComponent("codex-v9.json", isDirectory: false) + try FileManager.default.createDirectory( + at: legacyURL.deletingLastPathComponent(), + withIntermediateDirectories: true) + let producerKey = try #require(CostUsageCacheIO.currentProducerKey(provider: .codex)) + let legacy = """ + { + "version": 1, + "producerKey": "\(producerKey)", + "lastScanUnixMs": 999, + "files": { + "/tmp/session.jsonl": { + "mtimeUnixMs": 1, + "size": 100, + "days": {}, + "parsedBytes": 100 + } + }, + "days": {} + } + """ + try legacy.write(to: legacyURL, atomically: false, encoding: .utf8) + + let loaded = CostUsageCacheIO.load(provider: .codex, cacheRoot: root) + + #expect(loaded.lastScanUnixMs == 0) + #expect(loaded.files.isEmpty) + } + + @Test + func `Pi session cache ignores predecessor artifact with persisted offset`() throws { + let root = try self.makeTemporaryCacheRoot() + defer { try? FileManager.default.removeItem(at: root) } + + let legacyURL = root + .appendingPathComponent("cost-usage", isDirectory: true) + .appendingPathComponent("pi-sessions-v5.json", isDirectory: false) + try FileManager.default.createDirectory( + at: legacyURL.deletingLastPathComponent(), + withIntermediateDirectories: true) + var legacy = PiSessionCostCache(version: 5) + legacy.lastScanUnixMs = 999 + legacy.files = [ + "/tmp/session.jsonl": PiSessionFileUsage( + mtimeUnixMs: 1, + size: 100, + parsedBytes: 100, + lastModelContext: nil, + contributions: [:]), + ] + try JSONEncoder().encode(legacy).write(to: legacyURL) + + let loaded = PiSessionCostCacheIO.load(cacheRoot: root) + + #expect(loaded.version == 6) + #expect(loaded.lastScanUnixMs == 0) + #expect(loaded.files.isEmpty) } @Test diff --git a/Tests/CodexBarTests/CostUsageJsonlScannerTests.swift b/Tests/CodexBarTests/CostUsageJsonlScannerTests.swift index 0bf27a4cfa..926b623b02 100644 --- a/Tests/CodexBarTests/CostUsageJsonlScannerTests.swift +++ b/Tests/CodexBarTests/CostUsageJsonlScannerTests.swift @@ -58,6 +58,504 @@ struct CostUsageJsonlScannerTests { #expect(scanned[1].wasTruncated == true) } + @Test + func `jsonl scanner retries an incomplete final record after append`() throws { + let root = try self.makeTemporaryRoot() + defer { try? FileManager.default.removeItem(at: root) } + + let fileURL = root.appendingPathComponent("appending.jsonl", isDirectory: false) + let initial = #"{"type":"message","id":"partial"# + try initial.write(to: fileURL, atomically: true, encoding: .utf8) + + var firstPass: [String?] = [] + let resumeOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + maxLineBytes: 1024, + prefixBytes: 1024) + { line in + firstPass.append(String(bytes: line.bytes, encoding: .utf8)) + } + + #expect(firstPass.isEmpty) + #expect(resumeOffset == 0) + + let completion = #""}"# + "\n" + let handle = try FileHandle(forWritingTo: fileURL) + defer { try? handle.close() } + try handle.seekToEnd() + try handle.write(contentsOf: Data(completion.utf8)) + + var secondPass: [String?] = [] + let endOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + offset: resumeOffset, + maxLineBytes: 1024, + prefixBytes: 1024) + { line in + secondPass.append(String(bytes: line.bytes, encoding: .utf8)) + } + + #expect(secondPass == [initial + String(completion.dropLast())]) + #expect(endOffset == Int64(Data((initial + completion).utf8).count)) + } + + @Test + func `jsonl scanner accepts a complete final record without newline`() throws { + let root = try self.makeTemporaryRoot() + defer { try? FileManager.default.removeItem(at: root) } + + let fileURL = root.appendingPathComponent("final-record.jsonl", isDirectory: false) + let record = #"{"type":"message","id":"complete"}"# + try record.write(to: fileURL, atomically: true, encoding: .utf8) + + var scanned: [String?] = [] + let endOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + maxLineBytes: 1024, + prefixBytes: 1024) + { line in + scanned.append(String(bytes: line.bytes, encoding: .utf8)) + } + + #expect(scanned == [record]) + #expect(endOffset == Int64(Data(record.utf8).count)) + } + + @Test + func `jsonl scanner preserves a truncated final record`() throws { + let root = try self.makeTemporaryRoot() + defer { try? FileManager.default.removeItem(at: root) } + + let fileURL = root.appendingPathComponent("truncated-final-record.jsonl", isDirectory: false) + let record = #"{"message":"\#(String(repeating: "x", count: 256))"}"# + try record.write(to: fileURL, atomically: true, encoding: .utf8) + + var scanned: [CostUsageJsonl.Line] = [] + let endOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + maxLineBytes: 64, + prefixBytes: 64) + { line in + scanned.append(line) + } + + #expect(scanned.count == 1) + #expect(scanned[0].wasTruncated) + #expect(endOffset == Int64(Data(record.utf8).count)) + } + + @Test + func `jsonl scanner retries a truncated incomplete final record after append`() throws { + let root = try self.makeTemporaryRoot() + defer { try? FileManager.default.removeItem(at: root) } + + let fileURL = root.appendingPathComponent("truncated-appending.jsonl", isDirectory: false) + let initial = #"{"message":"\#(String(repeating: "x", count: 256))"# + try initial.write(to: fileURL, atomically: true, encoding: .utf8) + + var firstPass: [CostUsageJsonl.Line] = [] + let resumeOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + maxLineBytes: 64, + prefixBytes: 64) + { line in + firstPass.append(line) + } + + #expect(firstPass.isEmpty) + #expect(resumeOffset == 0) + + let completion = #""}"# + "\n" + let handle = try FileHandle(forWritingTo: fileURL) + defer { try? handle.close() } + try handle.seekToEnd() + try handle.write(contentsOf: Data(completion.utf8)) + + var secondPass: [CostUsageJsonl.Line] = [] + let endOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + offset: resumeOffset, + maxLineBytes: 64, + prefixBytes: 64) + { line in + secondPass.append(line) + } + + #expect(secondPass.count == 1) + #expect(secondPass[0].wasTruncated) + #expect(endOffset == Int64(Data((initial + completion).utf8).count)) + } + + @Test + func `jsonl scanner retries a truncated escape sequence`() throws { + let root = try self.makeTemporaryRoot() + defer { try? FileManager.default.removeItem(at: root) } + + let fileURL = root.appendingPathComponent("truncated-escape.jsonl", isDirectory: false) + let initial = #"{"message":""# + String(repeating: "x", count: 256) + #"\u12"# + try initial.write(to: fileURL, atomically: true, encoding: .utf8) + + var firstPass: [CostUsageJsonl.Line] = [] + let resumeOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + maxLineBytes: 64, + prefixBytes: 64) + { line in + firstPass.append(line) + } + + #expect(firstPass.isEmpty) + #expect(resumeOffset == 0) + + let completion = #"34"}"# + "\n" + let handle = try FileHandle(forWritingTo: fileURL) + defer { try? handle.close() } + try handle.seekToEnd() + try handle.write(contentsOf: Data(completion.utf8)) + + var secondPass: [CostUsageJsonl.Line] = [] + let endOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + offset: resumeOffset, + maxLineBytes: 64, + prefixBytes: 64) + { line in + secondPass.append(line) + } + + #expect(secondPass.count == 1) + #expect(secondPass[0].wasTruncated) + #expect(endOffset == Int64(Data((initial + completion).utf8).count)) + } + + @Test + func `jsonl scanner ignores nested delimiters inside strings`() throws { + let root = try self.makeTemporaryRoot() + defer { try? FileManager.default.removeItem(at: root) } + + let fileURL = root.appendingPathComponent("string-delimiters.jsonl", isDirectory: false) + let message = String(repeating: "{[", count: 64) + #""nested""# + let recordData = try JSONEncoder().encode(["message": message]) + try recordData.write(to: fileURL) + + var scanned: [CostUsageJsonl.Line] = [] + let endOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + maxLineBytes: 64, + prefixBytes: 64) + { line in + scanned.append(line) + } + + #expect(scanned.count == 1) + #expect(scanned[0].wasTruncated) + #expect(scanned[0].bytes.count == 64) + #expect(endOffset == Int64(recordData.count)) + } + + @Test + func `jsonl scanner commits only complete CRLF records`() throws { + let root = try self.makeTemporaryRoot() + defer { try? FileManager.default.removeItem(at: root) } + + let fileURL = root.appendingPathComponent("crlf.jsonl", isDirectory: false) + let firstRecord = #"{"id":1}"# + let partialRecord = #"{"id":"par"# + let initial = firstRecord + "\r\n" + partialRecord + try initial.write(to: fileURL, atomically: true, encoding: .utf8) + + var firstPass: [String?] = [] + let resumeOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + maxLineBytes: 1024, + prefixBytes: 1024) + { line in + firstPass.append(String(bytes: line.bytes, encoding: .utf8)) + } + + #expect(firstPass == [firstRecord + "\r"]) + #expect(resumeOffset == Int64(Data((firstRecord + "\r\n").utf8).count)) + + let completion = #"tial"}"# + "\r\n" + let handle = try FileHandle(forWritingTo: fileURL) + defer { try? handle.close() } + try handle.seekToEnd() + try handle.write(contentsOf: Data(completion.utf8)) + + var secondPass: [String?] = [] + let endOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + offset: resumeOffset, + maxLineBytes: 1024, + prefixBytes: 1024) + { line in + secondPass.append(String(bytes: line.bytes, encoding: .utf8)) + } + + let completedRecord = partialRecord + #"tial"}"# + "\r" + #expect(secondPass == [completedRecord]) + #expect(endOffset == Int64(Data((initial + completion).utf8).count)) + } + + @Test + func `jsonl scanner tracks an incomplete record across read chunks`() throws { + let root = try self.makeTemporaryRoot() + defer { try? FileManager.default.removeItem(at: root) } + + let fileURL = root.appendingPathComponent("multi-chunk-tail.jsonl", isDirectory: false) + let initial = #"{"message":""# + String(repeating: "x", count: 300_000) + try initial.write(to: fileURL, atomically: true, encoding: .utf8) + + var firstPass: [CostUsageJsonl.Line] = [] + let resumeOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + maxLineBytes: 64, + prefixBytes: 64) + { line in + firstPass.append(line) + } + + #expect(firstPass.isEmpty) + #expect(resumeOffset == 0) + + let completion = #""}"# + "\n" + let handle = try FileHandle(forWritingTo: fileURL) + defer { try? handle.close() } + try handle.seekToEnd() + try handle.write(contentsOf: Data(completion.utf8)) + + var secondPass: [CostUsageJsonl.Line] = [] + let endOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + offset: resumeOffset, + maxLineBytes: 64, + prefixBytes: 64) + { line in + secondPass.append(line) + } + + #expect(secondPass.count == 1) + #expect(secondPass[0].wasTruncated) + #expect(secondPass[0].bytes.count == 64) + #expect(endOffset == Int64(Data((initial + completion).utf8).count)) + } + + @Test + func `jsonl scanner retries truncated literal prefixes`() throws { + let root = try self.makeTemporaryRoot() + defer { try? FileManager.default.removeItem(at: root) } + + let cases = [("tru", "e"), ("fals", "e"), ("nul", "l")] + for (index, testCase) in cases.enumerated() { + let fileURL = root.appendingPathComponent("literal-\(index).jsonl", isDirectory: false) + let initial = String(repeating: " ", count: 128) + testCase.0 + try initial.write(to: fileURL, atomically: true, encoding: .utf8) + + var firstPass: [CostUsageJsonl.Line] = [] + let resumeOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + maxLineBytes: 64, + prefixBytes: 64) + { line in + firstPass.append(line) + } + + #expect(firstPass.isEmpty) + #expect(resumeOffset == 0) + + let completion = testCase.1 + "\n" + let handle = try FileHandle(forWritingTo: fileURL) + try handle.seekToEnd() + try handle.write(contentsOf: Data(completion.utf8)) + try handle.close() + + var secondPass: [CostUsageJsonl.Line] = [] + let endOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + offset: resumeOffset, + maxLineBytes: 64, + prefixBytes: 64) + { line in + secondPass.append(line) + } + + #expect(secondPass.count == 1) + #expect(secondPass[0].wasTruncated) + #expect(endOffset == Int64(Data((initial + completion).utf8).count)) + } + } + + @Test + func `jsonl scanner retries a truncated number exponent`() throws { + let root = try self.makeTemporaryRoot() + defer { try? FileManager.default.removeItem(at: root) } + + let fileURL = root.appendingPathComponent("number-exponent.jsonl", isDirectory: false) + let initial = String(repeating: "9", count: 300_000) + "e-" + try initial.write(to: fileURL, atomically: true, encoding: .utf8) + + var firstPass: [CostUsageJsonl.Line] = [] + let resumeOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + maxLineBytes: 64, + prefixBytes: 64) + { line in + firstPass.append(line) + } + + #expect(firstPass.isEmpty) + #expect(resumeOffset == 0) + + let completion = "2\n" + let handle = try FileHandle(forWritingTo: fileURL) + defer { try? handle.close() } + try handle.seekToEnd() + try handle.write(contentsOf: Data(completion.utf8)) + + var secondPass: [CostUsageJsonl.Line] = [] + let endOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + offset: resumeOffset, + maxLineBytes: 64, + prefixBytes: 64) + { line in + secondPass.append(line) + } + + #expect(secondPass.count == 1) + #expect(secondPass[0].wasTruncated) + #expect(endOffset == Int64(Data((initial + completion).utf8).count)) + } + + @Test + func `jsonl scanner retries a complete numeric prefix`() throws { + let root = try self.makeTemporaryRoot() + defer { try? FileManager.default.removeItem(at: root) } + + let fileURL = root.appendingPathComponent("numeric-prefix.jsonl", isDirectory: false) + let initial = "1" + try initial.write(to: fileURL, atomically: true, encoding: .utf8) + + var firstPass: [String?] = [] + let resumeOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + maxLineBytes: 64, + prefixBytes: 64) + { line in + firstPass.append(String(bytes: line.bytes, encoding: .utf8)) + } + + #expect(firstPass.isEmpty) + #expect(resumeOffset == 0) + + let completion = "2\n" + let handle = try FileHandle(forWritingTo: fileURL) + defer { try? handle.close() } + try handle.seekToEnd() + try handle.write(contentsOf: Data(completion.utf8)) + + var secondPass: [String?] = [] + let endOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + offset: resumeOffset, + maxLineBytes: 64, + prefixBytes: 64) + { line in + secondPass.append(String(bytes: line.bytes, encoding: .utf8)) + } + + #expect(secondPass == ["12"]) + #expect(endOffset == Int64(Data((initial + completion).utf8).count)) + } + + @Test + func `jsonl scanner retries a truncated complete numeric prefix`() throws { + let root = try self.makeTemporaryRoot() + defer { try? FileManager.default.removeItem(at: root) } + + let fileURL = root.appendingPathComponent("truncated-numeric-prefix.jsonl", isDirectory: false) + let initial = String(repeating: " ", count: 128) + "1" + try initial.write(to: fileURL, atomically: true, encoding: .utf8) + + var firstPass: [CostUsageJsonl.Line] = [] + let resumeOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + maxLineBytes: 64, + prefixBytes: 64) + { line in + firstPass.append(line) + } + + #expect(firstPass.isEmpty) + #expect(resumeOffset == 0) + + let completion = "2\n" + let handle = try FileHandle(forWritingTo: fileURL) + defer { try? handle.close() } + try handle.seekToEnd() + try handle.write(contentsOf: Data(completion.utf8)) + + var secondPass: [CostUsageJsonl.Line] = [] + let endOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + offset: resumeOffset, + maxLineBytes: 64, + prefixBytes: 64) + { line in + secondPass.append(line) + } + + #expect(secondPass.count == 1) + #expect(secondPass[0].wasTruncated) + #expect(endOffset == Int64(Data((initial + completion).utf8).count)) + } + + @Test + func `jsonl scanner accepts a number terminated by trailing whitespace`() throws { + let root = try self.makeTemporaryRoot() + defer { try? FileManager.default.removeItem(at: root) } + + let fileURL = root.appendingPathComponent("terminated-number.jsonl", isDirectory: false) + let record = "12 " + try record.write(to: fileURL, atomically: true, encoding: .utf8) + + var scanned: [String?] = [] + let endOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + maxLineBytes: 64, + prefixBytes: 64) + { line in + scanned.append(String(bytes: line.bytes, encoding: .utf8)) + } + + #expect(scanned == [record]) + #expect(endOffset == Int64(Data(record.utf8).count)) + } + + @Test + func `jsonl scanner commits complete EOF record larger than retained prefix`() throws { + let root = try self.makeTemporaryRoot() + defer { try? FileManager.default.removeItem(at: root) } + + let fileURL = root.appendingPathComponent("short-prefix.jsonl", isDirectory: false) + let record = #"{"message":"\#(String(repeating: "x", count: 128))"}"# + try record.write(to: fileURL, atomically: true, encoding: .utf8) + + var scanned: [CostUsageJsonl.Line] = [] + let endOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + maxLineBytes: 1024, + prefixBytes: 64) + { line in + scanned.append(line) + } + + #expect(scanned.count == 1) + #expect(scanned[0].wasTruncated) + #expect(scanned[0].bytes.count == 64) + #expect(endOffset == Int64(Data(record.utf8).count)) + } + private func makeTemporaryRoot() throws -> URL { let root = FileManager.default.temporaryDirectory.appendingPathComponent( "codexbar-cost-usage-jsonl-\(UUID().uuidString)", diff --git a/Tests/CodexBarTests/CostUsageScannerBreakdownTests.swift b/Tests/CodexBarTests/CostUsageScannerBreakdownTests.swift index 6d5fd1760c..dd9a6063b8 100644 --- a/Tests/CodexBarTests/CostUsageScannerBreakdownTests.swift +++ b/Tests/CodexBarTests/CostUsageScannerBreakdownTests.swift @@ -1870,7 +1870,7 @@ struct CostUsageScannerBreakdownTests { #expect(first.data[0].totalTokens == 132) let newCacheURL = CostUsageCacheIO.cacheFileURL(provider: .codex, cacheRoot: env.cacheRoot) - #expect(newCacheURL.lastPathComponent == "codex-v9.json") + #expect(newCacheURL.lastPathComponent == "codex-v10.json") #expect(FileManager.default.fileExists(atPath: newCacheURL.path)) #expect(FileManager.default.fileExists(atPath: oldCacheURL.path)) diff --git a/Tests/CodexBarTests/PiSessionCostScannerTests.swift b/Tests/CodexBarTests/PiSessionCostScannerTests.swift index e451cca1db..91b3c2c3b4 100644 --- a/Tests/CodexBarTests/PiSessionCostScannerTests.swift +++ b/Tests/CodexBarTests/PiSessionCostScannerTests.swift @@ -691,8 +691,8 @@ struct PiSessionCostScannerTests { #expect(FileManager.default.fileExists(atPath: newCacheURL.path)) let newCache = PiSessionCostCacheIO.load(cacheRoot: env.cacheRoot) let rebuilt = newCache.daysByProvider[UsageProvider.codex.rawValue]?[dayKey]?[model] - #expect(newCacheURL.lastPathComponent == "pi-sessions-v5.json") - #expect(newCache.version == 5) + #expect(newCacheURL.lastPathComponent == "pi-sessions-v6.json") + #expect(newCache.version == 6) #expect(rebuilt?.usageSampleCount == 1) #expect(rebuilt?.costSampleCount == 1) #expect(rebuilt?.costNanos == Int64((expectedCost * 1_000_000_000).rounded())) @@ -700,7 +700,7 @@ struct PiSessionCostScannerTests { @Test func `pi scanner ignores v4 cache with stale gpt56 cache write pricing`() throws { - // v4 stored complete costNanos before cache-write rates existed; v5 must reprice. + // v4 stored complete costNanos before cache-write rates existed; v6 must reprice. let env = try CostUsageTestEnvironment() defer { env.cleanup() } @@ -800,7 +800,7 @@ struct PiSessionCostScannerTests { let newCache = PiSessionCostCacheIO.load(cacheRoot: env.cacheRoot) let rebuilt = newCache.daysByProvider[UsageProvider.codex.rawValue]?[dayKey]?[model] - #expect(newCache.version == 5) + #expect(newCache.version == 6) #expect(rebuilt?.costNanos == Int64((expectedCost * 1_000_000_000).rounded())) } }