From b033a4aac70cc1a67a15f857995ad476eedbd9ba Mon Sep 17 00:00:00 2001 From: ShiroKSH Date: Tue, 14 Jul 2026 16:22:49 +0300 Subject: [PATCH 01/11] fix: retain incomplete jsonl tails --- CHANGELOG.md | 3 + .../Vendored/CostUsage/CostUsageJsonl.swift | 15 ++++- .../CostUsageJsonlScannerTests.swift | 63 +++++++++++++++++++ 3 files changed, 79 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b750f2ab8..2f24a8b6e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## 0.43.1 — Unreleased +### Fixed +- Cost usage: retain an incomplete trailing JSONL record for the next scan so active Codex, Claude, and Pi sessions do not lose newly appended token usage. + ## 0.43.0 — 2026-07-14 ### Added diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift index 35f58b446a..3bcbd8b123 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift @@ -47,6 +47,7 @@ enum CostUsageJsonl { var lineBytes = 0 var truncated = false var bytesRead: Int64 = 0 + var committedOffset = startOffset func appendSegment(_ bytes: UnsafePointer, count: Int) { guard count > 0 else { return } @@ -71,17 +72,26 @@ enum CostUsageJsonl { truncated = false } + func hasCompleteJSONTail() -> Bool { + guard !truncated, lineBytes == current.count else { return false } + return (try? JSONSerialization.jsonObject(with: current, options: [.fragmentsAllowed])) != nil + } + while true { try checkCancellation?() let reachedEOF = try autoreleasepool { let chunk = try handle.read(upToCount: 256 * 1024) ?? Data() if chunk.isEmpty { - flushLine() + if hasCompleteJSONTail() { + flushLine() + committedOffset = startOffset + bytesRead + } return true } try checkCancellation?() bytesRead += Int64(chunk.count) + let chunkStartOffset = startOffset + bytesRead - Int64(chunk.count) chunk.withUnsafeBytes { rawBuffer in guard let base = rawBuffer.bindMemory(to: UInt8.self).baseAddress else { return } var segmentStart = 0 @@ -90,6 +100,7 @@ enum CostUsageJsonl { if base[index] == 0x0A { appendSegment(base.advanced(by: segmentStart), count: index - segmentStart) flushLine() + committedOffset = chunkStartOffset + Int64(index + 1) segmentStart = index + 1 } index += 1 @@ -104,6 +115,6 @@ enum CostUsageJsonl { try checkCancellation?() } - return startOffset + bytesRead + return committedOffset } } diff --git a/Tests/CodexBarTests/CostUsageJsonlScannerTests.swift b/Tests/CodexBarTests/CostUsageJsonlScannerTests.swift index 0bf27a4cfa..433e136963 100644 --- a/Tests/CodexBarTests/CostUsageJsonlScannerTests.swift +++ b/Tests/CodexBarTests/CostUsageJsonlScannerTests.swift @@ -58,6 +58,69 @@ struct CostUsageJsonlScannerTests { #expect(scanned[1].wasTruncated == true) } + @Test + func `jsonl scanner retries an incomplete final record after append`() throws { + let root = try self.makeTemporaryRoot() + defer { try? FileManager.default.removeItem(at: root) } + + let fileURL = root.appendingPathComponent("appending.jsonl", isDirectory: false) + let initial = #"{"type":"message","id":"partial"# + try initial.write(to: fileURL, atomically: true, encoding: .utf8) + + var firstPass: [String] = [] + let resumeOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + maxLineBytes: 1024, + prefixBytes: 1024) + { line in + firstPass.append(String(decoding: line.bytes, as: UTF8.self)) + } + + #expect(firstPass.isEmpty) + #expect(resumeOffset == 0) + + let completion = #""}"# + "\n" + let handle = try FileHandle(forWritingTo: fileURL) + defer { try? handle.close() } + try handle.seekToEnd() + try handle.write(contentsOf: Data(completion.utf8)) + + var secondPass: [String] = [] + let endOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + offset: resumeOffset, + maxLineBytes: 1024, + prefixBytes: 1024) + { line in + secondPass.append(String(decoding: line.bytes, as: UTF8.self)) + } + + #expect(secondPass == [initial + String(completion.dropLast())]) + #expect(endOffset == Int64(Data((initial + completion).utf8).count)) + } + + @Test + func `jsonl scanner accepts a complete final record without newline`() throws { + let root = try self.makeTemporaryRoot() + defer { try? FileManager.default.removeItem(at: root) } + + let fileURL = root.appendingPathComponent("final-record.jsonl", isDirectory: false) + let record = #"{"type":"message","id":"complete"}"# + try record.write(to: fileURL, atomically: true, encoding: .utf8) + + var scanned: [String] = [] + let endOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + maxLineBytes: 1024, + prefixBytes: 1024) + { line in + scanned.append(String(decoding: line.bytes, as: UTF8.self)) + } + + #expect(scanned == [record]) + #expect(endOffset == Int64(Data(record.utf8).count)) + } + private func makeTemporaryRoot() throws -> URL { let root = FileManager.default.temporaryDirectory.appendingPathComponent( "codexbar-cost-usage-jsonl-\(UUID().uuidString)", From 1daa240a069d3529dede1183d5ba06fc33acfc64 Mon Sep 17 00:00:00 2001 From: ShiroKSH Date: Tue, 14 Jul 2026 22:00:41 +0300 Subject: [PATCH 02/11] fix: rebuild stale cost scan caches --- CHANGELOG.md | 3 - Sources/CodexBarCore/PiSessionCostCache.swift | 4 +- .../Vendored/CostUsage/CostUsageCache.swift | 4 +- .../Vendored/CostUsage/CostUsageJsonl.swift | 2 +- Tests/CodexBarTests/CostUsageCacheTests.swift | 72 ++++++++++++++++++- .../CostUsageJsonlScannerTests.swift | 23 ++++++ .../CostUsageScannerBreakdownTests.swift | 2 +- .../PiSessionCostScannerTests.swift | 8 +-- 8 files changed, 102 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f24a8b6e6..6b750f2ab8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,9 +2,6 @@ ## 0.43.1 — Unreleased -### Fixed -- Cost usage: retain an incomplete trailing JSONL record for the next scan so active Codex, Claude, and Pi sessions do not lose newly appended token usage. - ## 0.43.0 — 2026-07-14 ### Added diff --git a/Sources/CodexBarCore/PiSessionCostCache.swift b/Sources/CodexBarCore/PiSessionCostCache.swift index 355be9c18b..7f690c1025 100644 --- a/Sources/CodexBarCore/PiSessionCostCache.swift +++ b/Sources/CodexBarCore/PiSessionCostCache.swift @@ -2,7 +2,7 @@ import Foundation enum PiSessionCostCacheIO { /// Artifact schema version. Pricing changes are tracked separately by `pricingKey`. - private static let artifactVersion = 5 + private static let artifactVersion = 6 private static func defaultCacheRoot() -> URL { let root = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first! @@ -56,7 +56,7 @@ struct PiSessionCostCache: Codable { var daysByProvider: [String: [String: [String: PiPackedUsage]]] = [:] var files: [String: PiSessionFileUsage] = [:] - init(version: Int = 5) { + init(version: Int = 6) { self.version = version } } diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageCache.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageCache.swift index ce5cad9cf9..fb05f8b15a 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageCache.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageCache.swift @@ -9,9 +9,9 @@ enum CostUsageCacheIO { private static func artifactVersion(for provider: UsageProvider) -> Int { switch provider { case .codex: - 9 + 10 case .claude, .vertexai: - 4 + 5 default: 1 } diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift index 3bcbd8b123..c847ee5350 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift @@ -82,7 +82,7 @@ enum CostUsageJsonl { let reachedEOF = try autoreleasepool { let chunk = try handle.read(upToCount: 256 * 1024) ?? Data() if chunk.isEmpty { - if hasCompleteJSONTail() { + if truncated || hasCompleteJSONTail() { flushLine() committedOffset = startOffset + bytesRead } diff --git a/Tests/CodexBarTests/CostUsageCacheTests.swift b/Tests/CodexBarTests/CostUsageCacheTests.swift index 9cc6946f8d..54dd8e9df1 100644 --- a/Tests/CodexBarTests/CostUsageCacheTests.swift +++ b/Tests/CodexBarTests/CostUsageCacheTests.swift @@ -11,9 +11,75 @@ struct CostUsageCacheTests { let claudeURL = CostUsageCacheIO.cacheFileURL(provider: .claude, cacheRoot: root) let vertexURL = CostUsageCacheIO.cacheFileURL(provider: .vertexai, cacheRoot: root) - #expect(codexURL.lastPathComponent == "codex-v9.json") - #expect(claudeURL.lastPathComponent == "claude-v4.json") - #expect(vertexURL.lastPathComponent == "vertexai-v4.json") + #expect(codexURL.lastPathComponent == "codex-v10.json") + #expect(claudeURL.lastPathComponent == "claude-v5.json") + #expect(vertexURL.lastPathComponent == "vertexai-v5.json") + } + + @Test + func `cost cache ignores predecessor artifact with persisted offset`() throws { + let root = try self.makeTemporaryCacheRoot() + defer { try? FileManager.default.removeItem(at: root) } + + let legacyURL = root + .appendingPathComponent("cost-usage", isDirectory: true) + .appendingPathComponent("codex-v9.json", isDirectory: false) + try FileManager.default.createDirectory( + at: legacyURL.deletingLastPathComponent(), + withIntermediateDirectories: true) + let producerKey = try #require(CostUsageCacheIO.currentProducerKey(provider: .codex)) + let legacy = """ + { + "version": 1, + "producerKey": "\(producerKey)", + "lastScanUnixMs": 999, + "files": { + "/tmp/session.jsonl": { + "mtimeUnixMs": 1, + "size": 100, + "days": {}, + "parsedBytes": 100 + } + }, + "days": {} + } + """ + try legacy.write(to: legacyURL, atomically: false, encoding: .utf8) + + let loaded = CostUsageCacheIO.load(provider: .codex, cacheRoot: root) + + #expect(loaded.lastScanUnixMs == 0) + #expect(loaded.files.isEmpty) + } + + @Test + func `Pi session cache ignores predecessor artifact with persisted offset`() throws { + let root = try self.makeTemporaryCacheRoot() + defer { try? FileManager.default.removeItem(at: root) } + + let legacyURL = root + .appendingPathComponent("cost-usage", isDirectory: true) + .appendingPathComponent("pi-sessions-v5.json", isDirectory: false) + try FileManager.default.createDirectory( + at: legacyURL.deletingLastPathComponent(), + withIntermediateDirectories: true) + var legacy = PiSessionCostCache(version: 5) + legacy.lastScanUnixMs = 999 + legacy.files = [ + "/tmp/session.jsonl": PiSessionFileUsage( + mtimeUnixMs: 1, + size: 100, + parsedBytes: 100, + lastModelContext: nil, + contributions: [:]), + ] + try JSONEncoder().encode(legacy).write(to: legacyURL) + + let loaded = PiSessionCostCacheIO.load(cacheRoot: root) + + #expect(loaded.version == 6) + #expect(loaded.lastScanUnixMs == 0) + #expect(loaded.files.isEmpty) } @Test diff --git a/Tests/CodexBarTests/CostUsageJsonlScannerTests.swift b/Tests/CodexBarTests/CostUsageJsonlScannerTests.swift index 433e136963..b968ad5892 100644 --- a/Tests/CodexBarTests/CostUsageJsonlScannerTests.swift +++ b/Tests/CodexBarTests/CostUsageJsonlScannerTests.swift @@ -121,6 +121,29 @@ struct CostUsageJsonlScannerTests { #expect(endOffset == Int64(Data(record.utf8).count)) } + @Test + func `jsonl scanner preserves a truncated final record`() throws { + let root = try self.makeTemporaryRoot() + defer { try? FileManager.default.removeItem(at: root) } + + let fileURL = root.appendingPathComponent("truncated-final-record.jsonl", isDirectory: false) + let record = #"{"message":"\#(String(repeating: "x", count: 256))"}"# + try record.write(to: fileURL, atomically: true, encoding: .utf8) + + var scanned: [CostUsageJsonl.Line] = [] + let endOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + maxLineBytes: 64, + prefixBytes: 64) + { line in + scanned.append(line) + } + + #expect(scanned.count == 1) + #expect(scanned[0].wasTruncated) + #expect(endOffset == Int64(Data(record.utf8).count)) + } + private func makeTemporaryRoot() throws -> URL { let root = FileManager.default.temporaryDirectory.appendingPathComponent( "codexbar-cost-usage-jsonl-\(UUID().uuidString)", diff --git a/Tests/CodexBarTests/CostUsageScannerBreakdownTests.swift b/Tests/CodexBarTests/CostUsageScannerBreakdownTests.swift index 8a3b8ad4c3..afb21235cf 100644 --- a/Tests/CodexBarTests/CostUsageScannerBreakdownTests.swift +++ b/Tests/CodexBarTests/CostUsageScannerBreakdownTests.swift @@ -1870,7 +1870,7 @@ struct CostUsageScannerBreakdownTests { #expect(first.data[0].totalTokens == 132) let newCacheURL = CostUsageCacheIO.cacheFileURL(provider: .codex, cacheRoot: env.cacheRoot) - #expect(newCacheURL.lastPathComponent == "codex-v9.json") + #expect(newCacheURL.lastPathComponent == "codex-v10.json") #expect(FileManager.default.fileExists(atPath: newCacheURL.path)) #expect(FileManager.default.fileExists(atPath: oldCacheURL.path)) diff --git a/Tests/CodexBarTests/PiSessionCostScannerTests.swift b/Tests/CodexBarTests/PiSessionCostScannerTests.swift index e451cca1db..91b3c2c3b4 100644 --- a/Tests/CodexBarTests/PiSessionCostScannerTests.swift +++ b/Tests/CodexBarTests/PiSessionCostScannerTests.swift @@ -691,8 +691,8 @@ struct PiSessionCostScannerTests { #expect(FileManager.default.fileExists(atPath: newCacheURL.path)) let newCache = PiSessionCostCacheIO.load(cacheRoot: env.cacheRoot) let rebuilt = newCache.daysByProvider[UsageProvider.codex.rawValue]?[dayKey]?[model] - #expect(newCacheURL.lastPathComponent == "pi-sessions-v5.json") - #expect(newCache.version == 5) + #expect(newCacheURL.lastPathComponent == "pi-sessions-v6.json") + #expect(newCache.version == 6) #expect(rebuilt?.usageSampleCount == 1) #expect(rebuilt?.costSampleCount == 1) #expect(rebuilt?.costNanos == Int64((expectedCost * 1_000_000_000).rounded())) @@ -700,7 +700,7 @@ struct PiSessionCostScannerTests { @Test func `pi scanner ignores v4 cache with stale gpt56 cache write pricing`() throws { - // v4 stored complete costNanos before cache-write rates existed; v5 must reprice. + // v4 stored complete costNanos before cache-write rates existed; v6 must reprice. let env = try CostUsageTestEnvironment() defer { env.cleanup() } @@ -800,7 +800,7 @@ struct PiSessionCostScannerTests { let newCache = PiSessionCostCacheIO.load(cacheRoot: env.cacheRoot) let rebuilt = newCache.daysByProvider[UsageProvider.codex.rawValue]?[dayKey]?[model] - #expect(newCache.version == 5) + #expect(newCache.version == 6) #expect(rebuilt?.costNanos == Int64((expectedCost * 1_000_000_000).rounded())) } } From d364406c7f9baa77ebd6413d1e31eb2d7bbd4ee8 Mon Sep 17 00:00:00 2001 From: ShiroKSH Date: Tue, 14 Jul 2026 23:48:08 +0300 Subject: [PATCH 03/11] fix: retain incomplete oversized jsonl tails --- .../Vendored/CostUsage/CostUsageJsonl.swift | 63 ++++++++++++++++++- .../CostUsageJsonlScannerTests.swift | 42 +++++++++++++ 2 files changed, 103 insertions(+), 2 deletions(-) diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift index c847ee5350..391315e5df 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift @@ -6,6 +6,56 @@ enum CostUsageJsonl { let wasTruncated: Bool } + private struct JSONTailState { + private var containerDepth = 0 + private var insideString = false + private var escaping = false + private var sawNonWhitespace = false + + mutating func append(_ bytes: UnsafePointer, count: Int) { + for index in 0.., count: Int) { guard count > 0 else { return } lineBytes += count + jsonTailState.append(bytes, count: count) if current.count < prefixBytes { let appendCount = min(prefixBytes - current.count, count) if appendCount > 0 { @@ -70,10 +122,17 @@ enum CostUsageJsonl { current.removeAll(keepingCapacity: true) lineBytes = 0 truncated = false + jsonTailState.reset() } func hasCompleteJSONTail() -> Bool { - guard !truncated, lineBytes == current.count else { return false } + if truncated { + // The full record is intentionally not retained. Its structural state is enough + // to keep a still-open object or string retriable without changing the old + // behavior for complete records that exceed the safety limit. + return jsonTailState.isStructurallyComplete + } + guard lineBytes == current.count else { return false } return (try? JSONSerialization.jsonObject(with: current, options: [.fragmentsAllowed])) != nil } @@ -82,7 +141,7 @@ enum CostUsageJsonl { let reachedEOF = try autoreleasepool { let chunk = try handle.read(upToCount: 256 * 1024) ?? Data() if chunk.isEmpty { - if truncated || hasCompleteJSONTail() { + if hasCompleteJSONTail() { flushLine() committedOffset = startOffset + bytesRead } diff --git a/Tests/CodexBarTests/CostUsageJsonlScannerTests.swift b/Tests/CodexBarTests/CostUsageJsonlScannerTests.swift index b968ad5892..6df551c63a 100644 --- a/Tests/CodexBarTests/CostUsageJsonlScannerTests.swift +++ b/Tests/CodexBarTests/CostUsageJsonlScannerTests.swift @@ -144,6 +144,48 @@ struct CostUsageJsonlScannerTests { #expect(endOffset == Int64(Data(record.utf8).count)) } + @Test + func `jsonl scanner retries a truncated incomplete final record after append`() throws { + let root = try self.makeTemporaryRoot() + defer { try? FileManager.default.removeItem(at: root) } + + let fileURL = root.appendingPathComponent("truncated-appending.jsonl", isDirectory: false) + let initial = #"{"message":"\#(String(repeating: "x", count: 256))"# + try initial.write(to: fileURL, atomically: true, encoding: .utf8) + + var firstPass: [CostUsageJsonl.Line] = [] + let resumeOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + maxLineBytes: 64, + prefixBytes: 64) + { line in + firstPass.append(line) + } + + #expect(firstPass.isEmpty) + #expect(resumeOffset == 0) + + let completion = #""}"# + "\n" + let handle = try FileHandle(forWritingTo: fileURL) + defer { try? handle.close() } + try handle.seekToEnd() + try handle.write(contentsOf: Data(completion.utf8)) + + var secondPass: [CostUsageJsonl.Line] = [] + let endOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + offset: resumeOffset, + maxLineBytes: 64, + prefixBytes: 64) + { line in + secondPass.append(line) + } + + #expect(secondPass.count == 1) + #expect(secondPass[0].wasTruncated) + #expect(endOffset == Int64(Data((initial + completion).utf8).count)) + } + private func makeTemporaryRoot() throws -> URL { let root = FileManager.default.temporaryDirectory.appendingPathComponent( "codexbar-cost-usage-jsonl-\(UUID().uuidString)", From dba4ef49986002426a16aa3e4f1d97ea2f69e8e7 Mon Sep 17 00:00:00 2001 From: ShiroKSH Date: Tue, 14 Jul 2026 23:49:18 +0300 Subject: [PATCH 04/11] perf: avoid rescanning jsonl tail bytes --- .../Vendored/CostUsage/CostUsageJsonl.swift | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift index 391315e5df..73cc3b4b4a 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift @@ -12,12 +12,6 @@ enum CostUsageJsonl { private var escaping = false private var sawNonWhitespace = false - mutating func append(_ bytes: UnsafePointer, count: Int) { - for index in 0.., count: Int) { guard count > 0 else { return } lineBytes += count - jsonTailState.append(bytes, count: count) if current.count < prefixBytes { let appendCount = min(prefixBytes - current.count, count) if appendCount > 0 { @@ -161,6 +154,8 @@ enum CostUsageJsonl { flushLine() committedOffset = chunkStartOffset + Int64(index + 1) segmentStart = index + 1 + } else { + jsonTailState.append(base[index]) } index += 1 } From 12e1239430bdf15f91a966153fc6c6b131a5e878 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 16 Jul 2026 19:17:39 +0100 Subject: [PATCH 05/11] test: cover adversarial JSONL tails --- .../CostUsageJsonlScannerTests.swift | 154 ++++++++++++++++++ 1 file changed, 154 insertions(+) diff --git a/Tests/CodexBarTests/CostUsageJsonlScannerTests.swift b/Tests/CodexBarTests/CostUsageJsonlScannerTests.swift index 6df551c63a..0f6c2dba0f 100644 --- a/Tests/CodexBarTests/CostUsageJsonlScannerTests.swift +++ b/Tests/CodexBarTests/CostUsageJsonlScannerTests.swift @@ -186,6 +186,160 @@ struct CostUsageJsonlScannerTests { #expect(endOffset == Int64(Data((initial + completion).utf8).count)) } + @Test + func `jsonl scanner retries a truncated escape sequence`() throws { + let root = try self.makeTemporaryRoot() + defer { try? FileManager.default.removeItem(at: root) } + + let fileURL = root.appendingPathComponent("truncated-escape.jsonl", isDirectory: false) + let initial = #"{"message":""# + String(repeating: "x", count: 256) + #"\u12"# + try initial.write(to: fileURL, atomically: true, encoding: .utf8) + + var firstPass: [CostUsageJsonl.Line] = [] + let resumeOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + maxLineBytes: 64, + prefixBytes: 64) + { line in + firstPass.append(line) + } + + #expect(firstPass.isEmpty) + #expect(resumeOffset == 0) + + let completion = #"34"}"# + "\n" + let handle = try FileHandle(forWritingTo: fileURL) + defer { try? handle.close() } + try handle.seekToEnd() + try handle.write(contentsOf: Data(completion.utf8)) + + var secondPass: [CostUsageJsonl.Line] = [] + let endOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + offset: resumeOffset, + maxLineBytes: 64, + prefixBytes: 64) + { line in + secondPass.append(line) + } + + #expect(secondPass.count == 1) + #expect(secondPass[0].wasTruncated) + #expect(endOffset == Int64(Data((initial + completion).utf8).count)) + } + + @Test + func `jsonl scanner ignores nested delimiters inside strings`() throws { + let root = try self.makeTemporaryRoot() + defer { try? FileManager.default.removeItem(at: root) } + + let fileURL = root.appendingPathComponent("string-delimiters.jsonl", isDirectory: false) + let message = String(repeating: "{[", count: 64) + #""nested""# + let recordData = try JSONEncoder().encode(["message": message]) + try recordData.write(to: fileURL) + + var scanned: [CostUsageJsonl.Line] = [] + let endOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + maxLineBytes: 64, + prefixBytes: 64) + { line in + scanned.append(line) + } + + #expect(scanned.count == 1) + #expect(scanned[0].wasTruncated) + #expect(scanned[0].bytes.count == 64) + #expect(endOffset == Int64(recordData.count)) + } + + @Test + func `jsonl scanner commits only complete CRLF records`() throws { + let root = try self.makeTemporaryRoot() + defer { try? FileManager.default.removeItem(at: root) } + + let fileURL = root.appendingPathComponent("crlf.jsonl", isDirectory: false) + let firstRecord = #"{"id":1}"# + let partialRecord = #"{"id":"par"# + let initial = firstRecord + "\r\n" + partialRecord + try initial.write(to: fileURL, atomically: true, encoding: .utf8) + + var firstPass: [String] = [] + let resumeOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + maxLineBytes: 1024, + prefixBytes: 1024) + { line in + firstPass.append(String(decoding: line.bytes, as: UTF8.self)) + } + + #expect(firstPass == [firstRecord + "\r"]) + #expect(resumeOffset == Int64(Data((firstRecord + "\r\n").utf8).count)) + + let completion = #"tial"}"# + "\r\n" + let handle = try FileHandle(forWritingTo: fileURL) + defer { try? handle.close() } + try handle.seekToEnd() + try handle.write(contentsOf: Data(completion.utf8)) + + var secondPass: [String] = [] + let endOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + offset: resumeOffset, + maxLineBytes: 1024, + prefixBytes: 1024) + { line in + secondPass.append(String(decoding: line.bytes, as: UTF8.self)) + } + + let completedRecord = String(decoding: completion.utf8.dropLast(), as: UTF8.self) + #expect(secondPass == [partialRecord + completedRecord]) + #expect(endOffset == Int64(Data((initial + completion).utf8).count)) + } + + @Test + func `jsonl scanner tracks an incomplete record across read chunks`() throws { + let root = try self.makeTemporaryRoot() + defer { try? FileManager.default.removeItem(at: root) } + + let fileURL = root.appendingPathComponent("multi-chunk-tail.jsonl", isDirectory: false) + let initial = #"{"message":""# + String(repeating: "x", count: 300_000) + try initial.write(to: fileURL, atomically: true, encoding: .utf8) + + var firstPass: [CostUsageJsonl.Line] = [] + let resumeOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + maxLineBytes: 64, + prefixBytes: 64) + { line in + firstPass.append(line) + } + + #expect(firstPass.isEmpty) + #expect(resumeOffset == 0) + + let completion = #""}"# + "\n" + let handle = try FileHandle(forWritingTo: fileURL) + defer { try? handle.close() } + try handle.seekToEnd() + try handle.write(contentsOf: Data(completion.utf8)) + + var secondPass: [CostUsageJsonl.Line] = [] + let endOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + offset: resumeOffset, + maxLineBytes: 64, + prefixBytes: 64) + { line in + secondPass.append(line) + } + + #expect(secondPass.count == 1) + #expect(secondPass[0].wasTruncated) + #expect(secondPass[0].bytes.count == 64) + #expect(endOffset == Int64(Data((initial + completion).utf8).count)) + } + private func makeTemporaryRoot() throws -> URL { let root = FileManager.default.temporaryDirectory.appendingPathComponent( "codexbar-cost-usage-jsonl-\(UUID().uuidString)", From 518dd887ee77e9e06b650b13865931a37dd77143 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 16 Jul 2026 19:18:05 +0100 Subject: [PATCH 06/11] chore: refresh Codex parser hash --- Sources/CodexBarCore/Generated/CodexParserHash.generated.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift b/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift index e16a88f5cb..7358ef78c0 100644 --- a/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift +++ b/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift @@ -1,5 +1,5 @@ // Generated by Scripts/regenerate-codex-parser-hash.sh. Do not edit by hand. enum CodexParserHash { - static let value = "9b26fd821cf090dc" + static let value = "2aa413960a86ef2b" } From a1d1b31ecf2a1c6aeafe80326acb30a3a17cd68f Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 16 Jul 2026 19:20:37 +0100 Subject: [PATCH 07/11] test: validate JSONL tail UTF-8 --- .../CostUsageJsonlScannerTests.swift | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Tests/CodexBarTests/CostUsageJsonlScannerTests.swift b/Tests/CodexBarTests/CostUsageJsonlScannerTests.swift index 0f6c2dba0f..4e098cdaeb 100644 --- a/Tests/CodexBarTests/CostUsageJsonlScannerTests.swift +++ b/Tests/CodexBarTests/CostUsageJsonlScannerTests.swift @@ -67,13 +67,13 @@ struct CostUsageJsonlScannerTests { let initial = #"{"type":"message","id":"partial"# try initial.write(to: fileURL, atomically: true, encoding: .utf8) - var firstPass: [String] = [] + var firstPass: [String?] = [] let resumeOffset = try CostUsageJsonl.scan( fileURL: fileURL, maxLineBytes: 1024, prefixBytes: 1024) { line in - firstPass.append(String(decoding: line.bytes, as: UTF8.self)) + firstPass.append(String(bytes: line.bytes, encoding: .utf8)) } #expect(firstPass.isEmpty) @@ -85,14 +85,14 @@ struct CostUsageJsonlScannerTests { try handle.seekToEnd() try handle.write(contentsOf: Data(completion.utf8)) - var secondPass: [String] = [] + var secondPass: [String?] = [] let endOffset = try CostUsageJsonl.scan( fileURL: fileURL, offset: resumeOffset, maxLineBytes: 1024, prefixBytes: 1024) { line in - secondPass.append(String(decoding: line.bytes, as: UTF8.self)) + secondPass.append(String(bytes: line.bytes, encoding: .utf8)) } #expect(secondPass == [initial + String(completion.dropLast())]) @@ -108,13 +108,13 @@ struct CostUsageJsonlScannerTests { let record = #"{"type":"message","id":"complete"}"# try record.write(to: fileURL, atomically: true, encoding: .utf8) - var scanned: [String] = [] + var scanned: [String?] = [] let endOffset = try CostUsageJsonl.scan( fileURL: fileURL, maxLineBytes: 1024, prefixBytes: 1024) { line in - scanned.append(String(decoding: line.bytes, as: UTF8.self)) + scanned.append(String(bytes: line.bytes, encoding: .utf8)) } #expect(scanned == [record]) @@ -264,13 +264,13 @@ struct CostUsageJsonlScannerTests { let initial = firstRecord + "\r\n" + partialRecord try initial.write(to: fileURL, atomically: true, encoding: .utf8) - var firstPass: [String] = [] + var firstPass: [String?] = [] let resumeOffset = try CostUsageJsonl.scan( fileURL: fileURL, maxLineBytes: 1024, prefixBytes: 1024) { line in - firstPass.append(String(decoding: line.bytes, as: UTF8.self)) + firstPass.append(String(bytes: line.bytes, encoding: .utf8)) } #expect(firstPass == [firstRecord + "\r"]) @@ -282,18 +282,18 @@ struct CostUsageJsonlScannerTests { try handle.seekToEnd() try handle.write(contentsOf: Data(completion.utf8)) - var secondPass: [String] = [] + var secondPass: [String?] = [] let endOffset = try CostUsageJsonl.scan( fileURL: fileURL, offset: resumeOffset, maxLineBytes: 1024, prefixBytes: 1024) { line in - secondPass.append(String(decoding: line.bytes, as: UTF8.self)) + secondPass.append(String(bytes: line.bytes, encoding: .utf8)) } - let completedRecord = String(decoding: completion.utf8.dropLast(), as: UTF8.self) - #expect(secondPass == [partialRecord + completedRecord]) + let completedRecord = partialRecord + #"tial"}"# + "\r" + #expect(secondPass == [completedRecord]) #expect(endOffset == Int64(Data((initial + completion).utf8).count)) } From 0eeafd9527c7f5d304c0d0214b482d9629a44956 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 16 Jul 2026 19:38:21 +0100 Subject: [PATCH 08/11] fix: validate truncated JSONL scalars --- .../Generated/CodexParserHash.generated.swift | 2 +- .../Vendored/CostUsage/CostUsageJsonl.swift | 198 +++++++++++++++++- .../CostUsageJsonlScannerTests.swift | 87 ++++++++ 3 files changed, 280 insertions(+), 7 deletions(-) diff --git a/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift b/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift index 7358ef78c0..bf4df0dde9 100644 --- a/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift +++ b/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift @@ -1,5 +1,5 @@ // Generated by Scripts/regenerate-codex-parser-hash.sh. Do not edit by hand. enum CodexParserHash { - static let value = "2aa413960a86ef2b" + static let value = "f75e3eac764ca9fa" } diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift index 73cc3b4b4a..f709bf5855 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift @@ -7,20 +7,192 @@ enum CostUsageJsonl { } private struct JSONTailState { + private enum ScalarState { + case notScalar + case trueLiteral(Int) + case falseLiteral(Int) + case nullLiteral(Int) + case number(NumberState) + case invalid + } + + private enum NumberState { + case sign + case zero + case integer + case decimalPoint + case fraction + case exponentMarker + case exponentSign + case exponentDigits + case finished + case invalid + + var isComplete: Bool { + switch self { + case .zero, .integer, .fraction, .exponentDigits, .finished, .invalid: + true + case .sign, .decimalPoint, .exponentMarker, .exponentSign: + false + } + } + + func appending(_ byte: UInt8) -> Self { + switch self { + case .sign: + if byte == 0x30 { + return .zero + } + if (0x31...0x39).contains(byte) { + return .integer + } + case .zero: + if byte == 0x2E { + return .decimalPoint + } + if byte == 0x65 || byte == 0x45 { + return .exponentMarker + } + if JSONTailState.isWhitespace(byte) { + return .finished + } + case .integer: + if (0x30...0x39).contains(byte) { + return .integer + } + if byte == 0x2E { + return .decimalPoint + } + if byte == 0x65 || byte == 0x45 { + return .exponentMarker + } + if JSONTailState.isWhitespace(byte) { + return .finished + } + case .decimalPoint: + if (0x30...0x39).contains(byte) { + return .fraction + } + case .fraction: + if (0x30...0x39).contains(byte) { + return .fraction + } + if byte == 0x65 || byte == 0x45 { + return .exponentMarker + } + if JSONTailState.isWhitespace(byte) { + return .finished + } + case .exponentMarker: + if byte == 0x2B || byte == 0x2D { + return .exponentSign + } + if (0x30...0x39).contains(byte) { + return .exponentDigits + } + case .exponentSign: + if (0x30...0x39).contains(byte) { + return .exponentDigits + } + case .exponentDigits: + if (0x30...0x39).contains(byte) { + return .exponentDigits + } + if JSONTailState.isWhitespace(byte) { + return .finished + } + case .finished: + if JSONTailState.isWhitespace(byte) { + return .finished + } + case .invalid: + return .invalid + } + return .invalid + } + } + + private static let trueLiteral = Array("true".utf8) + private static let falseLiteral = Array("false".utf8) + private static let nullLiteral = Array("null".utf8) + private var containerDepth = 0 private var insideString = false private var escaping = false private var sawNonWhitespace = false + private var scalarState = ScalarState.notScalar mutating func reset() { self = Self() } var isStructurallyComplete: Bool { - self.sawNonWhitespace && !self.insideString && self.containerDepth == 0 + guard self.sawNonWhitespace else { return false } + switch self.scalarState { + case .notScalar: + return !self.insideString && self.containerDepth == 0 + case let .trueLiteral(matched): + return matched == Self.trueLiteral.count + case let .falseLiteral(matched): + return matched == Self.falseLiteral.count + case let .nullLiteral(matched): + return matched == Self.nullLiteral.count + case let .number(state): + return state.isComplete + case .invalid: + return true + } } mutating func append(_ byte: UInt8) { + if !self.sawNonWhitespace { + guard !Self.isWhitespace(byte) else { return } + self.sawNonWhitespace = true + switch byte { + case 0x22: + self.insideString = true + case 0x7B, 0x5B: + self.containerDepth = 1 + case 0x74: + self.scalarState = .trueLiteral(1) + case 0x66: + self.scalarState = .falseLiteral(1) + case 0x6E: + self.scalarState = .nullLiteral(1) + case 0x2D: + self.scalarState = .number(.sign) + case 0x30: + self.scalarState = .number(.zero) + case 0x31...0x39: + self.scalarState = .number(.integer) + default: + self.scalarState = .invalid + } + return + } + + switch self.scalarState { + case let .trueLiteral(matched): + self.scalarState = self.advanceLiteral(byte, expected: Self.trueLiteral, matched: matched) + .map(ScalarState.trueLiteral) ?? .invalid + return + case let .falseLiteral(matched): + self.scalarState = self.advanceLiteral(byte, expected: Self.falseLiteral, matched: matched) + .map(ScalarState.falseLiteral) ?? .invalid + return + case let .nullLiteral(matched): + self.scalarState = self.advanceLiteral(byte, expected: Self.nullLiteral, matched: matched) + .map(ScalarState.nullLiteral) ?? .invalid + return + case let .number(state): + self.scalarState = .number(state.appending(byte)) + return + case .invalid: + return + case .notScalar: + break + } + if self.insideString { if self.escaping { self.escaping = false @@ -36,18 +208,30 @@ enum CostUsageJsonl { case 0x20, 0x09, 0x0D: return case 0x22: - self.sawNonWhitespace = true self.insideString = true case 0x7B, 0x5B: - self.sawNonWhitespace = true self.containerDepth += 1 case 0x7D, 0x5D: - self.sawNonWhitespace = true self.containerDepth = max(0, self.containerDepth - 1) default: - self.sawNonWhitespace = true + break } } + + private func advanceLiteral( + _ byte: UInt8, + expected: [UInt8], + matched: Int) -> Int? + { + if matched < expected.count { + return byte == expected[matched] ? matched + 1 : nil + } + return Self.isWhitespace(byte) ? matched : nil + } + + private static func isWhitespace(_ byte: UInt8) -> Bool { + byte == 0x20 || byte == 0x09 || byte == 0x0D + } } @discardableResult @@ -165,7 +349,9 @@ enum CostUsageJsonl { } return false } - if reachedEOF { break } + if reachedEOF { + break + } try checkCancellation?() } diff --git a/Tests/CodexBarTests/CostUsageJsonlScannerTests.swift b/Tests/CodexBarTests/CostUsageJsonlScannerTests.swift index 4e098cdaeb..b221f53c95 100644 --- a/Tests/CodexBarTests/CostUsageJsonlScannerTests.swift +++ b/Tests/CodexBarTests/CostUsageJsonlScannerTests.swift @@ -340,6 +340,93 @@ struct CostUsageJsonlScannerTests { #expect(endOffset == Int64(Data((initial + completion).utf8).count)) } + @Test + func `jsonl scanner retries truncated literal prefixes`() throws { + let root = try self.makeTemporaryRoot() + defer { try? FileManager.default.removeItem(at: root) } + + let cases = [("tru", "e"), ("fals", "e"), ("nul", "l")] + for (index, testCase) in cases.enumerated() { + let fileURL = root.appendingPathComponent("literal-\(index).jsonl", isDirectory: false) + let initial = String(repeating: " ", count: 128) + testCase.0 + try initial.write(to: fileURL, atomically: true, encoding: .utf8) + + var firstPass: [CostUsageJsonl.Line] = [] + let resumeOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + maxLineBytes: 64, + prefixBytes: 64) + { line in + firstPass.append(line) + } + + #expect(firstPass.isEmpty) + #expect(resumeOffset == 0) + + let completion = testCase.1 + "\n" + let handle = try FileHandle(forWritingTo: fileURL) + try handle.seekToEnd() + try handle.write(contentsOf: Data(completion.utf8)) + try handle.close() + + var secondPass: [CostUsageJsonl.Line] = [] + let endOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + offset: resumeOffset, + maxLineBytes: 64, + prefixBytes: 64) + { line in + secondPass.append(line) + } + + #expect(secondPass.count == 1) + #expect(secondPass[0].wasTruncated) + #expect(endOffset == Int64(Data((initial + completion).utf8).count)) + } + } + + @Test + func `jsonl scanner retries a truncated number exponent`() throws { + let root = try self.makeTemporaryRoot() + defer { try? FileManager.default.removeItem(at: root) } + + let fileURL = root.appendingPathComponent("number-exponent.jsonl", isDirectory: false) + let initial = String(repeating: "9", count: 300_000) + "e-" + try initial.write(to: fileURL, atomically: true, encoding: .utf8) + + var firstPass: [CostUsageJsonl.Line] = [] + let resumeOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + maxLineBytes: 64, + prefixBytes: 64) + { line in + firstPass.append(line) + } + + #expect(firstPass.isEmpty) + #expect(resumeOffset == 0) + + let completion = "2\n" + let handle = try FileHandle(forWritingTo: fileURL) + defer { try? handle.close() } + try handle.seekToEnd() + try handle.write(contentsOf: Data(completion.utf8)) + + var secondPass: [CostUsageJsonl.Line] = [] + let endOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + offset: resumeOffset, + maxLineBytes: 64, + prefixBytes: 64) + { line in + secondPass.append(line) + } + + #expect(secondPass.count == 1) + #expect(secondPass[0].wasTruncated) + #expect(endOffset == Int64(Data((initial + completion).utf8).count)) + } + private func makeTemporaryRoot() throws -> URL { let root = FileManager.default.temporaryDirectory.appendingPathComponent( "codexbar-cost-usage-jsonl-\(UUID().uuidString)", From 74e81ef0bb4416dfac7f2ac7fc71f67322b0c4e2 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 16 Jul 2026 19:45:36 +0100 Subject: [PATCH 09/11] refactor: simplify JSONL scalar state --- .../Generated/CodexParserHash.generated.swift | 2 +- .../Vendored/CostUsage/CostUsageJsonl.swift | 183 ++++++++---------- 2 files changed, 85 insertions(+), 100 deletions(-) diff --git a/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift b/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift index bf4df0dde9..e1f4ce1780 100644 --- a/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift +++ b/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift @@ -1,5 +1,5 @@ // Generated by Scripts/regenerate-codex-parser-hash.sh. Do not edit by hand. enum CodexParserHash { - static let value = "f75e3eac764ca9fa" + static let value = "bbbb9cc285c67c8f" } diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift index f709bf5855..3ecad42d73 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift @@ -17,6 +17,28 @@ enum CostUsageJsonl { } private enum NumberState { + private enum ByteKind { + case zero + case digit + case decimalPoint + case exponentMarker + case sign + case whitespace + case other + + init(_ byte: UInt8) { + switch byte { + case 0x30: self = .zero + case 0x31...0x39: self = .digit + case 0x2E: self = .decimalPoint + case 0x65, 0x45: self = .exponentMarker + case 0x2B, 0x2D: self = .sign + case 0x20, 0x09, 0x0A, 0x0D: self = .whitespace + default: self = .other + } + } + } + case sign case zero case integer @@ -38,77 +60,29 @@ enum CostUsageJsonl { } func appending(_ byte: UInt8) -> Self { - switch self { - case .sign: - if byte == 0x30 { - return .zero - } - if (0x31...0x39).contains(byte) { - return .integer - } - case .zero: - if byte == 0x2E { - return .decimalPoint - } - if byte == 0x65 || byte == 0x45 { - return .exponentMarker - } - if JSONTailState.isWhitespace(byte) { - return .finished - } - case .integer: - if (0x30...0x39).contains(byte) { - return .integer - } - if byte == 0x2E { - return .decimalPoint - } - if byte == 0x65 || byte == 0x45 { - return .exponentMarker - } - if JSONTailState.isWhitespace(byte) { - return .finished - } - case .decimalPoint: - if (0x30...0x39).contains(byte) { - return .fraction - } - case .fraction: - if (0x30...0x39).contains(byte) { - return .fraction - } - if byte == 0x65 || byte == 0x45 { - return .exponentMarker - } - if JSONTailState.isWhitespace(byte) { - return .finished - } - case .exponentMarker: - if byte == 0x2B || byte == 0x2D { - return .exponentSign - } - if (0x30...0x39).contains(byte) { - return .exponentDigits - } - case .exponentSign: - if (0x30...0x39).contains(byte) { - return .exponentDigits - } - case .exponentDigits: - if (0x30...0x39).contains(byte) { - return .exponentDigits - } - if JSONTailState.isWhitespace(byte) { - return .finished - } - case .finished: - if JSONTailState.isWhitespace(byte) { - return .finished - } - case .invalid: - return .invalid + switch (self, ByteKind(byte)) { + case (.invalid, _): .invalid + case (.finished, .whitespace): .finished + case (.sign, .zero): .zero + case (.sign, .digit): .integer + case (.zero, .decimalPoint): .decimalPoint + case (.zero, .exponentMarker): .exponentMarker + case (.integer, .zero), (.integer, .digit): .integer + case (.integer, .decimalPoint): .decimalPoint + case (.integer, .exponentMarker): .exponentMarker + case (.decimalPoint, .zero), (.decimalPoint, .digit): .fraction + case (.fraction, .zero), (.fraction, .digit): .fraction + case (.fraction, .exponentMarker): .exponentMarker + case (.exponentMarker, .sign): .exponentSign + case (.exponentMarker, .zero), (.exponentMarker, .digit): .exponentDigits + case (.exponentSign, .zero), (.exponentSign, .digit): .exponentDigits + case (.exponentDigits, .zero), (.exponentDigits, .digit): .exponentDigits + case (.zero, .whitespace), + (.integer, .whitespace), + (.fraction, .whitespace), + (.exponentDigits, .whitespace): .finished + default: .invalid } - return .invalid } } @@ -146,53 +120,64 @@ enum CostUsageJsonl { mutating func append(_ byte: UInt8) { if !self.sawNonWhitespace { - guard !Self.isWhitespace(byte) else { return } - self.sawNonWhitespace = true - switch byte { - case 0x22: - self.insideString = true - case 0x7B, 0x5B: - self.containerDepth = 1 - case 0x74: - self.scalarState = .trueLiteral(1) - case 0x66: - self.scalarState = .falseLiteral(1) - case 0x6E: - self.scalarState = .nullLiteral(1) - case 0x2D: - self.scalarState = .number(.sign) - case 0x30: - self.scalarState = .number(.zero) - case 0x31...0x39: - self.scalarState = .number(.integer) - default: - self.scalarState = .invalid - } + self.start(byte) return } + guard !self.appendScalar(byte) else { return } + self.appendContainer(byte) + } + + private mutating func start(_ byte: UInt8) { + guard !Self.isWhitespace(byte) else { return } + self.sawNonWhitespace = true + switch byte { + case 0x22: + self.insideString = true + case 0x7B, 0x5B: + self.containerDepth = 1 + case 0x74: + self.scalarState = .trueLiteral(1) + case 0x66: + self.scalarState = .falseLiteral(1) + case 0x6E: + self.scalarState = .nullLiteral(1) + case 0x2D: + self.scalarState = .number(.sign) + case 0x30: + self.scalarState = .number(.zero) + case 0x31...0x39: + self.scalarState = .number(.integer) + default: + self.scalarState = .invalid + } + } + + private mutating func appendScalar(_ byte: UInt8) -> Bool { switch self.scalarState { case let .trueLiteral(matched): self.scalarState = self.advanceLiteral(byte, expected: Self.trueLiteral, matched: matched) .map(ScalarState.trueLiteral) ?? .invalid - return + return true case let .falseLiteral(matched): self.scalarState = self.advanceLiteral(byte, expected: Self.falseLiteral, matched: matched) .map(ScalarState.falseLiteral) ?? .invalid - return + return true case let .nullLiteral(matched): self.scalarState = self.advanceLiteral(byte, expected: Self.nullLiteral, matched: matched) .map(ScalarState.nullLiteral) ?? .invalid - return + return true case let .number(state): self.scalarState = .number(state.appending(byte)) - return + return true case .invalid: - return + return true case .notScalar: - break + return false } + } + private mutating func appendContainer(_ byte: UInt8) { if self.insideString { if self.escaping { self.escaping = false @@ -230,7 +215,7 @@ enum CostUsageJsonl { } private static func isWhitespace(_ byte: UInt8) -> Bool { - byte == 0x20 || byte == 0x09 || byte == 0x0D + byte == 0x20 || byte == 0x09 || byte == 0x0A || byte == 0x0D } } From 1f2d965ff0f425fb638ae09c22ddfb5cfc95ea38 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 16 Jul 2026 20:06:05 +0100 Subject: [PATCH 10/11] fix: retain extendable numeric JSONL tails --- .../Generated/CodexParserHash.generated.swift | 2 +- .../Vendored/CostUsage/CostUsageJsonl.swift | 17 +-- .../CostUsageJsonlScannerTests.swift | 105 ++++++++++++++++++ 3 files changed, 115 insertions(+), 9 deletions(-) diff --git a/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift b/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift index e1f4ce1780..3802864e25 100644 --- a/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift +++ b/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift @@ -1,5 +1,5 @@ // Generated by Scripts/regenerate-codex-parser-hash.sh. Do not edit by hand. enum CodexParserHash { - static let value = "bbbb9cc285c67c8f" + static let value = "577571954a2d36bc" } diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift index 3ecad42d73..fd17555f81 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift @@ -50,11 +50,12 @@ enum CostUsageJsonl { case finished case invalid - var isComplete: Bool { + var canCommitAtEOF: Bool { switch self { - case .zero, .integer, .fraction, .exponentDigits, .finished, .invalid: + case .finished, .invalid: true - case .sign, .decimalPoint, .exponentMarker, .exponentSign: + case .sign, .zero, .integer, .decimalPoint, .fraction, + .exponentMarker, .exponentSign, .exponentDigits: false } } @@ -112,7 +113,7 @@ enum CostUsageJsonl { case let .nullLiteral(matched): return matched == Self.nullLiteral.count case let .number(state): - return state.isComplete + return state.canCommitAtEOF case .invalid: return true } @@ -288,11 +289,11 @@ enum CostUsageJsonl { } func hasCompleteJSONTail() -> Bool { + guard jsonTailState.isStructurallyComplete else { return false } if truncated { - // The full record is intentionally not retained. Its structural state is enough - // to keep a still-open object or string retriable without changing the old - // behavior for complete records that exceed the safety limit. - return jsonTailState.isStructurallyComplete + // The full record is intentionally not retained. Its incremental state is enough + // to keep incomplete containers, strings, literals, and numbers retriable. + return true } guard lineBytes == current.count else { return false } return (try? JSONSerialization.jsonObject(with: current, options: [.fragmentsAllowed])) != nil diff --git a/Tests/CodexBarTests/CostUsageJsonlScannerTests.swift b/Tests/CodexBarTests/CostUsageJsonlScannerTests.swift index b221f53c95..475b275fa8 100644 --- a/Tests/CodexBarTests/CostUsageJsonlScannerTests.swift +++ b/Tests/CodexBarTests/CostUsageJsonlScannerTests.swift @@ -427,6 +427,111 @@ struct CostUsageJsonlScannerTests { #expect(endOffset == Int64(Data((initial + completion).utf8).count)) } + @Test + func `jsonl scanner retries a complete numeric prefix`() throws { + let root = try self.makeTemporaryRoot() + defer { try? FileManager.default.removeItem(at: root) } + + let fileURL = root.appendingPathComponent("numeric-prefix.jsonl", isDirectory: false) + let initial = "1" + try initial.write(to: fileURL, atomically: true, encoding: .utf8) + + var firstPass: [String?] = [] + let resumeOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + maxLineBytes: 64, + prefixBytes: 64) + { line in + firstPass.append(String(bytes: line.bytes, encoding: .utf8)) + } + + #expect(firstPass.isEmpty) + #expect(resumeOffset == 0) + + let completion = "2\n" + let handle = try FileHandle(forWritingTo: fileURL) + defer { try? handle.close() } + try handle.seekToEnd() + try handle.write(contentsOf: Data(completion.utf8)) + + var secondPass: [String?] = [] + let endOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + offset: resumeOffset, + maxLineBytes: 64, + prefixBytes: 64) + { line in + secondPass.append(String(bytes: line.bytes, encoding: .utf8)) + } + + #expect(secondPass == ["12"]) + #expect(endOffset == Int64(Data((initial + completion).utf8).count)) + } + + @Test + func `jsonl scanner retries a truncated complete numeric prefix`() throws { + let root = try self.makeTemporaryRoot() + defer { try? FileManager.default.removeItem(at: root) } + + let fileURL = root.appendingPathComponent("truncated-numeric-prefix.jsonl", isDirectory: false) + let initial = String(repeating: " ", count: 128) + "1" + try initial.write(to: fileURL, atomically: true, encoding: .utf8) + + var firstPass: [CostUsageJsonl.Line] = [] + let resumeOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + maxLineBytes: 64, + prefixBytes: 64) + { line in + firstPass.append(line) + } + + #expect(firstPass.isEmpty) + #expect(resumeOffset == 0) + + let completion = "2\n" + let handle = try FileHandle(forWritingTo: fileURL) + defer { try? handle.close() } + try handle.seekToEnd() + try handle.write(contentsOf: Data(completion.utf8)) + + var secondPass: [CostUsageJsonl.Line] = [] + let endOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + offset: resumeOffset, + maxLineBytes: 64, + prefixBytes: 64) + { line in + secondPass.append(line) + } + + #expect(secondPass.count == 1) + #expect(secondPass[0].wasTruncated) + #expect(endOffset == Int64(Data((initial + completion).utf8).count)) + } + + @Test + func `jsonl scanner accepts a number terminated by trailing whitespace`() throws { + let root = try self.makeTemporaryRoot() + defer { try? FileManager.default.removeItem(at: root) } + + let fileURL = root.appendingPathComponent("terminated-number.jsonl", isDirectory: false) + let record = "12 " + try record.write(to: fileURL, atomically: true, encoding: .utf8) + + var scanned: [String?] = [] + let endOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + maxLineBytes: 64, + prefixBytes: 64) + { line in + scanned.append(String(bytes: line.bytes, encoding: .utf8)) + } + + #expect(scanned == [record]) + #expect(endOffset == Int64(Data(record.utf8).count)) + } + private func makeTemporaryRoot() throws -> URL { let root = FileManager.default.temporaryDirectory.appendingPathComponent( "codexbar-cost-usage-jsonl-\(UUID().uuidString)", From 5788bd726d426ff07fe8c777bbff47b4219321f5 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 16 Jul 2026 20:17:57 +0100 Subject: [PATCH 11/11] test: prove short JSONL prefix commits --- .../CostUsageJsonlScannerTests.swift | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/Tests/CodexBarTests/CostUsageJsonlScannerTests.swift b/Tests/CodexBarTests/CostUsageJsonlScannerTests.swift index 475b275fa8..926b623b02 100644 --- a/Tests/CodexBarTests/CostUsageJsonlScannerTests.swift +++ b/Tests/CodexBarTests/CostUsageJsonlScannerTests.swift @@ -532,6 +532,30 @@ struct CostUsageJsonlScannerTests { #expect(endOffset == Int64(Data(record.utf8).count)) } + @Test + func `jsonl scanner commits complete EOF record larger than retained prefix`() throws { + let root = try self.makeTemporaryRoot() + defer { try? FileManager.default.removeItem(at: root) } + + let fileURL = root.appendingPathComponent("short-prefix.jsonl", isDirectory: false) + let record = #"{"message":"\#(String(repeating: "x", count: 128))"}"# + try record.write(to: fileURL, atomically: true, encoding: .utf8) + + var scanned: [CostUsageJsonl.Line] = [] + let endOffset = try CostUsageJsonl.scan( + fileURL: fileURL, + maxLineBytes: 1024, + prefixBytes: 64) + { line in + scanned.append(line) + } + + #expect(scanned.count == 1) + #expect(scanned[0].wasTruncated) + #expect(scanned[0].bytes.count == 64) + #expect(endOffset == Int64(Data(record.utf8).count)) + } + private func makeTemporaryRoot() throws -> URL { let root = FileManager.default.temporaryDirectory.appendingPathComponent( "codexbar-cost-usage-jsonl-\(UUID().uuidString)",