diff --git a/CHANGELOG.md b/CHANGELOG.md index 28a171b83d..7d8c1ad471 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ - Amp: parse bold Markdown usage labels from the latest CLI while preserving compatibility with older plain-text output (#3171). - OpenCode Go: expose a Monthly % lane token for custom menu bar layouts once the monthly window has been observed (#3175). +### Performance +- OpenCodex: parse only newly appended usage-log entries instead of rebuilding the entire cache on every refresh, cutting memory use while preserving compatibility with older app versions (#3140). Thanks @olddonkey! + ### Localization - Turkish: improve translations throughout settings and provider views, including the previously untranslated iCloud sync section (#3178). Thanks @husodrn46! diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageStore+CodexCache.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageStore+CodexCache.swift index 2957fd1bcb..074a0c2b79 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageStore+CodexCache.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageStore+CodexCache.swift @@ -86,7 +86,11 @@ extension CostUsageStore { requestedUntilDay: budgetProtectionWindow.untilKey, calendar: calendar) guard !result.catchUpRequired else { return result } - Self.identicalContentPreLockCheckpointForTesting?() + if let checkpoint = Self.identicalContentPreLockCheckpointForTesting, + checkpoint.databaseURL == self.databaseURL + { + checkpoint.checkpoint() + } guard self.beginSaveTransaction() else { var retry = result retry.catchUpRequired = true diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageStore.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageStore.swift index f639df7e72..7258adcaca 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageStore.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageStore.swift @@ -90,8 +90,10 @@ actor CostUsageStore { /// persisted file with the running count, so a crash-safety harness can SIGKILL the /// process at a deterministic mid-save point. Never set in production. nonisolated(unsafe) static var saveCycleCheckpointForTesting: ((Int) -> Void)? - /// Test-only interleaving point after optimistic identity succeeds and before its writer lock. - nonisolated(unsafe) static var identicalContentPreLockCheckpointForTesting: (() -> Void)? + /// Test-only interleaving point scoped to one database so parallel store fixtures stay isolated. + nonisolated(unsafe) static var identicalContentPreLockCheckpointForTesting: ( + databaseURL: URL, + checkpoint: () -> Void)? /// Test-only traversal proof for persisted Codex catch-up reconciliation. Never set in production. nonisolated(unsafe) static var codexCatchUpReconciliationVisitForTesting: (() -> Void)? diff --git a/Sources/CodexBarCore/Vendored/OpenCodexUsage/OpenCodexUsageParser.swift b/Sources/CodexBarCore/Vendored/OpenCodexUsage/OpenCodexUsageParser.swift index cff9b00742..607c8605ea 100644 --- a/Sources/CodexBarCore/Vendored/OpenCodexUsage/OpenCodexUsageParser.swift +++ b/Sources/CodexBarCore/Vendored/OpenCodexUsage/OpenCodexUsageParser.swift @@ -1,6 +1,56 @@ +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#elseif canImport(Musl) +import Musl +#endif +#if canImport(CryptoKit) +import CryptoKit +#else +import Crypto +#endif import Foundation public enum OpenCodexUsageParser { + private static let newline: UInt8 = 0x0A + /// Must match `OpenCodexUsageStore`'s prefix-digest window. + private static let prefixDigestByteLimit = 64 * 1024 + + struct ChangedUnderReadError: Error, Equatable { + let path: String + } + + @TaskLocal static var logReadRecorderForTesting: LogReadRecorder? + + final class LogReadRecorder: @unchecked Sendable { + private let lock = NSLock() + private var bytesRead: Int64 = 0 + private var completeLines = 0 + + func record(bytes: Int64, lines: Int) { + self.lock.lock() + self.bytesRead += bytes + self.completeLines += lines + self.lock.unlock() + } + + func snapshot() -> (bytesRead: Int64, completeLines: Int) { + self.lock.lock() + defer { self.lock.unlock() } + return (self.bytesRead, self.completeLines) + } + } + + static func withLogReadRecorderForTesting( + _ recorder: LogReadRecorder, + operation: () throws -> T) rethrows -> T + { + try self.$logReadRecorderForTesting.withValue(recorder) { + try operation() + } + } + public static func parseLine(_ line: String) -> OpenCodexUsageEntry? { let trimmed = line.trimmingCharacters(in: .whitespacesAndNewlines) guard !trimmed.isEmpty, let data = trimmed.data(using: .utf8) else { return nil } @@ -15,14 +65,191 @@ public enum OpenCodexUsageParser { } public static func parseLines(_ text: String) -> [OpenCodexUsageEntry] { - text.split(whereSeparator: \.isNewline).compactMap { self.parseLine(String($0)) } + self.parseJSONL(Data(text.utf8), baseOffset: 0).entries } public static func parse(fileURL: URL, fileManager: FileManager = .default) throws -> [OpenCodexUsageEntry] { - guard fileManager.fileExists(atPath: fileURL.path) else { return [] } - let data = try Data(contentsOf: fileURL) - guard let text = String(data: data, encoding: .utf8) else { return [] } - return self.parseLines(text) + try self.parseLog(fileURL: fileURL, from: 0, fileManager: fileManager).entries + } + + public static func parse( + fileURL: URL, + from offset: Int64, + fileManager: FileManager = .default) throws -> (entries: [OpenCodexUsageEntry], nextOffset: Int64) + { + let parsed = try self.parseLog(fileURL: fileURL, from: offset, fileManager: fileManager) + return (parsed.entries, parsed.nextOffset) + } + + static func parseLog( + fileURL: URL, + from offset: Int64, + fileManager _: FileManager) throws -> JSONLParseResult + { + let handle: FileHandle + do { + handle = try FileHandle(forReadingFrom: fileURL) + } catch { + if Self.isConfirmedAbsence(error) { + return JSONLParseResult( + entries: [], + nextOffset: max(0, offset), + bytesRead: 0, + completeLineCount: 0, + newlineTerminatedEntryCount: 0) + } + throw error + } + defer { try? handle.close() } + + var status = stat() + guard fstat(handle.fileDescriptor, &status) == 0 else { + throw Self.posixError(errno, path: fileURL.path) + } + let fileIdentity = "\(status.st_dev):\(status.st_ino)" + let size = Int64(status.st_size) + let startOffset = max(0, offset) + if startOffset > size { + throw ChangedUnderReadError(path: fileURL.path) + } + + // Hold the snapshotted bytes instead of `.mappedIfSafe`. A mapping can SIGBUS if the + // file is truncated before those pages are touched; Swift cannot catch that. Full parse + // holds the file (~42 MB on a heavy machine) and only runs on a cold cache or rebuild. + // The steady-state path reads only the appended tail (`parsedOffset.. JSONLParseResult { + var entries: [OpenCodexUsageEntry] = [] + var completeLineCount = 0 + var lineStart = data.startIndex + var lastCompleteEnd = data.startIndex + for newlineOffset in self.newlineOffsets(in: data) { + let newlineIndex = data.index(data.startIndex, offsetBy: newlineOffset) + if let entry = self.parseLineData(data[lineStart.. [Int] { + #if canImport(Darwin) || canImport(Glibc) || canImport(Musl) + let scanned: [Int]? = data.withUnsafeBytes { rawBuffer in + guard let baseAddress = rawBuffer.baseAddress else { return nil } + let count = rawBuffer.count + var offsets: [Int] = [] + offsets.reserveCapacity(max(1, count / 64)) + var searchStart = 0 + while searchStart < count { + guard let found = memchr( + baseAddress.advanced(by: searchStart), + Int32(Self.newline), + count - searchStart) + else { + break + } + let newlineOffset = baseAddress.distance(to: UnsafeRawPointer(found)) + offsets.append(newlineOffset) + searchStart = newlineOffset + 1 + } + return offsets + } + if let scanned { + return scanned + } + #endif + var offsets: [Int] = [] + var index = data.startIndex + while index < data.endIndex { + if data[index] == Self.newline { + offsets.append(data.distance(from: data.startIndex, to: index)) + } + index = data.index(after: index) + } + return offsets + } + + private static func parseLineData(_ line: Data) -> OpenCodexUsageEntry? { + guard let text = String(data: line, encoding: .utf8) else { return nil } + return self.parseLine(text) } private static func parse(_ object: [String: Any]) -> OpenCodexUsageEntry? { @@ -125,4 +352,62 @@ public enum OpenCodexUsageParser { } return nil } + + private static func prefixDigest( + handle: FileHandle, + fileData: Data?, + nextOffset: Int64, + path: String) throws -> String + { + let length = min(Int64(Self.prefixDigestByteLimit), max(0, nextOffset)) + let prefix: Data + if length == 0 { + prefix = Data() + } else if let fileData, Int64(fileData.count) >= length { + prefix = Data(fileData.prefix(Int(length))) + } else { + try handle.seek(toOffset: 0) + prefix = try Self.readExact(handle, byteCount: length, path: path) + } + return SHA256.hash(data: prefix).map { String(format: "%02x", $0) }.joined() + } + + private static func readExact(_ handle: FileHandle, byteCount: Int64, path: String) throws -> Data { + let count = Int(byteCount) + var data = Data() + data.reserveCapacity(count) + while data.count < count { + let chunk = try handle.read(upToCount: count - data.count) ?? Data() + if chunk.isEmpty { + throw ChangedUnderReadError(path: path) + } + data.append(chunk) + } + return data + } + + private static func isConfirmedAbsence(_ error: Error) -> Bool { + var current: Error? = error + while let err = current { + let nsError = err as NSError + if nsError.domain == NSPOSIXErrorDomain { + return nsError.code == Int(ENOENT) || nsError.code == Int(ENOTDIR) + } + if nsError.domain == NSCocoaErrorDomain, + nsError.code == CocoaError.fileReadNoSuchFile.rawValue + || nsError.code == CocoaError.fileNoSuchFile.rawValue + { + return true + } + current = nsError.userInfo[NSUnderlyingErrorKey] as? Error + } + return false + } + + private static func posixError(_ code: Int32, path: String) -> NSError { + NSError( + domain: NSPOSIXErrorDomain, + code: Int(code), + userInfo: [NSFilePathErrorKey: path]) + } } diff --git a/Sources/CodexBarCore/Vendored/OpenCodexUsage/OpenCodexUsageStore.swift b/Sources/CodexBarCore/Vendored/OpenCodexUsage/OpenCodexUsageStore.swift index 040b76d5d4..9fae10b9b2 100644 --- a/Sources/CodexBarCore/Vendored/OpenCodexUsage/OpenCodexUsageStore.swift +++ b/Sources/CodexBarCore/Vendored/OpenCodexUsage/OpenCodexUsageStore.swift @@ -3,12 +3,28 @@ import SQLite3 #elseif canImport(CSQLite3) import CSQLite3 #endif +#if canImport(CryptoKit) +import CryptoKit +#else +import Crypto +#endif +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#elseif canImport(Musl) +import Musl +#endif import Foundation /// Independent OpenCodex usage cache. Never writes Codex `cost-usage.sqlite`. public struct OpenCodexUsageStore: Sendable { - public static let databaseFilename = "opencodex-usage.sqlite" - private static let schemaVersion = 1 + /// Schema v2 lives in a versioned filename so a v1 build keeps using `opencodex-usage.sqlite`. + /// Leave that older file alone; do not delete it. + public static let databaseFilename = "opencodex-usage-v2.sqlite" + private static let schemaVersion = 2 + private static let cursorMetaKey = "parseCursor" + private static let prefixDigestByteLimit = 64 * 1024 private let databaseURL: URL @@ -16,6 +32,29 @@ public struct OpenCodexUsageStore: Sendable { self.databaseURL = cacheRoot.appendingPathComponent(Self.databaseFilename, isDirectory: false) } + /// Test-only. Unset in production; the optional call in `incrementalReload` is a no-op. + @TaskLocal private static var incrementalPostParseHookForTesting: (@Sendable () -> Void)? + + static func withLogReadRecorderForTesting( + _ recorder: OpenCodexUsageParser.LogReadRecorder, + operation: () throws -> T) rethrows -> T + { + try OpenCodexUsageParser.withLogReadRecorderForTesting(recorder, operation: operation) + } + + static func withIncrementalPostParseHookForTesting( + _ hook: @escaping @Sendable () -> Void, + operation: () throws -> T) rethrows -> T + { + try self.$incrementalPostParseHookForTesting.withValue(hook) { + try operation() + } + } + + static func incrementalPostParseHookInstalledForTesting() -> Bool { + self.incrementalPostParseHookForTesting != nil + } + public func loadSnapshot( logURL: URL, now: Date, @@ -34,91 +73,267 @@ public struct OpenCodexUsageStore: Sendable { } public func loadEntries(logURL: URL, fileManager: FileManager = .default) throws -> [OpenCodexUsageEntry] { - guard fileManager.fileExists(atPath: logURL.path) else { return [] } - let attributes = try fileManager.attributesOfItem(atPath: logURL.path) - let size = (attributes[.size] as? NSNumber)?.int64Value ?? 0 - let mtime = (attributes[.modificationDate] as? Date)?.timeIntervalSince1970 ?? 0 - let identity = "\(logURL.path)|\(size)|\(mtime)" - - if let cached = self.readCachedEntries(identity: identity), !cached.isEmpty { - return cached + var shouldRetryStaleWrite = true + while true { + guard let identity = try Self.statLog(at: logURL) else { return [] } + if let cached = self.readCachedState(), Self.canReuseCursor(cached.parseCursor, identity: identity) { + if identity.size == cached.parseCursor.parsedOffset { + return cached.entries + } + if let entries = try self.incrementalReload( + logURL: logURL, + identity: identity, + cursor: cached.parseCursor, + existing: cached.entries, + fileManager: fileManager) + { + return entries + } + if shouldRetryStaleWrite { + // Another loader changed the durable cursor. Re-run the cursor path once + // against that durable state, then fall back to a full reload. + shouldRetryStaleWrite = false + continue + } + return try self.fullReload(logURL: logURL, identity: identity, fileManager: fileManager) + } + return try self.fullReload(logURL: logURL, identity: identity, fileManager: fileManager) } + } - let parsed = try OpenCodexUsageParser.parse(fileURL: logURL, fileManager: fileManager) - var unique: [String: OpenCodexUsageEntry] = [:] - for entry in parsed { - unique[entry.requestID] = entry + func parseCursorForTesting() -> ( + parsedOffset: Int64, + prefixDigest: String, + fileIdentity: String, + path: String)? + { + guard let cursor = self.readCursor() else { return nil } + return (cursor.parsedOffset, cursor.prefixDigest, cursor.fileIdentity, cursor.path) + } + + func writeIncrementalEntriesForTesting( + _ entries: [OpenCodexUsageEntry], + path: String, + fileIdentity: String, + parsedOffset: Int64, + prefixDigest: String) + { + let cursor = ParseCursor( + path: path, + fileIdentity: fileIdentity, + parsedOffset: parsedOffset, + prefixDigest: prefixDigest) + _ = self.writeEntries(entries, cursor: cursor, replaceAll: false, baseCursor: cursor) + } + + private func fullReload( + logURL: URL, + identity: LogIdentity, + fileManager: FileManager, + allowRetry: Bool = true) throws -> [OpenCodexUsageEntry] + { + let parsed: OpenCodexUsageParser.JSONLParseResult + do { + parsed = try OpenCodexUsageParser.parseLog(fileURL: logURL, from: 0, fileManager: fileManager) + } catch { + if error is OpenCodexUsageParser.ChangedUnderReadError, allowRetry { + guard let current = try Self.statLog(at: logURL) else { return [] } + return try self.fullReload( + logURL: logURL, + identity: current, + fileManager: fileManager, + allowRetry: false) + } + throw error } - let deduped = unique.values.sorted { - if $0.timestamp != $1.timestamp { - return $0.timestamp < $1.timestamp + guard let parsedIdentity = parsed.fileIdentity else { return [] } + let pathIdentity: LogIdentity? + do { + pathIdentity = try Self.statLog(at: logURL) + } catch { + if allowRetry { + guard let current = try Self.statLog(at: logURL) else { return [] } + return try self.fullReload( + logURL: logURL, + identity: current, + fileManager: fileManager, + allowRetry: false) } - return $0.requestID < $1.requestID + throw error + } + guard let pathIdentity else { return [] } + if pathIdentity.fileIdentity != parsedIdentity { + if allowRetry { + return try self.fullReload( + logURL: logURL, + identity: pathIdentity, + fileManager: fileManager, + allowRetry: false) + } + return Self.dedupedAndSorted(parsed.entries) + } + let entries = Self.dedupedAndSorted(parsed.entries) + let cursor = ParseCursor( + path: identity.path, + fileIdentity: parsedIdentity, + parsedOffset: parsed.nextOffset, + prefixDigest: parsed.prefixDigest) + self.replaceCachedEntries(Self.dedupedAndSorted(parsed.newlineTerminatedEntries), cursor: cursor) + return entries + } + + /// Returns `nil` when the incremental write observed a durable cursor that is not `cursor`. + private func incrementalReload( + logURL: URL, + identity: LogIdentity, + cursor: ParseCursor, + existing: [OpenCodexUsageEntry], + fileManager: FileManager) throws -> [OpenCodexUsageEntry]? + { + let parsed: OpenCodexUsageParser.JSONLParseResult + do { + parsed = try OpenCodexUsageParser.parseLog( + fileURL: logURL, + from: cursor.parsedOffset, + fileManager: fileManager) + } catch { + if error is OpenCodexUsageParser.ChangedUnderReadError { + return try self.fullReload(logURL: logURL, identity: identity, fileManager: fileManager) + } + throw error + } + Self.incrementalPostParseHookForTesting?() + // Closes the TOCTOU window between the pre-read `canReuseCursor` check and this tail + // parse: a rotation or replacement in that window would otherwise merge cached rows from + // the old file with bytes from the new one and persist a cursor for a path that no longer + // names that file. A replacement that preserves path, st_dev, st_ino, size, AND the first + // min(64 KiB, parsedOffset) bytes remains undetectable. `parsed.fileIdentity` is the + // descriptor we actually read. + guard let parsedIdentity = parsed.fileIdentity else { return [] } + guard let postIdentity = try Self.statLog(at: logURL) else { return [] } + if parsedIdentity != identity.fileIdentity + || !Self.isSameLogAfterTailRead(preRead: identity, postRead: postIdentity, cursor: cursor) + { + return try self.fullReload(logURL: logURL, identity: postIdentity, fileManager: fileManager) + } + let nextOffset = parsed.nextOffset + let committed = parsed.newlineTerminatedEntries + let pending = parsed.pendingTrailingEntries + if committed.isEmpty, nextOffset == cursor.parsedOffset { + return Self.dedupedAndSorted(existing + pending) + } + let digest = nextOffset == cursor.parsedOffset ? cursor.prefixDigest : parsed.prefixDigest + if self.insertCachedEntries( + committed, + baseCursor: cursor, + cursor: ParseCursor( + path: identity.path, + fileIdentity: parsedIdentity, + parsedOffset: nextOffset, + prefixDigest: digest)) == .stale + { + return nil } - self.replaceCachedEntries(deduped, identity: identity) - return deduped + return Self.dedupedAndSorted(existing + committed + pending) } - private func readCachedEntries(identity: String) -> [OpenCodexUsageEntry]? { + private func readCachedState() -> (parseCursor: ParseCursor, entries: [OpenCodexUsageEntry])? { guard let db = self.open(readOnly: true) else { return nil } defer { sqlite3_close(db) } - guard Self.userVersion(db) == Self.schemaVersion, - Self.meta(db, key: "identity") == identity + guard Self.userVersion(db) == Self.schemaVersion else { return nil } + guard sqlite3_exec(db, "BEGIN", nil, nil, nil) == SQLITE_OK else { return nil } + defer { _ = sqlite3_exec(db, "ROLLBACK", nil, nil, nil) } + guard let cursor = Self.parseCursor(from: db), + let entries = Self.readEntries(from: db) else { return nil } + return (parseCursor: cursor, entries: entries) + } + + private static func readEntries(from db: OpaquePointer?) -> [OpenCodexUsageEntry]? { var statement: OpaquePointer? let sql = """ - SELECT request_id, timestamp, provider, model, usage_status, account_label, surface, conversation_id, payload + SELECT request_id, timestamp, provider, model, usage_status, account_label, surface, conversation_id, \ + input_tokens, output_tokens, cached_input_tokens, cache_read_input_tokens, \ + cache_creation_input_tokens, reasoning_output_tokens, usage_total_tokens, total_tokens FROM entries """ guard sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK else { return nil } defer { sqlite3_finalize(statement) } var entries: [OpenCodexUsageEntry] = [] - while sqlite3_step(statement) == SQLITE_ROW { - guard let payload = Self.text(statement, 8), - let data = payload.data(using: .utf8), - let entry = OpenCodexUsageParser.parse(data) - else { continue } - entries.append(entry) + var step = sqlite3_step(statement) + while step == SQLITE_ROW { + if let requestID = Self.text(statement, 0), + let provider = Self.text(statement, 2), + let model = Self.text(statement, 3), + let statusRaw = Self.text(statement, 4) + { + let usage = Self.tokenUsage( + inputTokens: Self.int(statement, 8), + outputTokens: Self.int(statement, 9), + cachedInputTokens: Self.int(statement, 10), + cacheReadInputTokens: Self.int(statement, 11), + cacheCreationInputTokens: Self.int(statement, 12), + reasoningOutputTokens: Self.int(statement, 13), + totalTokens: Self.int(statement, 14)) + entries.append(OpenCodexUsageEntry( + requestID: requestID, + timestamp: Date(timeIntervalSince1970: sqlite3_column_double(statement, 1)), + provider: provider, + model: model, + usageStatus: OpenCodexUsageStatus(rawValue: statusRaw) ?? .unreported, + accountLogLabel: Self.text(statement, 5), + surface: Self.text(statement, 6), + conversationID: Self.text(statement, 7), + usage: usage, + totalTokens: Self.int(statement, 15))) + } + step = sqlite3_step(statement) } - return entries + guard step == SQLITE_DONE else { return nil } + return Self.dedupedAndSorted(entries) } - private func replaceCachedEntries(_ entries: [OpenCodexUsageEntry], identity: String) { - guard let db = self.open(readOnly: false) else { return } + private func replaceCachedEntries(_ entries: [OpenCodexUsageEntry], cursor: ParseCursor) { + self.writeEntries(entries, cursor: cursor, replaceAll: true) + } + + private func insertCachedEntries( + _ entries: [OpenCodexUsageEntry], + baseCursor: ParseCursor, + cursor: ParseCursor) -> CachedWriteResult + { + self.writeEntries(entries, cursor: cursor, replaceAll: false, baseCursor: baseCursor) + } + + @discardableResult + private func writeEntries( + _ entries: [OpenCodexUsageEntry], + cursor: ParseCursor, + replaceAll: Bool, + baseCursor: ParseCursor? = nil) -> CachedWriteResult + { + guard let db = self.open(readOnly: false) else { return .applied } defer { sqlite3_close(db) } - _ = sqlite3_exec(db, "BEGIN IMMEDIATE", nil, nil, nil) - _ = sqlite3_exec(db, "DELETE FROM entries", nil, nil, nil) - Self.setMeta(db, key: "identity", value: identity) - var statement: OpaquePointer? - let sql = """ - INSERT OR REPLACE INTO entries( - request_id, timestamp, provider, model, usage_status, account_label, surface, conversation_id, payload - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) - """ - guard sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK else { + guard sqlite3_exec(db, "BEGIN IMMEDIATE", nil, nil, nil) == SQLITE_OK else { return .applied } + // Incremental appends only (`replaceAll == false`): after BEGIN IMMEDIATE, re-read the + // durable cursor. Reject the write whenever it differs from the exact base cursor this + // parse was derived from — another writer replaced the cache, including a different + // home/path. Full reloads re-derived the whole file and must still replace even a newer + // cursor (truncation, rotation, schema rebuild). + if !replaceAll, Self.isStaleIncrementalCursor(base: baseCursor, durable: Self.parseCursor(from: db)) { _ = sqlite3_exec(db, "ROLLBACK", nil, nil, nil) - return + return .stale } - defer { sqlite3_finalize(statement) } - for entry in entries { - sqlite3_reset(statement) - sqlite3_clear_bindings(statement) - Self.bind(statement, 1, entry.requestID) - sqlite3_bind_double(statement, 2, entry.timestamp.timeIntervalSince1970) - Self.bind(statement, 3, entry.provider) - Self.bind(statement, 4, entry.model) - Self.bind(statement, 5, entry.usageStatus.rawValue) - Self.bind(statement, 6, entry.accountLogLabel) - Self.bind(statement, 7, entry.surface) - Self.bind(statement, 8, entry.conversationID) - let payload = Self.payloadJSON(entry) - Self.bind(statement, 9, payload) - guard sqlite3_step(statement) == SQLITE_DONE else { - _ = sqlite3_exec(db, "ROLLBACK", nil, nil, nil) - return - } + if replaceAll { + _ = sqlite3_exec(db, "DELETE FROM entries", nil, nil, nil) + } + Self.setCursor(db, cursor) + guard Self.insertEntries(db, entries) else { + _ = sqlite3_exec(db, "ROLLBACK", nil, nil, nil) + return .applied } _ = sqlite3_exec(db, "COMMIT", nil, nil, nil) + return .applied } private func open(readOnly: Bool) -> OpaquePointer? { @@ -144,14 +359,59 @@ public struct OpenCodexUsageStore: Sendable { return db } + private func readCursor() -> ParseCursor? { + guard let db = self.open(readOnly: true) else { return nil } + defer { sqlite3_close(db) } + guard Self.userVersion(db) == Self.schemaVersion else { return nil } + return Self.parseCursor(from: db) + } + + private static func parseCursor(from db: OpaquePointer?) -> ParseCursor? { + guard let raw = self.meta(db, key: cursorMetaKey), + let data = raw.data(using: .utf8) + else { return nil } + return try? JSONDecoder().decode(ParseCursor.self, from: data) + } + + private static func isStaleIncrementalCursor(base: ParseCursor?, durable: ParseCursor?) -> Bool { + durable != base + } + + /// Threat model: the digest covers only the first min(64 KiB, parsedOffset) bytes, so an + /// in-place rewrite past 64 KiB that preserves size and inode is not detected. Rotation + /// changes the inode and truncation trips `size >= parsedOffset`. Acceptable because the log + /// is append-only. + private static func canReuseCursor(_ cursor: ParseCursor, identity: LogIdentity) -> Bool { + cursor.path == identity.path + && cursor.fileIdentity == identity.fileIdentity + && identity.size >= cursor.parsedOffset + && self.prefixDigest(fileURL: identity.url, parsedOffset: cursor.parsedOffset) == cursor.prefixDigest + } + + private static func isSameLogAfterTailRead( + preRead: LogIdentity, + postRead: LogIdentity, + cursor: ParseCursor) -> Bool + { + postRead.path == preRead.path + && postRead.fileIdentity == preRead.fileIdentity + && self.canReuseCursor(cursor, identity: postRead) + } + private static func ensureSchema(_ db: OpaquePointer?) { - guard self.userVersion(db) == 0 else { return } + // Schema v2 lives in `opencodex-usage-v2.sqlite` so a v1 build keeps using + // `opencodex-usage.sqlite`. Never delete that older file. + // `!= schemaVersion` still rebuilds THIS file if the version is wrong (corrupt header, + // leftover user_version 0, or a copied v1 payload). + guard self.userVersion(db) != self.schemaVersion else { return } let sql = """ - CREATE TABLE IF NOT EXISTS meta ( + DROP TABLE IF EXISTS entries; + DROP TABLE IF EXISTS meta; + CREATE TABLE meta ( key TEXT PRIMARY KEY, value TEXT NOT NULL ); - CREATE TABLE IF NOT EXISTS entries ( + CREATE TABLE entries ( request_id TEXT PRIMARY KEY, timestamp REAL NOT NULL, provider TEXT NOT NULL, @@ -160,13 +420,55 @@ public struct OpenCodexUsageStore: Sendable { account_label TEXT, surface TEXT, conversation_id TEXT, - payload TEXT NOT NULL + input_tokens INTEGER, + output_tokens INTEGER, + cached_input_tokens INTEGER, + cache_read_input_tokens INTEGER, + cache_creation_input_tokens INTEGER, + reasoning_output_tokens INTEGER, + usage_total_tokens INTEGER, + total_tokens INTEGER ); """ guard sqlite3_exec(db, sql, nil, nil, nil) == SQLITE_OK else { return } Self.setUserVersion(db, Self.schemaVersion) } + private static func insertEntries(_ db: OpaquePointer?, _ entries: [OpenCodexUsageEntry]) -> Bool { + var statement: OpaquePointer? + let sql = """ + INSERT OR REPLACE INTO entries( + request_id, timestamp, provider, model, usage_status, account_label, surface, conversation_id, + input_tokens, output_tokens, cached_input_tokens, cache_read_input_tokens, + cache_creation_input_tokens, reasoning_output_tokens, usage_total_tokens, total_tokens + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """ + guard sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK else { return false } + defer { sqlite3_finalize(statement) } + for entry in entries { + sqlite3_reset(statement) + sqlite3_clear_bindings(statement) + Self.bind(statement, 1, entry.requestID) + sqlite3_bind_double(statement, 2, entry.timestamp.timeIntervalSince1970) + Self.bind(statement, 3, entry.provider) + Self.bind(statement, 4, entry.model) + Self.bind(statement, 5, entry.usageStatus.rawValue) + Self.bind(statement, 6, entry.accountLogLabel) + Self.bind(statement, 7, entry.surface) + Self.bind(statement, 8, entry.conversationID) + Self.bind(statement, 9, entry.usage?.inputTokens) + Self.bind(statement, 10, entry.usage?.outputTokens) + Self.bind(statement, 11, entry.usage?.cachedInputTokens) + Self.bind(statement, 12, entry.usage?.cacheReadInputTokens) + Self.bind(statement, 13, entry.usage?.cacheCreationInputTokens) + Self.bind(statement, 14, entry.usage?.reasoningOutputTokens) + Self.bind(statement, 15, entry.usage?.totalTokens) + Self.bind(statement, 16, entry.totalTokens) + guard sqlite3_step(statement) == SQLITE_DONE else { return false } + } + return true + } + private static func userVersion(_ db: OpaquePointer?) -> Int { var statement: OpaquePointer? guard sqlite3_prepare_v2(db, "PRAGMA user_version", -1, &statement, nil) == SQLITE_OK else { return 0 } @@ -190,6 +492,13 @@ public struct OpenCodexUsageStore: Sendable { return Self.text(statement, 0) } + private static func setCursor(_ db: OpaquePointer?, _ cursor: ParseCursor) { + guard let data = try? JSONEncoder().encode(cursor), + let value = String(data: data, encoding: .utf8) + else { return } + Self.setMeta(db, key: Self.cursorMetaKey, value: value) + } + private static func setMeta(_ db: OpaquePointer?, key: String, value: String) { var statement: OpaquePointer? guard sqlite3_prepare_v2( @@ -213,59 +522,127 @@ public struct OpenCodexUsageStore: Sendable { sqlite3_bind_text(statement, index, value, -1, unsafeBitCast(-1, to: sqlite3_destructor_type.self)) } + private static func bind(_ statement: OpaquePointer?, _ index: Int32, _ value: Int?) { + guard let value else { + sqlite3_bind_null(statement, index) + return + } + sqlite3_bind_int64(statement, index, Int64(value)) + } + private static func text(_ statement: OpaquePointer?, _ index: Int32) -> String? { guard let pointer = sqlite3_column_text(statement, index) else { return nil } return String(cString: pointer) } - private static func payloadJSON(_ entry: OpenCodexUsageEntry) -> String { - var object: [String: Any] = [ - "requestId": entry.requestID, - "timestamp": entry.timestamp.timeIntervalSince1970 * 1000, - "provider": entry.provider, - "model": entry.model, - "usageStatus": entry.usageStatus.rawValue, - ] - if let accountLogLabel = entry.accountLogLabel { - object["accountLogLabel"] = accountLogLabel - } - if let surface = entry.surface { - object["surface"] = surface - } - if let conversationID = entry.conversationID { - object["conversationId"] = conversationID + private static func int(_ statement: OpaquePointer?, _ index: Int32) -> Int? { + guard sqlite3_column_type(statement, index) != SQLITE_NULL else { return nil } + return Int(sqlite3_column_int64(statement, index)) + } + + // Token fields are stored as independent nullable columns; keep the mapping explicit. + // swiftlint:disable:next function_parameter_count + private static func tokenUsage( + inputTokens: Int?, + outputTokens: Int?, + cachedInputTokens: Int?, + cacheReadInputTokens: Int?, + cacheCreationInputTokens: Int?, + reasoningOutputTokens: Int?, + totalTokens: Int?) -> OpenCodexTokenUsage? + { + if inputTokens == nil, + outputTokens == nil, + cachedInputTokens == nil, + cacheReadInputTokens == nil, + cacheCreationInputTokens == nil, + reasoningOutputTokens == nil, + totalTokens == nil + { + return nil } - if let totalTokens = entry.totalTokens { - object["totalTokens"] = totalTokens + return OpenCodexTokenUsage( + inputTokens: inputTokens, + outputTokens: outputTokens, + cachedInputTokens: cachedInputTokens, + cacheReadInputTokens: cacheReadInputTokens, + cacheCreationInputTokens: cacheCreationInputTokens, + reasoningOutputTokens: reasoningOutputTokens, + totalTokens: totalTokens) + } + + private static func dedupedAndSorted(_ entries: [OpenCodexUsageEntry]) -> [OpenCodexUsageEntry] { + var unique: [String: OpenCodexUsageEntry] = [:] + unique.reserveCapacity(entries.count) + for entry in entries { + unique[entry.requestID] = entry } - if let usage = entry.usage { - var usageObject: [String: Any] = [:] - if let inputTokens = usage.inputTokens { - usageObject["inputTokens"] = inputTokens - } - if let outputTokens = usage.outputTokens { - usageObject["outputTokens"] = outputTokens - } - if let cachedInputTokens = usage.cachedInputTokens { - usageObject["cachedInputTokens"] = cachedInputTokens - } - if let cacheReadInputTokens = usage.cacheReadInputTokens { - usageObject["cacheReadInputTokens"] = cacheReadInputTokens + return self.sortedEntries(Array(unique.values)) + } + + private static func sortedEntries(_ entries: [OpenCodexUsageEntry]) -> [OpenCodexUsageEntry] { + entries.sorted { + if $0.timestamp != $1.timestamp { + return $0.timestamp < $1.timestamp } - if let cacheCreationInputTokens = usage.cacheCreationInputTokens { - usageObject["cacheCreationInputTokens"] = cacheCreationInputTokens + return $0.requestID < $1.requestID + } + } + + private static func statLog(at url: URL) throws -> LogIdentity? { + try url.withUnsafeFileSystemRepresentation { pointer in + guard let pointer else { + throw POSIXError(.EINVAL) } - if let reasoningOutputTokens = usage.reasoningOutputTokens { - usageObject["reasoningOutputTokens"] = reasoningOutputTokens + var status = stat() + guard stat(pointer, &status) == 0 else { + let err = errno + if err == ENOENT || err == ENOTDIR { + return nil + } + throw POSIXError(POSIXErrorCode(rawValue: err) ?? .EIO) } - if let totalTokens = usage.totalTokens { - usageObject["totalTokens"] = totalTokens + return LogIdentity( + url: url, + path: url.path, + fileIdentity: "\(status.st_dev):\(status.st_ino)", + size: Int64(status.st_size)) + } + } + + /// Covers only the first min(64 KiB, parsedOffset) bytes; see `canReuseCursor` for the threat model. + private static func prefixDigest(fileURL: URL, parsedOffset: Int64) -> String? { + let length = min(Int64(Self.prefixDigestByteLimit), max(0, parsedOffset)) + let prefix: Data + if length == 0 { + prefix = Data() + } else { + guard let handle = try? FileHandle(forReadingFrom: fileURL) else { return nil } + defer { try? handle.close() } + guard let data = try? handle.read(upToCount: Int(length)), data.count == Int(length) else { + return nil } - object["usage"] = usageObject + prefix = data } - guard let data = try? JSONSerialization.data(withJSONObject: object, options: [.sortedKeys]), - let text = String(data: data, encoding: .utf8) - else { return "{}" } - return text + return SHA256.hash(data: prefix).map { String(format: "%02x", $0) }.joined() + } + + private enum CachedWriteResult: Equatable { + case applied + case stale + } + + private struct ParseCursor: Equatable, Sendable, Codable { + var path: String + var fileIdentity: String + var parsedOffset: Int64 + var prefixDigest: String + } + + private struct LogIdentity: Equatable, Sendable { + var url: URL + var path: String + var fileIdentity: String + var size: Int64 } } diff --git a/Tests/CodexBarTests/CostUsageStoreTests.swift b/Tests/CodexBarTests/CostUsageStoreTests.swift index 103f094112..e4f7e93682 100644 --- a/Tests/CodexBarTests/CostUsageStoreTests.swift +++ b/Tests/CodexBarTests/CostUsageStoreTests.swift @@ -594,13 +594,13 @@ extension CostUsageStoreTests { reread.lastScanUnixMs = 2000 let interloper = try SQLiteTestConnection(url: store.databaseURL) var checkpointError: Error? - CostUsageStore.identicalContentPreLockCheckpointForTesting = { + CostUsageStore.identicalContentPreLockCheckpointForTesting = (store.databaseURL, { do { try interloper.execute("UPDATE files SET parsed_bytes = 999 WHERE path = '\(path)'") } catch { checkpointError = error } - } + }) defer { CostUsageStore.identicalContentPreLockCheckpointForTesting = nil } let result = save(reread) diff --git a/Tests/CodexBarTests/OpenCodexUsageParserTests.swift b/Tests/CodexBarTests/OpenCodexUsageParserTests.swift index 2f405ab586..a264d4c136 100644 --- a/Tests/CodexBarTests/OpenCodexUsageParserTests.swift +++ b/Tests/CodexBarTests/OpenCodexUsageParserTests.swift @@ -1,3 +1,10 @@ +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#elseif canImport(Musl) +import Musl +#endif import Foundation import Testing @testable import CodexBarCore @@ -38,6 +45,34 @@ struct OpenCodexUsageParserTests { #expect(entries[0].accountLogLabel == nil) } + @Test + func `parseLines splits only on LF so CR and form feed are not record separators`() { + func record(_ id: String) -> String { + """ + {"requestId":"\(id)","timestamp":1784179200000,"provider":"openai","model":"gpt-5.4",\ + "usageStatus":"unreported"} + """ + } + + let crOnly = [record("a"), record("b"), record("c")].joined(separator: "\r") + #expect(OpenCodexUsageParser.parseLines(crOnly).isEmpty) + + let formFeed = [record("f"), record("g")].joined(separator: "\u{000C}") + #expect(OpenCodexUsageParser.parseLines(formFeed).isEmpty) + + let withLineSeparator = record("x") + "\n" + """ + {"requestId":"y","timestamp":1784179200000,"provider":"openai","model":"gpt-5.4",\ + "usageStatus":"unreported","note":"keep\u{2028}together"} + """ + #expect(OpenCodexUsageParser.parseLines(withLineSeparator).map(\.requestID) == ["x", "y"]) + + let withNextLine = record("x") + "\n" + """ + {"requestId":"y","timestamp":1784179200000,"provider":"openai","model":"gpt-5.4",\ + "usageStatus":"unreported","note":"keep\u{0085}together"} + """ + #expect(OpenCodexUsageParser.parseLines(withNextLine).map(\.requestID) == ["x", "y"]) + } + @Test func `does not resolve a default home while tests are running`() { #expect(OpenCodexUsageLog.usageLogURL(environment: ["TESTING_LIBRARY_VERSION": "1"]) == nil) @@ -97,8 +132,33 @@ struct OpenCodexUsageParserTests { #expect(snapshot.sessions[0].sessionID == "chat-1") #expect(snapshot.sessions[0].reasoningTokens == 3) #expect(snapshot.costProvenance == .listPriceEstimate) - #expect(OpenCodexUsageStore.databaseFilename == "opencodex-usage.sqlite") - #expect(FileManager.default.fileExists(atPath: root.appendingPathComponent("opencodex-usage.sqlite").path)) + #expect(OpenCodexUsageStore.databaseFilename == "opencodex-usage-v2.sqlite") + #expect(FileManager.default.fileExists( + atPath: root.appendingPathComponent("opencodex-usage-v2.sqlite").path)) + } + + @Test + func `missing usage log parses as empty`() throws { + let url = FileManager.default.temporaryDirectory + .appendingPathComponent("OpenCodexUsageParserMissing-\(UUID().uuidString).jsonl") + #expect(try OpenCodexUsageParser.parse(fileURL: url).isEmpty) + } + + @Test(.enabled(if: geteuid() != 0)) + func `unreadable usage log throws instead of returning empty`() throws { + let root = FileManager.default.temporaryDirectory + .appendingPathComponent("OpenCodexUsageParserUnreadable-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + let url = root.appendingPathComponent("usage.jsonl") + try Data("{\"requestId\":\"x\"}\n".utf8).write(to: url) + try FileManager.default.setAttributes([.posixPermissions: 0o000], ofItemAtPath: url.path) + defer { + try? FileManager.default.setAttributes([.posixPermissions: 0o600], ofItemAtPath: url.path) + try? FileManager.default.removeItem(at: root) + } + #expect(throws: (any Error).self) { + _ = try OpenCodexUsageParser.parse(fileURL: url) + } } @Test diff --git a/Tests/CodexBarTests/OpenCodexUsageStoreIncrementalTests.swift b/Tests/CodexBarTests/OpenCodexUsageStoreIncrementalTests.swift new file mode 100644 index 0000000000..a8dc8fe202 --- /dev/null +++ b/Tests/CodexBarTests/OpenCodexUsageStoreIncrementalTests.swift @@ -0,0 +1,804 @@ +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#elseif canImport(Musl) +import Musl +#endif +import Foundation +import SQLite3 +import Testing +@testable import CodexBarCore + +struct OpenCodexUsageStoreIncrementalTests { + @Test + func `incremental load matches a full parse after appended lines`() throws { + let harness = try Harness.make() + defer { harness.tearDown() } + + try harness.writeLines( + Harness.line(id: "req-1", input: 10), + Harness.line(id: "req-10", input: 4), + Harness.line(id: "req-ä", input: 7)) + let first = try harness.store.loadEntries(logURL: harness.log) + let firstExpected = try harness.referenceEntries() + #expect(first == firstExpected) + + try harness.appendLines( + Harness.line(id: "req-2", input: 3), + Harness.line(id: "req-b", input: 8), + Harness.line(id: "req-c", input: 1)) + let second = try harness.store.loadEntries(logURL: harness.log) + let secondExpected = try harness.referenceEntries() + #expect(second == secondExpected) + } + + @Test + func `appended lines parse only the tail and an unchanged file reads zero bytes`() throws { + let harness = try Harness.make() + defer { harness.tearDown() } + + try harness.writeLines( + Harness.line(id: "seed-1", input: 1), + Harness.line(id: "seed-2", input: 2)) + _ = try harness.store.loadEntries(logURL: harness.log) + + let appended = [ + Harness.line(id: "tail-1", input: 3), + Harness.line(id: "tail-2", input: 4), + Harness.line(id: "tail-3", input: 5), + ] + try harness.appendLines(appended) + + let tailRecorder = OpenCodexUsageParser.LogReadRecorder() + let afterAppend = try OpenCodexUsageStore.withLogReadRecorderForTesting(tailRecorder) { + try harness.store.loadEntries(logURL: harness.log) + } + let afterAppendExpected = try harness.referenceEntries() + #expect(afterAppend == afterAppendExpected) + #expect(tailRecorder.snapshot().completeLines == appended.count) + + let unchangedRecorder = OpenCodexUsageParser.LogReadRecorder() + let unchanged = try OpenCodexUsageStore.withLogReadRecorderForTesting(unchangedRecorder) { + try harness.store.loadEntries(logURL: harness.log) + } + #expect(unchanged == afterAppend) + #expect(unchangedRecorder.snapshot().bytesRead == 0) + #expect(unchangedRecorder.snapshot().completeLines == 0) + } + + @Test + func `partial trailing line is ignored until the newline arrives`() throws { + let harness = try Harness.make() + defer { harness.tearDown() } + + let complete = Harness.line(id: "complete", input: 4) + "\n" + try Data(complete.utf8).write(to: harness.log) + _ = try harness.store.loadEntries(logURL: harness.log) + let completeOffset = try #require(harness.store.parseCursorForTesting()?.parsedOffset) + #expect(completeOffset == Int64(complete.utf8.count)) + + let pending = Harness.line(id: "pending", input: 9) + let pendingBytes = Data(pending.utf8) + let splitIndex = pendingBytes.count / 2 + let head = String(data: pendingBytes.prefix(splitIndex), encoding: .utf8) ?? "" + let tail = String(data: pendingBytes.dropFirst(splitIndex), encoding: .utf8) ?? "" + try harness.append(head) + + let partial = try harness.store.loadEntries(logURL: harness.log) + #expect(partial.map(\.requestID) == ["complete"]) + #expect(harness.store.parseCursorForTesting()?.parsedOffset == completeOffset) + + try harness.append(tail + "\n") + let completed = try harness.store.loadEntries(logURL: harness.log) + #expect(completed.map(\.requestID) == ["complete", "pending"]) + let completedExpected = try harness.referenceEntries() + #expect(completed == completedExpected) + #expect(completed.count(where: { $0.requestID == "pending" }) == 1) + } + + @Test + func `later duplicate request ids win like a full parse`() throws { + let harness = try Harness.make() + defer { harness.tearDown() } + + try harness.writeLines(Harness.line(id: "dup", input: 1), Harness.line(id: "other", input: 2)) + _ = try harness.store.loadEntries(logURL: harness.log) + try harness.appendLines(Harness.line(id: "dup", input: 9)) + + let incremental = try harness.store.loadEntries(logURL: harness.log) + let incrementalExpected = try harness.referenceEntries() + #expect(incremental == incrementalExpected) + let duplicate = try #require(incremental.first { $0.requestID == "dup" }) + #expect(duplicate.usage?.inputTokens == 9) + #expect(incremental.count == 2) + } + + @Test + func `truncation and in-place rewrite fall back to a full parse`() throws { + let harness = try Harness.make() + defer { harness.tearDown() } + + let first = Harness.line(id: "id-A", input: 1) + let rewritten = Harness.line(id: "id-B", input: 9) + let width = max(first.utf8.count, rewritten.utf8.count) + let firstPadded = first.padding(toLength: width, withPad: " ", startingAt: 0) + let rewrittenPadded = rewritten.padding(toLength: width, withPad: " ", startingAt: 0) + #expect(Array(firstPadded.utf8) != Array(rewrittenPadded.utf8)) + #expect(firstPadded.utf8.count == rewrittenPadded.utf8.count) + let extra = Harness.line(id: "keep", input: 3) + try harness.writeLines(firstPadded, extra) + _ = try harness.store.loadEntries(logURL: harness.log) + + try harness.rewriteFirstLine(rewrittenPadded) + let afterRewrite = try harness.store.loadEntries(logURL: harness.log) + let rewriteExpected = try harness.referenceEntries() + #expect(afterRewrite == rewriteExpected) + #expect(afterRewrite.map(\.requestID) == ["id-B", "keep"]) + + let prefix = rewrittenPadded + "\n" + try harness.truncate(to: prefix.utf8.count) + let afterTruncate = try harness.store.loadEntries(logURL: harness.log) + let truncateExpected = try harness.referenceEntries() + #expect(afterTruncate == truncateExpected) + #expect(afterTruncate.map(\.requestID) == ["id-B"]) + } + + @Test + func `rotation with a shared 64 kib prefix falls back to a full parse`() throws { + let harness = try Harness.make() + defer { harness.tearDown() } + + let prefixLine = Harness.paddedLine(id: "pad", input: 1, padByteCount: Harness.prefixDigestByteLimit) + let firstTail = Harness.line(id: "id-A", input: 1) + let rotatedTail = Harness.line(id: "rotated", input: 6) + let width = max(firstTail.utf8.count, rotatedTail.utf8.count) + let firstPadded = firstTail.padding(toLength: width, withPad: " ", startingAt: 0) + let rotatedPadded = rotatedTail.padding(toLength: width, withPad: " ", startingAt: 0) + try harness.writeLines(prefixLine, firstPadded) + _ = try harness.store.loadEntries(logURL: harness.log) + let oldOffset = try #require(harness.store.parseCursorForTesting()?.parsedOffset) + + try FileManager.default.removeItem(at: harness.log) + try harness.writeLines(prefixLine, rotatedPadded) + let newSize = try FileManager.default.attributesOfItem(atPath: harness.log.path)[.size] as? Int64 + #expect(newSize ?? 0 >= oldOffset) + + let afterRotation = try harness.store.loadEntries(logURL: harness.log) + let rotationExpected = try harness.referenceEntries() + #expect(afterRotation == rotationExpected) + #expect(afterRotation.map(\.requestID) == ["pad", "rotated"]) + } + + @Test + func `in-place rewrite past the 64 kib digest window is not detected`() throws { + // The prefix digest only covers min(64 KiB, parsedOffset). An in-place rewrite past that + // window that preserves size and inode is invisible; the log is append-only. + let harness = try Harness.make() + defer { harness.tearDown() } + + let prefixLine = Harness.paddedLine(id: "pad", input: 1, padByteCount: Harness.prefixDigestByteLimit) + let firstTail = Harness.line(id: "id-A", input: 1) + let rewrittenTail = Harness.line(id: "id-B", input: 9) + let width = max(firstTail.utf8.count, rewrittenTail.utf8.count) + let firstPadded = firstTail.padding(toLength: width, withPad: " ", startingAt: 0) + let rewrittenPadded = rewrittenTail.padding(toLength: width, withPad: " ", startingAt: 0) + try harness.writeLines(prefixLine, firstPadded) + _ = try harness.store.loadEntries(logURL: harness.log) + + let prefixByteCount = prefixLine.utf8.count + 1 + try harness.rewriteLine(atByteOffset: prefixByteCount, rewrittenPadded) + let afterRewrite = try harness.store.loadEntries(logURL: harness.log) + #expect(afterRewrite.map(\.requestID) == ["id-A", "pad"]) + let full = try OpenCodexUsageParser.parse(fileURL: harness.log) + #expect(full.map(\.requestID) == ["pad", "id-B"]) + } + + @Test + func `schema v1 payload caches rebuild from the log`() throws { + let harness = try Harness.make() + defer { harness.tearDown() } + + try harness.writeLines(Harness.line(id: "live", input: 9, output: 2, total: 11)) + try Harness.writeV1Database( + at: harness.cacheRoot.appendingPathComponent(OpenCodexUsageStore.databaseFilename), + staleRequestID: "stale-v1") + let legacyURL = harness.cacheRoot.appendingPathComponent("opencodex-usage.sqlite") + try Harness.writeV1Database(at: legacyURL, staleRequestID: "legacy-v1") + + let entries = try harness.store.loadEntries(logURL: harness.log) + let schemaExpected = try harness.referenceEntries() + #expect(entries == schemaExpected) + #expect(entries.map(\.requestID) == ["live"]) + #expect(entries.contains { $0.requestID == "stale-v1" } == false) + #expect(entries.contains { $0.requestID == "legacy-v1" } == false) + #expect(entries[0].usage?.inputTokens == 9) + #expect(entries[0].usage?.outputTokens == 2) + #expect(entries[0].totalTokens == 11) + #expect(FileManager.default.fileExists(atPath: legacyURL.path)) + } + + @Test + func `CRLF file parse matches parseLines and offset parse skips a trailing partial line`() throws { + let harness = try Harness.make() + defer { harness.tearDown() } + + let text = """ + \(Harness.line(id: "one", input: 1))\r + \(Harness.line(id: "two", input: 2)) + + not-json + \(Harness.line(id: "three", input: 3)) + """ + try Data(text.utf8).write(to: harness.log) + + let fromFile = try OpenCodexUsageParser.parse(fileURL: harness.log) + #expect(fromFile.map(\.requestID) == OpenCodexUsageParser.parseLines(text).map(\.requestID)) + #expect(fromFile.map(\.requestID) == ["one", "two", "three"]) + + let complete = Harness.line(id: "full", input: 4) + "\n" + let partial = String(data: Data(Harness.line(id: "half", input: 5).utf8).prefix(12), encoding: .utf8) ?? "" + try Data((complete + partial).utf8).write(to: harness.log) + let sliced = try OpenCodexUsageParser.parse(fileURL: harness.log, from: 0) + #expect(sliced.entries.map(\.requestID) == ["full"]) + #expect(sliced.nextOffset == Int64(complete.utf8.count)) + } + + @Test + func `complete newline-less trailing record does not advance the cursor`() throws { + let harness = try Harness.make() + defer { harness.tearDown() } + + let complete = Harness.line(id: "head", input: 1) + "\n" + let trailing = Harness.line(id: "tail", input: 2) + try Data((complete + trailing).utf8).write(to: harness.log) + + let sliced = try OpenCodexUsageParser.parse(fileURL: harness.log, from: 0) + #expect(sliced.entries.map(\.requestID) == ["head", "tail"]) + #expect(sliced.nextOffset == Int64(complete.utf8.count)) + + let loaded = try harness.store.loadEntries(logURL: harness.log) + #expect(loaded.map(\.requestID) == ["head", "tail"]) + #expect(harness.store.parseCursorForTesting()?.parsedOffset == Int64(complete.utf8.count)) + } + + @Test + func `gluing bytes onto a newline-less trailing record matches a full parse`() throws { + let harness = try Harness.make() + defer { harness.tearDown() } + + let recordA = Harness.line(id: "a", input: 1) + let recordB = Harness.line(id: "b", input: 2) + try Data(recordA.utf8).write(to: harness.log) + let first = try harness.store.loadEntries(logURL: harness.log) + #expect(first.map(\.requestID) == ["a"]) + #expect(harness.store.parseCursorForTesting()?.parsedOffset == 0) + + try harness.append(recordB + "\n") + let incremental = try harness.store.loadEntries(logURL: harness.log) + let full = try harness.referenceEntries() + #expect(incremental == full) + #expect(full.isEmpty) + } + + @Test + func `schema v2 round trip preserves nil usage zero fields and mismatched totals`() throws { + let harness = try Harness.make() + defer { harness.tearDown() } + + try harness.writeLines( + Harness.lineWithoutUsage(id: "no-usage"), + Harness.lineWithZeroInputUsage(id: "zero-field"), + Harness.line(id: "mismatch", input: 10, output: 2, total: 99, usageTotal: 12)) + _ = try harness.store.loadEntries(logURL: harness.log) + let second = try harness.store.loadEntries(logURL: harness.log) + let expected = try harness.referenceEntries() + #expect(second == expected) + + let noUsage = try #require(second.first { $0.requestID == "no-usage" }) + #expect(noUsage.usage == nil) + let zeroField = try #require(second.first { $0.requestID == "zero-field" }) + #expect(zeroField.usage?.inputTokens == 0) + #expect(zeroField.usage?.outputTokens == nil) + #expect(zeroField.usage?.totalTokens == nil) + let mismatch = try #require(second.first { $0.requestID == "mismatch" }) + #expect(mismatch.totalTokens == 99) + #expect(mismatch.usage?.totalTokens == 12) + #expect(mismatch.usage?.inputTokens == 10) + } + + @Test + func `failed begin immediate leaves rows and cursor untouched`() throws { + let harness = try Harness.make() + defer { harness.tearDown() } + + try harness.writeLines(Harness.line(id: "keep", input: 1)) + _ = try harness.store.loadEntries(logURL: harness.log) + let cursorBefore = try #require(harness.store.parseCursorForTesting()) + let rowsBefore = try harness.sqliteRequestIDs() + #expect(rowsBefore == ["keep"]) + + try harness.appendLines(Harness.line(id: "new", input: 2)) + try harness.withExclusiveSQLiteWriteLock { + _ = try harness.store.loadEntries(logURL: harness.log) + let cursorAfter = try #require(harness.store.parseCursorForTesting()) + #expect(cursorAfter.parsedOffset == cursorBefore.parsedOffset) + #expect(cursorAfter.prefixDigest == cursorBefore.prefixDigest) + #expect(cursorAfter.fileIdentity == cursorBefore.fileIdentity) + #expect(try harness.sqliteRequestIDs() == rowsBefore) + } + } + + @Test + func `stale incremental write does not regress a newer durable cursor`() throws { + // Deterministic: commit a newer cursor, then force the write path with the older + // snapshot. No thread interleaving. + let harness = try Harness.make() + defer { harness.tearDown() } + + try harness.writeLines(Harness.line(id: "req-1", input: 1), Harness.line(id: "req-2", input: 2)) + let first = try harness.store.loadEntries(logURL: harness.log) + let firstCursor = try #require(harness.store.parseCursorForTesting()) + + try harness.appendLines(Harness.line(id: "req-3", input: 3), Harness.line(id: "req-4", input: 4)) + _ = try harness.store.loadEntries(logURL: harness.log) + let newerCursor = try #require(harness.store.parseCursorForTesting()) + #expect(newerCursor.parsedOffset > firstCursor.parsedOffset) + + harness.store.writeIncrementalEntriesForTesting( + first, + path: firstCursor.path, + fileIdentity: firstCursor.fileIdentity, + parsedOffset: firstCursor.parsedOffset, + prefixDigest: firstCursor.prefixDigest) + + let afterStale = try #require(harness.store.parseCursorForTesting()) + #expect(afterStale.parsedOffset == newerCursor.parsedOffset) + #expect(afterStale.prefixDigest == newerCursor.prefixDigest) + #expect(afterStale.fileIdentity == newerCursor.fileIdentity) + #expect(afterStale.path == newerCursor.path) + + let expected = try harness.referenceEntries() + #expect(try harness.sqliteEntryCount() == expected.count) + #expect(try harness.sqliteRequestIDs() == expected.map(\.requestID).sorted()) + + let loaded = try harness.store.loadEntries(logURL: harness.log) + #expect(loaded == expected) + } + + @Test + func `incremental reload falls back to a full parse when the cached rows cannot be read`() throws { + let harness = try Harness.make() + defer { harness.tearDown() } + + try harness.writeLines(Harness.line(id: "seed", input: 1)) + _ = try harness.store.loadEntries(logURL: harness.log) + try harness.dropEntriesTable() + try harness.appendLines(Harness.line(id: "tail", input: 2)) + + let recovered = try harness.store.loadEntries(logURL: harness.log) + let expected = try harness.referenceEntries() + #expect(recovered == expected) + #expect(recovered.map(\.requestID) == ["seed", "tail"]) + } + + @Test + func `incremental post-parse hook is unset by default`() { + #expect(OpenCodexUsageStore.incrementalPostParseHookInstalledForTesting() == false) + } + + @Test + func `post-parse replacement longer than the cursor falls back to a full parse`() throws { + try self.assertPostParseReplacementFallsBack(longerThanCursor: true) + } + + @Test + func `post-parse replacement shorter than the cursor falls back to a full parse`() throws { + try self.assertPostParseReplacementFallsBack(longerThanCursor: false) + } + + private func assertPostParseReplacementFallsBack(longerThanCursor: Bool) throws { + let harness = try Harness.make() + defer { harness.tearDown() } + + try harness.writeLines( + Harness.line(id: "seed-1", input: 1), + Harness.line(id: "seed-2", input: 2)) + _ = try harness.store.loadEntries(logURL: harness.log) + let firstCursor = try #require(harness.store.parseCursorForTesting()) + try harness.appendLines(Harness.line(id: "tail-1", input: 3)) + + let replacementLines: [String] = if longerThanCursor { + [ + Harness.line(id: "new-1", input: 11), + Harness.line(id: "new-2", input: 12), + Harness.paddedLine( + id: "new-pad", + input: 13, + padByteCount: max(64, Int(firstCursor.parsedOffset))), + ] + } else { + [Harness.line(id: "new-only", input: 5)] + } + let body = replacementLines.map { $0.hasSuffix("\n") ? $0 : $0 + "\n" }.joined() + let bodyCount = Int64(body.utf8.count) + if longerThanCursor { + #expect(bodyCount > firstCursor.parsedOffset) + } else { + #expect(bodyCount < firstCursor.parsedOffset) + } + #expect(OpenCodexUsageStore.incrementalPostParseHookInstalledForTesting() == false) + + let log = harness.log + let loaded = try OpenCodexUsageStore.withIncrementalPostParseHookForTesting { + try? FileManager.default.removeItem(at: log) + try? Data(body.utf8).write(to: log) + } operation: { + try harness.store.loadEntries(logURL: log) + } + + let expected = try harness.referenceEntries() + #expect(loaded == expected) + let loadedIDs = loaded.map(\.requestID) + #expect(Set(loadedIDs).count == loadedIDs.count) + #expect(Set(loadedIDs).isDisjoint(with: ["seed-1", "seed-2", "tail-1"])) + #expect(try harness.sqliteRequestIDs() == loadedIDs.sorted()) + + let cursor = try #require(harness.store.parseCursorForTesting()) + #expect(cursor.path == harness.log.path) + #expect(cursor.fileIdentity != firstCursor.fileIdentity) + #expect(cursor.parsedOffset == bodyCount) + + let recorder = OpenCodexUsageParser.LogReadRecorder() + let cached = try OpenCodexUsageStore.withLogReadRecorderForTesting(recorder) { + try harness.store.loadEntries(logURL: harness.log) + } + #expect(cached == loaded) + #expect(recorder.snapshot().bytesRead == 0) + } + + @Test + func `incremental load of log A does not keep another home's cached rows`() throws { + let harness = try Harness.make() + defer { harness.tearDown() } + + try harness.writeLines( + Harness.line(id: "a-seed-1", input: 1), + Harness.line(id: "a-seed-2", input: 2)) + _ = try harness.store.loadEntries(logURL: harness.log) + try harness.appendLines(Harness.line(id: "a-tail", input: 3)) + + let logB = harness.root.appendingPathComponent("usage-b.jsonl") + try Data([ + Harness.line(id: "b-1", input: 11) + "\n", + Harness.line(id: "b-2", input: 12) + "\n", + ].joined().utf8).write(to: logB) + + let store = harness.store + let loaded = try OpenCodexUsageStore.withIncrementalPostParseHookForTesting { + do { + _ = try store.loadEntries(logURL: logB) + } catch { + Issue.record(error) + } + } operation: { + try store.loadEntries(logURL: harness.log) + } + + let expected = try harness.referenceEntries() + #expect(loaded == expected) + #expect(loaded.map(\.requestID) == ["a-seed-1", "a-seed-2", "a-tail"]) + #expect(Set(loaded.map(\.requestID)).isDisjoint(with: ["b-1", "b-2"])) + #expect(try harness.sqliteRequestIDs() == ["a-seed-1", "a-seed-2", "a-tail"].sorted()) + #expect(try harness.sqliteRequestIDs().contains("b-1") == false) + #expect(try harness.sqliteRequestIDs().contains("b-2") == false) + } + + @Test + func `missing log returns an empty snapshot without throwing`() throws { + let harness = try Harness.make() + defer { harness.tearDown() } + #expect(try harness.store.loadEntries(logURL: harness.log) == []) + } + + @Test + func `stat failure other than absence throws instead of returning empty`() throws { + let harness = try Harness.make() + defer { harness.tearDown() } + let loop = harness.root.appendingPathComponent("loop.jsonl") + try FileManager.default.createSymbolicLink( + atPath: loop.path, + withDestinationPath: loop.lastPathComponent) + #expect(throws: (any Error).self) { + _ = try harness.store.loadEntries(logURL: loop) + } + } + + @Test(.enabled(if: geteuid() != 0)) + func `permission failure on the log directory throws instead of returning empty`() throws { + let harness = try Harness.make() + let logDir = harness.root.appendingPathComponent("logdir", isDirectory: true) + try FileManager.default.createDirectory(at: logDir, withIntermediateDirectories: true) + let log = logDir.appendingPathComponent("usage.jsonl") + try Data((Harness.line(id: "hidden", input: 1) + "\n").utf8).write(to: log) + try FileManager.default.setAttributes([.posixPermissions: 0o000], ofItemAtPath: logDir.path) + defer { + try? FileManager.default.setAttributes([.posixPermissions: 0o700], ofItemAtPath: logDir.path) + harness.tearDown() + } + #expect(throws: (any Error).self) { + _ = try harness.store.loadEntries(logURL: log) + } + } +} + +private struct Harness { + let root: URL + let cacheRoot: URL + let log: URL + let store: OpenCodexUsageStore + + static func make() throws -> Harness { + let root = FileManager.default.temporaryDirectory + .appendingPathComponent("OpenCodexUsageStoreIncremental-\(UUID().uuidString)", isDirectory: true) + let cacheRoot = root.appendingPathComponent("cache", isDirectory: true) + try FileManager.default.createDirectory(at: cacheRoot, withIntermediateDirectories: true) + return Harness( + root: root, + cacheRoot: cacheRoot, + log: root.appendingPathComponent("usage.jsonl"), + store: OpenCodexUsageStore(cacheRoot: cacheRoot)) + } + + func tearDown() { + try? FileManager.default.removeItem(at: self.root) + } + + static let prefixDigestByteLimit = 64 * 1024 + + static func line( + id: String, + input: Int, + output: Int = 1, + total: Int? = nil, + usageTotal: Int? = nil) -> String + { + let resolvedTotal = total ?? (input + output) + let resolvedUsageTotal = usageTotal ?? resolvedTotal + return """ + {"requestId":"\(id)","timestamp":1784179200000,"provider":"openai","model":"gpt-5.4",\ + "usageStatus":"reported","usage":{"inputTokens":\(input),"outputTokens":\(output),\ + "totalTokens":\(resolvedUsageTotal)},"totalTokens":\(resolvedTotal)} + """ + } + + static func lineWithoutUsage(id: String) -> String { + """ + {"requestId":"\(id)","timestamp":1784179200000,"provider":"openai","model":"gpt-5.4",\ + "usageStatus":"unreported"} + """ + } + + static func lineWithZeroInputUsage(id: String) -> String { + """ + {"requestId":"\(id)","timestamp":1784179200000,"provider":"openai","model":"gpt-5.4",\ + "usageStatus":"reported","usage":{"inputTokens":0}} + """ + } + + static func paddedLine(id: String, input: Int, padByteCount: Int) -> String { + let pad = String(repeating: "x", count: padByteCount) + return String(self.line(id: id, input: input).dropLast()) + ",\"pad\":\"\(pad)\"}" + } + + func writeLines(_ lines: String...) throws { + try self.writeLines(Array(lines)) + } + + func writeLines(_ lines: [String]) throws { + let body = lines.map { $0.hasSuffix("\n") ? $0 : $0 + "\n" }.joined() + try Data(body.utf8).write(to: self.log) + } + + func appendLines(_ lines: [String]) throws { + try self.append(lines.map { $0.hasSuffix("\n") ? $0 : $0 + "\n" }.joined()) + } + + func appendLines(_ lines: String...) throws { + try self.appendLines(Array(lines)) + } + + func append(_ text: String) throws { + let handle = try FileHandle(forUpdating: self.log) + defer { try? handle.close() } + _ = try handle.seekToEnd() + try handle.write(contentsOf: Data(text.utf8)) + } + + func rewriteFirstLine(_ line: String) throws { + try self.rewriteLine(atByteOffset: 0, line) + } + + func rewriteLine(atByteOffset offset: Int, _ line: String) throws { + let replacement = Data((line.hasSuffix("\n") ? line : line + "\n").utf8) + let handle = try FileHandle(forUpdating: self.log) + defer { try? handle.close() } + try handle.seek(toOffset: UInt64(offset)) + try handle.write(contentsOf: replacement) + } + + var databaseURL: URL { + self.cacheRoot.appendingPathComponent(OpenCodexUsageStore.databaseFilename) + } + + func sqliteEntryCount() throws -> Int { + var db: OpaquePointer? + guard sqlite3_open_v2(self.databaseURL.path, &db, SQLITE_OPEN_READONLY, nil) == SQLITE_OK else { + sqlite3_close(db) + throw FixtureError.sqlite + } + defer { sqlite3_close(db) } + var statement: OpaquePointer? + guard sqlite3_prepare_v2(db, "SELECT COUNT(*) FROM entries", -1, &statement, nil) == SQLITE_OK else { + throw FixtureError.sqlite + } + defer { sqlite3_finalize(statement) } + guard sqlite3_step(statement) == SQLITE_ROW else { throw FixtureError.sqlite } + return Int(sqlite3_column_int64(statement, 0)) + } + + func sqliteRequestIDs() throws -> [String] { + var db: OpaquePointer? + guard sqlite3_open_v2(self.databaseURL.path, &db, SQLITE_OPEN_READONLY, nil) == SQLITE_OK else { + sqlite3_close(db) + throw FixtureError.sqlite + } + defer { sqlite3_close(db) } + var statement: OpaquePointer? + guard sqlite3_prepare_v2( + db, + "SELECT request_id FROM entries ORDER BY request_id", + -1, + &statement, + nil) == SQLITE_OK + else { + throw FixtureError.sqlite + } + defer { sqlite3_finalize(statement) } + var ids: [String] = [] + while sqlite3_step(statement) == SQLITE_ROW { + if let pointer = sqlite3_column_text(statement, 0) { + ids.append(String(cString: pointer)) + } + } + return ids + } + + func dropEntriesTable() throws { + var db: OpaquePointer? + guard sqlite3_open(self.databaseURL.path, &db) == SQLITE_OK else { + sqlite3_close(db) + throw FixtureError.sqlite + } + defer { sqlite3_close(db) } + guard sqlite3_exec(db, "DROP TABLE entries", nil, nil, nil) == SQLITE_OK else { + throw FixtureError.sqlite + } + } + + func withExclusiveSQLiteWriteLock(_ body: () throws -> Void) throws { + var db: OpaquePointer? + guard sqlite3_open_v2(self.databaseURL.path, &db, SQLITE_OPEN_READWRITE, nil) == SQLITE_OK else { + sqlite3_close(db) + throw FixtureError.sqlite + } + defer { + _ = sqlite3_exec(db, "ROLLBACK", nil, nil, nil) + sqlite3_close(db) + } + sqlite3_busy_timeout(db, 0) + guard sqlite3_exec(db, "BEGIN IMMEDIATE", nil, nil, nil) == SQLITE_OK else { + throw FixtureError.sqlite + } + try body() + } + + func truncate(to byteCount: Int) throws { + let handle = try FileHandle(forUpdating: self.log) + defer { try? handle.close() } + try handle.truncate(atOffset: UInt64(byteCount)) + } + + func referenceEntries() throws -> [OpenCodexUsageEntry] { + let parsed = try OpenCodexUsageParser.parse(fileURL: self.log) + var unique: [String: OpenCodexUsageEntry] = [:] + for entry in parsed { + unique[entry.requestID] = entry + } + return unique.values.sorted { + if $0.timestamp != $1.timestamp { + return $0.timestamp < $1.timestamp + } + return $0.requestID < $1.requestID + } + } + + static func writeV1Database(at databaseURL: URL, staleRequestID: String) throws { + try FileManager.default.createDirectory( + at: databaseURL.deletingLastPathComponent(), + withIntermediateDirectories: true) + var db: OpaquePointer? + guard sqlite3_open(databaseURL.path, &db) == SQLITE_OK else { + sqlite3_close(db) + throw FixtureError.sqlite + } + defer { sqlite3_close(db) } + let schema = """ + CREATE TABLE meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + CREATE TABLE entries ( + request_id TEXT PRIMARY KEY, + timestamp REAL NOT NULL, + provider TEXT NOT NULL, + model TEXT NOT NULL, + usage_status TEXT NOT NULL, + account_label TEXT, + surface TEXT, + conversation_id TEXT, + payload TEXT NOT NULL + ); + PRAGMA user_version = 1; + """ + guard sqlite3_exec(db, schema, nil, nil, nil) == SQLITE_OK else { throw FixtureError.sqlite } + + try self.exec( + db, + sql: "INSERT INTO meta(key, value) VALUES(?, ?)", + bind: { statement in + self.bind(statement, 1, "identity") + self.bind(statement, 2, "/tmp/stale|1|1") + }) + + let payload = """ + {"requestId":"\(staleRequestID)","timestamp":1784179200000,"provider":"openai","model":"gpt-5.4",\ + "usageStatus":"reported","usage":{"inputTokens":1,"outputTokens":1,"totalTokens":2},"totalTokens":2} + """ + try self.exec( + db, + sql: """ + INSERT INTO entries( + request_id, timestamp, provider, model, usage_status, account_label, surface, conversation_id, payload + ) VALUES (?, ?, ?, ?, ?, NULL, NULL, NULL, ?) + """, + bind: { statement in + self.bind(statement, 1, staleRequestID) + sqlite3_bind_double(statement, 2, 1_784_179_200) + self.bind(statement, 3, "openai") + self.bind(statement, 4, "gpt-5.4") + self.bind(statement, 5, "reported") + self.bind(statement, 6, payload) + }) + } + + private static func exec( + _ db: OpaquePointer?, + sql: String, + bind: (OpaquePointer?) -> Void) throws + { + var statement: OpaquePointer? + guard sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK else { throw FixtureError.sqlite } + defer { sqlite3_finalize(statement) } + bind(statement) + guard sqlite3_step(statement) == SQLITE_DONE else { throw FixtureError.sqlite } + } + + private static func bind(_ statement: OpaquePointer?, _ index: Int32, _ value: String) { + sqlite3_bind_text(statement, index, value, -1, unsafeBitCast(-1, to: sqlite3_destructor_type.self)) + } + + private enum FixtureError: Error { + case sqlite + } +}