diff --git a/Sources/CodexBarCore/CodexLocalDataScope.swift b/Sources/CodexBarCore/CodexLocalDataScope.swift new file mode 100644 index 0000000000..b71c2b057f --- /dev/null +++ b/Sources/CodexBarCore/CodexLocalDataScope.swift @@ -0,0 +1,63 @@ +import Foundation + +/// Resolves all local Codex sources as one scope. A managed Codex home must +/// never silently borrow an ambient state database or cache identity. +struct CodexLocalDataScope: Sendable, Equatable { + let identifier: String + let codexHome: URL + let sessionsRoot: URL + let archivedSessionsRoot: URL + let stateDatabaseURL: URL + + static func resolve(options: CostUsageScanner.Options) -> CodexLocalDataScope { + if let sessionsRoot = options.codexSessionsRoot?.standardizedFileURL, + sessionsRoot.lastPathComponent == "sessions" + { + let home = sessionsRoot.deletingLastPathComponent() + return self.make(home: home) + } + + let environment = ProcessInfo.processInfo.environment + if let sqliteHome = Self.nonEmpty(environment["CODEX_SQLITE_HOME"]) { + let home = URL(fileURLWithPath: sqliteHome, isDirectory: true).standardizedFileURL + return self.make(home: home) + } + if let codexHome = Self.nonEmpty(environment["CODEX_HOME"]) { + let home = URL(fileURLWithPath: codexHome, isDirectory: true).standardizedFileURL + return self.make(home: home) + } + return self.make(home: FileManager.default.homeDirectoryForCurrentUser.appendingPathComponent(".codex")) + } + + func applying(to options: CostUsageScanner.Options) -> CostUsageScanner.Options { + var copy = options + copy.codexSessionsRoot = self.sessionsRoot + return copy + } + + private static func make(home: URL) -> CodexLocalDataScope { + let standardizedHome = home.standardizedFileURL + return CodexLocalDataScope( + identifier: "codex-workspaces:" + Self.scopeFingerprint(standardizedHome.path), + codexHome: standardizedHome, + sessionsRoot: standardizedHome.appendingPathComponent("sessions", isDirectory: true), + archivedSessionsRoot: standardizedHome.appendingPathComponent("archived_sessions", isDirectory: true), + stateDatabaseURL: standardizedHome.appendingPathComponent("state_5.sqlite", isDirectory: false)) + } + + /// Stable local identifier for cache partitioning. It is not a security + /// primitive; it only prevents raw home paths from entering cache metadata. + private static func scopeFingerprint(_ value: String) -> String { + var hash: UInt64 = 14_695_981_039_346_656_037 + for byte in value.utf8 { + hash ^= UInt64(byte) + hash &*= 1_099_511_628_211 + } + return String(hash, radix: 16) + } + + private static func nonEmpty(_ value: String?) -> String? { + let trimmed = value?.trimmingCharacters(in: .whitespacesAndNewlines) + return trimmed?.isEmpty == false ? trimmed : nil + } +} diff --git a/Sources/CodexBarCore/CodexLocalProjectRootResolver.swift b/Sources/CodexBarCore/CodexLocalProjectRootResolver.swift new file mode 100644 index 0000000000..4314accfa1 --- /dev/null +++ b/Sources/CodexBarCore/CodexLocalProjectRootResolver.swift @@ -0,0 +1,64 @@ +#if canImport(CryptoKit) +import CryptoKit +#else +import Crypto +#endif +import Foundation + +public enum CodexLocalProjectRootResolver { + public typealias ProjectIdentity = (id: String, displayName: String, path: String?) + + public static let chatsProjectId = "chats" + public static let chatsDisplayName = "Chats" + + public static func projectIdentity(for cwd: String?) -> ProjectIdentity { + guard let cwd = cwd?.trimmingCharacters(in: .whitespacesAndNewlines), !cwd.isEmpty else { + return (self.chatsProjectId, self.chatsDisplayName, nil) + } + + let root = self.resolveProjectRoot(from: URL(fileURLWithPath: cwd, isDirectory: true)) + let canonicalRoot = self.canonicalProjectURL(root) + let path = canonicalRoot.path + return ( + self.projectId(for: path), + canonicalRoot.lastPathComponent.isEmpty ? path : canonicalRoot.lastPathComponent, + path) + } + + public static func resolveProjectRoot(from cwd: URL) -> URL { + let fm = FileManager.default + let loggedCWD = cwd.standardizedFileURL + var current = loggedCWD + var isDirectory: ObjCBool = false + let cwdExists = fm.fileExists(atPath: current.path, isDirectory: &isDirectory) + // A missing CWD is historical evidence. Do not walk up to an existing + // parent repository because that would collapse a deleted worktree or + // folder into a different project identity. + guard cwdExists else { return loggedCWD } + if cwdExists, !isDirectory.boolValue { + current = current.deletingLastPathComponent() + } + + var candidate: URL? = current + while let url = candidate { + let gitURL = url.appendingPathComponent(".git") + if fm.fileExists(atPath: gitURL.path) { + return url.standardizedFileURL + } + let parent = url.deletingLastPathComponent() + candidate = parent.path == url.path ? nil : parent + } + + return current.standardizedFileURL + } + + private static func canonicalProjectURL(_ url: URL) -> URL { + url.resolvingSymlinksInPath().standardizedFileURL + } + + public static func projectId(for path: String) -> String { + let digest = SHA256.hash(data: Data(path.utf8)) + let hex = digest.map { String(format: "%02x", $0) }.joined() + return "project-\(hex.prefix(16))" + } +} diff --git a/Sources/CodexBarCore/CodexLocalProjectUsageIndexer.swift b/Sources/CodexBarCore/CodexLocalProjectUsageIndexer.swift new file mode 100644 index 0000000000..3deb393dcf --- /dev/null +++ b/Sources/CodexBarCore/CodexLocalProjectUsageIndexer.swift @@ -0,0 +1,1182 @@ +import Foundation + +enum CodexLocalProjectUsageIndexer { + enum IndexError: Error, Equatable { + case cacheScopeMismatch + } + + struct Options: Sendable { + var scannerOptions: CostUsageScanner.Options + + init(scannerOptions: CostUsageScanner.Options = CostUsageScanner.Options()) { + self.scannerOptions = scannerOptions + } + } + + static func cachedSnapshot( + now: Date = Date(), + historyDays: Int = 30, + options: Options = Options()) -> CodexLocalProjectUsageSnapshot? + { + _ = now + let clampedHistoryDays = max(1, min(365, historyDays)) + let stableScopeSignature = self.stableScopeSignature(options: options.scannerOptions) + let sidecar = CodexWorkspaceUsageSidecar(cacheRoot: options.scannerOptions.cacheRoot) + let catalogResult = CodexThreadCatalogReader.loadResult(options: options.scannerOptions) + let sourceStatus = CodexLocalProjectUsageSourceStatus(catalog: catalogResult.completeness) + if let snapshot = sidecar.loadLatestSnapshot( + scopeSignature: stableScopeSignature, + historyDays: clampedHistoryDays, + catalog: catalogResult.isComplete ? catalogResult.catalog : nil) + { + return self.projecting(snapshot, sourceStatus: sourceStatus) + } + return nil + } + + static func loadSnapshot( + now: Date = Date(), + historyDays: Int = 30, + forceRefresh: Bool = false, + options: Options = Options(), + progress: (@Sendable (CodexLocalProjectUsageIndexProgress) -> Void)? = nil, + checkCancellation: CostUsageScanner.CancellationCheck? = nil) throws -> CodexLocalProjectUsageSnapshot + { + let refreshSignpost = CodexModelsTelemetry.begin("IndexRefresh") + defer { CodexModelsTelemetry.end("IndexRefresh", id: refreshSignpost) } + let clampedHistoryDays = max(1, min(365, historyDays)) + let until = now + let since = Calendar.current.date(byAdding: .day, value: -(clampedHistoryDays - 1), to: now) ?? now + let comparisonSince = self.modelsAnalyticsScanStart(since: since, until: until) + var scannerOptions = options.scannerOptions + if forceRefresh { + scannerOptions.refreshMinIntervalSeconds = 0 + } + + progress?(CodexLocalProjectUsageIndexProgress(phase: .scanningLogs)) + _ = try CostUsageScanner.loadDailyReportCancellable( + provider: .codex, + since: comparisonSince, + until: until, + now: now, + options: scannerOptions, + checkCancellation: checkCancellation) + try checkCancellation?() + + let cache = CostUsageCacheIO.load(provider: .codex, cacheRoot: scannerOptions.cacheRoot) + let catalogResult = CodexThreadCatalogReader.loadResult(options: scannerOptions) + let catalog = catalogResult.catalog + let sourceStatus = CodexLocalProjectUsageSourceStatus(catalog: catalogResult.completeness) + let rootsFingerprint = self.rootsFingerprint(CostUsageScanner.codexRootsFingerprint(options: scannerOptions)) + let sidecar = CodexWorkspaceUsageSidecar(cacheRoot: scannerOptions.cacheRoot) + if !forceRefresh { + if let snapshot = sidecar.loadLatestSnapshot( + scopeSignature: self.stableScopeSignature(options: scannerOptions), + historyDays: clampedHistoryDays, + rootsFingerprint: rootsFingerprint, + cache: cache, + catalog: catalogResult.isComplete ? catalog : nil) + { + CodexModelsTelemetry.cacheHit(historyDays: clampedHistoryDays) + return self.projecting(snapshot, sourceStatus: sourceStatus) + } + } + + // Import only scanner-derived deltas, then aggregate from the + // sidecar's normalized rows. The raw cache remains the cursor and + // cumulative-token authority; it is no longer the aggregation source. + try sidecar.synchronizeSources( + cache: cache, + catalog: catalog, + catalogIsComplete: catalogResult.isComplete) + let sidecarCache = try sidecar.usageCache(roots: rootsFingerprint) + let snapshot = try self.buildSnapshotFromCostCache( + now: now, + historyDays: clampedHistoryDays, + since: since, + until: until, + options: scannerOptions, + cacheOverride: sidecarCache, + catalogOverride: catalog, + sourceStatus: sourceStatus, + progress: progress, + checkCancellation: checkCancellation) + progress?(CodexLocalProjectUsageIndexProgress(phase: .saving)) + try sidecar.synchronize( + snapshot: snapshot, + cache: cache, + catalog: catalog, + catalogIsComplete: catalogResult.isComplete, + rootsFingerprint: rootsFingerprint) + return snapshot + } + + private static func projecting( + _ snapshot: CodexLocalProjectUsageSnapshot, + sourceStatus: CodexLocalProjectUsageSourceStatus) -> CodexLocalProjectUsageSnapshot + { + guard snapshot.sourceStatus != sourceStatus else { return snapshot } + return CodexLocalProjectUsageSnapshot( + updatedAt: snapshot.updatedAt, + historyDays: snapshot.historyDays, + scopeSignature: snapshot.scopeSignature, + rootsFingerprint: snapshot.rootsFingerprint, + indexedFileCount: snapshot.indexedFileCount, + skippedFileCount: snapshot.skippedFileCount, + total: snapshot.total, + projects: snapshot.projects, + sessions: snapshot.sessions, + modelBreakdowns: snapshot.modelBreakdowns, + daily: snapshot.daily, + sourceStatus: sourceStatus, + modelsAnalytics: snapshot.modelsAnalytics) + } + + static func buildSnapshotFromCostCache( + now: Date = Date(), + historyDays: Int = 30, + since: Date, + until: Date, + options: CostUsageScanner.Options = CostUsageScanner.Options(), + cacheOverride: CostUsageCache? = nil, + catalogOverride: CodexThreadCatalog? = nil, + sourceStatus: CodexLocalProjectUsageSourceStatus = .complete, + progress: (@Sendable (CodexLocalProjectUsageIndexProgress) -> Void)? = nil, + checkCancellation: CostUsageScanner.CancellationCheck? = nil) throws -> CodexLocalProjectUsageSnapshot + { + let clampedHistoryDays = max(1, min(365, historyDays)) + let range = CostUsageScanner.CostUsageDayRange(since: since, until: until) + let cache = cacheOverride ?? CostUsageCacheIO.load(provider: .codex, cacheRoot: options.cacheRoot) + let catalog = catalogOverride ?? CodexThreadCatalogReader.load(options: options) + let expectedRoots = CostUsageScanner.codexRootsFingerprint(options: options) + let scopeSignature = self.stableScopeSignature(options: options) + let rootsFingerprint = self.rootsFingerprint(expectedRoots) + + guard cache.roots == expectedRoots else { + // The cost cache belongs to a different Codex home. Publishing an + // empty replacement would erase a still-valid last-complete view. + throw IndexError.cacheScopeMismatch + } + + let indexed = try self.sessionBuckets( + from: cache, + range: range, + catalog: catalog, + progress: progress, + checkCancellation: checkCancellation) + let indexedFiles = indexed.indexedFiles + let skippedFiles = indexed.skippedFiles + let sessionBuckets = indexed.sessionBuckets + + let sessions = sessionBuckets.values.map { bucket in + CodexLocalSessionUsage( + id: bucket.id, + projectId: bucket.projectId, + displayTitle: self.sessionTitle( + explicitTitle: bucket.title, + startedAt: bucket.startedAt, + latestActivity: bucket.latestActivity, + model: bucket.topModel), + cwd: bucket.cwd, + startedAt: bucket.startedAt, + latestActivity: bucket.latestActivity, + totals: bucket.total, + costEstimate: CodexLocalCostEstimate( + knownUSD: self.usd(fromNanos: bucket.costNanos) ?? 0, + unknownTokens: bucket.unknownCostTokens), + topModel: bucket.displayModel, + daily: self.dailyPoints(from: bucket.dailyTotals)) + }.sorted { lhs, rhs in + self.sortSessions(lhs, rhs) + } + + let projectBuckets = Dictionary(grouping: sessionBuckets.values, by: \.projectId) + let rawProjects = projectBuckets.values.map { buckets in + let sortedSessions = buckets.map { bucket in + CodexLocalSessionUsage( + id: bucket.id, + projectId: bucket.projectId, + displayTitle: self.sessionTitle( + explicitTitle: bucket.title, + startedAt: bucket.startedAt, + latestActivity: bucket.latestActivity, + model: bucket.topModel), + cwd: bucket.cwd, + startedAt: bucket.startedAt, + latestActivity: bucket.latestActivity, + totals: bucket.total, + costEstimate: CodexLocalCostEstimate( + knownUSD: self.usd(fromNanos: bucket.costNanos) ?? 0, + unknownTokens: bucket.unknownCostTokens), + topModel: bucket.displayModel, + daily: self.dailyPoints(from: bucket.dailyTotals)) + }.sorted { lhs, rhs in + self.sortSessions(lhs, rhs) + } + var total = CodexLocalUsageTotals.empty + var costNanos: Int64? + var unknownCostTokens = 0 + var latestActivity: Date? + var modelTotals: [String: ModelTotals] = [:] + var dailyTotals: [String: DailyTotals] = [:] + for bucket in buckets { + total = total.adding(bucket.total) + costNanos = self.addCost(costNanos, bucket.costNanos) + unknownCostTokens += bucket.unknownCostTokens + latestActivity = self.later(latestActivity, bucket.latestActivity) + for (day, daily) in bucket.dailyTotals { + self.mergeDailyTotals(&dailyTotals, day: day, totals: daily) + } + for (model, totals) in bucket.modelTotals { + self.mergeModelTotals(&modelTotals, model: model, totals: totals) + } + } + let first = buckets[0] + return CodexLocalProjectUsage( + id: first.projectId, + displayName: first.projectDisplayName, + path: first.projectPath, + totals: total, + costEstimate: CodexLocalCostEstimate( + knownUSD: self.usd(fromNanos: costNanos) ?? 0, + unknownTokens: unknownCostTokens), + sessionCount: buckets.count, + latestActivity: latestActivity, + topModel: self.topModel(from: modelTotals), + topSessions: Array(sortedSessions.prefix(5)), + modelBreakdowns: self.modelBreakdowns(from: modelTotals), + daily: self.dailyPoints(from: dailyTotals)) + }.sorted { lhs, rhs in + self.sortProjects(lhs, rhs) + } + let projects = rawProjects + + return try CodexLocalProjectUsageSnapshot( + updatedAt: now, + historyDays: clampedHistoryDays, + scopeSignature: scopeSignature, + rootsFingerprint: rootsFingerprint, + indexedFileCount: indexedFiles, + skippedFileCount: skippedFiles, + total: self.total(from: projects), + projects: projects, + sessions: sessions, + modelBreakdowns: self.globalModelBreakdowns(from: sessionBuckets.values), + daily: self.dailyPoints(from: cache.files.values, range: range), + sourceStatus: sourceStatus, + modelsAnalytics: self.modelsAnalyticsPayload( + context: ModelsAnalyticsContext( + currentBuckets: sessionBuckets, + cache: cache, + catalog: catalog, + projects: projects, + sourceStatus: sourceStatus), + window: ModelsAnalyticsWindow( + since: since, + until: until, + historyDays: clampedHistoryDays, + generatedAt: now), + identity: ModelsAnalyticsIdentity( + scopeSignature: scopeSignature, + rootsFingerprint: rootsFingerprint), + checkCancellation: checkCancellation)) + } +} + +extension CodexLocalProjectUsageIndexer { + fileprivate struct ModelsAnalyticsContext { + let currentBuckets: [String: SessionBucket] + let cache: CostUsageCache + let catalog: CodexThreadCatalog + let projects: [CodexLocalProjectUsage] + let sourceStatus: CodexLocalProjectUsageSourceStatus + } + + fileprivate struct ModelsAnalyticsWindow { + let since: Date + let until: Date + let historyDays: Int + let generatedAt: Date + } + + fileprivate struct ModelsAnalyticsIdentity { + let scopeSignature: String + let rootsFingerprint: [String: Int64] + } + + fileprivate struct FileTotals { + var totals: CodexLocalUsageTotals + var costNanos: Int64? + var unknownCostTokens: Int + var modelTotals: [String: ModelTotals] + var dailyTotals: [String: DailyTotals] + var modelDailyTotals: [String: [String: ModelDailyTotals]] + var topModel: String? + } + + fileprivate struct DailyTotals { + var totalTokens: Int + var cachedInputTokens: Int + var costNanos: Int64? + var unknownCostTokens: Int + } + + fileprivate struct ModelTotals { + var totalTokens: Int + var costNanos: Int64? + var unknownCostTokens: Int + } + + fileprivate struct ModelDailyTotals { + var inputTokens: Int + var cachedInputTokens: Int + var outputTokens: Int + var costNanos: Int64? + var unknownCostTokens: Int + } + + fileprivate struct SessionBucket { + var id: String + var projectId: String + var projectDisplayName: String + var projectPath: String? + var cwd: String? + var title: String? + var startedAt: Date? + var latestActivity: Date? + var catalogModel: String? + var modelTotals: [String: ModelTotals] + var dailyTotals: [String: DailyTotals] + var modelDailyTotals: [String: [String: ModelDailyTotals]] + var usageRows: [CostUsageScanner.CodexUsageRow] + var hasCompleteEventRows: Bool + var total: CodexLocalUsageTotals + var costNanos: Int64? + var unknownCostTokens: Int + + var topModel: String? { + CodexLocalProjectUsageIndexer.topModel(from: self.modelTotals) + } + + var displayModel: String? { + self.catalogModel ?? self.topModel + } + } + + fileprivate struct Metadata { + var sessionId: String? + var cwd: String? + var title: String? + var startedAt: Date? + var latestActivity: Date? + } + + fileprivate struct BucketMergeInput { + var sessionId: String + var projectIdentity: CodexLocalProjectRootResolver.ProjectIdentity + var metadata: Metadata + var started: Date? + var latest: Date? + var catalogModel: String? + var model: String? + var fileTotals: FileTotals + var usageRows: [CostUsageScanner.CodexUsageRow] + var hasCompleteEventRows: Bool + } + + fileprivate static func sessionBuckets( + from cache: CostUsageCache, + range: CostUsageScanner.CostUsageDayRange, + catalog: CodexThreadCatalog, + progress: (@Sendable (CodexLocalProjectUsageIndexProgress) -> Void)?, + checkCancellation: CostUsageScanner.CancellationCheck?) + throws -> (sessionBuckets: [String: SessionBucket], indexedFiles: Int, skippedFiles: Int) + { + var indexedFiles = 0 + var skippedFiles = 0 + var sessionBuckets: [String: SessionBucket] = [:] + let files = cache.files.sorted(by: { $0.key < $1.key }).filter { + $0.value.touchesCodexScanWindow(sinceKey: range.sinceKey, untilKey: range.untilKey) + } + progress?(CodexLocalProjectUsageIndexProgress( + phase: .indexingProjects, + processedFileCount: 0, + totalFileCount: files.count)) + + for (offset, entry) in files.enumerated() { + try checkCancellation?() + let path = entry.key + let usage = entry.value + guard let fileTotals = self.fileTotals(from: usage, range: range) else { + skippedFiles += 1 + self.reportProgressIfNeeded( + progress, + processedFiles: offset + 1, + totalFiles: files.count, + indexedFiles: indexedFiles, + skippedFiles: skippedFiles) + continue + } + indexedFiles += 1 + let fileURL = URL(fileURLWithPath: path) + let cachedMetadata = self.metadata(from: usage, catalogEntry: nil) + let catalogEntry = catalog.entry( + sessionId: cachedMetadata.sessionId ?? usage.sessionId, + rolloutPath: path) + let metadata = self.metadata(from: usage, catalogEntry: catalogEntry) + let sessionId = metadata.sessionId ?? usage.sessionId ?? fileURL.deletingPathExtension().lastPathComponent + let projectIdentity = CodexLocalProjectRootResolver.projectIdentity(for: metadata.cwd) + let latest = metadata.latestActivity ?? self.latestDate(from: usage.days.keys) + let started = metadata.startedAt ?? self.earliestDate(from: usage.days.keys) + let model = catalogEntry?.model ?? usage.lastModel ?? fileTotals.topModel + sessionBuckets[sessionId] = self.mergedBucket( + sessionBuckets[sessionId], + input: BucketMergeInput( + sessionId: sessionId, + projectIdentity: projectIdentity, + metadata: metadata, + started: started, + latest: latest, + catalogModel: catalogEntry?.model, + model: model, + fileTotals: fileTotals, + usageRows: (usage.codexRows ?? []).filter { + CostUsageScanner.CostUsageDayRange.isInRange( + dayKey: $0.day, + since: range.sinceKey, + until: range.untilKey) + }, + hasCompleteEventRows: usage.days.isEmpty || !(usage.codexRows?.isEmpty ?? true) + && + (usage.codexRows? + .allSatisfy { $0.eventIndex != nil && $0.timestampUnixMs != nil } ?? false))) + self.reportProgressIfNeeded( + progress, + processedFiles: offset + 1, + totalFiles: files.count, + indexedFiles: indexedFiles, + skippedFiles: skippedFiles) + } + + return (sessionBuckets, indexedFiles, skippedFiles) + } + + fileprivate static func reportProgressIfNeeded( + _ progress: (@Sendable (CodexLocalProjectUsageIndexProgress) -> Void)?, + processedFiles: Int, + totalFiles: Int, + indexedFiles: Int, + skippedFiles: Int) + { + guard processedFiles == 1 || processedFiles == totalFiles || processedFiles.isMultiple(of: 25) else { + return + } + progress?(CodexLocalProjectUsageIndexProgress( + phase: .indexingProjects, + processedFileCount: processedFiles, + totalFileCount: totalFiles, + indexedFileCount: indexedFiles, + skippedFileCount: skippedFiles)) + } + + fileprivate static func mergedBucket( + _ current: SessionBucket?, + input: BucketMergeInput) -> SessionBucket + { + var bucket = current ?? SessionBucket( + id: input.sessionId, + projectId: input.projectIdentity.id, + projectDisplayName: input.projectIdentity.displayName, + projectPath: input.projectIdentity.path, + cwd: input.metadata.cwd, + title: input.metadata.title, + startedAt: input.started, + latestActivity: input.latest, + catalogModel: input.catalogModel, + modelTotals: [:], + dailyTotals: [:], + modelDailyTotals: [:], + usageRows: [], + hasCompleteEventRows: true, + total: .empty, + costNanos: nil, + unknownCostTokens: 0) + if self.shouldUseProjectIdentity(input.projectIdentity, over: bucket, latestActivity: input.latest) { + bucket.projectId = input.projectIdentity.id + bucket.projectDisplayName = input.projectIdentity.displayName + bucket.projectPath = input.projectIdentity.path + } + bucket.cwd = bucket.cwd ?? input.metadata.cwd + bucket.title = input.metadata.title ?? bucket.title + bucket.catalogModel = input.catalogModel ?? bucket.catalogModel + bucket.startedAt = self.earlier(bucket.startedAt, input.started) + bucket.latestActivity = self.later(bucket.latestActivity, input.latest) + bucket.total = bucket.total.adding(input.fileTotals.totals) + bucket.usageRows.append(contentsOf: input.usageRows) + bucket.hasCompleteEventRows = bucket.hasCompleteEventRows && input.hasCompleteEventRows + bucket.costNanos = self.addCost(bucket.costNanos, input.fileTotals.costNanos) + bucket.unknownCostTokens += input.fileTotals.unknownCostTokens + for (modelName, totals) in input.fileTotals.modelTotals { + self.mergeModelTotals(&bucket.modelTotals, model: modelName, totals: totals) + } + for (day, totals) in input.fileTotals.dailyTotals { + self.mergeDailyTotals(&bucket.dailyTotals, day: day, totals: totals) + } + for (day, models) in input.fileTotals.modelDailyTotals { + for (model, totals) in models { + self.mergeModelDailyTotals(&bucket.modelDailyTotals, day: day, model: model, totals: totals) + } + } + if let model = input.model, !model.isEmpty, bucket.modelTotals[model] == nil { + bucket.modelTotals[model] = ModelTotals(totalTokens: 0, costNanos: nil, unknownCostTokens: 0) + } + return bucket + } + + fileprivate static func shouldUseProjectIdentity( + _ identity: CodexLocalProjectRootResolver.ProjectIdentity, + over bucket: SessionBucket, + latestActivity: Date?) -> Bool + { + guard identity.id != CodexLocalProjectRootResolver.chatsProjectId else { + return bucket.projectId == CodexLocalProjectRootResolver.chatsProjectId + } + guard bucket.projectId != CodexLocalProjectRootResolver.chatsProjectId else { + return true + } + let currentLatest = bucket.latestActivity ?? .distantPast + let incomingLatest = latestActivity ?? .distantPast + return incomingLatest >= currentLatest + } + + fileprivate static func fileTotals( + from usage: CostUsageFileUsage, + range: CostUsageScanner.CostUsageDayRange) -> FileTotals? + { + var input = 0 + var cached = 0 + var output = 0 + var costNanos: Int64? + var unknownCostTokens = 0 + var modelTotals: [String: ModelTotals] = [:] + var dailyTotals: [String: DailyTotals] = [:] + var modelDailyTotals: [String: [String: ModelDailyTotals]] = [:] + + for (day, models) in usage.days where CostUsageScanner.CostUsageDayRange + .isInRange(dayKey: day, since: range.sinceKey, until: range.untilKey) + { + for (model, values) in models { + let modelInput = max(0, values[safe: 0] ?? 0) + let modelCached = max(0, values[safe: 1] ?? 0) + let modelOutput = max(0, values[safe: 2] ?? 0) + let modelTotal = modelInput + modelOutput + guard modelTotal > 0 else { continue } + input += modelInput + cached += min(modelCached, modelInput) + output += modelOutput + let modelCostNanos = usage.codexCostNanos?[day]?[model] + let modelUnknownCostTokens = modelCostNanos == nil ? modelTotal : 0 + self.addModelTotals( + &modelTotals, + model: model, + tokens: modelTotal, + costNanos: modelCostNanos, + unknownCostTokens: modelUnknownCostTokens) + costNanos = self.addCost(costNanos, modelCostNanos) + unknownCostTokens += modelUnknownCostTokens + self.addDailyTotals( + &dailyTotals, + day: day, + totals: DailyTotals( + totalTokens: modelTotal, + cachedInputTokens: min(modelCached, modelInput), + costNanos: modelCostNanos, + unknownCostTokens: modelUnknownCostTokens)) + self.mergeModelDailyTotals( + &modelDailyTotals, + day: day, + model: model, + totals: ModelDailyTotals( + inputTokens: modelInput, + cachedInputTokens: min(modelCached, modelInput), + outputTokens: modelOutput, + costNanos: modelCostNanos, + unknownCostTokens: modelUnknownCostTokens)) + } + } + + guard input + output > 0 else { return nil } + return FileTotals( + totals: CodexLocalUsageTotals( + inputTokens: input, + cachedInputTokens: cached, + outputTokens: output, + reasoningOutputTokens: nil, + totalTokens: input + output), + costNanos: costNanos, + unknownCostTokens: unknownCostTokens, + modelTotals: modelTotals, + dailyTotals: dailyTotals, + modelDailyTotals: modelDailyTotals, + topModel: self.topModel(from: modelTotals)) + } + + static func modelsAnalyticsPeriods( + since: Date, + until: Date, + calendar: Calendar = .current) -> CodexModelsAnalyticsPeriods + { + let currentStart = calendar.startOfDay(for: since) + let currentEnd = calendar.date(byAdding: .day, value: 1, to: calendar.startOfDay(for: until)) ?? until + let current = DateInterval(start: currentStart, end: currentEnd) + let previousEnd = current.start + let previousStart = previousEnd.addingTimeInterval(-current.duration) + return CodexModelsAnalyticsPeriods( + current: current, + previous: DateInterval(start: previousStart, end: previousEnd)) + } + + static func modelsAnalyticsScanStart( + since: Date, + until: Date, + calendar: Calendar = .current) -> Date + { + let periods = self.modelsAnalyticsPeriods(since: since, until: until, calendar: calendar) + return calendar.startOfDay(for: periods.previous.start) + } + + fileprivate static func modelsAnalyticsPayload( + context: ModelsAnalyticsContext, + window: ModelsAnalyticsWindow, + identity: ModelsAnalyticsIdentity, + checkCancellation: CostUsageScanner.CancellationCheck?) throws -> CodexModelsAnalyticsPayload + { + let aggregationSignpost = CodexModelsTelemetry.begin("SnapshotAggregation") + defer { CodexModelsTelemetry.end("SnapshotAggregation", id: aggregationSignpost) } + let periods = self.modelsAnalyticsPeriods(since: window.since, until: window.until) + let previousInterval = periods.previous + let previousRange = CostUsageScanner.CostUsageDayRange( + since: previousInterval.start, + until: previousInterval.end.addingTimeInterval(-1)) + let previousBuckets = try self.sessionBuckets( + from: context.cache, + range: previousRange, + catalog: context.catalog, + progress: nil, + checkCancellation: checkCancellation).sessionBuckets + let currentFragments = self.analyticsFragments(from: context.currentBuckets.values) + let previousFragments = self.analyticsFragments(from: previousBuckets.values) + let currentBucketsByProject = Dictionary(grouping: context.currentBuckets.values, by: \.projectId) + let previousBucketsByProject = Dictionary(grouping: previousBuckets.values, by: \.projectId) + let currentFragmentsByProject = Dictionary(grouping: currentFragments, by: \.workspaceID) + let previousFragmentsByProject = Dictionary(grouping: previousFragments, by: \.workspaceID) + let revisionParts = identity.rootsFingerprint.sorted { $0.key < $1.key }.map { "\($0.key)=\($0.value)" } + let indexRevision = ([ + identity.scopeSignature, + context.cache.producerKey ?? "", + context.cache.codexPricingKey ?? "", + ] + revisionParts) + .joined(separator: "|") + let builder = CodexModelsAnalyticsBuilder() + let source = CodexModelsAnalyticsSource(current: currentFragments, previous: previousFragments) + let revision = CodexModelsAnalyticsRevision(generatedAt: window.generatedAt, indexRevision: indexRevision) + // Catalog failures can make workspace attribution incomplete, while aggregate-only + // cache rows cannot prove exact timestamp-boundary coverage. Treat either condition + // conservatively so an observed partial period is never presented as a full comparison. + let metadataIsComplete = context.sourceStatus == .complete + let currentIsComplete = metadataIsComplete + && context.currentBuckets.values.allSatisfy(\.hasCompleteEventRows) + let previousIsComplete = metadataIsComplete + && previousBuckets.values.allSatisfy(\.hasCompleteEventRows) + let all = builder.build(CodexModelsAnalyticsRequest( + source: source, + scopeID: nil, + periods: periods, + revision: revision, + legacy: self.legacyBaseline( + current: Array(context.currentBuckets.values), + previous: Array(previousBuckets.values)), + currentIsComplete: currentIsComplete, + previousIsComplete: previousIsComplete)) + + let workspaces = Dictionary(uniqueKeysWithValues: context.projects.map { project in + let projectBuckets = currentBucketsByProject[project.id] ?? [] + let previousProjectBuckets = previousBucketsByProject[project.id] ?? [] + let currentProjectIsComplete = metadataIsComplete + && projectBuckets.allSatisfy(\.hasCompleteEventRows) + let previousProjectIsComplete = metadataIsComplete + && previousProjectBuckets.allSatisfy(\.hasCompleteEventRows) + return ( + project.id, + builder.build(CodexModelsAnalyticsRequest( + source: CodexModelsAnalyticsSource( + current: currentFragmentsByProject[project.id] ?? [], + previous: previousFragmentsByProject[project.id] ?? []), + scopeID: project.id, + periods: periods, + revision: revision, + legacy: self.legacyBaseline( + current: Array(projectBuckets), + previous: Array(previousProjectBuckets)), + currentIsComplete: currentProjectIsComplete, + previousIsComplete: previousProjectIsComplete))) + }) + CodexModelsTelemetry.parity( + dimensions: all.diagnostics.mismatchDimensions ?? [], + rowCount: all.rows.count) + return CodexModelsAnalyticsPayload(allWorkspaces: all, workspaces: workspaces) + } + + fileprivate static func legacyBaseline( + current: [SessionBucket], + previous: [SessionBucket]) -> CodexModelsLegacyBaseline + { + struct LegacyModelAggregates { + var tokens: Int64 = 0 + var costNanos: Int64? + var unpricedTokens: Int64 = 0 + var sessionIDs: Set = [] + } + + struct LegacyAggregates { + let tokens: Int64 + let cost: Decimal + let pricedTokens: Int64 + let unpricedTokens: Int64 + let modelIDs: [String] + let topModelID: String? + let sessionReferences: Int + let models: [CodexModelsLegacyModelBaseline] + } + + func aggregates(_ buckets: [SessionBucket]) -> LegacyAggregates { + let tokens = Int64(buckets.reduce(0) { $0 + ($1.total.totalTokens ?? 0) }) + let costNanos = buckets.reduce(Int64.zero) { $0 + ($1.costNanos ?? 0) } + let unpricedTokens = Int64(buckets.reduce(0) { $0 + $1.unknownCostTokens }) + var models: [String: LegacyModelAggregates] = [:] + for bucket in buckets { + for (model, totals) in bucket.modelTotals { + guard totals.totalTokens > 0 else { continue } + let canonical = CostUsagePricing.normalizeCodexModel( + model.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()) + var values = models[canonical, default: LegacyModelAggregates()] + values.tokens += Int64(totals.totalTokens) + values.costNanos = self.addCost(values.costNanos, totals.costNanos) + values.unpricedTokens += Int64(totals.unknownCostTokens) + values.sessionIDs.insert(bucket.id) + models[canonical] = values + } + } + let modelIDs = models.keys.sorted() + let topModelID = models.max { + if $0.value.tokens != $1.value.tokens { + return $0.value.tokens < $1.value.tokens + } + return $0.key > $1.key + }?.key + let modelBaselines = models.sorted { $0.key < $1.key }.map { modelID, values in + CodexModelsLegacyModelBaseline( + modelID: modelID, + totalTokens: values.tokens, + knownCost: values.costNanos.map { Decimal($0) / 1_000_000_000 }, + pricedTokens: max(0, values.tokens - values.unpricedTokens), + unpricedTokens: values.unpricedTokens, + sessionReferences: values.sessionIDs.count) + } + return LegacyAggregates( + tokens: tokens, + cost: Decimal(costNanos) / 1_000_000_000, + pricedTokens: max(0, tokens - unpricedTokens), + unpricedTokens: unpricedTokens, + modelIDs: modelIDs, + topModelID: topModelID, + sessionReferences: models.values.reduce(0) { $0 + $1.sessionIDs.count }, + models: modelBaselines) + } + + let currentValues = aggregates(current) + let previousValues = aggregates(previous) + return CodexModelsLegacyBaseline( + totalTokens: currentValues.tokens, + modelIDs: currentValues.modelIDs, + knownCost: currentValues.cost, + pricedTokens: currentValues.pricedTokens, + unpricedTokens: currentValues.unpricedTokens, + activeModelCount: currentValues.modelIDs.count, + topModelID: currentValues.topModelID, + sessionReferenceTotal: currentValues.sessionReferences, + previousTotalTokens: previousValues.tokens, + previousKnownCost: previousValues.cost, + previousUnpricedTokens: previousValues.unpricedTokens, + previousSessionReferenceTotal: previousValues.sessionReferences, + currentModels: currentValues.models, + previousModels: previousValues.models) + } + + fileprivate static func analyticsFragments( + from buckets: Dictionary.Values) -> [CodexModelsUsageFragment] + { + buckets.flatMap { bucket in + if bucket.hasCompleteEventRows, !bucket.usageRows.isEmpty { + return bucket.usageRows.compactMap { row -> CodexModelsUsageFragment? in + guard let timestampUnixMs = row.timestampUnixMs else { return nil } + let timestamp = Date(timeIntervalSince1970: Double(timestampUnixMs) / 1000) + let day = Calendar.current.startOfDay(for: timestamp) + let inputTokens = max(0, row.input) + let outputTokens = max(0, row.output) + let totalTokens = Int64(inputTokens + outputTokens) + return CodexModelsUsageFragment( + workspaceID: bucket.projectId, + sessionID: bucket.id, + day: day, + timestamp: timestamp, + rawModelID: row.rawModel ?? row.model, + inputTokens: Int64(inputTokens), + cachedInputTokens: Int64(max(0, row.cached)), + outputTokens: Int64(outputTokens), + reasoningTokens: row.reasoning.map(Int64.init), + costNanos: row.knownCostNanos, + unpricedTokens: row.unpricedTokens.map(Int64.init) + ?? (row.knownCostNanos == nil ? totalTokens : 0)) + } + } + return bucket.modelDailyTotals.flatMap { day, models -> [CodexModelsUsageFragment] in + guard let date = CostUsageDateParser.parse(day) else { return [] } + return models.map { model, totals in + CodexModelsUsageFragment( + workspaceID: bucket.projectId, + sessionID: bucket.id, + day: date, + rawModelID: model, + inputTokens: Int64(totals.inputTokens), + cachedInputTokens: Int64(totals.cachedInputTokens), + outputTokens: Int64(totals.outputTokens), + reasoningTokens: nil, + costNanos: totals.costNanos, + unpricedTokens: Int64(totals.unknownCostTokens)) + } + } + } + } + + fileprivate static func metadata( + from usage: CostUsageFileUsage, + catalogEntry: CodexThreadCatalogEntry?) -> Metadata + { + let session = usage.codexSession + return Metadata( + sessionId: catalogEntry?.id ?? session?.sessionId ?? usage.sessionId, + cwd: catalogEntry?.cwd ?? session?.cwd, + title: catalogEntry?.title ?? catalogEntry?.preview ?? session?.title, + startedAt: self.date(fromUnixMilliseconds: catalogEntry?.createdAtUnixMs ?? session?.startedAtUnixMs), + latestActivity: self + .date(fromUnixMilliseconds: catalogEntry?.updatedAtUnixMs ?? session?.latestActivityUnixMs)) + } + + fileprivate static func date(fromUnixMilliseconds unixMs: Int64?) -> Date? { + guard let unixMs else { return nil } + return Date(timeIntervalSince1970: Double(unixMs) / 1000) + } + + fileprivate static func scopeSignature( + options: CostUsageScanner.Options, + cache: CostUsageCache, + catalogFingerprint: String? = nil) -> String + { + let roots = self.rootsFingerprint(CostUsageScanner.codexRootsFingerprint(options: options)) + var parts = roots.sorted { $0.key < $1.key }.map { "root:\($0.key)=\($0.value)" } + parts.append("producer=\(cache.producerKey ?? "")") + parts.append("pricing=\(cache.codexPricingKey ?? "")") + parts.append("priorityMetadata=\(cache.codexPriorityMetadataKey ?? "")") + if let catalogFingerprint { + parts.append("catalog=\(catalogFingerprint)") + } + return "codex-local-project:" + parts.joined(separator: "|") + } + + fileprivate static func stableScopeSignature(options: CostUsageScanner.Options) -> String { + CodexLocalDataScope.resolve(options: options).identifier + } + + fileprivate static func rootsFingerprint(_ roots: [String: Int64]) -> [String: Int64] { + Dictionary(uniqueKeysWithValues: roots.sorted { $0.key < $1.key }) + } + + fileprivate static func total(from projects: [CodexLocalProjectUsage]) -> CodexLocalUsageTotals { + projects.reduce(.empty) { partial, project in + partial.adding(project.totals) + } + } + + fileprivate static func dailyPoints( + from usages: Dictionary.Values, + range: CostUsageScanner.CostUsageDayRange) -> [CodexLocalUsageDailyPoint] + { + var buckets: [String: (tokens: Int, cachedInputTokens: Int, costNanos: Int64?)] = [:] + for usage in usages { + for (day, models) in usage.days where CostUsageScanner.CostUsageDayRange + .isInRange(dayKey: day, since: range.sinceKey, until: range.untilKey) + { + var bucket = buckets[day] ?? (0, 0, nil) + for (model, values) in models { + let input = max(0, values[safe: 0] ?? 0) + let cached = min(max(0, values[safe: 1] ?? 0), input) + let tokens = input + max(0, values[safe: 2] ?? 0) + bucket.tokens += tokens + bucket.cachedInputTokens += cached + if let nanos = usage.codexCostNanos?[day]?[model] { + bucket.costNanos = self.addCost(bucket.costNanos, nanos) + } + } + buckets[day] = bucket + } + } + return buckets.sorted { $0.key < $1.key }.map { + CodexLocalUsageDailyPoint( + day: $0.key, + totalTokens: $0.value.tokens, + cachedInputTokens: $0.value.cachedInputTokens, + estimatedCostUSD: self.usd(fromNanos: $0.value.costNanos)) + } + } + + fileprivate static func dailyPoints(from totals: [String: DailyTotals]) -> [CodexLocalUsageDailyPoint] { + totals.sorted { $0.key < $1.key }.map { + CodexLocalUsageDailyPoint( + day: $0.key, + totalTokens: $0.value.totalTokens, + cachedInputTokens: $0.value.cachedInputTokens, + estimatedCostUSD: self.usd(fromNanos: $0.value.costNanos)) + } + } + + fileprivate static func modelBreakdowns(from modelTotals: [String: ModelTotals]) + -> [CodexLocalUsageModelBreakdown] { + modelTotals.sorted { + if $0.value.totalTokens != $1.value.totalTokens { + return $0.value.totalTokens > $1.value.totalTokens + } + return $0.key < $1.key + }.map { + CodexLocalUsageModelBreakdown( + model: $0.key, + totals: CodexLocalUsageTotals( + inputTokens: nil, + cachedInputTokens: nil, + outputTokens: nil, + reasoningOutputTokens: nil, + totalTokens: $0.value.totalTokens), + costEstimate: CodexLocalCostEstimate( + knownUSD: self.usd(fromNanos: $0.value.costNanos) ?? 0, + unknownTokens: $0.value.unknownCostTokens)) + } + } + + fileprivate static func globalModelBreakdowns( + from sessions: Dictionary.Values) -> [CodexLocalUsageModelBreakdown] + { + var modelTotals: [String: ModelTotals] = [:] + for session in sessions { + for (model, totals) in session.modelTotals { + self.mergeModelTotals(&modelTotals, model: model, totals: totals) + } + } + return self.modelBreakdowns(from: modelTotals) + } + + fileprivate static func topModel(from modelTotals: [String: ModelTotals]) -> String? { + modelTotals.max { + if $0.value.totalTokens != $1.value.totalTokens { + return $0.value.totalTokens < $1.value.totalTokens + } + return $0.key > $1.key + }?.key + } + + fileprivate static func addModelTotals( + _ modelTotals: inout [String: ModelTotals], + model: String, + tokens: Int, + costNanos: Int64?, + unknownCostTokens: Int) + { + self.mergeModelTotals( + &modelTotals, + model: model, + totals: ModelTotals( + totalTokens: tokens, + costNanos: costNanos, + unknownCostTokens: unknownCostTokens)) + } + + fileprivate static func addDailyTotals( + _ dailyTotals: inout [String: DailyTotals], + day: String, + totals: DailyTotals) + { + self.mergeDailyTotals( + &dailyTotals, + day: day, + totals: totals) + } + + fileprivate static func mergeDailyTotals( + _ dailyTotals: inout [String: DailyTotals], + day: String, + totals: DailyTotals) + { + var current = dailyTotals[day] ?? DailyTotals( + totalTokens: 0, + cachedInputTokens: 0, + costNanos: nil, + unknownCostTokens: 0) + current.totalTokens += totals.totalTokens + current.cachedInputTokens += totals.cachedInputTokens + current.costNanos = self.addCost(current.costNanos, totals.costNanos) + current.unknownCostTokens += totals.unknownCostTokens + dailyTotals[day] = current + } + + fileprivate static func mergeModelDailyTotals( + _ modelDailyTotals: inout [String: [String: ModelDailyTotals]], + day: String, + model: String, + totals: ModelDailyTotals) + { + var current = modelDailyTotals[day]?[model] ?? ModelDailyTotals( + inputTokens: 0, + cachedInputTokens: 0, + outputTokens: 0, + costNanos: nil, + unknownCostTokens: 0) + current.inputTokens += totals.inputTokens + current.cachedInputTokens += totals.cachedInputTokens + current.outputTokens += totals.outputTokens + current.costNanos = self.addCost(current.costNanos, totals.costNanos) + current.unknownCostTokens += totals.unknownCostTokens + modelDailyTotals[day, default: [:]][model] = current + } + + fileprivate static func mergeModelTotals( + _ modelTotals: inout [String: ModelTotals], + model: String, + totals: ModelTotals) + { + var existing = modelTotals[model] ?? ModelTotals(totalTokens: 0, costNanos: nil, unknownCostTokens: 0) + existing.totalTokens += totals.totalTokens + existing.costNanos = self.addCost(existing.costNanos, totals.costNanos) + existing.unknownCostTokens += totals.unknownCostTokens + modelTotals[model] = existing + } + + fileprivate static func sortProjects(_ lhs: CodexLocalProjectUsage, _ rhs: CodexLocalProjectUsage) -> Bool { + let lTokens = lhs.totals.totalTokens ?? -1 + let rTokens = rhs.totals.totalTokens ?? -1 + if lTokens != rTokens { + return lTokens > rTokens + } + let lCost = lhs.estimatedCostUSD ?? -1 + let rCost = rhs.estimatedCostUSD ?? -1 + if lCost != rCost { + return lCost > rCost + } + if lhs.sessionCount != rhs.sessionCount { + return lhs.sessionCount > rhs.sessionCount + } + if lhs.latestActivity != rhs.latestActivity { + return (lhs.latestActivity ?? .distantPast) > (rhs.latestActivity ?? .distantPast) + } + return lhs.displayName.localizedStandardCompare(rhs.displayName) == .orderedAscending + } + + fileprivate static func sortSessions(_ lhs: CodexLocalSessionUsage, _ rhs: CodexLocalSessionUsage) -> Bool { + let lTokens = lhs.totals.totalTokens ?? -1 + let rTokens = rhs.totals.totalTokens ?? -1 + if lTokens != rTokens { + return lTokens > rTokens + } + let lCost = lhs.estimatedCostUSD ?? -1 + let rCost = rhs.estimatedCostUSD ?? -1 + if lCost != rCost { + return lCost > rCost + } + let lDate = lhs.latestActivity ?? .distantPast + let rDate = rhs.latestActivity ?? .distantPast + if lDate != rDate { + return lDate > rDate + } + return lhs.id < rhs.id + } + + fileprivate static func sessionTitle( + explicitTitle: String?, + startedAt: Date?, + latestActivity: Date?, + model: String?) -> String + { + if let explicitTitle = explicitTitle?.trimmingCharacters(in: .whitespacesAndNewlines), + !explicitTitle.isEmpty + { + return explicitTitle + } + let date = latestActivity ?? startedAt + var parts: [String] = [] + if let date { + let formatter = DateFormatter() + formatter.locale = Locale(identifier: "en_US_POSIX") + formatter.dateStyle = .medium + formatter.timeStyle = .short + parts.append(formatter.string(from: date)) + } + if let model, !model.isEmpty { + parts.append(model) + } + // Core stores a semantic fallback title, never a UI localization key. + return parts.isEmpty ? CodexLocalSessionUsage.localChatFallbackTitle : parts.joined(separator: " ยท ") + } + + fileprivate static func latestDate(from dayKeys: Dictionary.Keys) -> Date? { + dayKeys.compactMap(CostUsageDateParser.parse).max() + } + + fileprivate static func earliestDate(from dayKeys: Dictionary.Keys) -> Date? { + dayKeys.compactMap(CostUsageDateParser.parse).min() + } + + fileprivate static func addCost(_ lhs: Int64?, _ rhs: Int64?) -> Int64? { + guard let rhs else { return lhs } + guard let lhs else { return rhs } + return lhs + rhs + } + + fileprivate static func earlier(_ lhs: Date?, _ rhs: Date?) -> Date? { + switch (lhs, rhs) { + case let (lhs?, rhs?): + min(lhs, rhs) + case let (lhs?, nil): + lhs + case let (nil, rhs?): + rhs + case (nil, nil): + nil + } + } + + fileprivate static func later(_ lhs: Date?, _ rhs: Date?) -> Date? { + switch (lhs, rhs) { + case let (lhs?, rhs?): + max(lhs, rhs) + case let (lhs?, nil): + lhs + case let (nil, rhs?): + rhs + case (nil, nil): + nil + } + } + + fileprivate static func usd(fromNanos nanos: Int64?) -> Double? { + guard let nanos else { return nil } + return Double(nanos) / 1_000_000_000 + } +} diff --git a/Sources/CodexBarCore/CodexLocalProjectUsageModels.swift b/Sources/CodexBarCore/CodexLocalProjectUsageModels.swift new file mode 100644 index 0000000000..c32687bf7a --- /dev/null +++ b/Sources/CodexBarCore/CodexLocalProjectUsageModels.swift @@ -0,0 +1,676 @@ +import Foundation + +public struct CodexLocalProjectUsageIndexProgress: Sendable, Equatable { + public enum Phase: Sendable, Equatable { + case scanningLogs + case indexingProjects + case saving + } + + public let phase: Phase + public let processedFileCount: Int? + public let totalFileCount: Int? + public let indexedFileCount: Int + public let skippedFileCount: Int + + public init( + phase: Phase, + processedFileCount: Int? = nil, + totalFileCount: Int? = nil, + indexedFileCount: Int = 0, + skippedFileCount: Int = 0) + { + self.phase = phase + self.processedFileCount = processedFileCount + self.totalFileCount = totalFileCount + self.indexedFileCount = indexedFileCount + self.skippedFileCount = skippedFileCount + } +} + +public struct CodexLocalProjectUsageSnapshot: Sendable, Codable, Equatable { + public let updatedAt: Date + public let historyDays: Int + public let scopeSignature: String + public let rootsFingerprint: [String: Int64] + public let indexedFileCount: Int + public let skippedFileCount: Int + public let total: CodexLocalUsageTotals + public let projects: [CodexLocalProjectUsage] + public let sessions: [CodexLocalSessionUsage] + public let modelBreakdowns: [CodexLocalUsageModelBreakdown] + public let daily: [CodexLocalUsageDailyPoint] + public let sourceStatus: CodexLocalProjectUsageSourceStatus + public let modelsAnalytics: CodexModelsAnalyticsPayload? + + private enum CodingKeys: String, CodingKey { + case updatedAt + case historyDays + case scopeSignature + case rootsFingerprint + case indexedFileCount + case skippedFileCount + case total + case projects + case sessions + case modelBreakdowns + case daily + case sourceStatus + case modelsAnalytics + } + + public init( + updatedAt: Date, + historyDays: Int, + scopeSignature: String, + rootsFingerprint: [String: Int64], + indexedFileCount: Int, + skippedFileCount: Int, + total: CodexLocalUsageTotals, + projects: [CodexLocalProjectUsage], + sessions: [CodexLocalSessionUsage] = [], + modelBreakdowns: [CodexLocalUsageModelBreakdown] = [], + daily: [CodexLocalUsageDailyPoint], + sourceStatus: CodexLocalProjectUsageSourceStatus = .complete, + modelsAnalytics: CodexModelsAnalyticsPayload? = nil) + { + self.updatedAt = updatedAt + self.historyDays = historyDays + self.scopeSignature = scopeSignature + self.rootsFingerprint = rootsFingerprint + self.indexedFileCount = indexedFileCount + self.skippedFileCount = skippedFileCount + self.total = total + self.projects = projects + self.sessions = sessions + self.modelBreakdowns = modelBreakdowns + self.daily = daily + self.sourceStatus = sourceStatus + self.modelsAnalytics = modelsAnalytics + } + + public init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: Self.CodingKeys.self) + self.updatedAt = try container.decode(Date.self, forKey: .updatedAt) + self.historyDays = try container.decode(Int.self, forKey: .historyDays) + self.scopeSignature = try container.decode(String.self, forKey: .scopeSignature) + self.rootsFingerprint = try container.decode([String: Int64].self, forKey: .rootsFingerprint) + self.indexedFileCount = try container.decode(Int.self, forKey: .indexedFileCount) + self.skippedFileCount = try container.decode(Int.self, forKey: .skippedFileCount) + self.total = try container.decode(CodexLocalUsageTotals.self, forKey: .total) + self.projects = try container.decode([CodexLocalProjectUsage].self, forKey: .projects) + self.sessions = try container.decodeIfPresent([CodexLocalSessionUsage].self, forKey: .sessions) ?? [] + self.modelBreakdowns = try container.decodeIfPresent( + [CodexLocalUsageModelBreakdown].self, + forKey: .modelBreakdowns) ?? [] + self.daily = try container.decode([CodexLocalUsageDailyPoint].self, forKey: .daily) + self.sourceStatus = try container.decodeIfPresent( + CodexLocalProjectUsageSourceStatus.self, + forKey: .sourceStatus) ?? .complete + self.modelsAnalytics = try container.decodeIfPresent( + CodexModelsAnalyticsPayload.self, + forKey: .modelsAnalytics) + } + + /// An aggregate without per-project detail cannot render the inspector + /// truthfully. It should trigger a sidecar-backed refresh instead of + /// publishing empty charts and session lists beside valid totals. + public var hasInspectorDetail: Bool { + let projectsAreComplete = self.projects.allSatisfy { project in + guard (project.totals.totalTokens ?? 0) > 0 else { return true } + return !project.daily.isEmpty && (project.sessionCount == 0 || !project.topSessions.isEmpty) + } + let sessionsAreComplete = self.sessions.allSatisfy { session in + guard (session.totals.totalTokens ?? 0) > 0 else { return true } + return !session.daily.isEmpty + } + return projectsAreComplete && sessionsAreComplete + } +} + +extension CodexLocalProjectUsageSnapshot { + func hidingPersonalInformation(_ isEnabled: Bool) -> Self { + guard isEnabled else { return self } + return Self( + updatedAt: self.updatedAt, + historyDays: self.historyDays, + scopeSignature: self.scopeSignature, + rootsFingerprint: [:], + indexedFileCount: self.indexedFileCount, + skippedFileCount: self.skippedFileCount, + total: self.total, + projects: self.projects.map { $0.hidingPersonalInformation() }, + sessions: self.sessions.map { $0.hidingPersonalInformation() }, + modelBreakdowns: self.modelBreakdowns, + daily: self.daily, + sourceStatus: self.sourceStatus, + modelsAnalytics: self.modelsAnalytics) + } +} + +public enum CodexLocalProjectUsageSourceStatus: String, Sendable, Codable, Equatable { + case complete + case catalogMissing + case catalogLocked + case catalogCorrupt + case catalogIncompatible + case catalogUnreadable + + public var isPartial: Bool { + self != .complete + } + + init(catalog: CodexThreadCatalogCompleteness) { + switch catalog { + case .complete: + self = .complete + case let .unavailable(failure): + switch failure { + case .missing: self = .catalogMissing + case .locked: self = .catalogLocked + case .corrupt: self = .catalogCorrupt + case .incompatible: self = .catalogIncompatible + case .unreadable: self = .catalogUnreadable + } + } + } +} + +public enum CodexLocalUsageSeverity: String, Sendable, Codable, Equatable { + case normal + case elevated + case high +} + +public enum CodexLocalCostCoverage: String, Sendable, Codable, Equatable { + case known + case partial + case unavailable +} + +public struct CodexLocalCostEstimate: Sendable, Codable, Equatable { + public let knownUSD: Double + public let unknownTokens: Int + + public init(knownUSD: Double = 0, unknownTokens: Int = 0) { + self.knownUSD = max(0, knownUSD) + self.unknownTokens = max(0, unknownTokens) + } + + public var coverage: CodexLocalCostCoverage { + if self.unknownTokens == 0 { + return .known + } + return self.knownUSD > 0 ? .partial : .unavailable + } + + public var knownUSDOrNil: Double? { + self.knownUSD > 0 || self.unknownTokens == 0 ? self.knownUSD : nil + } +} + +public struct CodexLocalProjectUsage: Sendable, Codable, Identifiable, Equatable { + public let id: String + public let displayName: String + public let path: String? + public let totals: CodexLocalUsageTotals + public let costEstimate: CodexLocalCostEstimate + public let usageSeverity: CodexLocalUsageSeverity? + public let sessionCount: Int + public let latestActivity: Date? + public let topModel: String? + public let topSessions: [CodexLocalSessionUsage] + public let modelBreakdowns: [CodexLocalUsageModelBreakdown] + /// Raw daily totals belonging to this project. The presentation layer + /// applies cache inclusion and cost visibility without re-indexing. + public let daily: [CodexLocalUsageDailyPoint] + + public var severity: CodexLocalUsageSeverity { + self.usageSeverity ?? .normal + } + + public var estimatedCostUSD: Double? { + self.costEstimate.knownUSDOrNil + } + + public var hasUnknownCost: Bool { + self.costEstimate.unknownTokens > 0 + } + + public init( + id: String, + displayName: String, + path: String?, + totals: CodexLocalUsageTotals, + estimatedCostUSD: Double?, + hasUnknownCost: Bool, + sessionCount: Int, + latestActivity: Date?, + topModel: String?, + topSessions: [CodexLocalSessionUsage], + modelBreakdowns: [CodexLocalUsageModelBreakdown], + daily: [CodexLocalUsageDailyPoint] = [], + usageSeverity: CodexLocalUsageSeverity = .normal) + { + self.id = id + self.displayName = displayName + self.path = path + self.totals = totals + self.costEstimate = CodexLocalCostEstimate( + knownUSD: estimatedCostUSD ?? 0, + unknownTokens: hasUnknownCost ? totals.totalTokens ?? 0 : 0) + self.usageSeverity = usageSeverity + self.sessionCount = sessionCount + self.latestActivity = latestActivity + self.topModel = topModel + self.topSessions = topSessions + self.modelBreakdowns = modelBreakdowns + self.daily = daily + } + + public init( + id: String, + displayName: String, + path: String?, + totals: CodexLocalUsageTotals, + costEstimate: CodexLocalCostEstimate, + sessionCount: Int, + latestActivity: Date?, + topModel: String?, + topSessions: [CodexLocalSessionUsage], + modelBreakdowns: [CodexLocalUsageModelBreakdown], + daily: [CodexLocalUsageDailyPoint] = [], + usageSeverity: CodexLocalUsageSeverity? = nil) + { + self.id = id + self.displayName = displayName + self.path = path + self.totals = totals + self.costEstimate = costEstimate + self.usageSeverity = usageSeverity + self.sessionCount = sessionCount + self.latestActivity = latestActivity + self.topModel = topModel + self.topSessions = topSessions + self.modelBreakdowns = modelBreakdowns + self.daily = daily + } + + private enum CodingKeys: String, CodingKey { + case id + case displayName + case path + case totals + case costEstimate + case estimatedCostUSD + case hasUnknownCost + case sessionCount + case latestActivity + case topModel + case topSessions + case modelBreakdowns + case daily + } + + public init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + self.id = try container.decode(String.self, forKey: .id) + self.displayName = try container.decode(String.self, forKey: .displayName) + self.path = try container.decodeIfPresent(String.self, forKey: .path) + self.totals = try container.decode(CodexLocalUsageTotals.self, forKey: .totals) + self.costEstimate = try container.decodeIfPresent(CodexLocalCostEstimate.self, forKey: .costEstimate) + ?? CodexLocalCostEstimate( + knownUSD: container.decodeIfPresent(Double.self, forKey: .estimatedCostUSD) ?? 0, + unknownTokens: (container.decodeIfPresent(Bool.self, forKey: .hasUnknownCost) ?? false) + ? self.totals.totalTokens ?? 0 : 0) + // Severity is a display projection and intentionally is not persisted. + self.usageSeverity = nil + self.sessionCount = try container.decode(Int.self, forKey: .sessionCount) + self.latestActivity = try container.decodeIfPresent(Date.self, forKey: .latestActivity) + self.topModel = try container.decodeIfPresent(String.self, forKey: .topModel) + self.topSessions = try container.decodeIfPresent([CodexLocalSessionUsage].self, forKey: .topSessions) ?? [] + self.modelBreakdowns = try container.decodeIfPresent( + [CodexLocalUsageModelBreakdown].self, + forKey: .modelBreakdowns) ?? [] + self.daily = try container.decodeIfPresent([CodexLocalUsageDailyPoint].self, forKey: .daily) ?? [] + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(self.id, forKey: .id) + try container.encode(self.displayName, forKey: .displayName) + try container.encodeIfPresent(self.path, forKey: .path) + try container.encode(self.totals, forKey: .totals) + try container.encode(self.costEstimate, forKey: .costEstimate) + try container.encode(self.sessionCount, forKey: .sessionCount) + try container.encodeIfPresent(self.latestActivity, forKey: .latestActivity) + try container.encodeIfPresent(self.topModel, forKey: .topModel) + try container.encode(self.topSessions, forKey: .topSessions) + try container.encode(self.modelBreakdowns, forKey: .modelBreakdowns) + try container.encode(self.daily, forKey: .daily) + } +} + +extension CodexLocalProjectUsage { + fileprivate func hidingPersonalInformation() -> Self { + Self( + id: self.id, + displayName: self.id == CodexLocalProjectRootResolver.chatsProjectId + ? CodexLocalProjectRootResolver.chatsDisplayName + : "Workspace", + path: nil, + totals: self.totals, + costEstimate: self.costEstimate, + sessionCount: self.sessionCount, + latestActivity: self.latestActivity, + topModel: self.topModel, + topSessions: self.topSessions.map { $0.hidingPersonalInformation() }, + modelBreakdowns: self.modelBreakdowns, + daily: self.daily, + usageSeverity: self.usageSeverity) + } +} + +public struct CodexLocalSessionUsage: Sendable, Codable, Identifiable, Equatable { + /// Semantic fallback used when Codex did not persist a title. UI layers + /// localize this value instead of storing a localization key in Core. + public static let localChatFallbackTitle = "Local Codex chat" + + public let id: String + public let projectId: String + public let displayTitle: String + public let cwd: String? + public let startedAt: Date? + public let latestActivity: Date? + public let totals: CodexLocalUsageTotals + public let costEstimate: CodexLocalCostEstimate + public let topModel: String? + public let daily: [CodexLocalUsageDailyPoint] + + public init( + id: String, + projectId: String, + displayTitle: String, + cwd: String?, + startedAt: Date?, + latestActivity: Date?, + totals: CodexLocalUsageTotals, + estimatedCostUSD: Double?, + hasUnknownCost: Bool, + topModel: String?, + daily: [CodexLocalUsageDailyPoint] = []) + { + self.id = id + self.projectId = projectId + self.displayTitle = displayTitle + self.cwd = cwd + self.startedAt = startedAt + self.latestActivity = latestActivity + self.totals = totals + self.costEstimate = CodexLocalCostEstimate( + knownUSD: estimatedCostUSD ?? 0, + unknownTokens: hasUnknownCost ? totals.totalTokens ?? 0 : 0) + self.topModel = topModel + self.daily = daily + } + + public var estimatedCostUSD: Double? { + self.costEstimate.knownUSDOrNil + } + + public var hasUnknownCost: Bool { + self.costEstimate.unknownTokens > 0 + } + + public init( + id: String, + projectId: String, + displayTitle: String, + cwd: String?, + startedAt: Date?, + latestActivity: Date?, + totals: CodexLocalUsageTotals, + costEstimate: CodexLocalCostEstimate, + topModel: String?, + daily: [CodexLocalUsageDailyPoint] = []) + { + self.id = id + self.projectId = projectId + self.displayTitle = displayTitle + self.cwd = cwd + self.startedAt = startedAt + self.latestActivity = latestActivity + self.totals = totals + self.costEstimate = costEstimate + self.topModel = topModel + self.daily = daily + } + + private enum CodingKeys: String, CodingKey { + case id + case projectId + case displayTitle + case cwd + case startedAt + case latestActivity + case totals + case costEstimate + case estimatedCostUSD + case hasUnknownCost + case topModel + case daily + } + + public init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + self.id = try container.decode(String.self, forKey: .id) + self.projectId = try container.decode(String.self, forKey: .projectId) + self.displayTitle = try container.decode(String.self, forKey: .displayTitle) + self.cwd = try container.decodeIfPresent(String.self, forKey: .cwd) + self.startedAt = try container.decodeIfPresent(Date.self, forKey: .startedAt) + self.latestActivity = try container.decodeIfPresent(Date.self, forKey: .latestActivity) + self.totals = try container.decode(CodexLocalUsageTotals.self, forKey: .totals) + self.costEstimate = try container.decodeIfPresent(CodexLocalCostEstimate.self, forKey: .costEstimate) + ?? CodexLocalCostEstimate( + knownUSD: container.decodeIfPresent(Double.self, forKey: .estimatedCostUSD) ?? 0, + unknownTokens: (container.decodeIfPresent(Bool.self, forKey: .hasUnknownCost) ?? false) + ? self.totals.totalTokens ?? 0 : 0) + self.topModel = try container.decodeIfPresent(String.self, forKey: .topModel) + self.daily = try container.decodeIfPresent([CodexLocalUsageDailyPoint].self, forKey: .daily) ?? [] + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(self.id, forKey: .id) + try container.encode(self.projectId, forKey: .projectId) + try container.encode(self.displayTitle, forKey: .displayTitle) + try container.encodeIfPresent(self.cwd, forKey: .cwd) + try container.encodeIfPresent(self.startedAt, forKey: .startedAt) + try container.encodeIfPresent(self.latestActivity, forKey: .latestActivity) + try container.encode(self.totals, forKey: .totals) + try container.encode(self.costEstimate, forKey: .costEstimate) + try container.encodeIfPresent(self.topModel, forKey: .topModel) + try container.encode(self.daily, forKey: .daily) + } +} + +extension CodexLocalSessionUsage { + fileprivate func hidingPersonalInformation() -> Self { + Self( + id: self.id, + projectId: self.projectId, + displayTitle: Self.localChatFallbackTitle, + cwd: nil, + startedAt: self.startedAt, + latestActivity: self.latestActivity, + totals: self.totals, + costEstimate: self.costEstimate, + topModel: self.topModel, + daily: self.daily) + } +} + +public struct CodexLocalUsageTotals: Sendable, Codable, Equatable { + public let inputTokens: Int? + public let cachedInputTokens: Int? + public let outputTokens: Int? + public let reasoningOutputTokens: Int? + public let totalTokens: Int? + + public init( + inputTokens: Int?, + cachedInputTokens: Int?, + outputTokens: Int?, + reasoningOutputTokens: Int? = nil, + totalTokens: Int?) + { + self.inputTokens = inputTokens + self.cachedInputTokens = cachedInputTokens + self.outputTokens = outputTokens + self.reasoningOutputTokens = reasoningOutputTokens + self.totalTokens = totalTokens + } + + public static let unknown = CodexLocalUsageTotals( + inputTokens: nil, + cachedInputTokens: nil, + outputTokens: nil, + totalTokens: nil) + + public static let empty = CodexLocalUsageTotals( + inputTokens: 0, + cachedInputTokens: 0, + outputTokens: 0, + totalTokens: 0) + + public func adding(_ other: CodexLocalUsageTotals) -> CodexLocalUsageTotals { + CodexLocalUsageTotals( + inputTokens: Self.add(self.inputTokens, other.inputTokens), + cachedInputTokens: Self.add(self.cachedInputTokens, other.cachedInputTokens), + outputTokens: Self.add(self.outputTokens, other.outputTokens), + reasoningOutputTokens: Self.add(self.reasoningOutputTokens, other.reasoningOutputTokens), + totalTokens: Self.add(self.totalTokens, other.totalTokens)) + } + + private static func add(_ lhs: Int?, _ rhs: Int?) -> Int? { + switch (lhs, rhs) { + case let (lhs?, rhs?): + lhs + rhs + case let (lhs?, nil): + lhs + case let (nil, rhs?): + rhs + case (nil, nil): + nil + } + } +} + +public struct CodexLocalUsageModelBreakdown: Sendable, Codable, Identifiable, Equatable { + public var id: String { + self.model + } + + public let model: String + public let totals: CodexLocalUsageTotals + public let costEstimate: CodexLocalCostEstimate + + public init( + model: String, + totals: CodexLocalUsageTotals, + estimatedCostUSD: Double?, + hasUnknownCost: Bool) + { + self.model = model + self.totals = totals + self.costEstimate = CodexLocalCostEstimate( + knownUSD: estimatedCostUSD ?? 0, + unknownTokens: hasUnknownCost ? totals.totalTokens ?? 0 : 0) + } + + public var estimatedCostUSD: Double? { + self.costEstimate.knownUSDOrNil + } + + public var hasUnknownCost: Bool { + self.costEstimate.unknownTokens > 0 + } + + public init(model: String, totals: CodexLocalUsageTotals, costEstimate: CodexLocalCostEstimate) { + self.model = model + self.totals = totals + self.costEstimate = costEstimate + } + + private enum CodingKeys: String, CodingKey { + case model + case totals + case costEstimate + case estimatedCostUSD + case hasUnknownCost + } + + public init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + self.model = try container.decode(String.self, forKey: .model) + self.totals = try container.decode(CodexLocalUsageTotals.self, forKey: .totals) + self.costEstimate = try container.decodeIfPresent(CodexLocalCostEstimate.self, forKey: .costEstimate) + ?? CodexLocalCostEstimate( + knownUSD: container.decodeIfPresent(Double.self, forKey: .estimatedCostUSD) ?? 0, + unknownTokens: (container.decodeIfPresent(Bool.self, forKey: .hasUnknownCost) ?? false) + ? self.totals.totalTokens ?? 0 : 0) + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(self.model, forKey: .model) + try container.encode(self.totals, forKey: .totals) + try container.encode(self.costEstimate, forKey: .costEstimate) + } +} + +public struct CodexLocalUsageDailyPoint: Sendable, Codable, Identifiable, Equatable { + public var id: String { + self.day + } + + public let day: String + public let totalTokens: Int + public let cachedInputTokens: Int? + public let estimatedCostUSD: Double? + + public init( + day: String, + totalTokens: Int, + cachedInputTokens: Int? = nil, + estimatedCostUSD: Double?) + { + self.day = day + self.totalTokens = totalTokens + self.cachedInputTokens = cachedInputTokens + self.estimatedCostUSD = estimatedCostUSD + } + + private enum CodingKeys: String, CodingKey { + case day + case totalTokens + case cachedInputTokens + case estimatedCostUSD + } + + public init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + self.day = try container.decode(String.self, forKey: .day) + self.totalTokens = try container.decode(Int.self, forKey: .totalTokens) + self.cachedInputTokens = try container.decodeIfPresent(Int.self, forKey: .cachedInputTokens) + self.estimatedCostUSD = try container.decodeIfPresent(Double.self, forKey: .estimatedCostUSD) + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(self.day, forKey: .day) + try container.encode(self.totalTokens, forKey: .totalTokens) + try container.encodeIfPresent(self.cachedInputTokens, forKey: .cachedInputTokens) + try container.encodeIfPresent(self.estimatedCostUSD, forKey: .estimatedCostUSD) + } +} diff --git a/Sources/CodexBarCore/CodexLocalProjectUsageProjection.swift b/Sources/CodexBarCore/CodexLocalProjectUsageProjection.swift new file mode 100644 index 0000000000..d440ab5121 --- /dev/null +++ b/Sources/CodexBarCore/CodexLocalProjectUsageProjection.swift @@ -0,0 +1,98 @@ +import Foundation + +/// Transient presentation rules for a raw local-usage snapshot. +/// +/// The index persists only source-derived counters and cost coverage. Settings +/// such as cache inclusion and cost visibility are deliberately applied here, +/// so changing them never requires a corpus scan or sidecar rewrite. +public struct CodexLocalProjectUsageProjection: Sendable, Equatable { + public let includesCachedInput: Bool + public let showsEstimatedCost: Bool + + public init(includesCachedInput: Bool, showsEstimatedCost: Bool) { + self.includesCachedInput = includesCachedInput + self.showsEstimatedCost = showsEstimatedCost + } + + public func displayedTokens(for totals: CodexLocalUsageTotals) -> Int? { + guard let total = totals.totalTokens else { return nil } + return self.displayedTokens(totalTokens: total, cachedInputTokens: totals.cachedInputTokens) + } + + public func displayedTokens(totalTokens: Int, cachedInputTokens: Int?) -> Int { + guard !self.includesCachedInput else { return max(0, totalTokens) } + return max(0, totalTokens - (cachedInputTokens ?? 0)) + } + + public func rankedProjects(_ projects: [CodexLocalProjectUsage]) -> [CodexLocalProjectUsage] { + let severities = self.severities(for: projects) + return projects.sorted { lhs, rhs in + let lhsTokens = self.displayedTokens(for: lhs.totals) ?? -1 + let rhsTokens = self.displayedTokens(for: rhs.totals) ?? -1 + if lhsTokens != rhsTokens { return lhsTokens > rhsTokens } + + let lhsCost = lhs.costEstimate.knownUSD + let rhsCost = rhs.costEstimate.knownUSD + if lhsCost != rhsCost { return lhsCost > rhsCost } + if lhs.sessionCount != rhs.sessionCount { return lhs.sessionCount > rhs.sessionCount } + if lhs.latestActivity != rhs.latestActivity { + return (lhs.latestActivity ?? .distantPast) > (rhs.latestActivity ?? .distantPast) + } + return lhs.displayName.localizedStandardCompare(rhs.displayName) == .orderedAscending + }.map { project in + project.withDisplaySeverity(severities[project.id] ?? .normal) + } + } + + public func displayedCost(for estimate: CodexLocalCostEstimate) -> CodexLocalCostEstimate? { + self.showsEstimatedCost ? estimate : nil + } + + private func severities(for projects: [CodexLocalProjectUsage]) -> [String: CodexLocalUsageSeverity] { + let nonZero = projects.compactMap { self.displayedTokens(for: $0.totals) }.filter { $0 > 0 }.sorted() + let total = nonZero.reduce(0, +) + let median = self.percentile(nonZero, percentile: 0.5) + let p90 = self.percentile(nonZero, percentile: 0.9) + let outlierThreshold = max(p90, median * 5) + let maximum = nonZero.last ?? 0 + + return Dictionary(uniqueKeysWithValues: projects.map { project in + let tokens = self.displayedTokens(for: project.totals) ?? 0 + let severity: CodexLocalUsageSeverity = if tokens > 0, total > 0, + Double(tokens) >= Double(total) * 0.5 || + (tokens == maximum && tokens > outlierThreshold) + { + .high + } else if tokens > 0, median > 0, tokens >= median * 2 { + .elevated + } else { + .normal + } + return (project.id, severity) + }) + } + + private func percentile(_ values: [Int], percentile: Double) -> Int { + guard !values.isEmpty else { return 0 } + let index = Int((Double(values.count - 1) * percentile).rounded(.up)) + return values[min(max(index, 0), values.count - 1)] + } +} + +extension CodexLocalProjectUsage { + fileprivate func withDisplaySeverity(_ severity: CodexLocalUsageSeverity) -> Self { + CodexLocalProjectUsage( + id: self.id, + displayName: self.displayName, + path: self.path, + totals: self.totals, + costEstimate: self.costEstimate, + sessionCount: self.sessionCount, + latestActivity: self.latestActivity, + topModel: self.topModel, + topSessions: self.topSessions, + modelBreakdowns: self.modelBreakdowns, + daily: self.daily, + usageSeverity: severity) + } +} diff --git a/Sources/CodexBarCore/CodexModelsAnalyticsModels.swift b/Sources/CodexBarCore/CodexModelsAnalyticsModels.swift new file mode 100644 index 0000000000..556fd16fe5 --- /dev/null +++ b/Sources/CodexBarCore/CodexModelsAnalyticsModels.swift @@ -0,0 +1,883 @@ +import Foundation + +public enum CodexModelsRollout { + public static let featureFlagKey = "codex.models.revamp.enabled" + public static let defaultEnabled = true + + public static func isEnabled(defaults: UserDefaults = .standard) -> Bool { + defaults.object(forKey: self.featureFlagKey) as? Bool ?? self.defaultEnabled + } +} + +public enum CodexModelsMetric: String, CaseIterable, Codable, Identifiable, Sendable { + case tokens + case knownCost + case sessionReferences + + public var id: Self { + self + } + + public var title: String { + switch self { + case .tokens: "Tokens" + case .knownCost: "Cost" + case .sessionReferences: "Session refs" + } + } +} + +public enum CodexModelsGranularity: String, CaseIterable, Codable, Identifiable, Sendable { + case daily + case weekly + case monthly + + public var id: Self { + self + } + + public var title: String { + self.rawValue.capitalized + } +} + +public enum CodexModelsComparison: Codable, Equatable, Sendable { + case unavailable + case new + case ended + case unchanged + case percent(Double) + + public static func make(current: Double, previous: Double, previousIsComplete: Bool = true) -> Self { + guard previousIsComplete else { return .unavailable } + if current == 0, previous == 0 { return .unchanged } + if previous == 0 { return .new } + if current == 0 { return .ended } + let value = (current - previous) / previous + return value == 0 ? .unchanged : .percent(value) + } + + public var sortableValue: Double { + switch self { + case .unavailable: -.infinity + case .new: .infinity + case .ended: -1 + case .unchanged: 0 + case let .percent(value): value + } + } +} + +public struct CodexModelsCost: Codable, Equatable, Sendable { + public let knownAmount: Decimal + public let pricedTokens: Int64 + public let unpricedTokens: Int64 + public let currencyCode: String + + public init( + knownAmount: Decimal, + pricedTokens: Int64, + unpricedTokens: Int64, + currencyCode: String = "USD") + { + self.knownAmount = knownAmount + self.pricedTokens = pricedTokens + self.unpricedTokens = unpricedTokens + self.currencyCode = currencyCode + } + + public static let zero = Self(knownAmount: 0, pricedTokens: 0, unpricedTokens: 0) + + public var coverage: Double { + let total = self.pricedTokens + self.unpricedTokens + return total == 0 ? 1 : Double(self.pricedTokens) / Double(total) + } + + public func adding(_ other: Self) -> Self { + Self( + knownAmount: self.knownAmount + other.knownAmount, + pricedTokens: self.pricedTokens + other.pricedTokens, + unpricedTokens: self.unpricedTokens + other.unpricedTokens, + currencyCode: self.currencyCode) + } +} + +public struct CodexModelsUsageFragment: Equatable, Sendable { + public let workspaceID: String + public let sessionID: String + public let day: Date + public let timestamp: Date + public let rawModelID: String + public let inputTokens: Int64 + public let cachedInputTokens: Int64 + public let outputTokens: Int64 + public let reasoningTokens: Int64? + public let costNanos: Int64? + public let unpricedTokens: Int64 + + public init( + workspaceID: String, + sessionID: String, + day: Date, + timestamp: Date? = nil, + rawModelID: String, + inputTokens: Int64, + cachedInputTokens: Int64, + outputTokens: Int64, + reasoningTokens: Int64? = nil, + costNanos: Int64?, + unpricedTokens: Int64? = nil) + { + self.workspaceID = workspaceID + self.sessionID = sessionID + self.day = day + self.timestamp = timestamp ?? day + self.rawModelID = rawModelID + self.inputTokens = max(0, inputTokens) + self.cachedInputTokens = min(max(0, cachedInputTokens), max(0, inputTokens)) + self.outputTokens = max(0, outputTokens) + self.reasoningTokens = reasoningTokens.map { min(max(0, $0), max(0, outputTokens)) } + self.costNanos = costNanos + let totalTokens = self.inputTokens + self.outputTokens + let defaultUnpricedTokens = costNanos == nil ? totalTokens : 0 + self.unpricedTokens = min(max(0, unpricedTokens ?? defaultUnpricedTokens), totalTokens) + } + + public var totalTokens: Int64 { + self.inputTokens + self.outputTokens + } +} + +public struct CodexModelsDailyBucket: Codable, Equatable, Identifiable, Sendable { + public let day: Date + public let interval: DateInterval? + public let tokens: Int64 + public let sessionIDs: [String] + public let sessionReferenceIDs: [String] + public let cost: CodexModelsCost + + public var id: Date { + self.day + } + + public var sessionReferences: Int { + self.sessionReferenceIDs.count + } + + public init( + day: Date, + interval: DateInterval? = nil, + tokens: Int64, + sessionIDs: [String], + sessionReferenceIDs: [String]? = nil, + cost: CodexModelsCost) + { + self.day = day + self.interval = interval + self.tokens = tokens + self.sessionIDs = sessionIDs + self.sessionReferenceIDs = sessionReferenceIDs ?? sessionIDs + self.cost = cost + } + + private enum CodingKeys: String, CodingKey { + case day + case interval + case tokens + case sessionIDs + case sessionReferenceIDs + case cost + } + + public init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + self.day = try container.decode(Date.self, forKey: .day) + self.interval = try container.decodeIfPresent(DateInterval.self, forKey: .interval) + self.tokens = try container.decode(Int64.self, forKey: .tokens) + self.sessionIDs = try container.decode([String].self, forKey: .sessionIDs) + self.sessionReferenceIDs = try container.decodeIfPresent([String].self, forKey: .sessionReferenceIDs) + ?? self.sessionIDs + self.cost = try container.decode(CodexModelsCost.self, forKey: .cost) + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(self.day, forKey: .day) + try container.encodeIfPresent(self.interval, forKey: .interval) + try container.encode(self.tokens, forKey: .tokens) + try container.encode(self.sessionIDs, forKey: .sessionIDs) + try container.encode(self.sessionReferenceIDs, forKey: .sessionReferenceIDs) + try container.encode(self.cost, forKey: .cost) + } + + public var effectiveInterval: DateInterval { + self.interval ?? DateInterval(start: self.day, duration: 24 * 60 * 60) + } +} + +public struct CodexModelsRow: Codable, Equatable, Identifiable, Sendable { + public let id: String + public let displayName: String + public let rawAliases: [String] + public let inputTokens: Int64 + public let cachedInputTokens: Int64 + public let outputTokens: Int64 + public let reasoningTokens: Int64? + public let totalTokens: Int64 + public let share: Double + public let sessionReferences: Int + public let cost: CodexModelsCost + public let previousTotalTokens: Int64? + public let previousCost: CodexModelsCost? + public let previousSessionReferences: Int? + public let associatedSessionIDs: [String]? + public let tokenComparison: CodexModelsComparison + public let costComparison: CodexModelsComparison + public let sessionReferenceComparison: CodexModelsComparison + + public init( + id: String, + displayName: String, + rawAliases: [String], + inputTokens: Int64, + cachedInputTokens: Int64, + outputTokens: Int64, + reasoningTokens: Int64?, + totalTokens: Int64, + share: Double, + sessionReferences: Int, + cost: CodexModelsCost, + previousTotalTokens: Int64? = nil, + previousCost: CodexModelsCost? = nil, + previousSessionReferences: Int? = nil, + associatedSessionIDs: [String]? = nil, + tokenComparison: CodexModelsComparison, + costComparison: CodexModelsComparison, + sessionReferenceComparison: CodexModelsComparison) + { + self.id = id + self.displayName = displayName + self.rawAliases = rawAliases + self.inputTokens = inputTokens + self.cachedInputTokens = cachedInputTokens + self.outputTokens = outputTokens + self.reasoningTokens = reasoningTokens + self.totalTokens = totalTokens + self.share = share + self.sessionReferences = sessionReferences + self.cost = cost + self.previousTotalTokens = previousTotalTokens + self.previousCost = previousCost + self.previousSessionReferences = previousSessionReferences + self.associatedSessionIDs = associatedSessionIDs + self.tokenComparison = tokenComparison + self.costComparison = costComparison + self.sessionReferenceComparison = sessionReferenceComparison + } + + public func metricValue(_ metric: CodexModelsMetric) -> Double { + switch metric { + case .tokens: Double(self.totalTokens) + case .knownCost: NSDecimalNumber(decimal: self.cost.knownAmount).doubleValue + case .sessionReferences: Double(self.sessionReferences) + } + } + + public func comparison(_ metric: CodexModelsMetric) -> CodexModelsComparison { + switch metric { + case .tokens: self.tokenComparison + case .knownCost: self.costComparison + case .sessionReferences: self.sessionReferenceComparison + } + } +} + +public enum CodexModelsParityDimension: String, Codable, CaseIterable, Sendable { + case totalTokens = "total_tokens" + case modelIdentities = "model_identities" + case knownCost = "known_cost" + case pricingCoverage = "pricing_coverage" + case activeModelCount = "active_model_count" + case topModel = "top_model" + case comparisons + case sessionReferences = "session_references" + case modelTokens = "model_tokens" + case modelKnownCost = "model_known_cost" + case modelPricingCoverage = "model_pricing_coverage" + case modelSessionReferences = "model_session_references" +} + +public struct CodexModelsRolloutDiagnostics: Codable, Equatable, Sendable { + public let legacyTotalTokens: Int64 + public let revisedTotalTokens: Int64 + public let legacyModelIDs: [String] + public let revisedModelIDs: [String] + public let mismatches: [String] + public let mismatchDimensions: [CodexModelsParityDimension]? + + public init( + legacyTotalTokens: Int64, + revisedTotalTokens: Int64, + legacyModelIDs: [String], + revisedModelIDs: [String], + mismatches: [String], + mismatchDimensions: [CodexModelsParityDimension]? = nil) + { + self.legacyTotalTokens = legacyTotalTokens + self.revisedTotalTokens = revisedTotalTokens + self.legacyModelIDs = legacyModelIDs + self.revisedModelIDs = revisedModelIDs + self.mismatches = mismatches + self.mismatchDimensions = mismatchDimensions + } + + public var isMatched: Bool { + self.mismatches.isEmpty + } +} + +public struct CodexModelsAnalyticsSnapshot: Codable, Equatable, Sendable { + public let scopeID: String? + public let generatedAt: Date + public let indexRevision: String + public let currentInterval: DateInterval + public let previousInterval: DateInterval + /// Nil only when decoding a snapshot written before period completeness was persisted. + public let currentIsComplete: Bool? + /// Nil only when decoding a snapshot written before period completeness was persisted. + public let previousIsComplete: Bool? + public let totalTokens: Int64 + public let cost: CodexModelsCost + public let activeModelCount: Int + public let previousActiveModelCount: Int? + public let newlyActiveModelCount: Int? + public let uniqueSessionCount: Int + public let sessionReferenceTotal: Int + public let previousSessionReferenceTotal: Int? + public let tokenComparison: CodexModelsComparison + public let costComparison: CodexModelsComparison + public let sessionReferenceComparison: CodexModelsComparison? + public let rows: [CodexModelsRow] + public let daily: [CodexModelsDailyBucket] + public let dailyByModel: [String: [CodexModelsDailyBucket]] + public let diagnostics: CodexModelsRolloutDiagnostics + + public func totalValue(_ metric: CodexModelsMetric) -> Double { + switch metric { + case .tokens: Double(self.totalTokens) + case .knownCost: NSDecimalNumber(decimal: self.cost.knownAmount).doubleValue + case .sessionReferences: Double(self.sessionReferenceTotal) + } + } + + public func share(of row: CodexModelsRow, metric: CodexModelsMetric) -> Double { + let total = self.totalValue(metric) + return total == 0 ? 0 : row.metricValue(metric) / total + } + + public func comparison(_ metric: CodexModelsMetric) -> CodexModelsComparison { + switch metric { + case .tokens: self.tokenComparison + case .knownCost: self.costComparison + case .sessionReferences: self.sessionReferenceComparison ?? .unavailable + } + } + + public func invariantViolations() -> [String] { + var failures: [String] = [] + if self.rows.reduce(Int64.zero, { $0 + $1.totalTokens }) != self.totalTokens { + failures.append("summary_tokens") + } + let rowCost = self.rows.reduce(CodexModelsCost.zero) { $0.adding($1.cost) } + if rowCost != self.cost { failures.append("cost_coverage") } + if self.rows.count != self.activeModelCount { failures.append("active_models") } + if self.rows.reduce(0, { $0 + $1.sessionReferences }) != self.sessionReferenceTotal { + failures.append("session_references") + } + if self.daily.reduce(Int64.zero, { $0 + $1.tokens }) != self.totalTokens { + failures.append("timeline_tokens") + } + return failures + } +} + +public struct CodexModelsAnalyticsPayload: Codable, Equatable, Sendable { + public let allWorkspaces: CodexModelsAnalyticsSnapshot + public let workspaces: [String: CodexModelsAnalyticsSnapshot] + + public func snapshot(workspaceID: String?) -> CodexModelsAnalyticsSnapshot { + guard let workspaceID else { return self.allWorkspaces } + return self.workspaces[workspaceID] ?? self.allWorkspaces + } +} + +public struct CodexModelsAnalyticsSource: Sendable { + public let current: [CodexModelsUsageFragment] + public let previous: [CodexModelsUsageFragment] + + public init(current: [CodexModelsUsageFragment], previous: [CodexModelsUsageFragment]) { + self.current = current + self.previous = previous + } +} + +public struct CodexModelsAnalyticsPeriods: Sendable { + public let current: DateInterval + public let previous: DateInterval + + public init(current: DateInterval, previous: DateInterval) { + self.current = current + self.previous = previous + } +} + +public struct CodexModelsAnalyticsRevision: Sendable { + public let generatedAt: Date + public let indexRevision: String + + public init(generatedAt: Date, indexRevision: String) { + self.generatedAt = generatedAt + self.indexRevision = indexRevision + } +} + +public struct CodexModelsLegacyModelBaseline: Sendable { + public let modelID: String + public let totalTokens: Int64? + public let knownCost: Decimal? + public let pricedTokens: Int64? + public let unpricedTokens: Int64? + public let sessionReferences: Int? + + public init( + modelID: String, + totalTokens: Int64? = nil, + knownCost: Decimal? = nil, + pricedTokens: Int64? = nil, + unpricedTokens: Int64? = nil, + sessionReferences: Int? = nil) + { + self.modelID = modelID + self.totalTokens = totalTokens + self.knownCost = knownCost + self.pricedTokens = pricedTokens + self.unpricedTokens = unpricedTokens + self.sessionReferences = sessionReferences + } +} + +public struct CodexModelsLegacyBaseline: Sendable { + public let totalTokens: Int64 + public let modelIDs: [String] + public let knownCost: Decimal? + public let pricedTokens: Int64? + public let unpricedTokens: Int64? + public let activeModelCount: Int? + public let topModelID: String? + public let sessionReferenceTotal: Int? + public let previousTotalTokens: Int64? + public let previousKnownCost: Decimal? + public let previousUnpricedTokens: Int64? + public let previousSessionReferenceTotal: Int? + public let currentModels: [CodexModelsLegacyModelBaseline]? + public let previousModels: [CodexModelsLegacyModelBaseline]? + + public init( + totalTokens: Int64, + modelIDs: [String], + knownCost: Decimal? = nil, + pricedTokens: Int64? = nil, + unpricedTokens: Int64? = nil, + activeModelCount: Int? = nil, + topModelID: String? = nil, + sessionReferenceTotal: Int? = nil, + previousTotalTokens: Int64? = nil, + previousKnownCost: Decimal? = nil, + previousUnpricedTokens: Int64? = nil, + previousSessionReferenceTotal: Int? = nil, + currentModels: [CodexModelsLegacyModelBaseline]? = nil, + previousModels: [CodexModelsLegacyModelBaseline]? = nil) + { + self.totalTokens = totalTokens + self.modelIDs = modelIDs + self.knownCost = knownCost + self.pricedTokens = pricedTokens + self.unpricedTokens = unpricedTokens + self.activeModelCount = activeModelCount + self.topModelID = topModelID + self.sessionReferenceTotal = sessionReferenceTotal + self.previousTotalTokens = previousTotalTokens + self.previousKnownCost = previousKnownCost + self.previousUnpricedTokens = previousUnpricedTokens + self.previousSessionReferenceTotal = previousSessionReferenceTotal + self.currentModels = currentModels + self.previousModels = previousModels + } +} + +public struct CodexModelsAnalyticsRequest: Sendable { + public let source: CodexModelsAnalyticsSource + public let scopeID: String? + public let periods: CodexModelsAnalyticsPeriods + public let revision: CodexModelsAnalyticsRevision + public let legacy: CodexModelsLegacyBaseline + public let currentIsComplete: Bool + public let previousIsComplete: Bool + + public init( + source: CodexModelsAnalyticsSource, + scopeID: String?, + periods: CodexModelsAnalyticsPeriods, + revision: CodexModelsAnalyticsRevision, + legacy: CodexModelsLegacyBaseline, + currentIsComplete: Bool = true, + previousIsComplete: Bool = true) + { + self.source = source + self.scopeID = scopeID + self.periods = periods + self.revision = revision + self.legacy = legacy + self.currentIsComplete = currentIsComplete + self.previousIsComplete = previousIsComplete + } +} + +public struct CodexModelsAnalyticsBuilder: Sendable { + public init() {} + + public func build(_ request: CodexModelsAnalyticsRequest) -> CodexModelsAnalyticsSnapshot { + let current = self.filtered( + request.source.current, + scopeID: request.scopeID, + interval: request.periods.current) + let previous = self.filtered( + request.source.previous, + scopeID: request.scopeID, + interval: request.periods.previous) + let currentGroups = Dictionary(grouping: current) { self.canonicalID($0.rawModelID) } + let previousGroups = Dictionary(grouping: previous) { self.canonicalID($0.rawModelID) } + let comparisonIsComplete = request.currentIsComplete && request.previousIsComplete + let totalTokens = current.reduce(Int64.zero) { $0 + $1.totalTokens } + let previousTokens = previous.reduce(Int64.zero) { $0 + $1.totalTokens } + let previousSessionReferenceTotal = previousGroups.values.reduce(0) { partial, events in + partial + Set(events.map(\.sessionID)).count + } + let rows = self.makeRows( + currentGroups: currentGroups, + previousGroups: previousGroups, + totalTokens: totalTokens, + previousIsComplete: request.previousIsComplete, + comparisonIsComplete: comparisonIsComplete) + + let totalCost = rows.reduce(CodexModelsCost.zero) { $0.adding($1.cost) } + let previousCost = self.cost(previous) + let sessionReferenceTotal = rows.reduce(0) { $0 + $1.sessionReferences } + let tokenComparison = CodexModelsComparison.make( + current: Double(totalTokens), + previous: Double(previousTokens), + previousIsComplete: comparisonIsComplete) + let costComparison = CodexModelsComparison.make( + current: NSDecimalNumber(decimal: totalCost.knownAmount).doubleValue, + previous: NSDecimalNumber(decimal: previousCost.knownAmount).doubleValue, + previousIsComplete: comparisonIsComplete + && totalCost.unpricedTokens == 0 + && previousCost.unpricedTokens == 0) + let sessionReferenceComparison = CodexModelsComparison.make( + current: Double(sessionReferenceTotal), + previous: Double(previousSessionReferenceTotal), + previousIsComplete: comparisonIsComplete) + let newlyActiveModelCount: Int? = comparisonIsComplete + ? currentGroups.keys.count(where: { previousGroups[$0] == nil }) + : nil + let revisedIDs = rows.map(\.id).sorted() + let mismatchDimensions = self.parityMismatches(ParityInputs( + request: request, + currentGroups: currentGroups, + previousGroups: previousGroups, + rows: rows, + totalTokens: totalTokens, + totalCost: totalCost, + sessionReferenceTotal: sessionReferenceTotal, + comparisonIsComplete: comparisonIsComplete, + tokenComparison: tokenComparison, + costComparison: costComparison, + sessionReferenceComparison: sessionReferenceComparison)) + + let daily = self.dailyBuckets(current) + let dailyByModel = currentGroups.mapValues { self.dailyBuckets($0) } + return CodexModelsAnalyticsSnapshot( + scopeID: request.scopeID, + generatedAt: request.revision.generatedAt, + indexRevision: request.revision.indexRevision, + currentInterval: request.periods.current, + previousInterval: request.periods.previous, + currentIsComplete: request.currentIsComplete, + previousIsComplete: request.previousIsComplete, + totalTokens: totalTokens, + cost: totalCost, + activeModelCount: rows.count, + previousActiveModelCount: request.previousIsComplete ? previousGroups.count : nil, + newlyActiveModelCount: newlyActiveModelCount, + uniqueSessionCount: Set(current.map(\.sessionID)).count, + sessionReferenceTotal: sessionReferenceTotal, + previousSessionReferenceTotal: request.previousIsComplete ? previousSessionReferenceTotal : nil, + tokenComparison: tokenComparison, + costComparison: costComparison, + sessionReferenceComparison: sessionReferenceComparison, + rows: rows, + daily: daily, + dailyByModel: dailyByModel, + diagnostics: CodexModelsRolloutDiagnostics( + legacyTotalTokens: request.legacy.totalTokens, + revisedTotalTokens: totalTokens, + legacyModelIDs: request.legacy.modelIDs.sorted(), + revisedModelIDs: revisedIDs, + mismatches: mismatchDimensions.map(\.rawValue), + mismatchDimensions: mismatchDimensions)) + } + + private func makeRows( + currentGroups: [String: [CodexModelsUsageFragment]], + previousGroups: [String: [CodexModelsUsageFragment]], + totalTokens: Int64, + previousIsComplete: Bool, + comparisonIsComplete: Bool) -> [CodexModelsRow] + { + currentGroups.map { modelID, events in + let earlier = previousGroups[modelID, default: []] + let tokens = events.reduce(Int64.zero) { $0 + $1.totalTokens } + let previousModelTokens = earlier.reduce(Int64.zero) { $0 + $1.totalTokens } + let cost = self.cost(events) + let previousCost = self.cost(earlier) + let sessionReferences = Set(events.map(\.sessionID)).count + let previousSessionReferences = Set(earlier.map(\.sessionID)).count + let reasoningValues = events.compactMap(\.reasoningTokens) + return CodexModelsRow( + id: modelID, + displayName: CostUsagePricing.codexDisplayLabel(model: modelID) ?? modelID, + rawAliases: Array(Set(events.map(\.rawModelID))).sorted(), + inputTokens: events.reduce(0) { $0 + $1.inputTokens }, + cachedInputTokens: events.reduce(0) { $0 + $1.cachedInputTokens }, + outputTokens: events.reduce(0) { $0 + $1.outputTokens }, + reasoningTokens: reasoningValues.isEmpty ? nil : reasoningValues.reduce(0, +), + totalTokens: tokens, + share: totalTokens == 0 ? 0 : Double(tokens) / Double(totalTokens), + sessionReferences: sessionReferences, + cost: cost, + previousTotalTokens: previousIsComplete ? previousModelTokens : nil, + previousCost: previousIsComplete ? previousCost : nil, + previousSessionReferences: previousIsComplete ? previousSessionReferences : nil, + associatedSessionIDs: Array(Set(events.map(\.sessionID))).sorted(), + tokenComparison: .make( + current: Double(tokens), + previous: Double(previousModelTokens), + previousIsComplete: comparisonIsComplete), + costComparison: .make( + current: NSDecimalNumber(decimal: cost.knownAmount).doubleValue, + previous: NSDecimalNumber(decimal: previousCost.knownAmount).doubleValue, + previousIsComplete: comparisonIsComplete + && cost.unpricedTokens == 0 + && previousCost.unpricedTokens == 0), + sessionReferenceComparison: .make( + current: Double(sessionReferences), + previous: Double(previousSessionReferences), + previousIsComplete: comparisonIsComplete)) + } + .sorted { + if $0.totalTokens != $1.totalTokens { return $0.totalTokens > $1.totalTokens } + if $0.sessionReferences != $1.sessionReferences { return $0.sessionReferences > $1.sessionReferences } + return $0.id < $1.id + } + } + + private struct ParityInputs { + let request: CodexModelsAnalyticsRequest + let currentGroups: [String: [CodexModelsUsageFragment]] + let previousGroups: [String: [CodexModelsUsageFragment]] + let rows: [CodexModelsRow] + let totalTokens: Int64 + let totalCost: CodexModelsCost + let sessionReferenceTotal: Int + let comparisonIsComplete: Bool + let tokenComparison: CodexModelsComparison + let costComparison: CodexModelsComparison + let sessionReferenceComparison: CodexModelsComparison + } + + private func parityMismatches(_ inputs: ParityInputs) -> [CodexModelsParityDimension] { + var dimensions: [CodexModelsParityDimension] = [] + let request = inputs.request + if request.legacy.totalTokens != inputs.totalTokens { dimensions.append(.totalTokens) } + let legacyIDs = Array(Set(request.legacy.modelIDs.map { self.canonicalID($0) })).sorted() + if legacyIDs != inputs.rows.map(\.id).sorted() { dimensions.append(.modelIdentities) } + if let knownCost = request.legacy.knownCost, knownCost != inputs.totalCost.knownAmount { + dimensions.append(.knownCost) + } + if let priced = request.legacy.pricedTokens, + let unpriced = request.legacy.unpricedTokens, + priced != inputs.totalCost.pricedTokens || unpriced != inputs.totalCost.unpricedTokens + { + dimensions.append(.pricingCoverage) + } + if let count = request.legacy.activeModelCount, count != inputs.rows.count { + dimensions.append(.activeModelCount) + } + if let topModelID = request.legacy.topModelID, + self.canonicalID(topModelID) != inputs.rows.first?.id + { + dimensions.append(.topModel) + } + if let references = request.legacy.sessionReferenceTotal, references != inputs.sessionReferenceTotal { + dimensions.append(.sessionReferences) + } + if inputs.comparisonIsComplete, + self.legacyComparisonsMismatch( + request.legacy, + tokenComparison: inputs.tokenComparison, + costComparison: inputs.costComparison, + sessionReferenceComparison: inputs.sessionReferenceComparison) + { + dimensions.append(.comparisons) + } + let modelDimensions = self.modelParityMismatches( + legacy: request.currentIsComplete ? request.legacy.currentModels : nil, + revised: inputs.currentGroups) + .union(self.modelParityMismatches( + legacy: request.previousIsComplete ? request.legacy.previousModels : nil, + revised: inputs.previousGroups)) + for dimension in [ + CodexModelsParityDimension.modelTokens, + .modelKnownCost, + .modelPricingCoverage, + .modelSessionReferences, + ] where modelDimensions.contains(dimension) { + dimensions.append(dimension) + } + return dimensions + } + + private func legacyComparisonsMismatch( + _ legacy: CodexModelsLegacyBaseline, + tokenComparison: CodexModelsComparison, + costComparison: CodexModelsComparison, + sessionReferenceComparison: CodexModelsComparison) -> Bool + { + guard let previousTokens = legacy.previousTotalTokens else { return false } + let legacyToken = CodexModelsComparison.make( + current: Double(legacy.totalTokens), + previous: Double(previousTokens)) + let legacyCost = legacy.knownCost.flatMap { currentCost in + legacy.previousKnownCost.map { previousCost in + CodexModelsComparison.make( + current: NSDecimalNumber(decimal: currentCost).doubleValue, + previous: NSDecimalNumber(decimal: previousCost).doubleValue, + previousIsComplete: legacy.previousUnpricedTokens == 0) + } + } + let legacySessions = legacy.sessionReferenceTotal.flatMap { currentReferences in + legacy.previousSessionReferenceTotal.map { previousReferences in + CodexModelsComparison.make(current: Double(currentReferences), previous: Double(previousReferences)) + } + } + return legacyToken != tokenComparison + || legacyCost.map { $0 != costComparison } == true + || legacySessions.map { $0 != sessionReferenceComparison } == true + } + + public func canonicalID(_ rawID: String) -> String { + CostUsagePricing.normalizeCodexModel( + rawID.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()) + } + + private func modelParityMismatches( + legacy: [CodexModelsLegacyModelBaseline]?, + revised: [String: [CodexModelsUsageFragment]]) -> Set + { + guard let legacy else { return [] } + let normalized = Dictionary(grouping: legacy) { self.canonicalID($0.modelID) } + var mismatches: Set = [] + for (modelID, legacyRows) in normalized { + guard let revisedFragments = revised[modelID] else { continue } + let revisedTokens = revisedFragments.reduce(Int64.zero) { $0 + $1.totalTokens } + let revisedCost = self.cost(revisedFragments) + let revisedSessionReferences = Set(revisedFragments.map(\.sessionID)).count + let legacyTokens = self.sumIfComplete(legacyRows.map(\.totalTokens)) + let legacyKnownCost = self.sumIfComplete(legacyRows.map(\.knownCost)) + let legacyPricedTokens = self.sumIfComplete(legacyRows.map(\.pricedTokens)) + let legacyUnpricedTokens = self.sumIfComplete(legacyRows.map(\.unpricedTokens)) + // Distinct session counts cannot be safely summed across alias rows because + // the underlying session sets may overlap. Production baselines canonicalize + // first, so an exact value remains available there. + let legacySessionReferences = legacyRows.count == 1 ? legacyRows[0].sessionReferences : nil + + if let legacyTokens, legacyTokens != revisedTokens { + mismatches.insert(.modelTokens) + } + if let legacyKnownCost, legacyKnownCost != revisedCost.knownAmount { + mismatches.insert(.modelKnownCost) + } + if legacyPricedTokens.map({ $0 != revisedCost.pricedTokens }) == true + || legacyUnpricedTokens.map({ $0 != revisedCost.unpricedTokens }) == true + { + mismatches.insert(.modelPricingCoverage) + } + if let legacySessionReferences, legacySessionReferences != revisedSessionReferences { + mismatches.insert(.modelSessionReferences) + } + } + return mismatches + } + + private func sumIfComplete(_ values: [Int64?]) -> Int64? { + let available = values.compactMap(\.self) + return available.count == values.count ? available.reduce(0, +) : nil + } + + private func sumIfComplete(_ values: [Decimal?]) -> Decimal? { + let available = values.compactMap(\.self) + return available.count == values.count ? available.reduce(0, +) : nil + } + + private func filtered( + _ fragments: [CodexModelsUsageFragment], + scopeID: String?, + interval: DateInterval) -> [CodexModelsUsageFragment] + { + fragments.filter { fragment in + (scopeID == nil || fragment.workspaceID == scopeID) + && fragment.timestamp >= interval.start + && fragment.timestamp < interval.end + } + } + + private func cost(_ fragments: [CodexModelsUsageFragment]) -> CodexModelsCost { + fragments.reduce(.zero) { partial, fragment in + let unpriced = fragment.unpricedTokens + let priced = fragment.totalTokens - unpriced + let amount = fragment.costNanos.map { Decimal($0) / 1_000_000_000 } ?? 0 + return partial.adding(CodexModelsCost( + knownAmount: amount, + pricedTokens: priced, + unpricedTokens: unpriced)) + } + } + + private func dailyBuckets(_ fragments: [CodexModelsUsageFragment]) -> [CodexModelsDailyBucket] { + let calendar = Calendar.current + return Dictionary(grouping: fragments, by: \.day).map { day, values in + let start = calendar.startOfDay(for: day) + let end = calendar.date(byAdding: .day, value: 1, to: start) ?? start.addingTimeInterval(24 * 60 * 60) + return CodexModelsDailyBucket( + day: start, + interval: DateInterval(start: start, end: end), + tokens: values.reduce(0) { $0 + $1.totalTokens }, + sessionIDs: Array(Set(values.map(\.sessionID))).sorted(), + sessionReferenceIDs: Array(Set(values.map { + "\($0.sessionID)\u{1F}\(self.canonicalID($0.rawModelID))" + })).sorted(), + cost: self.cost(values)) + } + .sorted { $0.day < $1.day } + } +} diff --git a/Sources/CodexBarCore/CodexModelsCSVExporter.swift b/Sources/CodexBarCore/CodexModelsCSVExporter.swift new file mode 100644 index 0000000000..50c7672a10 --- /dev/null +++ b/Sources/CodexBarCore/CodexModelsCSVExporter.swift @@ -0,0 +1,124 @@ +import Foundation + +public enum CodexModelsCSVExporter { + private static let columns = [ + "query_scope", + "period_start", + "period_end", + "canonical_model_id", + "display_name", + "raw_aliases", + "associated_session_ids", + "active_metric", + "tokens", + "input_tokens", + "cached_input_tokens", + "output_tokens", + "reasoning_tokens", + "share", + "session_references", + "previous_tokens", + "previous_session_references", + "known_cost", + "previous_known_cost", + "cost_status", + "cost_coverage", + "previous_cost_status", + "previous_cost_coverage", + "currency", + "priced_tokens", + "unpriced_tokens", + "previous_priced_tokens", + "previous_unpriced_tokens", + "comparison_kind", + "comparison_value", + "index_revision", + "snapshot_generated_at", + ] + + public static func export( + snapshot: CodexModelsAnalyticsSnapshot, + rows: [CodexModelsRow]? = nil, + metric: CodexModelsMetric = .tokens) -> String + { + let iso8601 = ISO8601DateFormatter() + let rows = rows ?? snapshot.rows + let body = rows.map { row in + let comparison = self.comparisonFields(row.comparison(metric)) + let share = String(format: "%.17g", snapshot.share(of: row, metric: metric)) + let associatedSessionIDs = (row.associatedSessionIDs ?? []).joined(separator: "|") + let previousTokens = row.previousTotalTokens.map(String.init) ?? "" + let previousSessionReferences = row.previousSessionReferences.map(String.init) ?? "" + let knownCost = row.cost.pricedTokens == 0 + ? "" + : NSDecimalNumber(decimal: row.cost.knownAmount).stringValue + let previousKnownCost = row.previousCost.flatMap { cost in + cost.pricedTokens == 0 ? nil : NSDecimalNumber(decimal: cost.knownAmount).stringValue + } ?? "" + let costStatus = self.costStatus(row.cost, usageTokens: row.totalTokens) + let previousCostStatus = row.previousCost.map { + self.costStatus($0, usageTokens: $0.pricedTokens + $0.unpricedTokens) + } ?? "unavailable" + let fields: [String] = [ + snapshot.scopeID ?? "all_workspaces", + iso8601.string(from: snapshot.currentInterval.start), + iso8601.string(from: snapshot.currentInterval.end), + row.id, + row.displayName, + row.rawAliases.joined(separator: "|"), + associatedSessionIDs, + metric.rawValue, + String(row.totalTokens), + String(row.inputTokens), + String(row.cachedInputTokens), + String(row.outputTokens), + row.reasoningTokens.map(String.init) ?? "", + share, + String(row.sessionReferences), + previousTokens, + previousSessionReferences, + knownCost, + previousKnownCost, + costStatus, + String(format: "%.17g", row.cost.coverage), + previousCostStatus, + row.previousCost.map { String(format: "%.17g", $0.coverage) } ?? "", + row.cost.currencyCode, + String(row.cost.pricedTokens), + String(row.cost.unpricedTokens), + row.previousCost.map { String($0.pricedTokens) } ?? "", + row.previousCost.map { String($0.unpricedTokens) } ?? "", + comparison.kind, + comparison.value, + snapshot.indexRevision, + iso8601.string(from: snapshot.generatedAt), + ] + return fields.map(self.escape).joined(separator: ",") + } + return ([self.columns.joined(separator: ",")] + body).joined(separator: "\n") + "\n" + } + + private static func comparisonFields(_ comparison: CodexModelsComparison) -> (kind: String, value: String) { + switch comparison { + case .unavailable: ("unavailable", "") + case .new: ("new", "") + case .ended: ("ended", "") + case .unchanged: ("unchanged", "0") + case let .percent(value): ("percent", String(format: "%.17g", value)) + } + } + + private static func costStatus(_ cost: CodexModelsCost, usageTokens: Int64) -> String { + if usageTokens == 0 { return "no_usage" } + if cost.pricedTokens == 0 { return "unavailable" } + if cost.unpricedTokens > 0 { return "partial" } + return "known" + } + + private static func escape(_ value: String) -> String { + guard value.contains(",") || value.contains("\"") || value.contains("\n") || value.contains("\r") else { + return value + } + return "\"" + value.replacingOccurrences(of: "\"", with: "\"\"") + "\"" + } +} diff --git a/Sources/CodexBarCore/CodexModelsTelemetry.swift b/Sources/CodexBarCore/CodexModelsTelemetry.swift new file mode 100644 index 0000000000..aba571cdac --- /dev/null +++ b/Sources/CodexBarCore/CodexModelsTelemetry.swift @@ -0,0 +1,57 @@ +import Foundation + +#if canImport(os) +import os + +typealias CodexModelsSignpostID = OSSignpostID + +enum CodexModelsTelemetry { + private static let log = OSLog(subsystem: "com.steipete.codexbar", category: "ModelsAnalytics") + + static func begin(_ name: StaticString) -> CodexModelsSignpostID { + let id = OSSignpostID(log: self.log) + os_signpost(.begin, log: self.log, name: name, signpostID: id) + return id + } + + static func end(_ name: StaticString, id: OSSignpostID) { + os_signpost(.end, log: self.log, name: name, signpostID: id) + } + + static func cacheHit(historyDays: Int) { + os_signpost(.event, log: self.log, name: "SnapshotCacheHit", "historyDays=%{public}d", historyDays) + } + + static func parity(dimensions: [CodexModelsParityDimension], rowCount: Int) { + let mask = dimensions.reduce(0) { partial, dimension in + guard let index = CodexModelsParityDimension.allCases.firstIndex(of: dimension) else { return partial } + return partial | (1 << index) + } + os_signpost( + .event, + log: self.log, + name: "DualRunParity", + "mismatches=%{public}d mask=%{public}d rows=%{public}d", + dimensions.count, + mask, + rowCount) + } +} + +#else + +struct CodexModelsSignpostID: Sendable {} + +enum CodexModelsTelemetry { + static func begin(_: StaticString) -> CodexModelsSignpostID { + CodexModelsSignpostID() + } + + static func end(_: StaticString, id _: CodexModelsSignpostID) {} + + static func cacheHit(historyDays _: Int) {} + + static func parity(dimensions _: [CodexModelsParityDimension], rowCount _: Int) {} +} + +#endif diff --git a/Sources/CodexBarCore/CodexThreadCatalogReader.swift b/Sources/CodexBarCore/CodexThreadCatalogReader.swift new file mode 100644 index 0000000000..37f94282ec --- /dev/null +++ b/Sources/CodexBarCore/CodexThreadCatalogReader.swift @@ -0,0 +1,261 @@ +import Foundation +#if canImport(SQLite3) +import SQLite3 +#elseif canImport(CSQLite3) +import CSQLite3 +#endif + +struct CodexThreadCatalog: Sendable { + static let empty = CodexThreadCatalog(entriesById: [:], entriesByRolloutPath: [:], fingerprint: nil) + + let entriesById: [String: CodexThreadCatalogEntry] + let entriesByRolloutPath: [String: CodexThreadCatalogEntry] + let fingerprint: String? + + func entry(sessionId: String?, rolloutPath: String) -> CodexThreadCatalogEntry? { + if let sessionId, let entry = self.entriesById[sessionId] { + return entry + } + return self.entriesByRolloutPath[URL(fileURLWithPath: rolloutPath).standardizedFileURL.path] + } +} + +enum CodexThreadCatalogCompleteness: Sendable, Equatable { + case complete + case unavailable(CodexThreadCatalogFailure) +} + +enum CodexThreadCatalogFailure: Sendable, Equatable { + case missing + case locked + case corrupt + case incompatible + case unreadable +} + +struct CodexThreadCatalogReadResult: Sendable { + let catalog: CodexThreadCatalog + let databaseURL: URL + let completeness: CodexThreadCatalogCompleteness + + var isComplete: Bool { + if case .complete = self.completeness { return true } + return false + } +} + +struct CodexThreadCatalogEntry: Sendable, Equatable { + let id: String + let rolloutPath: String + let cwd: String? + let title: String? + let preview: String? + let modelProvider: String? + let model: String? + let reasoningEffort: String? + let createdAtUnixMs: Int64? + let updatedAtUnixMs: Int64? + let archived: Bool +} + +enum CodexThreadCatalogReader { + static func load(options: CostUsageScanner.Options) -> CodexThreadCatalog { + self.loadResult(options: options).catalog + } + + static func loadResult(options: CostUsageScanner.Options) -> CodexThreadCatalogReadResult { + let url = self.databaseURL(options: options) + guard FileManager.default.fileExists(atPath: url.path) else { + return CodexThreadCatalogReadResult( + catalog: .empty, + databaseURL: url, + completeness: .unavailable(.missing)) + } + + #if canImport(SQLite3) || canImport(CSQLite3) + var db: OpaquePointer? + let openResult = sqlite3_open_v2(url.path, &db, SQLITE_OPEN_READONLY, nil) + guard openResult == SQLITE_OK else { + let failure: CodexThreadCatalogFailure = openResult == SQLITE_BUSY || openResult == SQLITE_LOCKED + ? .locked + : openResult == SQLITE_CORRUPT || openResult == SQLITE_NOTADB ? .corrupt : .unreadable + sqlite3_close(db) + return CodexThreadCatalogReadResult( + catalog: .empty, + databaseURL: url, + completeness: .unavailable(failure)) + } + defer { sqlite3_close(db) } + sqlite3_busy_timeout(db, 250) + sqlite3_exec(db, "PRAGMA query_only = ON", nil, nil, nil) + + guard let hasThreadsTable = self.hasThreadsTable(db) else { + let code = sqlite3_errcode(db) + let failure: CodexThreadCatalogFailure = code == SQLITE_BUSY || code == SQLITE_LOCKED + ? .locked + : code == SQLITE_CORRUPT || code == SQLITE_NOTADB ? .corrupt : .unreadable + return CodexThreadCatalogReadResult( + catalog: .empty, + databaseURL: url, + completeness: .unavailable(failure)) + } + guard hasThreadsTable else { + return CodexThreadCatalogReadResult( + catalog: .empty, + databaseURL: url, + completeness: .unavailable(.incompatible)) + } + guard let entries = self.readEntries(db) else { + let code = sqlite3_errcode(db) + let failure: CodexThreadCatalogFailure = code == SQLITE_BUSY || code == SQLITE_LOCKED + ? .locked + : code == SQLITE_CORRUPT || code == SQLITE_NOTADB ? .corrupt : .unreadable + return CodexThreadCatalogReadResult( + catalog: .empty, + databaseURL: url, + completeness: .unavailable(failure)) + } + let fingerprint = self.fingerprint(databaseURL: url, db: db) + let catalog = CodexThreadCatalog( + entriesById: Dictionary(uniqueKeysWithValues: entries.map { ($0.id, $0) }), + entriesByRolloutPath: Dictionary(uniqueKeysWithValues: entries.map { + (URL(fileURLWithPath: $0.rolloutPath).standardizedFileURL.path, $0) + }), + fingerprint: fingerprint) + return CodexThreadCatalogReadResult(catalog: catalog, databaseURL: url, completeness: .complete) + #else + return CodexThreadCatalogReadResult( + catalog: .empty, + databaseURL: url, + completeness: .unavailable(.incompatible)) + #endif + } + + private static func databaseURL(options: CostUsageScanner.Options) -> URL { + CodexLocalDataScope.resolve(options: options).stateDatabaseURL + } + + #if canImport(SQLite3) || canImport(CSQLite3) + private static func hasThreadsTable(_ db: OpaquePointer?) -> Bool? { + var stmt: OpaquePointer? + guard sqlite3_prepare_v2( + db, + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'threads' LIMIT 1", + -1, + &stmt, + nil) == SQLITE_OK + else { return nil } + defer { sqlite3_finalize(stmt) } + switch sqlite3_step(stmt) { + case SQLITE_ROW: + return true + case SQLITE_DONE: + return false + default: + return nil + } + } + + private static func readEntries(_ db: OpaquePointer?) -> [CodexThreadCatalogEntry]? { + let query = """ + SELECT id, rollout_path, cwd, title, preview, model_provider, model, reasoning_effort, + created_at_ms, updated_at_ms, created_at, updated_at, archived + FROM threads + """ + var stmt: OpaquePointer? + guard sqlite3_prepare_v2(db, query, -1, &stmt, nil) == SQLITE_OK else { return nil } + defer { sqlite3_finalize(stmt) } + + var entries: [CodexThreadCatalogEntry] = [] + while true { + let step = sqlite3_step(stmt) + if step == SQLITE_DONE { break } + guard step == SQLITE_ROW else { return nil } + guard let id = self.text(stmt, 0), + let rolloutPath = self.text(stmt, 1) + else { continue } + entries.append(CodexThreadCatalogEntry( + id: id, + rolloutPath: rolloutPath, + cwd: self.text(stmt, 2), + title: self.nonEmptyText(stmt, 3), + preview: self.nonEmptyText(stmt, 4), + modelProvider: self.nonEmptyText(stmt, 5), + model: self.nonEmptyText(stmt, 6), + reasoningEffort: self.nonEmptyText(stmt, 7), + createdAtUnixMs: self.integer(stmt, preferredColumn: 8, fallbackColumn: 10), + updatedAtUnixMs: self.integer(stmt, preferredColumn: 9, fallbackColumn: 11), + archived: sqlite3_column_int(stmt, 12) != 0)) + } + return entries + } + + private static func fingerprint(databaseURL: URL, db: OpaquePointer?) -> String { + let attributes = (try? FileManager.default.attributesOfItem(atPath: databaseURL.path)) ?? [:] + let size = (attributes[.size] as? NSNumber)?.int64Value ?? -1 + let mtime = (attributes[.modificationDate] as? Date)?.timeIntervalSince1970 ?? -1 + let walURL = URL(fileURLWithPath: databaseURL.path + "-wal") + let walAttributes = (try? FileManager.default.attributesOfItem(atPath: walURL.path)) ?? [:] + let walSize = (walAttributes[.size] as? NSNumber)?.int64Value ?? -1 + let walMtime = (walAttributes[.modificationDate] as? Date)?.timeIntervalSince1970 ?? -1 + let summary = self.catalogSummary(db) + return [ + databaseURL.standardizedFileURL.path, + "size=\(size)", + "mtime=\(Int64(mtime * 1000))", + "walSize=\(walSize)", + "walMtime=\(Int64(walMtime * 1000))", + "rows=\(summary.rowCount)", + "maxUpdated=\(summary.maxUpdatedAtUnixMs ?? -1)", + ].joined(separator: "|") + } + + private static func catalogSummary(_ db: OpaquePointer?) -> (rowCount: Int64, maxUpdatedAtUnixMs: Int64?) { + var stmt: OpaquePointer? + guard sqlite3_prepare_v2( + db, + "SELECT COUNT(*), MAX(CASE WHEN updated_at_ms IS NOT NULL THEN updated_at_ms " + + "ELSE updated_at * 1000 END) FROM threads", + -1, + &stmt, + nil) == SQLITE_OK + else { return (0, nil) } + defer { sqlite3_finalize(stmt) } + guard sqlite3_step(stmt) == SQLITE_ROW else { return (0, nil) } + let rowCount = sqlite3_column_int64(stmt, 0) + let maxUpdated = sqlite3_column_type(stmt, 1) == SQLITE_NULL ? nil : sqlite3_column_int64(stmt, 1) + return (rowCount, maxUpdated) + } + + private static func nonEmptyText(_ stmt: OpaquePointer?, _ index: Int32) -> String? { + self.text(stmt, index)?.trimmingCharacters(in: .whitespacesAndNewlines).nilIfEmpty + } + + private static func text(_ stmt: OpaquePointer?, _ index: Int32) -> String? { + guard sqlite3_column_type(stmt, index) != SQLITE_NULL, + let cString = sqlite3_column_text(stmt, index) + else { return nil } + return String(cString: cString) + } + + private static func integer( + _ stmt: OpaquePointer?, + preferredColumn: Int32, + fallbackColumn: Int32) -> Int64? + { + if sqlite3_column_type(stmt, preferredColumn) != SQLITE_NULL { + return sqlite3_column_int64(stmt, preferredColumn) + } + if sqlite3_column_type(stmt, fallbackColumn) != SQLITE_NULL { + return sqlite3_column_int64(stmt, fallbackColumn) * 1000 + } + return nil + } + #endif +} + +extension String { + fileprivate var nilIfEmpty: String? { + self.isEmpty ? nil : self + } +} diff --git a/Sources/CodexBarCore/CodexWorkspaceUsageFingerprint.swift b/Sources/CodexBarCore/CodexWorkspaceUsageFingerprint.swift new file mode 100644 index 0000000000..4602f8810e --- /dev/null +++ b/Sources/CodexBarCore/CodexWorkspaceUsageFingerprint.swift @@ -0,0 +1,66 @@ +#if canImport(CryptoKit) +import CryptoKit +#else +import Crypto +#endif +import Foundation + +/// Stable sidecar identity for a normalized Codex rollout. +/// Scanner cursors are intentionally excluded: they do not alter the sidecar's +/// persisted usage, but rows, attribution, and cost maps do. +struct CodexWorkspaceUsageFingerprintPayload: Encodable { + let days: [String: [String: [Int]]] + let lastModel: String? + let sessionID: String? + let forkedFromID: String? + let projectPath: String? + let canonicalProjectPath: String? + let session: CostUsageCodexSessionMetadata? + let costNanos: [String: [String: Int64]]? + let prioritySurchargeNanos: [String: [String: Int64]]? + let standardCostNanos: [String: [String: Int64]]? + let priorityCostNanos: [String: [String: Int64]]? + let standardTokens: [String: [String: Int]]? + let priorityTokens: [String: [String: Int]]? + let rows: [CostUsageScanner.CodexUsageRow]? + + init(usage: CostUsageFileUsage) { + self.days = usage.days + self.lastModel = usage.lastModel + self.sessionID = usage.sessionId + self.forkedFromID = usage.forkedFromId + self.projectPath = usage.projectPath + self.canonicalProjectPath = usage.canonicalProjectPath + self.session = usage.codexSession + self.costNanos = usage.codexCostNanos + self.prioritySurchargeNanos = usage.codexPrioritySurchargeNanos + self.standardCostNanos = usage.codexStandardCostNanos + self.priorityCostNanos = usage.codexPriorityCostNanos + self.standardTokens = usage.codexStandardTokens + self.priorityTokens = usage.codexPriorityTokens + self.rows = usage.codexRows + } +} + +enum CodexWorkspaceUsageFingerprint { + static func make(for usage: CostUsageFileUsage) -> String { + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + guard let data = try? encoder.encode(CodexWorkspaceUsageFingerprintPayload(usage: usage)) else { + return "unavailable" + } + return SHA256.hash(data: data).map { String(format: "%02x", $0) }.joined() + } +} + +extension CostUsageFileUsage { + func codexWorkspaceUsageFingerprintValue() -> String { + self.codexWorkspaceContentFingerprint ?? CodexWorkspaceUsageFingerprint.make(for: self) + } + + func refreshingCodexWorkspaceUsageFingerprint() -> Self { + var updated = self + updated.codexWorkspaceContentFingerprint = CodexWorkspaceUsageFingerprint.make(for: updated) + return updated + } +} diff --git a/Sources/CodexBarCore/CodexWorkspaceUsageSidecar.swift b/Sources/CodexBarCore/CodexWorkspaceUsageSidecar.swift new file mode 100644 index 0000000000..28f58a518c --- /dev/null +++ b/Sources/CodexBarCore/CodexWorkspaceUsageSidecar.swift @@ -0,0 +1,983 @@ +import Foundation + +#if canImport(SQLite3) +import SQLite3 +#elseif canImport(CSQLite3) +import CSQLite3 +#endif + +// swiftlint:disable type_body_length +/// CodexBar-owned persistence for local Workspaces attribution. This database +/// never attaches to or writes Codex's state database; it only imports typed +/// catalog/cache values after those sources have been read successfully. +struct CodexWorkspaceUsageSidecar: Sendable { + private static let schemaVersion = 5 + private static let snapshotPayloadFormatVersion = 3 + private let cacheRoot: URL? + + private struct RolloutSourceIdentity: Equatable { + let mtimeUnixMs: Int64 + let size: Int64 + let parsedBytes: Int64 + let sessionID: String + let producerKey: String + let pricingKey: String + let contentFingerprint: String + + init( + mtimeUnixMs: Int64, + size: Int64, + parsedBytes: Int64, + sessionID: String, + producerKey: String, + pricingKey: String, + contentFingerprint: String) + { + self.mtimeUnixMs = mtimeUnixMs + self.size = size + self.parsedBytes = parsedBytes + self.sessionID = sessionID + self.producerKey = producerKey + self.pricingKey = pricingKey + self.contentFingerprint = contentFingerprint + } + + init(usage: CostUsageFileUsage, cache: CostUsageCache) { + self.mtimeUnixMs = usage.mtimeUnixMs + self.size = usage.size + self.parsedBytes = usage.parsedBytes ?? -1 + self.sessionID = usage.codexSession?.sessionId ?? usage.sessionId ?? "" + self.producerKey = cache.producerKey ?? "" + self.pricingKey = cache.codexPricingKey ?? "" + self.contentFingerprint = usage.codexWorkspaceUsageFingerprintValue() + } + + var legacyFingerprint: String { + [ + "version=3", + "mtime=\(self.mtimeUnixMs)", + "size=\(self.size)", + "parsed=\(self.parsedBytes)", + "session=\(self.sessionID)", + "producer=\(self.producerKey)", + "pricing=\(self.pricingKey)", + "content=\(self.contentFingerprint)", + ].joined(separator: "|") + } + } + + init(cacheRoot: URL? = nil) { + self.cacheRoot = cacheRoot + } + + func loadLatestSnapshot( + scopeSignature: String, + historyDays: Int, + rootsFingerprint: [String: Int64]? = nil, + cache: CostUsageCache? = nil, + catalog: CodexThreadCatalog? = nil) -> CodexLocalProjectUsageSnapshot? + { + #if canImport(SQLite3) || canImport(CSQLite3) + guard let db = self.open(readOnly: true) else { return nil } + defer { sqlite3_close(db) } + let sql = """ + SELECT snapshot_payloads.payload, + index_state.roots_fingerprint, + index_state.catalog_fingerprint, + index_state.cache_producer_key, + index_state.pricing_key, + index_state.cache_fingerprint, + snapshot_payloads.payload_format_version + FROM snapshot_payloads + JOIN index_state ON index_state.scope_signature = snapshot_payloads.scope_signature + WHERE snapshot_payloads.scope_signature = ? + AND snapshot_payloads.history_days = ? + AND snapshot_payloads.is_complete = 1 + ORDER BY updated_at_ms DESC + LIMIT 1 + """ + guard let statement = Self.prepare(db, sql) else { return nil } + defer { sqlite3_finalize(statement) } + Self.bind(scopeSignature, to: statement, at: 1) + sqlite3_bind_int64(statement, 2, Int64(historyDays)) + guard sqlite3_step(statement) == SQLITE_ROW, + let bytes = sqlite3_column_blob(statement, 0) + else { return nil } + let length = Int(sqlite3_column_bytes(statement, 0)) + let data = Data(bytes: bytes, count: length) + let payloadFormatVersion = Int(sqlite3_column_int(statement, 6)) + guard payloadFormatVersion == Self.snapshotPayloadFormatVersion, + let snapshot = try? JSONDecoder.codexLocalProjectUsageSidecar.decode( + CodexLocalProjectUsageSnapshot.self, + from: data) + else { return nil } + // Snapshots written before project-level detail was persisted can still + // contain valid totals, but cannot render a selected project's chart or + // sessions. Keep the sidecar's normalized rows and rebuild only this + // presentation payload on the next background refresh. + guard snapshot.hasInspectorDetail, snapshot.modelsAnalytics != nil else { return nil } + if let rootsFingerprint { + guard let rootsBytes = sqlite3_column_blob(statement, 1) else { return nil } + let rootsLength = Int(sqlite3_column_bytes(statement, 1)) + guard let persistedRoots = try? JSONDecoder().decode( + [String: Int64].self, + from: Data(bytes: rootsBytes, count: rootsLength)), + persistedRoots == rootsFingerprint + else { return nil } + } + if let cache { + guard Self.columnString(statement, at: 3) == cache.producerKey, + Self.columnString(statement, at: 4) == cache.codexPricingKey, + Self.columnString(statement, at: 5) == Self.cacheFingerprint(cache) + else { return nil } + } + if let catalog, Self.columnString(statement, at: 2) != catalog.fingerprint { + return nil + } + return snapshot + #else + return nil + #endif + } + + func synchronize( + snapshot: CodexLocalProjectUsageSnapshot, + cache: CostUsageCache, + catalog: CodexThreadCatalog, + catalogIsComplete: Bool = true, + rootsFingerprint: [String: Int64]) throws + { + #if canImport(SQLite3) || canImport(CSQLite3) + guard let db = self.open(readOnly: false) else { + throw SidecarError.openFailed + } + defer { sqlite3_close(db) } + try Self.ensureSchema(db) + try Self.begin(db) + do { + let generation = UUID().uuidString + try self.upsertCatalog(catalog, generation: generation, db: db) + if catalogIsComplete { + try self.pruneCatalog(generation: generation, db: db) + } + try self.importChangedRollouts(cache, catalog: catalog, generation: generation, db: db) + try self.markMissingRollouts(generation: generation, db: db) + try self.storeSnapshot(snapshot, cache: cache, catalog: catalog, rootsFingerprint: rootsFingerprint, db: db) + try Self.commit(db) + } catch { + Self.rollback(db) + throw error + } + #else + _ = snapshot + _ = cache + _ = catalog + _ = catalogIsComplete + _ = rootsFingerprint + #endif + } + + /// Imports only source deltas. Callers can then aggregate from + /// `usageCache(roots:)` before committing a new complete snapshot. + func synchronizeSources( + cache: CostUsageCache, + catalog: CodexThreadCatalog, + catalogIsComplete: Bool = true) throws + { + #if canImport(SQLite3) || canImport(CSQLite3) + guard let db = self.open(readOnly: false) else { throw SidecarError.openFailed } + defer { sqlite3_close(db) } + try Self.ensureSchema(db) + try Self.begin(db) + do { + let generation = UUID().uuidString + try self.upsertCatalog(catalog, generation: generation, db: db) + if catalogIsComplete { + try self.pruneCatalog(generation: generation, db: db) + } + try self.importChangedRollouts(cache, catalog: catalog, generation: generation, db: db) + try self.markMissingRollouts(generation: generation, db: db) + try Self.commit(db) + } catch { + Self.rollback(db) + throw error + } + #else + _ = cache + _ = catalog + _ = catalogIsComplete + #endif + } + + /// Rehydrates only the attribution fields and daily usage rows needed by + /// the existing project aggregator. The scanner remains authoritative for + /// JSONL cursors and token deltas; this avoids another walk of its cache. + func usageCache(roots: [String: Int64]) throws -> CostUsageCache { + #if canImport(SQLite3) || canImport(CSQLite3) + guard let db = self.open(readOnly: true) else { throw SidecarError.openFailed } + defer { sqlite3_close(db) } + let sql = """ + SELECT r.rollout_path, + COALESCE(c.id, r.session_id), + COALESCE(c.cwd, r.cwd), + COALESCE(c.title, c.preview, r.title), + COALESCE(c.created_at_ms, r.started_at_ms), + COALESCE(c.updated_at_ms, r.latest_activity_ms), + COALESCE(c.model, r.last_model), + r.forked_from_id, + r.project_path, + r.canonical_project_path, + d.day, + d.model, + d.input_tokens, + d.cached_input_tokens, + d.output_tokens, + d.cost_nanos, + d.standard_tokens, + d.priority_tokens, + d.standard_cost_nanos, + d.priority_cost_nanos, + d.priority_surcharge_nanos + FROM usage_rollouts r + LEFT JOIN catalog_threads c ON c.id = r.session_id OR c.rollout_path = r.rollout_path + JOIN usage_daily d ON d.rollout_path = r.rollout_path + WHERE r.is_present = 1 + ORDER BY r.rollout_path, d.day, d.model + """ + guard let statement = Self.prepare(db, sql) else { throw SidecarError.statementFailed } + defer { sqlite3_finalize(statement) } + + var cache = CostUsageCache() + cache.roots = roots + while sqlite3_step(statement) == SQLITE_ROW { + guard let path = Self.columnString(statement, at: 0), + let day = Self.columnString(statement, at: 10), + let model = Self.columnString(statement, at: 11) + else { continue } + + var usage = cache.files[path] ?? Self.emptyFileUsage( + sessionId: Self.columnString(statement, at: 1), + cwd: Self.columnString(statement, at: 2), + title: Self.columnString(statement, at: 3), + startedAtUnixMs: Self.columnInt64(statement, at: 4), + latestActivityUnixMs: Self.columnInt64(statement, at: 5), + lastModel: Self.columnString(statement, at: 6), + forkedFromId: Self.columnString(statement, at: 7), + projectPath: Self.columnString(statement, at: 8), + canonicalProjectPath: Self.columnString(statement, at: 9)) + usage.days[day, default: [:]][model] = [ + Int(sqlite3_column_int64(statement, 12)), + Int(sqlite3_column_int64(statement, 13)), + Int(sqlite3_column_int64(statement, 14)), + ] + Self.assign(Self.columnInt64(statement, at: 15), to: &usage.codexCostNanos, day: day, model: model) + Self.assign(Self.columnInt64(statement, at: 16), to: &usage.codexStandardTokens, day: day, model: model) + Self.assign(Self.columnInt64(statement, at: 17), to: &usage.codexPriorityTokens, day: day, model: model) + Self.assign(Self.columnInt64(statement, at: 18), to: &usage.codexStandardCostNanos, day: day, model: model) + Self.assign(Self.columnInt64(statement, at: 19), to: &usage.codexPriorityCostNanos, day: day, model: model) + Self.assign( + Self.columnInt64(statement, at: 20), + to: &usage.codexPrioritySurchargeNanos, + day: day, + model: model) + cache.files[path] = usage + } + let eventSQL = """ + SELECT e.rollout_path, e.day, e.canonical_model, e.raw_model, e.turn_id, e.event_index, + e.timestamp_ms, e.input_tokens, e.cached_input_tokens, e.output_tokens, + e.known_cost_nanos, e.unpriced_tokens, e.pricing_model, e.pricing_mode, + e.reasoning_tokens + FROM usage_events e + JOIN usage_rollouts r ON r.rollout_path = e.rollout_path + WHERE r.is_present = 1 AND r.event_detail_complete = 1 + ORDER BY e.rollout_path, e.event_index + """ + guard let eventStatement = Self.prepare(db, eventSQL) else { throw SidecarError.statementFailed } + defer { sqlite3_finalize(eventStatement) } + while sqlite3_step(eventStatement) == SQLITE_ROW { + guard let path = Self.columnString(eventStatement, at: 0), + var usage = cache.files[path], + let day = Self.columnString(eventStatement, at: 1), + let canonicalModel = Self.columnString(eventStatement, at: 2) + else { continue } + var rows = usage.codexRows ?? [] + rows.append(CostUsageScanner.CodexUsageRow( + day: day, + model: canonicalModel, + rawModel: Self.columnString(eventStatement, at: 3), + turnID: Self.columnString(eventStatement, at: 4), + eventIndex: Self.columnInt64(eventStatement, at: 5).map(Int.init), + timestampUnixMs: Self.columnInt64(eventStatement, at: 6), + input: Int(sqlite3_column_int64(eventStatement, 7)), + cached: Int(sqlite3_column_int64(eventStatement, 8)), + output: Int(sqlite3_column_int64(eventStatement, 9)), + reasoning: Self.columnInt64(eventStatement, at: 14).map(Int.init), + knownCostNanos: Self.columnInt64(eventStatement, at: 10), + unpricedTokens: Self.columnInt64(eventStatement, at: 11).map(Int.init), + pricingModel: Self.columnString(eventStatement, at: 12), + pricingMode: Self.columnString(eventStatement, at: 13))) + usage.codexRows = rows + cache.files[path] = usage + } + return cache + #else + _ = roots + return CostUsageCache() + #endif + } + + func clear() { + try? FileManager.default.removeItem(at: self.databaseURL()) + try? FileManager.default.removeItem(at: URL(fileURLWithPath: self.databaseURL().path + "-wal")) + try? FileManager.default.removeItem(at: URL(fileURLWithPath: self.databaseURL().path + "-shm")) + } + + private func databaseURL() -> URL { + let root = self.cacheRoot ?? FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first! + return root + .appendingPathComponent("local-usage", isDirectory: true) + .appendingPathComponent("codex-workspaces-v1.sqlite", isDirectory: false) + } + + #if canImport(SQLite3) || canImport(CSQLite3) + private func open(readOnly: Bool) -> OpaquePointer? { + let url = self.databaseURL() + if !readOnly { + try? FileManager.default.createDirectory( + at: url.deletingLastPathComponent(), + withIntermediateDirectories: true) + } + var db: OpaquePointer? + let flags = readOnly ? SQLITE_OPEN_READONLY : SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE + guard sqlite3_open_v2(url.path, &db, flags, nil) == SQLITE_OK else { + sqlite3_close(db) + return nil + } + sqlite3_busy_timeout(db, 250) + if !readOnly { + guard sqlite3_exec(db, "PRAGMA journal_mode = WAL", nil, nil, nil) == SQLITE_OK, + sqlite3_exec(db, "PRAGMA synchronous = NORMAL", nil, nil, nil) == SQLITE_OK + else { + sqlite3_close(db) + return nil + } + } + return db + } + + private static func ensureSchema(_ db: OpaquePointer?) throws { + let current = Self.userVersion(db) + guard current == 0 || current == Self.schemaVersion else { + throw SidecarError.incompatibleSchema + } + guard current == 0 else { return } + try Self.execute(db, """ + CREATE TABLE IF NOT EXISTS schema_meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS catalog_threads ( + id TEXT PRIMARY KEY, + rollout_path TEXT NOT NULL, + cwd TEXT, + title TEXT, + preview TEXT, + model_provider TEXT, + model TEXT, + reasoning_effort TEXT, + created_at_ms INTEGER, + updated_at_ms INTEGER, + archived INTEGER NOT NULL, + source_fingerprint TEXT, + last_seen_generation TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS usage_rollouts ( + rollout_path TEXT PRIMARY KEY, + fingerprint TEXT NOT NULL, + session_id TEXT, + cwd TEXT, + title TEXT, + started_at_ms INTEGER, + latest_activity_ms INTEGER, + last_model TEXT, + project_path TEXT, + canonical_project_path TEXT, + forked_from_id TEXT, + source_mtime_ms INTEGER, + source_size INTEGER, + source_parsed_bytes INTEGER, + source_session_id TEXT, + source_producer_key TEXT, + source_pricing_key TEXT, + content_fingerprint TEXT, + event_detail_complete INTEGER NOT NULL DEFAULT 0, + is_present INTEGER NOT NULL DEFAULT 1, + last_seen_generation TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS usage_daily ( + rollout_path TEXT NOT NULL, + day TEXT NOT NULL, + model TEXT NOT NULL, + input_tokens INTEGER NOT NULL, + cached_input_tokens INTEGER NOT NULL, + output_tokens INTEGER NOT NULL, + cost_nanos INTEGER, + standard_tokens INTEGER, + priority_tokens INTEGER, + standard_cost_nanos INTEGER, + priority_cost_nanos INTEGER, + priority_surcharge_nanos INTEGER, + PRIMARY KEY (rollout_path, day, model) + ); + CREATE TABLE IF NOT EXISTS usage_events ( + rollout_path TEXT NOT NULL, + event_index INTEGER NOT NULL, + timestamp_ms INTEGER NOT NULL, + day TEXT NOT NULL, + canonical_model TEXT NOT NULL, + raw_model TEXT, + turn_id TEXT, + input_tokens INTEGER NOT NULL, + cached_input_tokens INTEGER NOT NULL, + output_tokens INTEGER NOT NULL, + known_cost_nanos INTEGER, + unpriced_tokens INTEGER NOT NULL, + pricing_model TEXT, + pricing_mode TEXT, + reasoning_tokens INTEGER, + PRIMARY KEY (rollout_path, event_index) + ); + CREATE TABLE IF NOT EXISTS snapshot_payloads ( + scope_signature TEXT NOT NULL, + history_days INTEGER NOT NULL, + updated_at_ms INTEGER NOT NULL, + is_complete INTEGER NOT NULL, + payload_format_version INTEGER NOT NULL DEFAULT 1, + payload BLOB NOT NULL, + PRIMARY KEY (scope_signature, history_days) + ); + CREATE TABLE IF NOT EXISTS index_state ( + scope_signature TEXT PRIMARY KEY, + roots_fingerprint TEXT NOT NULL, + catalog_fingerprint TEXT, + cache_producer_key TEXT, + pricing_key TEXT, + cache_fingerprint TEXT, + last_success_ms INTEGER NOT NULL + ); + CREATE INDEX IF NOT EXISTS usage_daily_rollout_day ON usage_daily (rollout_path, day); + CREATE INDEX IF NOT EXISTS usage_events_timestamp ON usage_events (timestamp_ms); + CREATE INDEX IF NOT EXISTS usage_events_model_timestamp ON usage_events (canonical_model, timestamp_ms); + CREATE INDEX IF NOT EXISTS usage_events_turn ON usage_events (turn_id); + CREATE INDEX IF NOT EXISTS usage_rollouts_session ON usage_rollouts (session_id); + CREATE INDEX IF NOT EXISTS catalog_threads_rollout ON catalog_threads (rollout_path); + """) + try Self.execute(db, "PRAGMA user_version = \(Self.schemaVersion)") + } + + private func upsertCatalog( + _ catalog: CodexThreadCatalog, + generation: String, + db: OpaquePointer?) throws + { + let sql = """ + INSERT INTO catalog_threads ( + id, rollout_path, cwd, title, preview, model_provider, model, reasoning_effort, + created_at_ms, updated_at_ms, archived, source_fingerprint, last_seen_generation + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + rollout_path = excluded.rollout_path, + cwd = excluded.cwd, + title = excluded.title, + preview = excluded.preview, + model_provider = excluded.model_provider, + model = excluded.model, + reasoning_effort = excluded.reasoning_effort, + created_at_ms = excluded.created_at_ms, + updated_at_ms = excluded.updated_at_ms, + archived = excluded.archived, + source_fingerprint = excluded.source_fingerprint, + last_seen_generation = excluded.last_seen_generation + """ + guard let statement = Self.prepare(db, sql) else { throw SidecarError.statementFailed } + defer { sqlite3_finalize(statement) } + for entry in catalog.entriesById.values { + sqlite3_reset(statement) + sqlite3_clear_bindings(statement) + Self.bind(entry.id, to: statement, at: 1) + Self.bind(entry.rolloutPath, to: statement, at: 2) + Self.bind(entry.cwd, to: statement, at: 3) + Self.bind(entry.title, to: statement, at: 4) + Self.bind(entry.preview, to: statement, at: 5) + Self.bind(entry.modelProvider, to: statement, at: 6) + Self.bind(entry.model, to: statement, at: 7) + Self.bind(entry.reasoningEffort, to: statement, at: 8) + Self.bind(entry.createdAtUnixMs, to: statement, at: 9) + Self.bind(entry.updatedAtUnixMs, to: statement, at: 10) + sqlite3_bind_int(statement, 11, entry.archived ? 1 : 0) + Self.bind(catalog.fingerprint, to: statement, at: 12) + Self.bind(generation, to: statement, at: 13) + guard sqlite3_step(statement) == SQLITE_DONE else { throw SidecarError.writeFailed } + } + } + + /// A complete catalog generation is authoritative: entries absent from it + /// must no longer override rollout-derived metadata. Unavailable reads + /// deliberately skip this deletion so the last-good attribution remains. + private func pruneCatalog(generation: String, db: OpaquePointer?) throws { + guard let statement = Self.prepare( + db, + "DELETE FROM catalog_threads WHERE last_seen_generation != ?") + else { throw SidecarError.statementFailed } + defer { sqlite3_finalize(statement) } + Self.bind(generation, to: statement, at: 1) + guard sqlite3_step(statement) == SQLITE_DONE else { throw SidecarError.writeFailed } + } + + private func importChangedRollouts( + _ cache: CostUsageCache, + catalog: CodexThreadCatalog, + generation: String, + db: OpaquePointer?) throws + { + let existing = try Self.existingRolloutSourceIdentities(db: db) + guard let touchStatement = Self.prepare( + db, + "UPDATE usage_rollouts SET is_present = 1, last_seen_generation = ? WHERE rollout_path = ?") + else { throw SidecarError.statementFailed } + defer { sqlite3_finalize(touchStatement) } + + for (path, usage) in cache.files { + let identity = RolloutSourceIdentity(usage: usage, cache: cache) + if existing[path] == identity { + try Self.touchRollout(path: path, generation: generation, statement: touchStatement) + continue + } + let catalogEntry = catalog.entry( + sessionId: usage.codexSession?.sessionId ?? usage.sessionId, + rolloutPath: path) + try Self.deleteUsage(path: path, db: db) + try Self.upsertRollout( + path: path, + usage: usage, + catalogEntry: catalogEntry, + identity: identity, + generation: generation, + db: db) + try Self.insertDaily(path: path, usage: usage, db: db) + try Self.insertEvents(path: path, usage: usage, db: db) + } + } + + private func markMissingRollouts(generation: String, db: OpaquePointer?) throws { + guard let statement = Self.prepare( + db, + "UPDATE usage_rollouts SET is_present = 0 WHERE last_seen_generation != ?") + else { throw SidecarError.statementFailed } + defer { sqlite3_finalize(statement) } + Self.bind(generation, to: statement, at: 1) + guard sqlite3_step(statement) == SQLITE_DONE else { throw SidecarError.writeFailed } + } + + private func storeSnapshot( + _ snapshot: CodexLocalProjectUsageSnapshot, + cache: CostUsageCache, + catalog: CodexThreadCatalog, + rootsFingerprint: [String: Int64], + db: OpaquePointer?) throws + { + let data = try JSONEncoder.codexLocalProjectUsageSidecar.encode(snapshot) + let sql = """ + INSERT INTO snapshot_payloads ( + scope_signature, history_days, updated_at_ms, is_complete, payload_format_version, payload + ) VALUES (?, ?, ?, 1, ?, ?) + ON CONFLICT(scope_signature, history_days) DO UPDATE SET + updated_at_ms = excluded.updated_at_ms, + is_complete = 1, + payload_format_version = excluded.payload_format_version, + payload = excluded.payload + """ + guard let statement = Self.prepare(db, sql) else { throw SidecarError.statementFailed } + defer { sqlite3_finalize(statement) } + Self.bind(snapshot.scopeSignature, to: statement, at: 1) + sqlite3_bind_int64(statement, 2, Int64(snapshot.historyDays)) + sqlite3_bind_int64(statement, 3, Int64((snapshot.updatedAt.timeIntervalSince1970 * 1000).rounded())) + sqlite3_bind_int(statement, 4, Int32(Self.snapshotPayloadFormatVersion)) + Self.bind(data, to: statement, at: 5) + guard sqlite3_step(statement) == SQLITE_DONE else { throw SidecarError.writeFailed } + + let stateSQL = """ + INSERT INTO index_state ( + scope_signature, roots_fingerprint, catalog_fingerprint, cache_producer_key, pricing_key, cache_fingerprint, + last_success_ms + ) VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(scope_signature) DO UPDATE SET + roots_fingerprint = excluded.roots_fingerprint, + catalog_fingerprint = COALESCE(excluded.catalog_fingerprint, index_state.catalog_fingerprint), + cache_producer_key = excluded.cache_producer_key, + pricing_key = excluded.pricing_key, + cache_fingerprint = excluded.cache_fingerprint, + last_success_ms = excluded.last_success_ms + """ + guard let state = Self.prepare(db, stateSQL) else { throw SidecarError.statementFailed } + defer { sqlite3_finalize(state) } + Self.bind(snapshot.scopeSignature, to: state, at: 1) + let rootsData = try JSONEncoder().encode(rootsFingerprint) + Self.bind(rootsData, to: state, at: 2) + Self.bind(catalog.fingerprint, to: state, at: 3) + Self.bind(cache.producerKey, to: state, at: 4) + Self.bind(cache.codexPricingKey, to: state, at: 5) + Self.bind(Self.cacheFingerprint(cache), to: state, at: 6) + sqlite3_bind_int64(state, 7, Int64((snapshot.updatedAt.timeIntervalSince1970 * 1000).rounded())) + guard sqlite3_step(state) == SQLITE_DONE else { throw SidecarError.writeFailed } + } + + // Source fields remain explicit so the sidecar never reconstructs parser state. + // swiftlint:disable:next function_parameter_count + private static func upsertRollout( + path: String, + usage: CostUsageFileUsage, + catalogEntry: CodexThreadCatalogEntry?, + identity: RolloutSourceIdentity, + generation: String, + db: OpaquePointer?) throws + { + let session = usage.codexSession + let sql = """ + INSERT INTO usage_rollouts ( + rollout_path, fingerprint, session_id, cwd, title, started_at_ms, latest_activity_ms, last_model, + project_path, canonical_project_path, forked_from_id, + source_mtime_ms, source_size, source_parsed_bytes, source_session_id, source_producer_key, + source_pricing_key, content_fingerprint, event_detail_complete, is_present, last_seen_generation + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?) + ON CONFLICT(rollout_path) DO UPDATE SET + fingerprint = excluded.fingerprint, + session_id = COALESCE(excluded.session_id, usage_rollouts.session_id), + cwd = COALESCE(excluded.cwd, usage_rollouts.cwd), + title = COALESCE(excluded.title, usage_rollouts.title), + started_at_ms = COALESCE(excluded.started_at_ms, usage_rollouts.started_at_ms), + latest_activity_ms = COALESCE(excluded.latest_activity_ms, usage_rollouts.latest_activity_ms), + last_model = COALESCE(excluded.last_model, usage_rollouts.last_model), + project_path = COALESCE(excluded.project_path, usage_rollouts.project_path), + canonical_project_path = COALESCE(excluded.canonical_project_path, usage_rollouts.canonical_project_path), + forked_from_id = COALESCE(excluded.forked_from_id, usage_rollouts.forked_from_id), + source_mtime_ms = excluded.source_mtime_ms, + source_size = excluded.source_size, + source_parsed_bytes = excluded.source_parsed_bytes, + source_session_id = excluded.source_session_id, + source_producer_key = excluded.source_producer_key, + source_pricing_key = excluded.source_pricing_key, + content_fingerprint = excluded.content_fingerprint, + event_detail_complete = excluded.event_detail_complete, + is_present = 1, + last_seen_generation = excluded.last_seen_generation + """ + guard let statement = Self.prepare(db, sql) else { throw SidecarError.statementFailed } + defer { sqlite3_finalize(statement) } + Self.bind(path, to: statement, at: 1) + Self.bind(identity.legacyFingerprint, to: statement, at: 2) + Self.bind(catalogEntry?.id ?? session?.sessionId ?? usage.sessionId, to: statement, at: 3) + Self.bind(catalogEntry?.cwd ?? session?.cwd, to: statement, at: 4) + Self.bind(catalogEntry?.title ?? session?.title, to: statement, at: 5) + Self.bind(catalogEntry?.createdAtUnixMs ?? session?.startedAtUnixMs, to: statement, at: 6) + Self.bind(catalogEntry?.updatedAtUnixMs ?? session?.latestActivityUnixMs, to: statement, at: 7) + Self.bind(catalogEntry?.model ?? usage.lastModel, to: statement, at: 8) + Self.bind(usage.projectPath, to: statement, at: 9) + Self.bind(usage.canonicalProjectPath, to: statement, at: 10) + Self.bind( + catalogEntry == nil ? session?.forkedFromId ?? usage.forkedFromId : usage.forkedFromId, + to: statement, + at: 11) + sqlite3_bind_int64(statement, 12, identity.mtimeUnixMs) + sqlite3_bind_int64(statement, 13, identity.size) + sqlite3_bind_int64(statement, 14, identity.parsedBytes) + Self.bind(identity.sessionID, to: statement, at: 15) + Self.bind(identity.producerKey, to: statement, at: 16) + Self.bind(identity.pricingKey, to: statement, at: 17) + Self.bind(identity.contentFingerprint, to: statement, at: 18) + sqlite3_bind_int(statement, 19, Self.hasCompleteEventDetail(usage) ? 1 : 0) + Self.bind(generation, to: statement, at: 20) + guard sqlite3_step(statement) == SQLITE_DONE else { throw SidecarError.writeFailed } + } + + private static func insertDaily(path: String, usage: CostUsageFileUsage, db: OpaquePointer?) throws { + let sql = """ + INSERT INTO usage_daily ( + rollout_path, day, model, input_tokens, cached_input_tokens, output_tokens, cost_nanos, + standard_tokens, priority_tokens, standard_cost_nanos, priority_cost_nanos, priority_surcharge_nanos + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """ + guard let statement = Self.prepare(db, sql) else { throw SidecarError.statementFailed } + defer { sqlite3_finalize(statement) } + for (day, models) in usage.days { + for (model, values) in models { + sqlite3_reset(statement) + sqlite3_clear_bindings(statement) + Self.bind(path, to: statement, at: 1) + Self.bind(day, to: statement, at: 2) + Self.bind(model, to: statement, at: 3) + sqlite3_bind_int64(statement, 4, Int64(max(0, values[safe: 0] ?? 0))) + sqlite3_bind_int64(statement, 5, Int64(max(0, values[safe: 1] ?? 0))) + sqlite3_bind_int64(statement, 6, Int64(max(0, values[safe: 2] ?? 0))) + Self.bind(usage.codexCostNanos?[day]?[model], to: statement, at: 7) + Self.bind(usage.codexStandardTokens?[day]?[model], to: statement, at: 8) + Self.bind(usage.codexPriorityTokens?[day]?[model], to: statement, at: 9) + Self.bind(usage.codexStandardCostNanos?[day]?[model], to: statement, at: 10) + Self.bind(usage.codexPriorityCostNanos?[day]?[model], to: statement, at: 11) + Self.bind(usage.codexPrioritySurchargeNanos?[day]?[model], to: statement, at: 12) + guard sqlite3_step(statement) == SQLITE_DONE else { throw SidecarError.writeFailed } + } + } + } + + private static func insertEvents(path: String, usage: CostUsageFileUsage, db: OpaquePointer?) throws { + guard self.hasCompleteEventDetail(usage), let rows = usage.codexRows else { return } + let sql = """ + INSERT INTO usage_events ( + rollout_path, event_index, timestamp_ms, day, canonical_model, raw_model, turn_id, + input_tokens, cached_input_tokens, output_tokens, known_cost_nanos, unpriced_tokens, + pricing_model, pricing_mode, reasoning_tokens + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """ + guard let statement = Self.prepare(db, sql) else { throw SidecarError.statementFailed } + defer { sqlite3_finalize(statement) } + for row in rows { + guard let eventIndex = row.eventIndex, let timestampUnixMs = row.timestampUnixMs else { continue } + sqlite3_reset(statement) + sqlite3_clear_bindings(statement) + Self.bind(path, to: statement, at: 1) + sqlite3_bind_int64(statement, 2, Int64(eventIndex)) + sqlite3_bind_int64(statement, 3, timestampUnixMs) + Self.bind(row.day, to: statement, at: 4) + Self.bind(row.model, to: statement, at: 5) + Self.bind(row.rawModel, to: statement, at: 6) + Self.bind(row.turnID, to: statement, at: 7) + sqlite3_bind_int64(statement, 8, Int64(max(0, row.input))) + sqlite3_bind_int64(statement, 9, Int64(max(0, row.cached))) + sqlite3_bind_int64(statement, 10, Int64(max(0, row.output))) + Self.bind(row.knownCostNanos, to: statement, at: 11) + sqlite3_bind_int64(statement, 12, Int64(max(0, row.unpricedTokens ?? 0))) + Self.bind(row.pricingModel, to: statement, at: 13) + Self.bind(row.pricingMode, to: statement, at: 14) + Self.bind(row.reasoning.map(Int64.init), to: statement, at: 15) + guard sqlite3_step(statement) == SQLITE_DONE else { throw SidecarError.writeFailed } + } + } + + private static func deleteUsage(path: String, db: OpaquePointer?) throws { + for table in ["usage_daily", "usage_events"] { + guard let statement = prepare(db, "DELETE FROM \(table) WHERE rollout_path = ?") + else { throw SidecarError.statementFailed } + defer { sqlite3_finalize(statement) } + Self.bind(path, to: statement, at: 1) + guard sqlite3_step(statement) == SQLITE_DONE else { throw SidecarError.writeFailed } + } + } + + private static func hasCompleteEventDetail(_ usage: CostUsageFileUsage) -> Bool { + guard let rows = usage.codexRows, !rows.isEmpty else { return usage.days.isEmpty } + return rows.allSatisfy { $0.eventIndex != nil && $0.timestampUnixMs != nil } + } + + private static func touchRollout(path: String, generation: String, statement: OpaquePointer?) throws { + sqlite3_reset(statement) + sqlite3_clear_bindings(statement) + self.bind(generation, to: statement, at: 1) + self.bind(path, to: statement, at: 2) + guard sqlite3_step(statement) == SQLITE_DONE else { throw SidecarError.writeFailed } + } + + private static func existingRolloutSourceIdentities( + db: OpaquePointer?) throws -> [String: RolloutSourceIdentity] + { + guard let statement = prepare( + db, + """ + SELECT rollout_path, source_mtime_ms, source_size, source_parsed_bytes, source_session_id, + source_producer_key, source_pricing_key, content_fingerprint + FROM usage_rollouts + WHERE event_detail_complete = 1 + """) + else { throw SidecarError.statementFailed } + defer { sqlite3_finalize(statement) } + var identities: [String: RolloutSourceIdentity] = [:] + while sqlite3_step(statement) == SQLITE_ROW { + guard let path = Self.columnString(statement, at: 0), + let contentFingerprint = Self.columnString(statement, at: 7) + else { continue } + identities[path] = RolloutSourceIdentity( + mtimeUnixMs: Self.columnInt64(statement, at: 1) ?? Int64.min, + size: Self.columnInt64(statement, at: 2) ?? Int64.min, + parsedBytes: Self.columnInt64(statement, at: 3) ?? Int64.min, + sessionID: Self.columnString(statement, at: 4) ?? "", + producerKey: Self.columnString(statement, at: 5) ?? "", + pricingKey: Self.columnString(statement, at: 6) ?? "", + contentFingerprint: contentFingerprint) + } + return identities + } + + private static func cacheFingerprint(_ cache: CostUsageCache) -> String { + var hash: UInt64 = 14_695_981_039_346_656_037 + for (path, usage) in cache.files.sorted(by: { $0.key < $1.key }) { + let identity = RolloutSourceIdentity(usage: usage, cache: cache) + for byte in "\(path)|\(identity.legacyFingerprint)\n".utf8 { + hash ^= UInt64(byte) + hash &*= 1_099_511_628_211 + } + } + return String(hash, radix: 16) + } + + private static func columnString(_ statement: OpaquePointer?, at index: Int32) -> String? { + guard let text = sqlite3_column_text(statement, index) else { return nil } + return String(cString: text) + } + + private static func columnInt64(_ statement: OpaquePointer?, at index: Int32) -> Int64? { + sqlite3_column_type(statement, index) == SQLITE_NULL ? nil : sqlite3_column_int64(statement, index) + } + + private static func assign( + _ value: Int64?, + to map: inout [String: [String: T]]?, + day: String, + model: String) + { + guard let value else { return } + var values = map ?? [:] + values[day, default: [:]][model] = T(value) + map = values + } + + // A SQL row carries scanner metadata as individual columns. Keeping the + // parameters explicit avoids a lossy intermediate representation. + // swiftlint:disable:next function_parameter_count + private static func emptyFileUsage( + sessionId: String?, + cwd: String?, + title: String?, + startedAtUnixMs: Int64?, + latestActivityUnixMs: Int64?, + lastModel: String?, + forkedFromId: String?, + projectPath: String?, + canonicalProjectPath: String?) -> CostUsageFileUsage + { + CostUsageFileUsage( + mtimeUnixMs: 0, + size: 0, + days: [:], + parsedBytes: nil, + lastModel: lastModel, + lastTotals: nil, + lastCountedTotals: nil, + lastRawTotalsBaseline: nil, + hasDivergentTotals: nil, + lastCodexTurnID: nil, + sessionId: sessionId, + forkedFromId: forkedFromId, + projectPath: projectPath, + canonicalProjectPath: canonicalProjectPath, + codexCostCacheComplete: true, + codexSession: CostUsageCodexSessionMetadata( + sessionId: sessionId, + forkedFromId: forkedFromId, + cwd: cwd, + title: title, + startedAtUnixMs: startedAtUnixMs, + latestActivityUnixMs: latestActivityUnixMs), + codexCostNanos: nil, + codexPrioritySurchargeNanos: nil, + codexStandardCostNanos: nil, + codexPriorityCostNanos: nil, + codexStandardTokens: nil, + codexPriorityTokens: nil, + codexTurnIDs: nil, + codexRows: nil, + claudeRows: nil) + } + + private static func begin(_ db: OpaquePointer?) throws { + try self.execute(db, "BEGIN IMMEDIATE") + } + + private static func commit(_ db: OpaquePointer?) throws { + try self.execute(db, "COMMIT") + } + + private static func rollback(_ db: OpaquePointer?) { + try? self.execute(db, "ROLLBACK") + } + + private static func execute(_ db: OpaquePointer?, _ sql: String) throws { + guard sqlite3_exec(db, sql, nil, nil, nil) == SQLITE_OK else { throw SidecarError.writeFailed } + } + + private static func prepare(_ db: OpaquePointer?, _ sql: String) -> OpaquePointer? { + var statement: OpaquePointer? + guard sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK else { return nil } + return statement + } + + private static func bind(_ value: String?, to statement: OpaquePointer?, at index: Int32) { + guard let value else { + sqlite3_bind_null(statement, index) + return + } + sqlite3_bind_text(statement, index, value, -1, Self.sqliteTransient) + } + + private static func bind(_ value: Int64?, to statement: OpaquePointer?, at index: Int32) { + guard let value else { + sqlite3_bind_null(statement, index) + return + } + sqlite3_bind_int64(statement, index, value) + } + + private static func bind(_ value: Int?, to statement: OpaquePointer?, at index: Int32) { + self.bind(value.map(Int64.init), to: statement, at: index) + } + + private static func bind(_ value: Data, to statement: OpaquePointer?, at index: Int32) { + _ = value.withUnsafeBytes { bytes in + sqlite3_bind_blob(statement, index, bytes.baseAddress, Int32(value.count), Self.sqliteTransient) + } + } + + private static func userVersion(_ db: OpaquePointer?) -> Int32 { + guard let statement = prepare(db, "PRAGMA user_version") else { return 0 } + defer { sqlite3_finalize(statement) } + guard sqlite3_step(statement) == SQLITE_ROW else { return 0 } + return sqlite3_column_int(statement, 0) + } + + private static let sqliteTransient = unsafeBitCast(-1, to: sqlite3_destructor_type.self) + + private enum SidecarError: Error { + case openFailed + case incompatibleSchema + case statementFailed + case writeFailed + } + #endif +} + +extension JSONDecoder { + static var codexLocalProjectUsageSidecar: JSONDecoder { + let decoder = JSONDecoder() + decoder.dateDecodingStrategy = .millisecondsSince1970 + return decoder + } +} + +extension JSONEncoder { + static var codexLocalProjectUsageSidecar: JSONEncoder { + let encoder = JSONEncoder() + encoder.dateEncodingStrategy = .millisecondsSince1970 + return encoder + } +} + +// swiftlint:enable type_body_length diff --git a/Sources/CodexBarCore/CostUsageFetcher.swift b/Sources/CodexBarCore/CostUsageFetcher.swift index 295fa75e06..5573575855 100644 --- a/Sources/CodexBarCore/CostUsageFetcher.swift +++ b/Sources/CodexBarCore/CostUsageFetcher.swift @@ -26,6 +26,7 @@ public enum CostUsageError: LocalizedError, Sendable { } } +// swiftlint:disable:next type_body_length public struct CostUsageFetcher: Sendable { package struct CachedCodexTokenSnapshotResult: Sendable { package let snapshot: CostUsageTokenSnapshot @@ -66,6 +67,45 @@ public struct CostUsageFetcher: Sendable { scannerOptions: self.scannerOptionsOverride()) } + public func loadCachedCodexLocalProjectUsageSnapshot( + now: Date = Date(), + codexHomePath: String? = nil, + historyDays: Int = 30, + hidePersonalInfo: Bool) async -> CodexLocalProjectUsageSnapshot? + { + await Self.loadCachedCodexLocalProjectUsageSnapshot( + now: now, + codexHomePath: codexHomePath, + historyDays: historyDays, + hidePersonalInfo: hidePersonalInfo, + scannerOptions: self.scannerOptionsOverride()) + } + + public func loadCodexLocalProjectUsageSnapshot( + now: Date = Date(), + forceRefresh: Bool = false, + codexHomePath: String? = nil, + historyDays: Int = 30, + hidePersonalInfo: Bool, + progress: (@Sendable (CodexLocalProjectUsageIndexProgress) -> Void)? = nil) + async throws -> CodexLocalProjectUsageSnapshot + { + try await Self.loadCodexLocalProjectUsageSnapshot( + now: now, + forceRefresh: forceRefresh, + codexHomePath: codexHomePath, + historyDays: historyDays, + hidePersonalInfo: hidePersonalInfo, + progress: progress, + scannerOptions: self.scannerOptionsOverride()) + } + + public func clearCachedCodexLocalProjectUsageSnapshot(codexHomePath: String? = nil) async { + await Self.clearCachedCodexLocalProjectUsageSnapshot( + codexHomePath: codexHomePath, + scannerOptions: self.scannerOptionsOverride()) + } + public func loadTokenSnapshot( provider: UsageProvider, environment: [String: String] = ProcessInfo.processInfo.environment, @@ -571,6 +611,77 @@ public struct CostUsageFetcher: Sendable { } } + static func loadCachedCodexLocalProjectUsageSnapshot( + now: Date = Date(), + codexHomePath: String? = nil, + historyDays: Int = 30, + hidePersonalInfo: Bool, + scannerOptions overrideScannerOptions: CostUsageScanner.Options? = nil) async -> CodexLocalProjectUsageSnapshot? + { + let cachedSnapshot: CodexLocalProjectUsageSnapshot?? = try? await CostUsageScanExecutor.run { _ in + let options = Self.codexLocalScannerOptions( + codexHomePath: codexHomePath, + overrideScannerOptions: overrideScannerOptions) + return CodexLocalProjectUsageIndexer.cachedSnapshot( + now: now, + historyDays: historyDays, + options: CodexLocalProjectUsageIndexer.Options(scannerOptions: options)) + } + return cachedSnapshot.flatMap(\.self)?.hidingPersonalInformation(hidePersonalInfo) + } + + static func loadCodexLocalProjectUsageSnapshot( + now: Date = Date(), + forceRefresh: Bool = false, + codexHomePath: String? = nil, + historyDays: Int = 30, + hidePersonalInfo: Bool, + progress: (@Sendable (CodexLocalProjectUsageIndexProgress) -> Void)? = nil, + scannerOptions overrideScannerOptions: CostUsageScanner + .Options? = nil) async throws -> CodexLocalProjectUsageSnapshot + { + let options = Self.codexLocalScannerOptions( + codexHomePath: codexHomePath, + overrideScannerOptions: overrideScannerOptions) + let scanOptions = options + let snapshot = try await CostUsageScanExecutor.run { checkCancellation in + try CodexLocalProjectUsageIndexer.loadSnapshot( + now: now, + historyDays: historyDays, + forceRefresh: forceRefresh, + options: CodexLocalProjectUsageIndexer.Options(scannerOptions: scanOptions), + progress: progress, + checkCancellation: checkCancellation) + } + return snapshot.hidingPersonalInformation(hidePersonalInfo) + } + + static func clearCachedCodexLocalProjectUsageSnapshot( + codexHomePath: String? = nil, + scannerOptions overrideScannerOptions: CostUsageScanner.Options? = nil) async + { + _ = try? await CostUsageScanExecutor.run { _ in + let options = Self.codexLocalScannerOptions( + codexHomePath: codexHomePath, + overrideScannerOptions: overrideScannerOptions) + CodexWorkspaceUsageSidecar(cacheRoot: options.cacheRoot).clear() + } + } + + private static func codexLocalScannerOptions( + codexHomePath: String?, + overrideScannerOptions: CostUsageScanner.Options?) -> CostUsageScanner.Options + { + var options = overrideScannerOptions ?? CostUsageScanner.Options() + if let codexHomePath = codexHomePath?.trimmingCharacters(in: .whitespacesAndNewlines), + !codexHomePath.isEmpty + { + options.codexSessionsRoot = URL(fileURLWithPath: codexHomePath, isDirectory: true) + .appendingPathComponent("sessions", isDirectory: true) + } + return CodexLocalDataScope.resolve(options: options).applying(to: options) + } + private static func loadBedrockDailyReport( environment: [String: String], since: Date, diff --git a/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift b/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift index 0f885bdbcb..4d8866a768 100644 --- a/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift +++ b/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift @@ -1,5 +1,5 @@ // Generated by Scripts/regenerate-codex-parser-hash.sh. Do not edit by hand. enum CodexParserHash { - static let value = "e8496ca631793ba0" + static let value = "a15a1040092b4a62" } diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageCache.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageCache.swift index 7323a07c43..a4a6ec7c1b 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageCache.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageCache.swift @@ -11,7 +11,7 @@ enum CostUsageCacheIO { private static func artifactVersion(for provider: UsageProvider) -> Int { switch provider { case .codex: - 10 + 11 case .claude, .vertexai: 5 default: @@ -147,6 +147,7 @@ struct CostUsageFileUsage: Codable { var projectPath: String? var canonicalProjectPath: String? var codexCostCacheComplete: Bool? + var codexSession: CostUsageCodexSessionMetadata? var codexCostNanos: [String: [String: Int64]]? var codexPrioritySurchargeNanos: [String: [String: Int64]]? var codexStandardCostNanos: [String: [String: Int64]]? @@ -154,12 +155,68 @@ struct CostUsageFileUsage: Codable { var codexStandardTokens: [String: [String: Int]]? var codexPriorityTokens: [String: [String: Int]]? var codexTurnIDs: [String]? + /// Refreshed by Codex normalization paths, never by sidecar cache validation. + var codexWorkspaceContentFingerprint: String? var codexRows: [CostUsageScanner.CodexUsageRow]? var claudeRows: [CostUsageScanner.ClaudeUsageRow]? } +struct CostUsageCodexSessionMetadata: Codable, Equatable { + var sessionId: String? + var forkedFromId: String? + var cwd: String? + var title: String? + var startedAtUnixMs: Int64? + var latestActivityUnixMs: Int64? + + var isEmpty: Bool { + self.sessionId == nil + && self.forkedFromId == nil + && self.cwd == nil + && self.title == nil + && self.startedAtUnixMs == nil + && self.latestActivityUnixMs == nil + } + + func merging(_ newer: CostUsageCodexSessionMetadata) -> CostUsageCodexSessionMetadata { + CostUsageCodexSessionMetadata( + sessionId: newer.sessionId ?? self.sessionId, + forkedFromId: newer.forkedFromId ?? self.forkedFromId, + cwd: newer.cwd ?? self.cwd, + title: newer.title ?? self.title, + startedAtUnixMs: Self.earlier(self.startedAtUnixMs, newer.startedAtUnixMs), + latestActivityUnixMs: Self.later(self.latestActivityUnixMs, newer.latestActivityUnixMs)) + } + + private static func earlier(_ lhs: Int64?, _ rhs: Int64?) -> Int64? { + switch (lhs, rhs) { + case let (lhs?, rhs?): min(lhs, rhs) + case let (lhs?, nil): lhs + case let (nil, rhs?): rhs + case (nil, nil): nil + } + } + + private static func later(_ lhs: Int64?, _ rhs: Int64?) -> Int64? { + switch (lhs, rhs) { + case let (lhs?, rhs?): max(lhs, rhs) + case let (lhs?, nil): lhs + case let (nil, rhs?): rhs + case (nil, nil): nil + } + } +} + struct CostUsageCodexTotals: Codable, Equatable { var input: Int var cached: Int var output: Int + var reasoning: Int? + + init(input: Int, cached: Int, output: Int, reasoning: Int? = nil) { + self.input = input + self.cached = cached + self.output = output + self.reasoning = reasoning + } } diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner+CacheHelpers.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner+CacheHelpers.swift index 8bdbeec25d..377c48e18d 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner+CacheHelpers.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner+CacheHelpers.swift @@ -1,3 +1,5 @@ +// swiftlint:disable file_length + import Foundation #if canImport(Musl) import Musl @@ -291,6 +293,7 @@ extension CostUsageScanner { projectPath: String? = nil, canonicalProjectPath: String? = nil, codexCostCacheComplete: Bool? = true, + codexSession: CostUsageCodexSessionMetadata? = nil, codexCostNanos: [String: [String: Int64]]? = nil, codexPrioritySurchargeNanos: [String: [String: Int64]]? = nil, codexStandardCostNanos: [String: [String: Int64]]? = nil, @@ -321,6 +324,7 @@ extension CostUsageScanner { projectPath: projectPath, canonicalProjectPath: canonicalProjectPath, codexCostCacheComplete: codexCostCacheComplete, + codexSession: codexSession, codexCostNanos: codexCostNanos, codexPrioritySurchargeNanos: codexPrioritySurchargeNanos, codexStandardCostNanos: codexStandardCostNanos, @@ -424,8 +428,59 @@ extension CostUsageScanner { splitMaps.priorityTokens) updated.codexCostCacheComplete = true updated.codexTurnIDs = Self.mergeCodexTurnIDs(usage.codexTurnIDs, rows: migratedRows) - updated.codexRows = rows - return updated + updated.codexRows = Self.codexRowsWithPricingAudit( + rows, + priorityTurns: priorityTurns, + modelsDevCatalog: modelsDevCatalog, + modelsDevCacheRoot: modelsDevCacheRoot) + return updated.refreshingCodexWorkspaceUsageFingerprint() + } + + static func codexRowsWithPricingAudit( + _ rows: [CodexUsageRow], + priorityTurns: [String: CodexPriorityTurnMetadata], + modelsDevCatalog: ModelsDevCatalog?, + modelsDevCacheRoot: URL?) -> [CodexUsageRow] + { + rows.map { row in + let priorityMetadata = row.turnID.flatMap { priorityTurns[$0] } + let pricedModel = priorityMetadata.map { Self.codexPriorityPricingModel(for: row, priorityMetadata: $0) } + ?? row.model + let baseCost = CostUsagePricing.codexCostUSD( + model: pricedModel, + inputTokens: row.input, + cachedInputTokens: row.cached, + outputTokens: row.output, + modelsDevCatalog: modelsDevCatalog, + modelsDevCacheRoot: modelsDevCacheRoot) + let exactCost: Double? = if priorityMetadata != nil, + let priorityCost = CostUsagePricing.codexPriorityCostUSD( + model: pricedModel, + inputTokens: row.input, + cachedInputTokens: row.cached, + outputTokens: row.output) + { + max(priorityCost, baseCost ?? priorityCost) + } else { + baseCost + } + let totalTokens = max(0, row.input) + max(0, row.output) + return CodexUsageRow( + day: row.day, + model: row.model, + rawModel: row.rawModel, + turnID: row.turnID, + eventIndex: row.eventIndex, + timestampUnixMs: row.timestampUnixMs, + input: row.input, + cached: row.cached, + output: row.output, + reasoning: row.reasoning, + knownCostNanos: exactCost.map { Int64(($0 * Self.costScale).rounded()) }, + unpricedTokens: exactCost == nil ? totalTokens : 0, + pricingModel: pricedModel, + pricingMode: priorityMetadata == nil ? "standard" : "priority") + } } static func codexMergedCostMap( @@ -741,6 +796,7 @@ extension CostUsageScanner { splitMaps.priorityTokens), codexTurnIDs: Self.mergeCodexTurnIDs(nil, rows: rows), codexRows: rows) + .refreshingCodexWorkspaceUsageFingerprint() } static func mergeCostMaps( @@ -964,6 +1020,7 @@ extension CostUsageScanner { return !(Set(cached.codexTurnIDs ?? []).isDisjoint(with: context.changedPriorityTurnIDs)) } + // swiftlint:disable:next function_body_length static func appendCodexFileIncrementIfPossible( input: CodexFileScanInput, context: CodexFileScanContext, @@ -1018,7 +1075,16 @@ extension CostUsageScanner { if delta.forkedFromId != nil { return false } - let sessionId = delta.sessionId ?? cached.sessionId + let migrated = Self.codexFileUsageWithCostCache(cached, context: context) + let cachedSessionMetadata = migrated.codexSession ?? CostUsageCodexSessionMetadata( + sessionId: migrated.sessionId, + forkedFromId: migrated.forkedFromId, + cwd: nil, + title: nil, + startedAtUnixMs: nil, + latestActivityUnixMs: nil) + let codexSession = cachedSessionMetadata.merging(delta.codexSession) + let sessionId = codexSession.sessionId ?? delta.sessionId ?? cached.sessionId let projectPath = delta.projectPath ?? cached.projectPath let canonicalProjectPath = delta.projectPath.map { context.resources.projectPathResolver.canonicalProjectPath(for: $0) @@ -1046,7 +1112,6 @@ extension CostUsageScanner { fileIdentity: input.metadata.path, state: &state) - let migrated = Self.codexFileUsageWithCostCache(cached, context: context) let migratedCached = sessionAlreadyContributed ? Self.codexFileUsageByFilteringRows(migrated, rows: retainedCachedRows, context: context) : migrated @@ -1087,9 +1152,10 @@ extension CostUsageScanner { hasInterleavedTotals: delta.hasInterleavedTotals, lastCodexTurnID: delta.lastCodexTurnID, sessionId: sessionId, - forkedFromId: delta.forkedFromId ?? migratedCached.forkedFromId, + forkedFromId: codexSession.forkedFromId ?? delta.forkedFromId ?? migratedCached.forkedFromId, projectPath: projectPath, canonicalProjectPath: canonicalProjectPath, + codexSession: codexSession.isEmpty ? nil : codexSession, codexCostNanos: Self.codexMergedCostMap( migratedCached.codexCostNanos, deltaRows: uniqueRows, @@ -1111,7 +1177,12 @@ extension CostUsageScanner { migratedCached.codexPriorityTokens, splitMaps.priorityTokens), codexTurnIDs: Self.mergeCodexTurnIDs(migratedCached.codexTurnIDs, rows: uniqueRows), - codexRows: Self.mergeCodexRows(retainedCachedRows, rows: uniqueRows, sessionId: sessionId)) + codexRows: Self.codexRowsWithPricingAudit( + Self.mergeCodexRows(retainedCachedRows, rows: uniqueRows, sessionId: sessionId) ?? [], + priorityTurns: context.resources.priorityTurns, + modelsDevCatalog: context.resources.modelsDevCatalog, + modelsDevCacheRoot: context.resources.modelsDevCacheRoot)) + .refreshingCodexWorkspaceUsageFingerprint() Self.rememberScannedCodexFile( input: input, session: CodexScannedSession(id: sessionId, days: mergedDays), @@ -1145,7 +1216,15 @@ extension CostUsageScanner { parentSessionId: parsed.forkedFromId, dependsOnParentTotals: parsed.dependsOnParentTotals, inheritedResolver: context.resources.inheritedResolver) - let sessionId = parsed.sessionId ?? input.cached?.sessionId + let cachedSessionMetadata = input.cached?.codexSession ?? CostUsageCodexSessionMetadata( + sessionId: input.cached?.sessionId, + forkedFromId: input.cached?.forkedFromId, + cwd: nil, + title: nil, + startedAtUnixMs: nil, + latestActivityUnixMs: nil) + let parsedCodexSession = cachedSessionMetadata.merging(parsed.codexSession) + let sessionId = parsedCodexSession.sessionId ?? parsed.sessionId ?? input.cached?.sessionId let projectPath = parsed.projectPath ?? input.cached?.projectPath let canonicalProjectPath = parsed.projectPath.map { context.resources.projectPathResolver.canonicalProjectPath(for: $0) @@ -1188,10 +1267,11 @@ extension CostUsageScanner { hasInterleavedTotals: parsed.hasInterleavedTotals, lastCodexTurnID: parsed.lastCodexTurnID, sessionId: sessionId, - forkedFromId: parsed.forkedFromId, + forkedFromId: parsedCodexSession.forkedFromId ?? parsed.forkedFromId, forkBaselineDependencyKey: forkBaselineDependencyKey, projectPath: projectPath, canonicalProjectPath: canonicalProjectPath, + codexSession: parsedCodexSession.isEmpty ? nil : parsedCodexSession, codexCostNanos: Self.mergeCostMaps( context.dropDeferredCodexRows ? nil @@ -1236,7 +1316,12 @@ extension CostUsageScanner { : Self.mergeCodexTurnIDs(migratedCached?.codexTurnIDs, rows: uniqueRows), codexRows: context.dropDeferredCodexRows ? nil - : Self.mergeCodexRows(migratedCached?.codexRows, rows: uniqueRows, sessionId: sessionId)) + : Self.codexRowsWithPricingAudit( + Self.mergeCodexRows(migratedCached?.codexRows, rows: uniqueRows, sessionId: sessionId) ?? [], + priorityTurns: context.resources.priorityTurns, + modelsDevCatalog: context.resources.modelsDevCatalog, + modelsDevCacheRoot: context.resources.modelsDevCacheRoot)) + .refreshingCodexWorkspaceUsageFingerprint() Self.applyFileDays(cache: &cache, fileDays: cache.files[input.metadata.path]?.days ?? [:], sign: 1) Self.rememberScannedCodexFile( input: input, diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift index 08b9651994..a7ccc8a790 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift @@ -67,17 +67,57 @@ enum CostUsageScanner { let forkedFromId: String? let dependsOnParentTotals: Bool let projectPath: String? + let codexSession: CostUsageCodexSessionMetadata let rows: [CodexUsageRow] } struct CodexUsageRow: Codable, Equatable { let day: String let model: String + let rawModel: String? let turnID: String? let eventIndex: Int? + let timestampUnixMs: Int64? let input: Int let cached: Int let output: Int + let reasoning: Int? + let knownCostNanos: Int64? + let unpricedTokens: Int? + let pricingModel: String? + let pricingMode: String? + + init( + day: String, + model: String, + rawModel: String? = nil, + turnID: String?, + eventIndex: Int?, + timestampUnixMs: Int64? = nil, + input: Int, + cached: Int, + output: Int, + reasoning: Int? = nil, + knownCostNanos: Int64? = nil, + unpricedTokens: Int? = nil, + pricingModel: String? = nil, + pricingMode: String? = nil) + { + self.day = day + self.model = model + self.rawModel = rawModel + self.turnID = turnID + self.eventIndex = eventIndex + self.timestampUnixMs = timestampUnixMs + self.input = input + self.cached = cached + self.output = output + self.reasoning = reasoning.map { min(max(0, $0), max(0, output)) } + self.knownCostNanos = knownCostNanos + self.unpricedTokens = unpricedTokens + self.pricingModel = pricingModel + self.pricingMode = pricingMode + } } struct CodexScanState { @@ -138,7 +178,8 @@ enum CostUsageScanner { CostUsageCodexTotals( input: lhs.input + rhs.input, cached: lhs.cached + rhs.cached, - output: lhs.output + rhs.output) + output: lhs.output + rhs.output, + reasoning: self.codexAddOptional(lhs.reasoning, rhs.reasoning)) } private static func codexMinTotals( @@ -148,18 +189,24 @@ enum CostUsageScanner { CostUsageCodexTotals( input: min(lhs.input, rhs.input), cached: min(lhs.cached, rhs.cached), - output: min(lhs.output, rhs.output)) + output: min(lhs.output, rhs.output), + reasoning: self.codexMinOptional(lhs.reasoning, rhs.reasoning)) } private static func codexTotalDelta( from baseline: CostUsageCodexTotals?, to current: CostUsageCodexTotals) -> CostUsageCodexTotals { + let reasoning = Self.codexOptionalDelta( + from: baseline?.reasoning, + to: current.reasoning, + hasBaseline: baseline != nil) let baseline = baseline ?? .init(input: 0, cached: 0, output: 0) return CostUsageCodexTotals( input: max(0, current.input - baseline.input), cached: max(0, current.cached - baseline.cached), - output: max(0, current.output - baseline.output)) + output: max(0, current.output - baseline.output), + reasoning: reasoning) } private static func codexDivergentTotalDelta( @@ -180,7 +227,11 @@ enum CostUsageScanner { return CostUsageCodexTotals( input: delta(raw: rawBaseline.input, counted: countedBaseline.input, current: current.input), cached: delta(raw: rawBaseline.cached, counted: countedBaseline.cached, current: current.cached), - output: delta(raw: rawBaseline.output, counted: countedBaseline.output, current: current.output)) + output: delta(raw: rawBaseline.output, counted: countedBaseline.output, current: current.output), + reasoning: Self.codexDivergentOptionalDelta( + raw: rawBaseline.reasoning, + counted: countedBaseline.reasoning, + current: current.reasoning)) } private static func codexMaxTotals( @@ -191,7 +242,8 @@ enum CostUsageScanner { return CostUsageCodexTotals( input: max(lhs.input, rhs.input), cached: max(lhs.cached, rhs.cached), - output: max(lhs.output, rhs.output)) + output: max(lhs.output, rhs.output), + reasoning: Self.codexMaxOptional(lhs.reasoning, rhs.reasoning)) } /// Post-latch totals containment for interleaved cumulative counters (issue #2037 Phase 1). @@ -218,7 +270,58 @@ enum CostUsageScanner { return CostUsageCodexTotals( input: component(water: watermark.input, counted: counted.input, current: current.input), cached: component(water: watermark.cached, counted: counted.cached, current: current.cached), - output: component(water: watermark.output, counted: counted.output, current: current.output)) + output: component(water: watermark.output, counted: counted.output, current: current.output), + reasoning: Self.codexContainedOptionalDelta( + water: watermark.reasoning, + counted: counted.reasoning, + current: current.reasoning)) + } + + private static func codexAddOptional(_ lhs: Int?, _ rhs: Int?) -> Int? { + guard let lhs, let rhs else { return nil } + return lhs + rhs + } + + private static func codexMinOptional(_ lhs: Int?, _ rhs: Int?) -> Int? { + guard let lhs, let rhs else { return nil } + return min(lhs, rhs) + } + + private static func codexMaxOptional(_ lhs: Int?, _ rhs: Int?) -> Int? { + switch (lhs, rhs) { + case let (lhs?, rhs?): max(lhs, rhs) + case let (lhs?, nil): lhs + case let (nil, rhs?): rhs + case (nil, nil): nil + } + } + + private static func codexSubtractOptional(_ value: Int?, _ baseline: Int?) -> Int? { + guard let value, let baseline else { return nil } + return max(0, value - baseline) + } + + private static func codexOptionalDelta(from baseline: Int?, to current: Int?, hasBaseline: Bool) -> Int? { + guard let current else { return nil } + if !hasBaseline { return current } + guard let baseline else { return nil } + return max(0, current - baseline) + } + + private static func codexDivergentOptionalDelta(raw: Int?, counted: Int?, current: Int?) -> Int? { + guard let raw, let counted, let current else { return nil } + if current >= raw { + return max(0, current - raw) + } + return max(0, current - counted) + } + + private static func codexContainedOptionalDelta(water: Int?, counted: Int?, current: Int?) -> Int? { + guard let water, let counted, let current else { return nil } + if current >= water { + return max(0, current - max(water, counted)) + } + return max(0, current - counted) } /// Post-latch event delta: contained totals growth, optionally capped by `last`. @@ -264,7 +367,7 @@ enum CostUsageScanner { } func isSeen(_ totals: CostUsageCodexTotals) -> Bool { - self.seenRawTotals.contains(totals) + self.seenRawTotals.contains { CostUsageScanner.codexTotalsEqual($0, totals) } } /// Latches interleaved mode when any component of an observed cumulative snapshot drops @@ -284,7 +387,7 @@ enum CostUsageScanner { /// value for best-effort re-emission suppression. Call after computing the event's delta. mutating func commitObserved(_ totals: CostUsageCodexTotals) { self.raiseWatermark(to: totals) - if !self.seenRawTotals.contains(totals) { + if !self.seenRawTotals.contains(where: { CostUsageScanner.codexTotalsEqual($0, totals) }) { self.seenRawTotals.append(totals) if self.seenRawTotals.count > Self.seenRawTotalsLimit { self.seenRawTotals.removeFirst(self.seenRawTotals.count - Self.seenRawTotalsLimit) @@ -313,7 +416,12 @@ enum CostUsageScanner { last: CostUsageCodexTotals?, total: CostUsageCodexTotals?) -> CostUsageCodexTotals { - let base = self.countedTotals ?? .init(input: 0, cached: 0, output: 0) + let hasReasoning = last?.reasoning != nil || total?.reasoning != nil + let base = self.countedTotals ?? .init( + input: 0, + cached: 0, + output: 0, + reasoning: hasReasoning ? 0 : nil) if let total { // Best-effort exact re-emission suppression (precision only; containment is load-bearing). if self.tracker.isSeen(total) { @@ -1299,6 +1407,13 @@ enum CostUsageScanner { let isSubagentThread: Bool } + private struct CodexTurnContextMetadata { + let timestamp: String? + let model: String? + let cwd: String? + let title: String? + } + private struct CodexTokenCountRecord { let timestamp: String let model: String? @@ -1309,7 +1424,7 @@ enum CostUsageScanner { private enum CodexFastLine { case sessionMeta(CodexSessionMetadata) - case turnContext(model: String?) + case turnContext(CodexTurnContextMetadata) case interAgentCommunication(triggerTurn: Bool) case taskStarted(turnID: String?) case tokenCount(CodexTokenCountRecord) @@ -1340,6 +1455,7 @@ enum CostUsageScanner { private static let codexJSONFieldModel = Array("model".utf8) private static let codexJSONFieldModelName = Array("model_name".utf8) private static let codexJSONFieldOutputTokens = Array("output_tokens".utf8) + private static let codexJSONFieldReasoningOutputTokens = Array("reasoning_output_tokens".utf8) private static let codexJSONFieldParentSessionId = Array("parent_session_id".utf8) private static let codexJSONFieldParentSessionIdCamel = Array("parentSessionId".utf8) private static let codexJSONFieldPayload = Array("payload".utf8) @@ -1348,12 +1464,16 @@ enum CostUsageScanner { private static let codexJSONFieldSessionId = Array("session_id".utf8) private static let codexJSONFieldSessionIdCamel = Array("sessionId".utf8) private static let codexJSONFieldTimestamp = Array("timestamp".utf8) + private static let codexJSONFieldTitle = Array("title".utf8) + private static let codexJSONFieldName = Array("name".utf8) private static let codexJSONFieldTotalTokenUsage = Array("total_token_usage".utf8) private static let codexJSONFieldTriggerTurn = Array("trigger_turn".utf8) private static let codexJSONFieldTurnId = Array("turn_id".utf8) private static let codexJSONFieldTurnIdCamel = Array("turnId".utf8) private static let codexJSONFieldType = Array("type".utf8) private static let codexJSONFieldCwd = Array("cwd".utf8) + private static let codexJSONFieldCurrentWorkingDirectory = Array("current_working_directory".utf8) + private static let codexJSONFieldCurrentWorkingDirectoryCamel = Array("currentWorkingDirectory".utf8) static func codexModelEvidence(_ raw: String?) -> String? { guard let trimmed = raw?.trimmingCharacters(in: .whitespacesAndNewlines), !trimmed.isEmpty else { return nil } @@ -1533,7 +1653,12 @@ enum CostUsageScanner { Self .extractJSONByteIntField(Self.codexJSONFieldOutputTokens, from: bytes, in: objectRange, atDepth: 1) ?? 0) - return CostUsageCodexTotals(input: input, cached: cached, output: output) + let reasoning = Self.extractJSONByteIntField( + Self.codexJSONFieldReasoningOutputTokens, + from: bytes, + in: objectRange, + atDepth: 1).map { min(max(0, $0), output) } + return CostUsageCodexTotals(input: input, cached: cached, output: output, reasoning: reasoning) } private static func codexInterAgentCommunication( @@ -1554,6 +1679,7 @@ enum CostUsageScanner { return .interAgentCommunication(triggerTurn: triggerTurn) } + // swiftlint:disable:next function_body_length private static func parseCodexFastLine(_ bytes: Data) -> CodexFastLine? { bytes.withUnsafeBytes { rawBytes in let rawBuffer = rawBytes.bindMemory(to: UInt8.self) @@ -1593,12 +1719,23 @@ enum CostUsageScanner { } ?? false)) case "turn_context": + let timestamp = Self.extractJSONByteStringField( + Self.codexJSONFieldTimestamp, + from: rawBuffer, + in: objectRange, + atDepth: 1) guard let payloadRange = Self.extractJSONByteObjectField( Self.codexJSONFieldPayload, from: rawBuffer, in: objectRange, atDepth: 1) - else { return .turnContext(model: nil) } + else { + return .turnContext(CodexTurnContextMetadata( + timestamp: timestamp, + model: nil, + cwd: nil, + title: nil)) + } let infoRange = Self.extractJSONByteObjectField( Self.codexJSONFieldInfo, from: rawBuffer, @@ -1629,7 +1766,36 @@ enum CostUsageScanner { in: $0, atDepth: 1) }) - return .turnContext(model: model) + let cwd = Self.extractJSONByteStringField( + Self.codexJSONFieldCwd, + from: rawBuffer, + in: payloadRange, + atDepth: 1) + ?? Self.extractJSONByteStringField( + Self.codexJSONFieldCurrentWorkingDirectory, + from: rawBuffer, + in: payloadRange, + atDepth: 1) + ?? Self.extractJSONByteStringField( + Self.codexJSONFieldCurrentWorkingDirectoryCamel, + from: rawBuffer, + in: payloadRange, + atDepth: 1) + let title = Self.extractJSONByteStringField( + Self.codexJSONFieldTitle, + from: rawBuffer, + in: payloadRange, + atDepth: 1) + ?? Self.extractJSONByteStringField( + Self.codexJSONFieldName, + from: rawBuffer, + in: payloadRange, + atDepth: 1) + return .turnContext(CodexTurnContextMetadata( + timestamp: timestamp, + model: model, + cwd: cwd, + title: title)) case "inter_agent_communication_metadata": // Compact Codex JSONL uses this exact spelling. Whitespace/escaped variants fall @@ -1941,16 +2107,22 @@ enum CostUsageScanner { } let total = (info["total_token_usage"] as? [String: Any]).map { - CostUsageCodexTotals( + let output = toInt($0["output_tokens"]) + return CostUsageCodexTotals( input: toInt($0["input_tokens"]), cached: toInt($0["cached_input_tokens"] ?? $0["cache_read_input_tokens"]), - output: toInt($0["output_tokens"])) + output: output, + reasoning: ($0["reasoning_output_tokens"] as? NSNumber) + .map { min(max(0, $0.intValue), max(0, output)) }) } let last = (info["last_token_usage"] as? [String: Any]).map { - CostUsageCodexTotals( + let output = max(0, toInt($0["output_tokens"])) + return CostUsageCodexTotals( input: max(0, toInt($0["input_tokens"])), cached: max(0, toInt($0["cached_input_tokens"] ?? $0["cache_read_input_tokens"])), - output: max(0, toInt($0["output_tokens"]))) + output: output, + reasoning: ($0["reasoning_output_tokens"] as? NSNumber) + .map { min(max(0, $0.intValue), output) }) } appendSnapshot(timestamp: timestamp, last: last, total: total) } @@ -2010,6 +2182,13 @@ enum CostUsageScanner { forkedFromId: nil, dependsOnParentTotals: false, projectPath: nil, + codexSession: CostUsageCodexSessionMetadata( + sessionId: nil, + forkedFromId: nil, + cwd: nil, + title: nil, + startedAtUnixMs: nil, + latestActivityUnixMs: nil), rows: []) } @@ -2043,6 +2222,13 @@ enum CostUsageScanner { var candidateBoundaryDependsOnParentTotals = false var parentConfirmedLocalBoundary = false var suppressUnownedCopiedPrefix = false + var codexSession = CostUsageCodexSessionMetadata( + sessionId: nil, + forkedFromId: nil, + cwd: nil, + title: nil, + startedAtUnixMs: nil, + latestActivityUnixMs: nil) var inheritedTotals: CostUsageCodexTotals? var remainingInheritedTotals: CostUsageCodexTotals? var forkBaselineResolved = false @@ -2075,6 +2261,41 @@ enum CostUsageScanner { days[dayKey] = dayModels } + func sanitizedString(_ value: String?) -> String? { + guard let value else { return nil } + let trimmed = value.trimmingCharacters(in: .whitespacesAndNewlines) + return trimmed.isEmpty ? nil : trimmed + } + + func unixMilliseconds(from timestamp: String?) -> Int64? { + guard let timestamp, + let date = Self.dateFromTimestamp(timestamp) + else { return nil } + return Int64((date.timeIntervalSince1970 * 1000).rounded()) + } + + func observeTimestamp(_ timestamp: String?) { + guard let unixMs = unixMilliseconds(from: timestamp) else { return } + codexSession.startedAtUnixMs = switch codexSession.startedAtUnixMs { + case let current?: min(current, unixMs) + case nil: unixMs + } + codexSession.latestActivityUnixMs = switch codexSession.latestActivityUnixMs { + case let current?: max(current, unixMs) + case nil: unixMs + } + } + + func observeCwd(_ value: String?) { + guard let value = sanitizedString(value) else { return } + codexSession.cwd = value + } + + func observeTitle(_ value: String?) { + guard let value = sanitizedString(value) else { return } + codexSession.title = value + } + func resolveForkBaseline(parentSessionId: String, forkedAt: String) throws { guard !forkBaselineResolved else { return } guard let inheritedTotalsResolver else { return } @@ -2115,12 +2336,17 @@ enum CostUsageScanner { guard CodexSubagentRolloutShape.sameConcreteSessionID(metadata.sessionId, sessionId) else { return } if forkedFromId == nil, let enrichedParentID = metadata.forkedFromId { forkedFromId = enrichedParentID + codexSession.forkedFromId = enrichedParentID forkTimestamp = metadata.forkTimestamp ?? forkTimestamp try configureForkAccountingIfReady() } if projectPath == nil { projectPath = metadata.projectPath } + observeTimestamp(metadata.forkTimestamp) + if codexSession.cwd == nil { + observeCwd(metadata.projectPath) + } return } didCaptureLeafMetadata = true @@ -2128,12 +2354,17 @@ enum CostUsageScanner { forkedFromId = metadata.forkedFromId forkTimestamp = metadata.forkTimestamp projectPath = metadata.projectPath + codexSession.sessionId = metadata.sessionId + codexSession.forkedFromId = metadata.forkedFromId + observeTimestamp(metadata.forkTimestamp) + observeCwd(metadata.projectPath) isSubagentThread = metadata.isSubagentThread try configureForkAccountingIfReady() } // swiftlint:disable:next function_body_length func handleTokenCount(_ record: CodexTokenCountRecord) throws { + observeTimestamp(record.timestamp) guard let dayKey = Self.dayKeyFromTimestamp(record.timestamp) ?? Self.dayKeyFromParsedISO(record.timestamp) else { return } guard !suppressUnownedCopiedPrefix else { return } @@ -2147,6 +2378,7 @@ enum CostUsageScanner { var deltaInput = 0 var deltaCached = 0 var deltaOutput = 0 + var deltaReasoning: Int? func adjustedLastDelta(_ rawDelta: CostUsageCodexTotals) -> CostUsageCodexTotals { guard var remaining = remainingInheritedTotals else { return rawDelta } @@ -2154,11 +2386,13 @@ enum CostUsageScanner { let adjusted = CostUsageCodexTotals( input: max(0, rawDelta.input - remaining.input), cached: max(0, rawDelta.cached - remaining.cached), - output: max(0, rawDelta.output - remaining.output)) + output: max(0, rawDelta.output - remaining.output), + reasoning: Self.codexSubtractOptional(rawDelta.reasoning, remaining.reasoning)) remaining.input = max(0, remaining.input - rawDelta.input) remaining.cached = max(0, remaining.cached - rawDelta.cached) remaining.output = max(0, remaining.output - rawDelta.output) + remaining.reasoning = Self.codexSubtractOptional(remaining.reasoning, rawDelta.reasoning) remainingInheritedTotals = if remaining.input == 0, remaining.cached == 0, remaining.output == 0 { @@ -2177,7 +2411,8 @@ enum CostUsageScanner { return CostUsageCodexTotals( input: max(0, rawTotals.input - inheritedTotals.input), cached: max(0, rawTotals.cached - inheritedTotals.cached), - output: max(0, rawTotals.output - inheritedTotals.output)) + output: max(0, rawTotals.output - inheritedTotals.output), + reasoning: Self.codexSubtractOptional(rawTotals.reasoning, inheritedTotals.reasoning)) } if let adjustedTotal { @@ -2216,7 +2451,12 @@ enum CostUsageScanner { deltaInput = delta.input deltaCached = delta.cached deltaOutput = delta.output - let prev = previousTotals ?? .init(input: 0, cached: 0, output: 0) + deltaReasoning = delta.reasoning + let prev = previousTotals ?? .init( + input: 0, + cached: 0, + output: 0, + reasoning: delta.reasoning == nil ? nil : 0) previousTotals = Self.codexAddTotals(prev, delta) rawTotalsBaseline = rawBaseline if !Self.codexTotalsEqual(rawTotalsBaseline, previousTotals) { @@ -2244,7 +2484,12 @@ enum CostUsageScanner { deltaInput = adjustedDelta.input deltaCached = adjustedDelta.cached deltaOutput = adjustedDelta.output - let prev = previousTotals ?? .init(input: 0, cached: 0, output: 0) + deltaReasoning = adjustedDelta.reasoning + let prev = previousTotals ?? .init( + input: 0, + cached: 0, + output: 0, + reasoning: adjustedDelta.reasoning == nil ? nil : 0) previousTotals = Self.codexAddTotals(prev, adjustedDelta) rawTotalsBaseline = previousTotals } @@ -2271,7 +2516,11 @@ enum CostUsageScanner { let rawDelta = last let hadRemainingInheritedTotals = remainingInheritedTotals != nil var adjustedDelta = adjustedLastDelta(rawDelta) - let prev = previousTotals ?? .init(input: 0, cached: 0, output: 0) + let prev = previousTotals ?? .init( + input: 0, + cached: 0, + output: 0, + reasoning: adjustedDelta.reasoning == nil ? nil : 0) if let currentTotals = adjustedTotal, !hasUnresolvedForkBaseline { if tracker.sawInterleavedTotals { @@ -2301,6 +2550,7 @@ enum CostUsageScanner { deltaInput = adjustedDelta.input deltaCached = adjustedDelta.cached deltaOutput = adjustedDelta.output + deltaReasoning = adjustedDelta.reasoning previousTotals = countedTotals rawTotalsBaseline = countedTotals tracker.raiseWatermark(to: countedTotals) @@ -2332,11 +2582,14 @@ enum CostUsageScanner { rows.append(CodexUsageRow( day: dayKey, model: normModel, + rawModel: model, turnID: record.turnID ?? currentTurnID, eventIndex: eventIndex, + timestampUnixMs: unixMilliseconds(from: record.timestamp), input: deltaInput, cached: deltaCached, - output: deltaOutput)) + output: deltaOutput, + reasoning: deltaReasoning)) } } @@ -2344,9 +2597,13 @@ enum CostUsageScanner { switch fastLine { case let .sessionMeta(metadata): try handleSessionMetadata(metadata) - case let .turnContext(model): - if let model { - currentModel = model + case let .turnContext(metadata): + observeTimestamp(metadata.timestamp) + observeCwd(metadata.cwd) + observeTitle(metadata.title) + if let model = metadata.model { + // An explicitly blank context clears stale model evidence; an omitted field preserves it. + currentModel = sanitizedString(model) } case .interAgentCommunication: break @@ -2407,7 +2664,11 @@ enum CostUsageScanner { if truncatedTurnContext.isValid { do { try routeFastLine( - .turnContext(model: truncatedTurnContext.model), + .turnContext(CodexTurnContextMetadata( + timestamp: nil, + model: truncatedTurnContext.model, + cwd: nil, + title: nil)), lineIndex: lineIndex) } catch { deferredError = error @@ -2500,17 +2761,27 @@ enum CostUsageScanner { } if type == "turn_context" { - var model: String? + var metadata = CodexTurnContextMetadata( + timestamp: tsText, + model: nil, + cwd: nil, + title: nil) if let payload = obj["payload"] as? [String: Any] { let info = payload["info"] as? [String: Any] - model = Self.codexTurnContextModel( - payloadModel: payload["model"] as? String, - payloadModelName: payload["model_name"] as? String, - infoModel: info?["model"] as? String, - infoModelName: info?["model_name"] as? String) + metadata = CodexTurnContextMetadata( + timestamp: tsText, + model: Self.codexTurnContextModel( + payloadModel: payload["model"] as? String, + payloadModelName: payload["model_name"] as? String, + infoModel: info?["model"] as? String, + infoModelName: info?["model_name"] as? String), + cwd: payload["cwd"] as? String + ?? payload["current_working_directory"] as? String + ?? payload["currentWorkingDirectory"] as? String, + title: payload["title"] as? String ?? payload["name"] as? String) } do { - try routeFastLine(.turnContext(model: model), lineIndex: lineIndex) + try routeFastLine(.turnContext(metadata), lineIndex: lineIndex) } catch { deferredError = error } @@ -2545,10 +2816,13 @@ enum CostUsageScanner { } func tokenTotals(_ usage: [String: Any]) -> CostUsageCodexTotals { - CostUsageCodexTotals( + let output = max(0, toInt(usage["output_tokens"])) + return CostUsageCodexTotals( input: max(0, toInt(usage["input_tokens"])), cached: max(0, toInt(usage["cached_input_tokens"] ?? usage["cache_read_input_tokens"])), - output: max(0, toInt(usage["output_tokens"]))) + output: output, + reasoning: (usage["reasoning_output_tokens"] as? NSNumber) + .map { min(max(0, $0.intValue), output) }) } let record = CodexTokenCountRecord( @@ -2578,11 +2852,16 @@ enum CostUsageScanner { else { continue } if forkedFromId == nil, let enrichedParentID = metadata.forkedFromId { forkedFromId = enrichedParentID + codexSession.forkedFromId = enrichedParentID forkTimestamp = metadata.forkTimestamp ?? forkTimestamp } if projectPath == nil { projectPath = metadata.projectPath } + observeTimestamp(metadata.forkTimestamp) + if codexSession.cwd == nil { + observeCwd(metadata.projectPath) + } } let observations = pendingSubagentLines.compactMap { buffered -> CodexSubagentRolloutShape .Observation? in @@ -2700,6 +2979,7 @@ enum CostUsageScanner { && (candidateBoundaryDependsOnParentTotals || (subagentCounterSemantics != .independent && !usesLocalSubagentBoundary)), projectPath: projectPath, + codexSession: codexSession, rows: rows) } diff --git a/Tests/CodexBarTests/CodexLocalProjectUsageTests.swift b/Tests/CodexBarTests/CodexLocalProjectUsageTests.swift new file mode 100644 index 0000000000..2127cbe690 --- /dev/null +++ b/Tests/CodexBarTests/CodexLocalProjectUsageTests.swift @@ -0,0 +1,1736 @@ +import Foundation +#if canImport(SQLite3) +import SQLite3 +#endif +import Testing +@testable import CodexBar +@testable import CodexBarCore + +@MainActor +// Shared JSONL/SQLite fixtures make the attribution and sidecar assertions +// readable without duplicating test environments across several files. +// swiftlint:disable file_length +// swiftlint:disable type_body_length +struct CodexLocalProjectUsageTests { + private final class ProgressRecorder: @unchecked Sendable { + private let lock = NSLock() + private var events: [CodexLocalProjectUsageIndexProgress] = [] + + func append(_ progress: CodexLocalProjectUsageIndexProgress) { + self.lock.lock() + defer { self.lock.unlock() } + self.events.append(progress) + } + + var snapshot: [CodexLocalProjectUsageIndexProgress] { + self.lock.lock() + defer { self.lock.unlock() } + return self.events + } + } + + private struct CodexUsageFixture { + var filename: String + var sessionID: String + var cwd: String? + var input: Int + var cached: Int + var output: Int + } + + @Test + func `project root resolver keeps sibling paths distinct`() throws { + let root = FileManager.default.temporaryDirectory + .appendingPathComponent("codexbar-project-root-\(UUID().uuidString)", isDirectory: true) + defer { try? FileManager.default.removeItem(at: root) } + let app = root.appendingPathComponent("app", isDirectory: true) + let appOld = root.appendingPathComponent("app-old", isDirectory: true) + let appSource = app.appendingPathComponent("Sources", isDirectory: true) + let appOldSource = appOld.appendingPathComponent("Sources", isDirectory: true) + try FileManager.default.createDirectory( + at: app.appendingPathComponent(".git", isDirectory: true), + withIntermediateDirectories: true) + try FileManager.default.createDirectory( + at: appOld.appendingPathComponent(".git", isDirectory: true), + withIntermediateDirectories: true) + try FileManager.default.createDirectory(at: appSource, withIntermediateDirectories: true) + try FileManager.default.createDirectory(at: appOldSource, withIntermediateDirectories: true) + + let appIdentity = CodexLocalProjectRootResolver.projectIdentity(for: appSource.path) + let appOldIdentity = CodexLocalProjectRootResolver.projectIdentity(for: appOldSource.path) + + #expect(appIdentity.path == app.standardizedFileURL.path) + #expect(appOldIdentity.path == appOld.standardizedFileURL.path) + #expect(appIdentity.id != appOldIdentity.id) + } + + @Test + func `local data scope avoids persisting raw Codex home paths`() { + let home = FileManager.default.temporaryDirectory + .appendingPathComponent("workspaces-private-home", isDirectory: true) + let scope = CodexLocalDataScope.resolve(options: CostUsageScanner.Options( + codexSessionsRoot: home.appendingPathComponent("sessions", isDirectory: true))) + + #expect(scope.codexHome == home.standardizedFileURL) + #expect(scope.identifier.hasPrefix("codex-workspaces:")) + #expect(!scope.identifier.contains(home.path)) + } + + @Test + func `v10 cache remains untouched while v11 rebuilds and then refreshes incrementally`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + + let day = Date() + let costCacheRoot = env.cacheRoot.appendingPathComponent("cost-usage", isDirectory: true) + try FileManager.default.createDirectory(at: costCacheRoot, withIntermediateDirectories: true) + let v10URL = costCacheRoot.appendingPathComponent("codex-v10.json", isDirectory: false) + let v10Bytes = Data("recoverable-v10-cursor".utf8) + try v10Bytes.write(to: v10URL) + let v11URL = CostUsageCacheIO.cacheFileURL(provider: .codex, cacheRoot: env.cacheRoot) + #expect(!FileManager.default.fileExists(atPath: v11URL.path)) + + try self.writeCodexUsageFile( + env: env, + day: day, + fixture: CodexUsageFixture( + filename: "upgrade-first.jsonl", + sessionID: "upgrade-first", + cwd: env.root.path, + input: 100, + cached: 20, + output: 30)) + var options = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + cacheRoot: env.cacheRoot) + options.refreshMinIntervalSeconds = 0 + let first = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day, + options: options) + + #expect(first.data.first?.totalTokens == 130) + #expect(try Data(contentsOf: v10URL) == v10Bytes) + #expect(FileManager.default.fileExists(atPath: v11URL.path)) + #expect(CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot).files.count == 1) + + try self.writeCodexUsageFile( + env: env, + day: day, + fixture: CodexUsageFixture( + filename: "upgrade-second.jsonl", + sessionID: "upgrade-second", + cwd: env.root.path, + input: 40, + cached: 5, + output: 10)) + let warmNow = Date().addingTimeInterval(1) + let warm = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: warmNow, + now: warmNow, + options: options) + + #expect(warm.data.first?.totalTokens == 180) + #expect(CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot).files.count == 2) + #expect(try Data(contentsOf: v10URL) == v10Bytes) + } + + @Test + func `project root resolver treats git file worktree as most specific project`() throws { + let root = FileManager.default.temporaryDirectory + .appendingPathComponent("codexbar-project-worktree-\(UUID().uuidString)", isDirectory: true) + defer { try? FileManager.default.removeItem(at: root) } + let app = root.appendingPathComponent("app", isDirectory: true) + let worktree = app.appendingPathComponent("worktree", isDirectory: true) + let source = worktree.appendingPathComponent("Sources", isDirectory: true) + try FileManager.default.createDirectory( + at: app.appendingPathComponent(".git", isDirectory: true), + withIntermediateDirectories: true) + try FileManager.default.createDirectory(at: source, withIntermediateDirectories: true) + try "gitdir: ../.git/worktrees/worktree\n".write( + to: worktree.appendingPathComponent(".git", isDirectory: false), + atomically: true, + encoding: .utf8) + + let identity = CodexLocalProjectRootResolver.projectIdentity(for: source.path) + + #expect(identity.path == worktree.standardizedFileURL.path) + #expect(identity.displayName == "worktree") + } + + @Test + func `project root resolver preserves missing logged CWD when no git root exists`() throws { + let root = FileManager.default.temporaryDirectory + .appendingPathComponent("codexbar-missing-cwd-\(UUID().uuidString)", isDirectory: true) + defer { try? FileManager.default.removeItem(at: root) } + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + let loggedCWD = root.appendingPathComponent("deleted-project/Sources", isDirectory: true) + + let identity = CodexLocalProjectRootResolver.projectIdentity(for: loggedCWD.path) + + #expect(identity.path == loggedCWD.standardizedFileURL.path) + #expect(identity.displayName == "Sources") + } + + @Test + func `project root resolver preserves a deleted CWD below an existing repository`() throws { + let root = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString, isDirectory: true) + defer { try? FileManager.default.removeItem(at: root) } + let repository = root.appendingPathComponent("project", isDirectory: true) + let deletedCWD = repository.appendingPathComponent("removed-worktree", isDirectory: true) + try FileManager.default.createDirectory( + at: repository.appendingPathComponent(".git", isDirectory: true), + withIntermediateDirectories: true) + + let identity = CodexLocalProjectRootResolver.projectIdentity(for: deletedCWD.path) + + #expect(identity.path == deletedCWD.standardizedFileURL.path) + #expect(identity.displayName == "removed-worktree") + } + + @Test + func `project root resolver canonicalizes a live symlinked repository`() throws { + let root = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString, isDirectory: true) + defer { try? FileManager.default.removeItem(at: root) } + let repository = root.appendingPathComponent("project", isDirectory: true) + let source = repository.appendingPathComponent("Sources", isDirectory: true) + let symlink = root.appendingPathComponent("project-link", isDirectory: true) + try FileManager.default.createDirectory( + at: repository.appendingPathComponent(".git", isDirectory: true), + withIntermediateDirectories: true) + try FileManager.default.createDirectory(at: source, withIntermediateDirectories: true) + try FileManager.default.createSymbolicLink(atPath: symlink.path, withDestinationPath: repository.path) + + let direct = CodexLocalProjectRootResolver.projectIdentity(for: source.path) + let linked = CodexLocalProjectRootResolver.projectIdentity( + for: symlink.appendingPathComponent("Sources", isDirectory: true).path) + + #expect(linked.id == direct.id) + #expect(linked.path == repository.standardizedFileURL.path) + } + + @Test + func `project usage index aggregates projects and chats from existing codex scan cache`() async throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + + let day = Date() + let project = env.root.appendingPathComponent("CodexBar", isDirectory: true) + let projectSource = project.appendingPathComponent("Sources", isDirectory: true) + try FileManager.default.createDirectory( + at: project.appendingPathComponent(".git", isDirectory: true), + withIntermediateDirectories: true) + try FileManager.default.createDirectory(at: projectSource, withIntermediateDirectories: true) + + try self.writeCodexUsageFile( + env: env, + day: day, + fixture: CodexUsageFixture( + filename: "project.jsonl", + sessionID: "project-session", + cwd: projectSource.path, + input: 100, + cached: 20, + output: 30)) + try self.writeCodexUsageFile( + env: env, + day: day, + fixture: CodexUsageFixture( + filename: "chat.jsonl", + sessionID: "chat-session", + cwd: nil, + input: 50, + cached: 5, + output: 10)) + + let options = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + cacheRoot: env.cacheRoot) + let snapshot = try await CostUsageFetcher.loadCodexLocalProjectUsageSnapshot( + now: day, + forceRefresh: true, + historyDays: 2, + hidePersonalInfo: false, + scannerOptions: options) + + #expect(snapshot.indexedFileCount == 2) + #expect(snapshot.projects.map(\.displayName) == ["CodexBar", "Chats"]) + #expect(snapshot.projects.first?.totals.totalTokens == 130) + #expect(snapshot.projects.first?.totals.cachedInputTokens == 20) + #expect(snapshot.projects.first?.path == project.standardizedFileURL.path) + #expect(snapshot.projects.first?.estimatedCostUSD != nil) + #expect(snapshot.projects.first?.modelBreakdowns.first?.estimatedCostUSD != nil) + #expect(snapshot.projects.first?.modelBreakdowns.first?.hasUnknownCost == false) + #expect(snapshot.projects.first?.daily.first?.totalTokens == 130) + #expect(snapshot.projects.last?.id == CodexLocalProjectRootResolver.chatsProjectId) + #expect(snapshot.projects.last?.daily.first?.totalTokens == 60) + #expect(snapshot.total.totalTokens == 190) + #expect(snapshot.sessions.count == 2) + #expect(snapshot.daily.first?.totalTokens == 190) + let hiddenSnapshot = try #require(await CostUsageFetcher.loadCachedCodexLocalProjectUsageSnapshot( + now: day, + historyDays: 2, + hidePersonalInfo: true, + scannerOptions: options)) + #expect(hiddenSnapshot.rootsFingerprint.isEmpty) + #expect(hiddenSnapshot.projects.first?.displayName == "Workspace") + #expect(hiddenSnapshot.projects.first?.path == nil) + #expect(hiddenSnapshot.projects.first?.topSessions.first?.displayTitle + == CodexLocalSessionUsage.localChatFallbackTitle) + #expect(hiddenSnapshot.projects.first?.topSessions.first?.cwd == nil) + #expect(hiddenSnapshot.sessions.allSatisfy { + $0.displayTitle == CodexLocalSessionUsage.localChatFallbackTitle && $0.cwd == nil + }) + #expect(snapshot.projects.first?.displayName == "CodexBar") + #expect(snapshot.projects.first?.path == project.standardizedFileURL.path) + let allModels = try #require(snapshot.modelsAnalytics?.allWorkspaces) + #if canImport(SQLite3) || canImport(CSQLite3) + #expect(snapshot.sourceStatus == .catalogMissing) + #expect(allModels.currentIsComplete == false) + #expect(allModels.previousIsComplete == false) + #else + #expect(snapshot.sourceStatus == .complete) + #expect(allModels.currentIsComplete == true) + #expect(allModels.previousIsComplete == true) + #endif + } + + @Test + func `project usage index uses cached codex session metadata without reading jsonl`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + + let day = Date() + let project = env.root.appendingPathComponent("CodexBar", isDirectory: true) + try FileManager.default.createDirectory( + at: project.appendingPathComponent(".git", isDirectory: true), + withIntermediateDirectories: true) + let fixture = CodexUsageFixture( + filename: "missing.jsonl", + sessionID: "cached-session", + cwd: project.path, + input: 100, + cached: 20, + output: 30) + let options = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + cacheRoot: env.cacheRoot) + let dayKey = CostUsageScanner.CostUsageDayRange.dayKey(from: day) + let missingFileURL = env.root.appendingPathComponent("missing-session.jsonl", isDirectory: false) + var cache = CostUsageCache() + cache.scanSinceKey = dayKey + cache.scanUntilKey = dayKey + cache.roots = CostUsageScanner.codexRootsFingerprint(options: options) + cache.files[missingFileURL.path] = self.makeCachedFileUsage( + dayKey: dayKey, + fixture: fixture, + costNanos: 1) + CostUsageCacheIO.save(provider: .codex, cache: cache, cacheRoot: env.cacheRoot) + + let snapshot = try CodexLocalProjectUsageIndexer.buildSnapshotFromCostCache( + now: day, + historyDays: 1, + since: day, + until: day, + options: options) + + #expect(FileManager.default.fileExists(atPath: missingFileURL.path) == false) + #expect(snapshot.projects.count == 1) + #expect(snapshot.projects.first?.displayName == "CodexBar") + #expect(snapshot.projects.first?.path == project.standardizedFileURL.path) + #expect(snapshot.projects.first?.totals.totalTokens == 130) + } + + @Test + func `project usage index uses codex state database catalog when available`() throws { + #if canImport(SQLite3) + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + + let day = Date(timeIntervalSince1970: 1_800_000_000) + let project = env.root.appendingPathComponent("CatalogProject", isDirectory: true) + let projectSource = project.appendingPathComponent("Sources", isDirectory: true) + try FileManager.default.createDirectory( + at: project.appendingPathComponent(".git", isDirectory: true), + withIntermediateDirectories: true) + try FileManager.default.createDirectory(at: projectSource, withIntermediateDirectories: true) + + let dayKey = CostUsageScanner.CostUsageDayRange.dayKey(from: day) + let rolloutURL = env.codexSessionsRoot.appendingPathComponent("catalog-session.jsonl", isDirectory: false) + let stateDatabaseURL = env.codexHomeRoot.appendingPathComponent("state_5.sqlite", isDirectory: false) + try self.writeCodexStateDatabase( + at: stateDatabaseURL, + thread: CodexStateThreadFixture( + id: "catalog-session", + rolloutPath: rolloutURL.path, + cwd: projectSource.path, + title: "Catalog title", + preview: "Catalog preview", + model: "openai/gpt-5.4-catalog", + createdAtUnixMs: 1_800_000_000_000, + updatedAtUnixMs: 1_800_000_120_000)) + + var cache = CostUsageCache() + let options = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + cacheRoot: env.cacheRoot) + cache.scanSinceKey = dayKey + cache.scanUntilKey = dayKey + cache.roots = CostUsageScanner.codexRootsFingerprint(options: options) + cache.files[rolloutURL.path] = self.makeCachedFileUsage( + dayKey: dayKey, + fixture: CodexUsageFixture( + filename: "catalog-session.jsonl", + sessionID: "catalog-session", + cwd: nil, + input: 100, + cached: 10, + output: 25), + costNanos: 1) + CostUsageCacheIO.save(provider: .codex, cache: cache, cacheRoot: env.cacheRoot) + + let snapshot = try CodexLocalProjectUsageIndexer.buildSnapshotFromCostCache( + now: day, + historyDays: 1, + since: day, + until: day, + options: options) + + #expect(snapshot.projects.map(\.displayName) == ["CatalogProject"]) + #expect(snapshot.projects.first?.path == project.standardizedFileURL.path) + #expect(snapshot.sessions.first?.displayTitle == "Catalog title") + #expect(snapshot.sessions.first?.cwd == projectSource.path) + #expect(snapshot.sessions.first?.latestActivity == Date(timeIntervalSince1970: 1_800_000_120)) + #expect(snapshot.sessions.first?.topModel == "openai/gpt-5.4-catalog") + #expect(snapshot.total.totalTokens == 125) + #else + #expect(Bool(true)) + #endif + } + + @Test + func `catalog reader normalizes legacy seconds timestamps to milliseconds`() throws { + #if canImport(SQLite3) + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + + let rolloutURL = env.codexSessionsRoot.appendingPathComponent("legacy-timestamp.jsonl", isDirectory: false) + try self.writeCodexStateDatabase( + at: env.codexHomeRoot.appendingPathComponent("state_5.sqlite", isDirectory: false), + thread: CodexStateThreadFixture( + id: "legacy-timestamp", + rolloutPath: rolloutURL.path, + cwd: env.root.path, + title: "Legacy timestamp", + preview: "Legacy timestamp preview", + model: "openai/gpt-5.4", + createdAtUnixMs: 1_800_000_000_000, + updatedAtUnixMs: 1_800_000_120_000, + usesLegacyTimestampColumns: true)) + + let options = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + cacheRoot: env.cacheRoot) + let entry = try #require(CodexThreadCatalogReader.load(options: options).entriesById["legacy-timestamp"]) + + #expect(entry.createdAtUnixMs == 1_800_000_000_000) + #expect(entry.updatedAtUnixMs == 1_800_000_120_000) + #else + #expect(Bool(true)) + #endif + } + + @Test + func `cached refresh surfaces catalog degradation while retaining last good usage`() throws { + #if canImport(SQLite3) + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + + let day = Date(timeIntervalSince1970: 1_800_000_000) + let project = env.root.appendingPathComponent("CachedCatalogProject", isDirectory: true) + let source = project.appendingPathComponent("Sources", isDirectory: true) + try FileManager.default.createDirectory( + at: project.appendingPathComponent(".git", isDirectory: true), + withIntermediateDirectories: true) + try FileManager.default.createDirectory(at: source, withIntermediateDirectories: true) + let rolloutURL = env.codexSessionsRoot.appendingPathComponent("cached-catalog.jsonl", isDirectory: false) + let catalogURL = env.codexHomeRoot.appendingPathComponent("state_5.sqlite", isDirectory: false) + try self.writeCodexStateDatabase( + at: catalogURL, + thread: CodexStateThreadFixture( + id: "cached-catalog", + rolloutPath: rolloutURL.path, + cwd: source.path, + title: "Cached catalog title", + preview: "Cached catalog preview", + model: "openai/gpt-5.4-catalog", + createdAtUnixMs: 1_800_000_000_000, + updatedAtUnixMs: 1_800_000_120_000)) + try self.writeCodexUsageFile( + env: env, + day: day, + fixture: CodexUsageFixture( + filename: "cached-catalog.jsonl", + sessionID: "cached-catalog", + cwd: nil, + input: 100, + cached: 10, + output: 25)) + var options = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + cacheRoot: env.cacheRoot) + options.refreshMinIntervalSeconds = 0 + + let complete = try CodexLocalProjectUsageIndexer.loadSnapshot( + now: day, + historyDays: 1, + forceRefresh: true, + options: .init(scannerOptions: options)) + try FileManager.default.removeItem(at: catalogURL) + let degraded = try CodexLocalProjectUsageIndexer.loadSnapshot( + now: day, + historyDays: 1, + options: .init(scannerOptions: options)) + + #expect(complete.sourceStatus == .complete) + #expect(degraded.sourceStatus == .catalogMissing) + #expect(degraded.total == complete.total) + #expect(degraded.projects == complete.projects) + #expect(degraded.sessions == complete.sessions) + #else + #expect(Bool(true)) + #endif + } + + @Test + func `sidecar retains catalog metadata when a sparse rollout update arrives during catalog failure`() throws { + #if canImport(SQLite3) + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + + let day = Date(timeIntervalSince1970: 1_800_000_000) + let dayKey = CostUsageScanner.CostUsageDayRange.dayKey(from: day) + let rolloutURL = env.codexSessionsRoot.appendingPathComponent("retained-metadata.jsonl", isDirectory: false) + let project = env.root.appendingPathComponent("RetainedCatalogProject", isDirectory: true) + try FileManager.default.createDirectory(at: project, withIntermediateDirectories: true) + try self.writeCodexStateDatabase( + at: env.codexHomeRoot.appendingPathComponent("state_5.sqlite", isDirectory: false), + thread: CodexStateThreadFixture( + id: "retained-session", + rolloutPath: rolloutURL.path, + cwd: project.path, + title: "Retained catalog title", + preview: "Retained catalog preview", + model: "openai/gpt-5.4-catalog", + createdAtUnixMs: 1_800_000_000_000, + updatedAtUnixMs: 1_800_000_120_000)) + + let options = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + cacheRoot: env.cacheRoot) + let sparseFixture = CodexUsageFixture( + filename: "retained-metadata.jsonl", + sessionID: "retained-session", + cwd: nil, + input: 100, + cached: 0, + output: 20) + var cache = CostUsageCache() + cache.roots = CostUsageScanner.codexRootsFingerprint(options: options) + cache.files[rolloutURL.path] = self.makeCachedFileUsage( + dayKey: dayKey, + fixture: sparseFixture, + costNanos: 1) + let sidecar = CodexWorkspaceUsageSidecar(cacheRoot: env.cacheRoot) + try sidecar.synchronizeSources(cache: cache, catalog: CodexThreadCatalogReader.load(options: options)) + + var changedFixture = sparseFixture + changedFixture.input = 110 + var changedUsage = self.makeCachedFileUsage( + dayKey: dayKey, + fixture: changedFixture, + costNanos: 1) + changedUsage.mtimeUnixMs = 1 + cache.files[rolloutURL.path] = changedUsage + try sidecar.synchronizeSources(cache: cache, catalog: .empty, catalogIsComplete: false) + + let rehydrated = try sidecar.usageCache(roots: cache.roots ?? [:]) + let metadata = rehydrated.files[rolloutURL.path]?.codexSession + #expect(metadata?.cwd == project.path) + #expect(metadata?.title == "Retained catalog title") + #expect(rehydrated.files[rolloutURL.path]?.days[dayKey]?["openai/gpt-5.4"]?[0] == 110) + #else + #expect(Bool(true)) + #endif + } + + @Test + func `sidecar prunes catalog metadata absent from a complete generation`() throws { + #if canImport(SQLite3) + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + + let rolloutURL = env.codexSessionsRoot.appendingPathComponent("pruned-catalog.jsonl", isDirectory: false) + let entry = CodexThreadCatalogEntry( + id: "pruned-catalog", + rolloutPath: rolloutURL.path, + cwd: "/catalog/cwd", + title: "Catalog title", + preview: "Catalog preview", + modelProvider: "openai", + model: "openai/gpt-5.4-catalog", + reasoningEffort: "high", + createdAtUnixMs: 1_800_000_000_000, + updatedAtUnixMs: 1_800_000_120_000, + archived: false) + let catalog = CodexThreadCatalog( + entriesById: [entry.id: entry], + entriesByRolloutPath: [rolloutURL.standardizedFileURL.path: entry], + fingerprint: "complete-generation-1") + var cache = CostUsageCache() + cache.files[rolloutURL.path] = self.makeCachedFileUsage( + dayKey: "2027-01-15", + fixture: CodexUsageFixture( + filename: "pruned-catalog.jsonl", + sessionID: entry.id, + cwd: "/rollout/cwd", + input: 100, + cached: 0, + output: 20), + costNanos: 1) + let sidecar = CodexWorkspaceUsageSidecar(cacheRoot: env.cacheRoot) + try sidecar.synchronizeSources(cache: cache, catalog: catalog) + #expect(try sidecar.usageCache(roots: [:]).files[rolloutURL.path]?.codexSession?.cwd == "/catalog/cwd") + + try sidecar.synchronizeSources(cache: cache, catalog: .empty) + + #expect(try sidecar.usageCache(roots: [:]).files[rolloutURL.path]?.codexSession?.cwd == "/rollout/cwd") + #else + #expect(Bool(true)) + #endif + } + + @Test + func `catalog reader distinguishes missing and corrupt state databases`() throws { + #if canImport(SQLite3) + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + let options = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + cacheRoot: env.cacheRoot) + + #expect(CodexThreadCatalogReader.loadResult(options: options).completeness == .unavailable(.missing)) + + let databaseURL = env.codexHomeRoot.appendingPathComponent("state_5.sqlite", isDirectory: false) + try "not a SQLite database".write(to: databaseURL, atomically: true, encoding: .utf8) + #expect(CodexThreadCatalogReader.loadResult(options: options).completeness == .unavailable(.corrupt)) + #else + #expect(Bool(true)) + #endif + } + + @Test + func `cached project usage snapshot preserves last complete data when pricing changes`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + + let day = Date() + let project = env.root.appendingPathComponent("CodexBar", isDirectory: true) + try FileManager.default.createDirectory( + at: project.appendingPathComponent(".git", isDirectory: true), + withIntermediateDirectories: true) + let fixture = CodexUsageFixture( + filename: "project.jsonl", + sessionID: "cached-session", + cwd: project.path, + input: 100, + cached: 20, + output: 30) + let options = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + cacheRoot: env.cacheRoot) + let dayKey = CostUsageScanner.CostUsageDayRange.dayKey(from: day) + var cache = CostUsageCache() + cache.scanSinceKey = dayKey + cache.scanUntilKey = dayKey + cache.codexPricingKey = "pricing-a" + cache.roots = CostUsageScanner.codexRootsFingerprint(options: options) + cache.files[env.root.appendingPathComponent("project.jsonl").path] = self.makeCachedFileUsage( + dayKey: dayKey, + fixture: fixture, + costNanos: 1) + CostUsageCacheIO.save(provider: .codex, cache: cache, cacheRoot: env.cacheRoot) + let snapshot = try CodexLocalProjectUsageIndexer.buildSnapshotFromCostCache( + now: day, + historyDays: 1, + since: day, + until: day, + options: options) + let catalog = CodexThreadCatalogReader.load(options: options) + try CodexWorkspaceUsageSidecar(cacheRoot: env.cacheRoot).synchronize( + snapshot: snapshot, + cache: cache, + catalog: catalog, + rootsFingerprint: CostUsageScanner.codexRootsFingerprint(options: options)) + + #expect(CodexLocalProjectUsageIndexer.cachedSnapshot(now: day, historyDays: 1, options: .init( + scannerOptions: options)) != nil) + + cache.codexPricingKey = "pricing-b" + CostUsageCacheIO.save(provider: .codex, cache: cache, cacheRoot: env.cacheRoot) + + #expect(CodexLocalProjectUsageIndexer.cachedSnapshot(now: day, historyDays: 1, options: .init( + scannerOptions: options))?.total.totalTokens == 130) + } + + @Test + func `project usage severity separates high usage from unknown cost coverage`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + + let day = Date() + let highProject = env.root.appendingPathComponent("high", isDirectory: true) + let normalProject = env.root.appendingPathComponent("normal", isDirectory: true) + let partialProject = env.root.appendingPathComponent("partial", isDirectory: true) + for project in [highProject, normalProject, partialProject] { + try FileManager.default.createDirectory( + at: project.appendingPathComponent(".git", isDirectory: true), + withIntermediateDirectories: true) + } + let options = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + cacheRoot: env.cacheRoot) + let dayKey = CostUsageScanner.CostUsageDayRange.dayKey(from: day) + var cache = CostUsageCache() + cache.scanSinceKey = dayKey + cache.scanUntilKey = dayKey + cache.roots = CostUsageScanner.codexRootsFingerprint(options: options) + cache.files[env.root.appendingPathComponent("high.jsonl").path] = self.makeCachedFileUsage( + dayKey: dayKey, + fixture: CodexUsageFixture( + filename: "high.jsonl", + sessionID: "high", + cwd: highProject.path, + input: 800, + cached: 0, + output: 200), + costNanos: 1) + cache.files[env.root.appendingPathComponent("normal.jsonl").path] = self.makeCachedFileUsage( + dayKey: dayKey, + fixture: CodexUsageFixture( + filename: "normal.jsonl", + sessionID: "normal", + cwd: normalProject.path, + input: 80, + cached: 0, + output: 20), + costNanos: 1) + cache.files[env.root.appendingPathComponent("partial.jsonl").path] = self.makeCachedFileUsage( + dayKey: dayKey, + fixture: CodexUsageFixture( + filename: "partial.jsonl", + sessionID: "partial", + cwd: partialProject.path, + input: 80, + cached: 0, + output: 20), + costNanos: nil) + CostUsageCacheIO.save(provider: .codex, cache: cache, cacheRoot: env.cacheRoot) + + let snapshot = try CodexLocalProjectUsageIndexer.buildSnapshotFromCostCache( + now: day, + historyDays: 1, + since: day, + until: day, + options: options) + let projected = CodexLocalProjectUsageProjection( + includesCachedInput: true, + showsEstimatedCost: true) + .rankedProjects(snapshot.projects) + let projectsByName = Dictionary(uniqueKeysWithValues: projected.map { ($0.displayName, $0) }) + + #expect(projectsByName["high"]?.severity == .high) + #expect(projectsByName["normal"]?.severity == .normal) + #expect(projectsByName["partial"]?.hasUnknownCost == true) + #expect(projectsByName["partial"]?.severity == .normal) + #expect(projectsByName["partial"]?.costEstimate.unknownTokens == 100) + } + + @Test + func `project usage index does not downgrade known session project to chats`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + + let day = Date() + let project = env.root.appendingPathComponent("CodexBar", isDirectory: true) + let projectSource = project.appendingPathComponent("Sources", isDirectory: true) + try FileManager.default.createDirectory( + at: project.appendingPathComponent(".git", isDirectory: true), + withIntermediateDirectories: true) + try FileManager.default.createDirectory(at: projectSource, withIntermediateDirectories: true) + + let projectFixture = CodexUsageFixture( + filename: "a-project-fragment.jsonl", + sessionID: "split-session", + cwd: projectSource.path, + input: 100, + cached: 20, + output: 30) + let chatFixture = CodexUsageFixture( + filename: "z-chat-fragment.jsonl", + sessionID: "split-session", + cwd: nil, + input: 50, + cached: 5, + output: 10) + let projectFileURL = try self.writeCodexUsageFile( + env: env, + day: day, + fixture: projectFixture) + let chatFileURL = try self.writeCodexUsageFile( + env: env, + day: day, + fixture: chatFixture) + + let options = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + cacheRoot: env.cacheRoot) + let dayKey = CostUsageScanner.CostUsageDayRange.dayKey(from: day) + var cache = CostUsageCache() + cache.scanSinceKey = dayKey + cache.scanUntilKey = dayKey + cache.roots = CostUsageScanner.codexRootsFingerprint(options: options) + cache.files[projectFileURL.path] = self.makeCachedFileUsage( + dayKey: dayKey, + fixture: projectFixture, + costNanos: 1) + cache.files[chatFileURL.path] = self.makeCachedFileUsage( + dayKey: dayKey, + fixture: chatFixture, + costNanos: 1) + CostUsageCacheIO.save(provider: .codex, cache: cache, cacheRoot: env.cacheRoot) + + let snapshot = try CodexLocalProjectUsageIndexer.buildSnapshotFromCostCache( + now: day, + historyDays: 1, + since: day, + until: day, + options: options) + + #expect(snapshot.projects.count == 1) + #expect(snapshot.projects.first?.displayName == "CodexBar") + #expect(snapshot.projects.first?.path == project.standardizedFileURL.path) + #expect(snapshot.projects.first?.sessionCount == 1) + #expect(snapshot.projects.first?.totals.totalTokens == 190) + #expect(snapshot.sessions.first?.projectId != CodexLocalProjectRootResolver.chatsProjectId) + } + + @Test + func `project usage index reports remaining file progress`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + + let day = Date() + let firstFixture = CodexUsageFixture( + filename: "first.jsonl", + sessionID: "first-session", + cwd: nil, + input: 100, + cached: 20, + output: 30) + let secondFixture = CodexUsageFixture( + filename: "second.jsonl", + sessionID: "second-session", + cwd: nil, + input: 50, + cached: 5, + output: 10) + let firstFileURL = try self.writeCodexUsageFile(env: env, day: day, fixture: firstFixture) + let secondFileURL = try self.writeCodexUsageFile(env: env, day: day, fixture: secondFixture) + let options = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + cacheRoot: env.cacheRoot) + let dayKey = CostUsageScanner.CostUsageDayRange.dayKey(from: day) + var cache = CostUsageCache() + cache.scanSinceKey = dayKey + cache.scanUntilKey = dayKey + cache.roots = CostUsageScanner.codexRootsFingerprint(options: options) + cache.files[firstFileURL.path] = self.makeCachedFileUsage(dayKey: dayKey, fixture: firstFixture, costNanos: 1) + cache.files[secondFileURL.path] = self.makeCachedFileUsage(dayKey: dayKey, fixture: secondFixture, costNanos: 1) + CostUsageCacheIO.save(provider: .codex, cache: cache, cacheRoot: env.cacheRoot) + + let recorder = ProgressRecorder() + _ = try CodexLocalProjectUsageIndexer.buildSnapshotFromCostCache( + now: day, + historyDays: 1, + since: day, + until: day, + options: options, + progress: { progress in + recorder.append(progress) + }) + let events = recorder.snapshot + + #expect(events.first?.phase == .indexingProjects) + #expect(events.first?.processedFileCount == 0) + #expect(events.first?.totalFileCount == 2) + #expect(events.last?.processedFileCount == 2) + #expect(events.last?.totalFileCount == 2) + #expect(events.last?.indexedFileCount == 2) + } + + @discardableResult + private func writeCodexUsageFile( + env: CostUsageTestEnvironment, + day: Date, + fixture: CodexUsageFixture) throws + -> URL + { + var turnPayload: [String: Any] = [ + "model": "openai/gpt-5.4", + ] + if let cwd = fixture.cwd { + turnPayload["cwd"] = cwd + } + let objects: [[String: Any]] = [ + [ + "type": "session_meta", + "timestamp": env.isoString(for: day), + "payload": ["id": fixture.sessionID], + ], + [ + "type": "turn_context", + "timestamp": env.isoString(for: day), + "payload": turnPayload, + ], + [ + "type": "event_msg", + "timestamp": env.isoString(for: day.addingTimeInterval(1)), + "payload": [ + "type": "token_count", + "info": [ + "last_token_usage": [ + "input_tokens": fixture.input, + "cached_input_tokens": fixture.cached, + "output_tokens": fixture.output, + ], + "model": "openai/gpt-5.4", + ], + ], + ], + ] + return try env.writeCodexSessionFile(day: day, filename: fixture.filename, contents: env.jsonl(objects)) + } + + private func makeCachedFileUsage( + dayKey: String, + fixture: CodexUsageFixture, + costNanos: Int64?) -> CostUsageFileUsage + { + let model = "openai/gpt-5.4" + return CostUsageFileUsage( + mtimeUnixMs: 0, + size: 0, + days: [dayKey: [model: [fixture.input, fixture.cached, fixture.output]]], + parsedBytes: nil, + lastModel: model, + lastTotals: nil, + lastCountedTotals: nil, + lastRawTotalsBaseline: nil, + hasDivergentTotals: nil, + lastCodexTurnID: nil, + sessionId: fixture.sessionID, + forkedFromId: nil, + codexSession: CostUsageCodexSessionMetadata( + sessionId: fixture.sessionID, + forkedFromId: nil, + cwd: fixture.cwd, + title: nil, + startedAtUnixMs: nil, + latestActivityUnixMs: nil), + codexCostNanos: costNanos.map { [dayKey: [model: $0]] }, + codexPrioritySurchargeNanos: nil, + codexStandardCostNanos: nil, + codexPriorityCostNanos: nil, + codexStandardTokens: nil, + codexPriorityTokens: nil, + codexTurnIDs: nil, + codexRows: nil, + claudeRows: nil) + } + + @Test + func `workspace fingerprint covers sidecar semantics but not scanner cursors`() throws { + let fixture = CodexUsageFixture( + filename: "rollout.jsonl", + sessionID: "fingerprint-session", + cwd: "/tmp/fingerprint-project", + input: 12, + cached: 3, + output: 4) + let day = "2026-07-25" + let baseline = self.makeCachedFileUsage(dayKey: day, fixture: fixture, costNanos: 42) + .refreshingCodexWorkspaceUsageFingerprint() + let fingerprint = try #require(baseline.codexWorkspaceContentFingerprint) + + var cursorOnly = baseline + cursorOnly.lastRawTotalsWatermark = CostUsageCodexTotals(input: 1000, cached: 900, output: 100) + #expect(cursorOnly.codexWorkspaceUsageFingerprintValue() == fingerprint) + + var changedDaily = baseline + changedDaily.days[day]?["openai/gpt-5.4"] = [24, 6, 8] + changedDaily = changedDaily.refreshingCodexWorkspaceUsageFingerprint() + #expect(changedDaily.codexWorkspaceUsageFingerprintValue() != fingerprint) + + var changedCost = baseline + changedCost.codexCostNanos?[day]?["openai/gpt-5.4"] = 84 + changedCost = changedCost.refreshingCodexWorkspaceUsageFingerprint() + #expect(changedCost.codexWorkspaceUsageFingerprintValue() != fingerprint) + + var changedProject = baseline + changedProject.projectPath = "/tmp/another-project" + changedProject = changedProject.refreshingCodexWorkspaceUsageFingerprint() + #expect(changedProject.codexWorkspaceUsageFingerprintValue() != fingerprint) + } + + #if canImport(SQLite3) + private struct CodexStateThreadFixture { + var id: String + var rolloutPath: String + var cwd: String + var title: String + var preview: String + var model: String + var createdAtUnixMs: Int64 + var updatedAtUnixMs: Int64 + var usesLegacyTimestampColumns = false + } + + private func writeCodexStateDatabase(at url: URL, thread: CodexStateThreadFixture) throws { + try FileManager.default.createDirectory(at: url.deletingLastPathComponent(), withIntermediateDirectories: true) + var db: OpaquePointer? + guard sqlite3_open(url.path, &db) == SQLITE_OK else { + sqlite3_close(db) + throw NSError(domain: "CodexLocalProjectUsageTests", code: 1) + } + defer { sqlite3_close(db) } + try self.execSQLite(db, """ + CREATE TABLE threads ( + id TEXT PRIMARY KEY, + rollout_path TEXT NOT NULL, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + source TEXT NOT NULL, + model_provider TEXT NOT NULL, + cwd TEXT NOT NULL, + title TEXT NOT NULL, + sandbox_policy TEXT NOT NULL, + approval_mode TEXT NOT NULL, + tokens_used INTEGER NOT NULL DEFAULT 0, + archived INTEGER NOT NULL DEFAULT 0, + model TEXT, + reasoning_effort TEXT, + created_at_ms INTEGER, + updated_at_ms INTEGER, + preview TEXT NOT NULL DEFAULT '' + ) + """) + var stmt: OpaquePointer? + let insert = """ + INSERT INTO threads ( + id, rollout_path, created_at, updated_at, source, model_provider, cwd, title, + sandbox_policy, approval_mode, tokens_used, archived, model, reasoning_effort, + created_at_ms, updated_at_ms, preview + ) VALUES (?, ?, ?, ?, 'codex', 'openai', ?, ?, 'workspace-write', 'never', 0, 0, ?, 'high', ?, ?, ?) + """ + guard sqlite3_prepare_v2(db, insert, -1, &stmt, nil) == SQLITE_OK else { + throw NSError(domain: "CodexLocalProjectUsageTests", code: 2) + } + defer { sqlite3_finalize(stmt) } + let transient = unsafeBitCast(-1, to: sqlite3_destructor_type.self) + sqlite3_bind_text(stmt, 1, thread.id, -1, transient) + sqlite3_bind_text(stmt, 2, thread.rolloutPath, -1, transient) + sqlite3_bind_int64(stmt, 3, thread.createdAtUnixMs / 1000) + sqlite3_bind_int64(stmt, 4, thread.updatedAtUnixMs / 1000) + sqlite3_bind_text(stmt, 5, thread.cwd, -1, transient) + sqlite3_bind_text(stmt, 6, thread.title, -1, transient) + sqlite3_bind_text(stmt, 7, thread.model, -1, transient) + if thread.usesLegacyTimestampColumns { + sqlite3_bind_null(stmt, 8) + sqlite3_bind_null(stmt, 9) + } else { + sqlite3_bind_int64(stmt, 8, thread.createdAtUnixMs) + sqlite3_bind_int64(stmt, 9, thread.updatedAtUnixMs) + } + sqlite3_bind_text(stmt, 10, thread.preview, -1, transient) + guard sqlite3_step(stmt) == SQLITE_DONE else { + throw NSError(domain: "CodexLocalProjectUsageTests", code: 3) + } + } + + private func execSQLite(_ db: OpaquePointer?, _ sql: String) throws { + var error: UnsafeMutablePointer? + guard sqlite3_exec(db, sql, nil, nil, &error) == SQLITE_OK else { + sqlite3_free(error) + throw NSError(domain: "CodexLocalProjectUsageTests", code: 4) + } + } + #endif + + private func makeProject( + id: String, + name: String, + input: Int, + cached: Int, + output: Int) -> CodexLocalProjectUsage + { + CodexLocalProjectUsage( + id: id, + displayName: name, + path: "/tmp/\(name)", + totals: CodexLocalUsageTotals( + inputTokens: input, + cachedInputTokens: cached, + outputTokens: output, + totalTokens: input + output), + estimatedCostUSD: Double(input + output) / 1000, + hasUnknownCost: false, + sessionCount: 1, + latestActivity: nil, + topModel: "gpt-5.4", + topSessions: [], + modelBreakdowns: []) + } +} + +extension CodexLocalProjectUsageTests { + @Test + func `daily usage projection follows cached input setting`() { + let point = CodexLocalUsageDailyPoint( + day: "2026-07-12", + totalTokens: 100, + cachedInputTokens: 40, + estimatedCostUSD: 1) + let includeCache = CodexLocalProjectUsageProjection(includesCachedInput: true, showsEstimatedCost: true) + let excludeCache = CodexLocalProjectUsageProjection(includesCachedInput: false, showsEstimatedCost: true) + + #expect(includeCache.displayedTokens( + totalTokens: point.totalTokens, + cachedInputTokens: point.cachedInputTokens) == 100) + #expect(excludeCache.displayedTokens( + totalTokens: point.totalTokens, + cachedInputTokens: point.cachedInputTokens) == 60) + } + + @Test + func `ranking projects preserves daily usage`() throws { + let daily = CodexLocalUsageDailyPoint( + day: "2026-07-12", + totalTokens: 100, + cachedInputTokens: 40, + estimatedCostUSD: 1) + let project = CodexLocalProjectUsage( + id: "project", + displayName: "Project", + path: "/tmp/Project", + totals: CodexLocalUsageTotals( + inputTokens: 80, + cachedInputTokens: 40, + outputTokens: 20, + totalTokens: 100), + costEstimate: CodexLocalCostEstimate(knownUSD: 1, unknownTokens: 0), + sessionCount: 1, + latestActivity: nil, + topModel: "gpt-5.4", + topSessions: [], + modelBreakdowns: [], + daily: [daily]) + let projection = CodexLocalProjectUsageProjection( + includesCachedInput: true, + showsEstimatedCost: true) + + let ranked = try #require(projection.rankedProjects([project]).first) + #expect(ranked.daily == [daily]) + } + + @Test + func `projection derives severity from displayed tokens not price`() { + let projection = CodexLocalProjectUsageProjection( + includesCachedInput: true, + showsEstimatedCost: true) + let dominant = self.makeProject( + id: "dominant", + name: "Dominant", + input: 1000, + cached: 0, + output: 0) + let costlySmall = CodexLocalProjectUsage( + id: "costly-small", + displayName: "CostlySmall", + path: "/tmp/CostlySmall", + totals: CodexLocalUsageTotals( + inputTokens: 1, + cachedInputTokens: 0, + outputTokens: 0, + totalTokens: 1), + costEstimate: CodexLocalCostEstimate(knownUSD: 10000, unknownTokens: 0), + sessionCount: 1, + latestActivity: nil, + topModel: "gpt-5.4", + topSessions: [], + modelBreakdowns: []) + + let ranked = projection.rankedProjects([costlySmall, dominant]) + #expect(ranked.map(\.id) == ["dominant", "costly-small"]) + #expect(ranked.first?.severity == .high) + #expect(ranked.last?.severity == .normal) + } + + @Test + func `cost coverage retains the number of unpriced tokens`() { + let estimate = CodexLocalCostEstimate(knownUSD: 2.5, unknownTokens: 37) + + #expect(estimate.coverage == .partial) + #expect(estimate.knownUSD == 2.5) + #expect(estimate.unknownTokens == 37) + } + + @Test + func `persisted projects exclude display severity`() throws { + let project = CodexLocalProjectUsage( + id: "project", + displayName: "Project", + path: "/tmp/project", + totals: .empty, + costEstimate: CodexLocalCostEstimate(knownUSD: 1, unknownTokens: 0), + sessionCount: 1, + latestActivity: nil, + topModel: nil, + topSessions: [], + modelBreakdowns: [], + usageSeverity: .high) + + let encoded = try JSONEncoder.codexLocalProjectUsageSidecar.encode(project) + let json = try #require(String(data: encoded, encoding: .utf8)) + let decoded = try JSONDecoder.codexLocalProjectUsageSidecar.decode(CodexLocalProjectUsage.self, from: encoded) + + #expect(!json.contains("usageSeverity")) + #expect(decoded.severity == .normal) + } + + @Test + func `snapshot ignores legacy process state without persisting it again`() throws { + let snapshot = CodexLocalProjectUsageSnapshot( + updatedAt: Date(timeIntervalSince1970: 1_700_000_000), + historyDays: 30, + scopeSignature: "scope", + rootsFingerprint: ["sessions": 1], + indexedFileCount: 1, + skippedFileCount: 0, + total: .empty, + projects: [], + daily: []) + + let encoded = try JSONEncoder.codexLocalProjectUsageSidecar.encode(snapshot) + let currentJSON = try #require(String(data: encoded, encoding: .utf8)) + var legacyObject = try #require(JSONSerialization.jsonObject(with: encoded) as? [String: Any]) + legacyObject["stale"] = true + legacyObject["indexing"] = true + legacyObject["errorMessage"] = "previous refresh failed" + let legacyData = try JSONSerialization.data(withJSONObject: legacyObject) + let decoded = try JSONDecoder.codexLocalProjectUsageSidecar.decode( + CodexLocalProjectUsageSnapshot.self, + from: legacyData) + + #expect(decoded == snapshot) + #expect(!currentJSON.contains("\"stale\"")) + #expect(!currentJSON.contains("\"indexing\"")) + #expect(!currentJSON.contains("\"errorMessage\"")) + } + + @Test + func `snapshot persists source completeness and defaults legacy payloads to complete`() throws { + let snapshot = CodexLocalProjectUsageSnapshot( + updatedAt: Date(timeIntervalSince1970: 1_700_000_000), + historyDays: 30, + scopeSignature: "scope", + rootsFingerprint: ["sessions": 1], + indexedFileCount: 1, + skippedFileCount: 0, + total: .empty, + projects: [], + daily: [], + sourceStatus: .catalogLocked) + + let encoded = try JSONEncoder.codexLocalProjectUsageSidecar.encode(snapshot) + let decoded = try JSONDecoder.codexLocalProjectUsageSidecar.decode( + CodexLocalProjectUsageSnapshot.self, + from: encoded) + #expect(decoded.sourceStatus == .catalogLocked) + + var legacyObject = try #require(JSONSerialization.jsonObject(with: encoded) as? [String: Any]) + legacyObject.removeValue(forKey: "sourceStatus") + let legacyData = try JSONSerialization.data(withJSONObject: legacyObject) + let decodedLegacy = try JSONDecoder.codexLocalProjectUsageSidecar.decode( + CodexLocalProjectUsageSnapshot.self, + from: legacyData) + #expect(decodedLegacy.sourceStatus == .complete) + } + + @Test + func `aggregate only snapshots are rejected until inspector detail is available`() { + let now = Date(timeIntervalSince1970: 1_700_000_000) + let dailyPoint = CodexLocalUsageDailyPoint( + day: "2023-11-14", + totalTokens: 100, + estimatedCostUSD: 1) + let totals = CodexLocalUsageTotals( + inputTokens: 80, + cachedInputTokens: 20, + outputTokens: 20, + totalTokens: 100) + let incompleteProject = CodexLocalProjectUsage( + id: "project", + displayName: "Project", + path: "/tmp/project", + totals: totals, + costEstimate: CodexLocalCostEstimate(knownUSD: 1, unknownTokens: 0), + sessionCount: 1, + latestActivity: now, + topModel: "gpt-5.4", + topSessions: [], + modelBreakdowns: []) + let incomplete = CodexLocalProjectUsageSnapshot( + updatedAt: now, + historyDays: 1, + scopeSignature: "scope", + rootsFingerprint: [:], + indexedFileCount: 1, + skippedFileCount: 0, + total: totals, + projects: [incompleteProject], + daily: []) + #expect(!incomplete.hasInspectorDetail) + + let session = CodexLocalSessionUsage( + id: "session", + projectId: "project", + displayTitle: "Project session", + cwd: "/tmp/project", + startedAt: now, + latestActivity: now, + totals: totals, + estimatedCostUSD: 1, + hasUnknownCost: false, + topModel: "gpt-5.4", + daily: [dailyPoint]) + let completeProject = CodexLocalProjectUsage( + id: "project", + displayName: "Project", + path: "/tmp/project", + totals: totals, + costEstimate: CodexLocalCostEstimate(knownUSD: 1, unknownTokens: 0), + sessionCount: 1, + latestActivity: now, + topModel: "gpt-5.4", + topSessions: [session], + modelBreakdowns: [], + daily: [dailyPoint]) + let complete = CodexLocalProjectUsageSnapshot( + updatedAt: now, + historyDays: 1, + scopeSignature: "scope", + rootsFingerprint: [:], + indexedFileCount: 1, + skippedFileCount: 0, + total: totals, + projects: [completeProject], + sessions: [session], + daily: [dailyPoint]) + #expect(complete.hasInspectorDetail) + } + + @Test + func `session daily attribution round trips and legacy payload defaults to empty`() throws { + let daily = [ + CodexLocalUsageDailyPoint( + day: "2026-07-12", + totalTokens: 100, + cachedInputTokens: 20, + estimatedCostUSD: 1.25), + CodexLocalUsageDailyPoint( + day: "2026-07-13", + totalTokens: 250, + cachedInputTokens: 50, + estimatedCostUSD: 2.5), + ] + let session = CodexLocalSessionUsage( + id: "session", + projectId: "project", + displayTitle: "A chat spanning two days", + cwd: "/tmp/project", + startedAt: nil, + latestActivity: nil, + totals: CodexLocalUsageTotals( + inputTokens: 280, + cachedInputTokens: 70, + outputTokens: 70, + totalTokens: 350), + estimatedCostUSD: 3.75, + hasUnknownCost: false, + topModel: "gpt-5.4", + daily: daily) + + let encoded = try JSONEncoder.codexLocalProjectUsageSidecar.encode(session) + let decoded = try JSONDecoder.codexLocalProjectUsageSidecar.decode( + CodexLocalSessionUsage.self, + from: encoded) + #expect(decoded.daily == daily) + + var legacyObject = try #require(JSONSerialization.jsonObject(with: encoded) as? [String: Any]) + legacyObject.removeValue(forKey: "daily") + let legacyData = try JSONSerialization.data(withJSONObject: legacyObject) + let decodedLegacy = try JSONDecoder.codexLocalProjectUsageSidecar.decode( + CodexLocalSessionUsage.self, + from: legacyData) + #expect(decodedLegacy.daily.isEmpty) + } + + @Test + func `sidecar rejects unreleased schema versions`() throws { + #if canImport(SQLite3) + for version in [2, 3, 4] { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + let databaseDirectory = env.cacheRoot.appendingPathComponent("local-usage", isDirectory: true) + try FileManager.default.createDirectory(at: databaseDirectory, withIntermediateDirectories: true) + let databaseURL = databaseDirectory.appendingPathComponent("codex-workspaces-v1.sqlite") + var database: OpaquePointer? + #expect(sqlite3_open(databaseURL.path, &database) == SQLITE_OK) + try self.execSQLite(database, "PRAGMA user_version = \(version);") + sqlite3_close(database) + + #expect(throws: (any Error).self) { + try CodexWorkspaceUsageSidecar(cacheRoot: env.cacheRoot).synchronizeSources( + cache: CostUsageCache(), + catalog: .empty) + } + + database = nil + #expect(sqlite3_open(databaseURL.path, &database) == SQLITE_OK) + var statement: OpaquePointer? + #expect(sqlite3_prepare_v2(database, "PRAGMA user_version", -1, &statement, nil) == SQLITE_OK) + #expect(sqlite3_step(statement) == SQLITE_ROW) + #expect(sqlite3_column_int(statement, 0) == Int32(version)) + sqlite3_finalize(statement) + sqlite3_close(database) + } + #endif + } + + @Test + func `aggregate models analytics preserves priced zero and unavailable coverage`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + + let now = Date() + let dayKey = CostUsageScanner.CostUsageDayRange.dayKey(from: now) + let project = env.root.appendingPathComponent("Project", isDirectory: true) + try FileManager.default.createDirectory( + at: project.appendingPathComponent(".git", isDirectory: true), + withIntermediateDirectories: true) + let options = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + cacheRoot: env.cacheRoot) + let pricedFixture = CodexUsageFixture( + filename: "priced.jsonl", + sessionID: "priced-fallback", + cwd: project.path, + input: 10, + cached: 0, + output: 0) + let unavailableFixture = CodexUsageFixture( + filename: "unavailable.jsonl", + sessionID: "unavailable-fallback", + cwd: project.path, + input: 20, + cached: 0, + output: 0) + var unavailableUsage = self.makeCachedFileUsage( + dayKey: dayKey, + fixture: unavailableFixture, + costNanos: nil) + let unavailableModel = "openai/gpt-5.4-mini" + unavailableUsage.days = [dayKey: [unavailableModel: [20, 0, 0]]] + unavailableUsage.lastModel = unavailableModel + + var cache = CostUsageCache() + cache.roots = CostUsageScanner.codexRootsFingerprint(options: options) + cache.files[env.root.appendingPathComponent("priced.jsonl").path] = self.makeCachedFileUsage( + dayKey: dayKey, + fixture: pricedFixture, + costNanos: 0) + cache.files[env.root.appendingPathComponent("unavailable.jsonl").path] = unavailableUsage + + let snapshot = try CodexLocalProjectUsageIndexer.buildSnapshotFromCostCache( + now: now, + historyDays: 1, + since: now, + until: now, + options: options, + cacheOverride: cache, + catalogOverride: .empty) + let analytics = try #require(snapshot.modelsAnalytics?.allWorkspaces) + let priced = try #require(analytics.rows.first { $0.id == "gpt-5.4" }) + let unavailable = try #require(analytics.rows.first { $0.id == "gpt-5.4-mini" }) + + #expect(priced.cost.knownAmount == 0) + #expect(priced.cost.pricedTokens == 10) + #expect(priced.cost.unpricedTokens == 0) + #expect(unavailable.cost.knownAmount == 0) + #expect(unavailable.cost.pricedTokens == 0) + #expect(unavailable.cost.unpricedTokens == 20) + #expect(analytics.cost.knownAmount == 0) + #expect(analytics.cost.pricedTokens == 10) + #expect(analytics.cost.unpricedTokens == 20) + #expect(analytics.diagnostics.isMatched) + } + + @Test + func `sidecar rehydrates cache-backed project aggregates`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + + let now = Date() + let dayKey = CostUsageScanner.CostUsageDayRange.dayKey(from: now) + let project = env.root.appendingPathComponent("Project", isDirectory: true) + try FileManager.default.createDirectory( + at: project.appendingPathComponent(".git", isDirectory: true), + withIntermediateDirectories: true) + let options = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + cacheRoot: env.cacheRoot) + let path = env.root.appendingPathComponent("rollout.jsonl").path + let fixture = CodexUsageFixture( + filename: "rollout.jsonl", + sessionID: "sidecar-session", + cwd: project.path, + input: 120, + cached: 20, + output: 30) + var cache = CostUsageCache() + cache.roots = CostUsageScanner.codexRootsFingerprint(options: options) + var cachedUsage = self.makeCachedFileUsage( + dayKey: dayKey, + fixture: fixture, + costNanos: 42) + cachedUsage.codexRows = [CostUsageScanner.CodexUsageRow( + day: dayKey, + model: "openai/gpt-5.4", + rawModel: "GPT-5.4", + turnID: "turn-1", + eventIndex: 0, + timestampUnixMs: Int64(now.timeIntervalSince1970 * 1000), + input: fixture.input, + cached: fixture.cached, + output: fixture.output, + reasoning: 12, + knownCostNanos: 42, + unpricedTokens: 0, + pricingModel: "openai/gpt-5.4", + pricingMode: "standard")] + cache.files[path] = cachedUsage + let catalog = CodexThreadCatalog.empty + let baseline = try CodexLocalProjectUsageIndexer.buildSnapshotFromCostCache( + now: now, + historyDays: 1, + since: now, + until: now, + options: options, + cacheOverride: cache, + catalogOverride: catalog) + let baselineModel = try #require(baseline.modelsAnalytics?.allWorkspaces.rows.first) + let expectedKnownCost = try #require(Decimal(string: "0.000000042")) + #expect(baselineModel.reasoningTokens == 12) + #expect(baselineModel.cost.knownAmount == expectedKnownCost) + #expect(baselineModel.cost.pricedTokens == Int64(fixture.input + fixture.output)) + #expect(baselineModel.cost.unpricedTokens == 0) + #expect(baseline.modelsAnalytics?.allWorkspaces.diagnostics.isMatched == true) + let sidecar = CodexWorkspaceUsageSidecar(cacheRoot: env.cacheRoot) + try sidecar.synchronize( + snapshot: baseline, + cache: cache, + catalog: catalog, + rootsFingerprint: cache.roots ?? [:]) + #if canImport(SQLite3) + let sidecarURL = env.cacheRoot + .appendingPathComponent("local-usage", isDirectory: true) + .appendingPathComponent("codex-workspaces-v1.sqlite", isDirectory: false) + var database: OpaquePointer? + #expect(sqlite3_open(sidecarURL.path, &database) == SQLITE_OK) + defer { sqlite3_close(database) } + var statement: OpaquePointer? + #expect(sqlite3_prepare_v2( + database, + "SELECT payload_format_version, payload FROM snapshot_payloads", + -1, + &statement, + nil) == SQLITE_OK) + #expect(sqlite3_step(statement) == SQLITE_ROW) + #expect(sqlite3_column_int(statement, 0) == 3) + let numericPayloadBytes = try #require(sqlite3_column_blob(statement, 1)) + let numericPayload = Data( + bytes: numericPayloadBytes, + count: Int(sqlite3_column_bytes(statement, 1))) + sqlite3_finalize(statement) + let numericObject = try #require(JSONSerialization.jsonObject(with: numericPayload) as? [String: Any]) + #expect(numericObject["updatedAt"] is NSNumber) + + statement = nil + #expect(sqlite3_prepare_v2( + database, + "UPDATE snapshot_payloads SET payload_format_version = 2 WHERE scope_signature = ? AND history_days = ?", + -1, + &statement, + nil) == SQLITE_OK) + let transient = unsafeBitCast(-1, to: sqlite3_destructor_type.self) + sqlite3_bind_text(statement, 1, baseline.scopeSignature, -1, transient) + sqlite3_bind_int64(statement, 2, Int64(baseline.historyDays)) + #expect(sqlite3_step(statement) == SQLITE_DONE) + sqlite3_finalize(statement) + #expect(sidecar.loadLatestSnapshot( + scopeSignature: baseline.scopeSignature, + historyDays: baseline.historyDays) == nil) + #expect(try sidecar.usageCache(roots: cache.roots ?? [:]).files[path]?.codexRows?.first?.knownCostNanos == 42) + try sidecar.synchronize( + snapshot: baseline, + cache: cache, + catalog: catalog, + rootsFingerprint: cache.roots ?? [:]) + statement = nil + #expect(sqlite3_prepare_v2( + database, + "SELECT payload_format_version FROM snapshot_payloads", + -1, + &statement, + nil) == SQLITE_OK) + #expect(sqlite3_step(statement) == SQLITE_ROW) + #expect(sqlite3_column_int(statement, 0) == 3) + sqlite3_finalize(statement) + #expect(sidecar.loadLatestSnapshot( + scopeSignature: baseline.scopeSignature, + historyDays: baseline.historyDays)?.total == baseline.total) + #endif + let rehydratedCache = try sidecar.usageCache(roots: cache.roots ?? [:]) + let rehydratedEvent = try #require(rehydratedCache.files[path]?.codexRows?.first) + let rehydrated = try CodexLocalProjectUsageIndexer.buildSnapshotFromCostCache( + now: now, + historyDays: 1, + since: now, + until: now, + options: options, + cacheOverride: rehydratedCache, + catalogOverride: catalog) + + #expect(rehydrated.total == baseline.total) + #expect(rehydrated.projects.map(\.id) == baseline.projects.map(\.id)) + #expect(rehydrated.projects.first?.costEstimate == baseline.projects.first?.costEstimate) + #expect(rehydratedEvent.rawModel == "GPT-5.4") + #expect(rehydratedEvent.timestampUnixMs == Int64(now.timeIntervalSince1970 * 1000)) + #expect(rehydratedEvent.reasoning == 12) + #expect(rehydratedEvent.knownCostNanos == 42) + #expect(rehydratedEvent.pricingMode == "standard") + let rehydratedModel = try #require(rehydrated.modelsAnalytics?.allWorkspaces.rows.first) + #expect(rehydratedModel.reasoningTokens == 12) + #expect(rehydratedModel.cost.knownAmount == expectedKnownCost) + #expect(rehydratedModel.cost.pricedTokens == Int64(fixture.input + fixture.output)) + #expect(rehydratedModel.cost.unpricedTokens == 0) + #expect(rehydrated.modelsAnalytics?.allWorkspaces.diagnostics.isMatched == true) + } + + @Test + func `sidecar refreshes changed usage when rollout metadata is unchanged`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + + let dayKey = CostUsageScanner.CostUsageDayRange.dayKey(from: Date()) + let path = env.root.appendingPathComponent("rollout.jsonl").path + let fixture = CodexUsageFixture( + filename: "rollout.jsonl", + sessionID: "sidecar-content-change", + cwd: env.root.path, + input: 120, + cached: 20, + output: 30) + var cache = CostUsageCache() + let timestampUnixMs = Int64(Date().timeIntervalSince1970 * 1000) + var initialUsage = self.makeCachedFileUsage( + dayKey: dayKey, + fixture: fixture, + costNanos: 42) + initialUsage.codexRows = [CostUsageScanner.CodexUsageRow( + day: dayKey, + model: "openai/gpt-5.4", + rawModel: "GPT-5.4", + turnID: "turn-1", + eventIndex: 0, + timestampUnixMs: timestampUnixMs, + input: 120, + cached: 20, + output: 30, + knownCostNanos: 42, + unpricedTokens: 0, + pricingModel: "openai/gpt-5.4", + pricingMode: "standard")] + initialUsage = initialUsage.refreshingCodexWorkspaceUsageFingerprint() + cache.files[path] = initialUsage + let sidecar = CodexWorkspaceUsageSidecar(cacheRoot: env.cacheRoot) + try sidecar.synchronizeSources(cache: cache, catalog: .empty) + + let model = "openai/gpt-5.4" + var changedUsage = try #require(cache.files[path]) + changedUsage.days[dayKey]?[model] = [240, 40, 60] + changedUsage.codexCostNanos?[dayKey]?[model] = 84 + changedUsage.codexRows = [CostUsageScanner.CodexUsageRow( + day: dayKey, + model: model, + rawModel: "GPT-5.4", + turnID: "turn-1", + eventIndex: 0, + timestampUnixMs: timestampUnixMs, + input: 240, + cached: 40, + output: 60, + knownCostNanos: 84, + unpricedTokens: 0, + pricingModel: model, + pricingMode: "priority")] + changedUsage = changedUsage.refreshingCodexWorkspaceUsageFingerprint() + cache.files[path] = changedUsage + try sidecar.synchronizeSources(cache: cache, catalog: .empty) + + let updated = try #require(sidecar.usageCache(roots: [:]).files[path]) + #expect(updated.days[dayKey]?[model] == [240, 40, 60]) + #expect(updated.codexCostNanos?[dayKey]?[model] == 84) + #expect(updated.codexRows?.first?.input == 240) + #expect(updated.codexRows?.first?.knownCostNanos == 84) + #expect(updated.codexRows?.first?.pricingMode == "priority") + + changedUsage.days = [:] + changedUsage.codexCostNanos = [:] + changedUsage.codexRows = [] + changedUsage = changedUsage.refreshingCodexWorkspaceUsageFingerprint() + cache.files[path] = changedUsage + try sidecar.synchronizeSources(cache: cache, catalog: .empty) + + #expect(try sidecar.usageCache(roots: [:]).files[path] == nil) + } +} + +// swiftlint:enable type_body_length diff --git a/Tests/CodexBarTests/CodexModelsAnalyticsParityTests.swift b/Tests/CodexBarTests/CodexModelsAnalyticsParityTests.swift new file mode 100644 index 0000000000..6f6ecd4d01 --- /dev/null +++ b/Tests/CodexBarTests/CodexModelsAnalyticsParityTests.swift @@ -0,0 +1,95 @@ +import Foundation +import Testing +@testable import CodexBarCore + +@Suite("Codex Models analytics parity") +struct CodexModelsAnalyticsParityTests { + @Test + func `per model parity canonicalizes aliases with audited cost`() throws { + let current = DateInterval( + start: Date(timeIntervalSince1970: 1_000_000), + duration: 7 * 24 * 60 * 60) + let previous = DateInterval( + start: current.start.addingTimeInterval(-current.duration), + duration: current.duration) + let currentFragments = [ + self.fragment( + day: current.start, + model: "GPT-5.4", + input: 6, + session: "current", + costNanos: 30), + self.fragment( + day: current.start, + model: "openai/gpt-5.4", + input: 4, + session: "current", + costNanos: 20), + ] + let previousFragments = [self.fragment( + day: previous.start, + model: "gpt-5.4", + input: 5, + session: "previous", + costNanos: 10)] + let currentKnownCost = try #require(Decimal(string: "0.00000005")) + let previousKnownCost = try #require(Decimal(string: "0.00000001")) + let snapshot = CodexModelsAnalyticsBuilder().build(CodexModelsAnalyticsRequest( + source: CodexModelsAnalyticsSource(current: currentFragments, previous: previousFragments), + scopeID: nil, + periods: CodexModelsAnalyticsPeriods(current: current, previous: previous), + revision: CodexModelsAnalyticsRevision(generatedAt: current.end, indexRevision: "fixture"), + legacy: CodexModelsLegacyBaseline( + totalTokens: 10, + modelIDs: ["gpt-5.4"], + knownCost: currentKnownCost, + pricedTokens: 10, + unpricedTokens: 0, + activeModelCount: 1, + topModelID: "gpt-5.4", + sessionReferenceTotal: 1, + previousTotalTokens: 5, + previousKnownCost: previousKnownCost, + previousUnpricedTokens: 0, + previousSessionReferenceTotal: 1, + currentModels: [CodexModelsLegacyModelBaseline( + modelID: "gpt-5.4", + totalTokens: 10, + knownCost: currentKnownCost, + pricedTokens: 10, + unpricedTokens: 0, + sessionReferences: 1)], + previousModels: [CodexModelsLegacyModelBaseline( + modelID: "gpt-5.4", + totalTokens: 5, + knownCost: previousKnownCost, + pricedTokens: 5, + unpricedTokens: 0, + sessionReferences: 1)]))) + + let row = try #require(snapshot.rows.first) + #expect(row.id == "gpt-5.4") + #expect(row.cost.knownAmount == currentKnownCost) + #expect(row.cost.pricedTokens == 10) + #expect(row.cost.unpricedTokens == 0) + #expect(snapshot.diagnostics.isMatched) + } + + private func fragment( + day: Date, + model: String, + input: Int64, + session: String, + costNanos: Int64) -> CodexModelsUsageFragment + { + CodexModelsUsageFragment( + workspaceID: "workspace", + sessionID: session, + day: day, + rawModelID: model, + inputTokens: input, + cachedInputTokens: 0, + outputTokens: 0, + costNanos: costNanos) + } +} diff --git a/Tests/CodexBarTests/CodexModelsAnalyticsTests.swift b/Tests/CodexBarTests/CodexModelsAnalyticsTests.swift new file mode 100644 index 0000000000..568f0da17a --- /dev/null +++ b/Tests/CodexBarTests/CodexModelsAnalyticsTests.swift @@ -0,0 +1,528 @@ +import Foundation +import Testing +@testable import CodexBar +@testable import CodexBarCore + +@Suite("Codex Models analytics") +struct CodexModelsAnalyticsTests { + private let calendar: Calendar = { + var calendar = Calendar(identifier: .gregorian) + calendar.locale = Locale(identifier: "en_US_POSIX") + calendar.timeZone = TimeZone(identifier: "America/Los_Angeles")! + return calendar + }() + + @Test + func `canonical aliases merge while raw aliases remain auditable`() { + let intervals = self.intervals(days: 7) + let current = [ + self.fragment(day: intervals.current.start, model: "GPT-5", input: 10, output: 2, session: "one"), + self.fragment(day: intervals.current.start, model: "gpt-5", input: 7, output: 1, session: "two"), + self.fragment(day: intervals.current.start, model: " OpenAI/GPT-5 ", input: 3, output: 0, session: "three"), + ] + let snapshot = self.build(current: current, previous: [], intervals: intervals, legacyTotal: 23) + + #expect(snapshot.rows.count == 1) + #expect(snapshot.rows[0].id == "gpt-5") + #expect(snapshot.rows[0].rawAliases == [" OpenAI/GPT-5 ", "GPT-5", "gpt-5"]) + #expect(snapshot.rows[0].sessionReferences == 3) + #expect(snapshot.rows[0].associatedSessionIDs == ["one", "three", "two"]) + } + + @Test + func `cached input and reasoning detail are not double counted`() { + let intervals = self.intervals(days: 7) + let fragment = CodexModelsUsageFragment( + workspaceID: "workspace", + sessionID: "one", + day: intervals.current.start, + rawModelID: "model-a", + inputTokens: 100, + cachedInputTokens: 80, + outputTokens: 40, + reasoningTokens: 30, + costNanos: 1_500_000_000) + let snapshot = self.build(current: [fragment], previous: [], intervals: intervals, legacyTotal: 140) + let row = snapshot.rows[0] + + #expect(row.totalTokens == 140) + #expect(row.cachedInputTokens == 80) + #expect(row.reasoningTokens == 30) + #expect(snapshot.invariantViolations().isEmpty) + } + + @Test + func `pricing distinguishes complete partial unavailable and known zero`() throws { + let intervals = self.intervals(days: 7) + let current = [ + self.fragment(day: intervals.current.start, model: "model-a", input: 10, output: 0, costNanos: 0), + self.fragment(day: intervals.current.start, model: "model-b", input: 20, output: 0, costNanos: nil), + CodexModelsUsageFragment( + workspaceID: "workspace", + sessionID: "partial", + day: intervals.current.start, + rawModelID: "model-a", + inputTokens: 25, + cachedInputTokens: 0, + outputTokens: 0, + costNanos: 250_000_000, + unpricedTokens: 20), + ] + let snapshot = self.build(current: current, previous: [], intervals: intervals, legacyTotal: 55) + let priced = try #require(snapshot.rows.first { $0.id == "model-a" }) + let unavailable = try #require(snapshot.rows.first { $0.id == "model-b" }) + + #expect(priced.cost.knownAmount == Decimal(string: "0.25")) + #expect(priced.cost.pricedTokens == 15) + #expect(priced.cost.unpricedTokens == 20) + #expect(unavailable.cost.knownAmount == 0) + #expect(unavailable.cost.pricedTokens == 0) + #expect(unavailable.cost.unpricedTokens == 20) + #expect(snapshot.cost.pricedTokens + snapshot.cost.unpricedTokens == snapshot.totalTokens) + } + + @Test + func `current and previous periods are equal duration across DST`() throws { + let end = try #require(self.calendar.date(from: DateComponents(year: 2026, month: 3, day: 12))) + let currentStart = try #require(self.calendar.date(byAdding: .day, value: -7, to: end)) + let currentInterval = DateInterval(start: currentStart, end: end) + let previousStart = currentStart.addingTimeInterval(-currentInterval.duration) + let intervals = ( + current: currentInterval, + previous: DateInterval(start: previousStart, end: currentStart)) + let current = [self.fragment(day: currentStart, model: "model-a", input: 20, output: 0)] + let previous = [self.fragment(day: previousStart, model: "model-a", input: 10, output: 0)] + let snapshot = self.build(current: current, previous: previous, intervals: intervals, legacyTotal: 20) + + #expect(intervals.current.duration == intervals.previous.duration) + #expect(snapshot.rows[0].tokenComparison == .percent(1)) + } + + @Test + func `indexer periods are adjacent equal seconds across both DST transitions`() throws { + let springSince = try #require(self.calendar.date(from: DateComponents(year: 2026, month: 3, day: 6))) + let springUntil = try #require(self.calendar.date(from: DateComponents(year: 2026, month: 3, day: 12))) + let fallSince = try #require(self.calendar.date(from: DateComponents(year: 2026, month: 10, day: 30))) + let fallUntil = try #require(self.calendar.date(from: DateComponents(year: 2026, month: 11, day: 5))) + + for periods in [ + CodexLocalProjectUsageIndexer.modelsAnalyticsPeriods( + since: springSince, + until: springUntil, + calendar: self.calendar), + CodexLocalProjectUsageIndexer.modelsAnalyticsPeriods( + since: fallSince, + until: fallUntil, + calendar: self.calendar), + ] { + #expect(periods.current.duration == periods.previous.duration) + #expect(periods.previous.end == periods.current.start) + } + + let fallPeriods = CodexLocalProjectUsageIndexer.modelsAnalyticsPeriods( + since: fallSince, + until: fallUntil, + calendar: self.calendar) + let scanStart = CodexLocalProjectUsageIndexer.modelsAnalyticsScanStart( + since: fallSince, + until: fallUntil, + calendar: self.calendar) + let calendarDaySubtraction = try #require( + self.calendar.date(byAdding: .day, value: -7, to: fallPeriods.current.start)) + #expect(calendarDaySubtraction > fallPeriods.previous.start) + #expect(scanStart <= fallPeriods.previous.start) + #expect(fallPeriods.previous.start.timeIntervalSince(scanStart) < 24 * 60 * 60) + } + + @Test + func `timestamp filtering uses half open current and previous boundaries`() { + let intervals = self.intervals(days: 2) + let current = [ + self.fragment( + day: intervals.current.start, + timestamp: intervals.current.start, + model: "model-a", + input: 10, + output: 0), + self.fragment( + day: intervals.current.start, + timestamp: intervals.current.end.addingTimeInterval(-0.001), + model: "model-a", + input: 20, + output: 0), + self.fragment( + day: intervals.current.end, + timestamp: intervals.current.end, + model: "model-a", + input: 40, + output: 0), + ] + let previous = [ + self.fragment( + day: intervals.previous.start, + timestamp: intervals.previous.start, + model: "model-a", + input: 5, + output: 0), + self.fragment( + day: intervals.previous.start, + timestamp: intervals.previous.end.addingTimeInterval(-0.001), + model: "model-a", + input: 7, + output: 0), + self.fragment( + day: intervals.previous.end, + timestamp: intervals.previous.end, + model: "model-a", + input: 9, + output: 0), + ] + let snapshot = self.build(current: current, previous: previous, intervals: intervals, legacyTotal: 30) + + #expect(snapshot.totalTokens == 30) + #expect(snapshot.rows[0].previousTotalTokens == 12) + #expect(snapshot.rows[0].tokenComparison == .percent(1.5)) + } + + @Test + func `session references count distinct session model pairs`() throws { + let intervals = self.intervals(days: 7) + let secondDay = try #require(self.calendar.date(byAdding: .day, value: 1, to: intervals.current.start)) + let current = [ + self.fragment(day: intervals.current.start, model: "model-a", input: 1, output: 0, session: "same"), + self.fragment(day: secondDay, model: "model-a", input: 1, output: 0, session: "same"), + self.fragment(day: secondDay, model: "model-b", input: 1, output: 0, session: "same"), + ] + let snapshot = self.build(current: current, previous: [], intervals: intervals, legacyTotal: 3) + + #expect(snapshot.uniqueSessionCount == 1) + #expect(snapshot.sessionReferenceTotal == 2) + #expect(snapshot.rows.first { $0.id == "model-a" }?.sessionReferences == 1) + #expect(snapshot.rows.first { $0.id == "model-b" }?.sessionReferences == 1) + let allModelsBucket = try #require(snapshot.daily.last) + #expect(allModelsBucket.sessionIDs == ["same"]) + #expect(allModelsBucket.sessionReferenceIDs.count == 2) + #expect(allModelsBucket.sessionReferences == 2) + #expect(snapshot.dailyByModel["model-a"]?.last?.sessionReferences == 1) + #expect(snapshot.dailyByModel["model-b"]?.last?.sessionReferences == 1) + } + + @Test + func `incomplete previous coverage suppresses comparisons and newly active semantics`() { + let intervals = self.intervals(days: 7) + let current = [ + self.fragment(day: intervals.current.start, model: "model-a", input: 20, output: 0, session: "one"), + self.fragment(day: intervals.current.start, model: "model-b", input: 10, output: 0, session: "two"), + ] + let previous = [ + self.fragment(day: intervals.previous.start, model: "model-a", input: 10, output: 0, session: "old"), + ] + let complete = self.build(current: current, previous: previous, intervals: intervals, legacyTotal: 30) + let incomplete = self.build( + current: current, + previous: previous, + intervals: intervals, + legacyTotal: 30, + previousIsComplete: false) + + #expect(complete.previousActiveModelCount == 1) + #expect(complete.newlyActiveModelCount == 1) + #expect(complete.rows.first { $0.id == "model-a" }?.previousTotalTokens == 10) + + #expect(incomplete.currentIsComplete == true) + #expect(incomplete.previousIsComplete == false) + #expect(incomplete.previousActiveModelCount == nil) + #expect(incomplete.newlyActiveModelCount == nil) + #expect(incomplete.previousSessionReferenceTotal == nil) + #expect(incomplete.tokenComparison == .unavailable) + #expect(incomplete.costComparison == .unavailable) + #expect(incomplete.sessionReferenceComparison == .unavailable) + for row in incomplete.rows { + #expect(row.previousTotalTokens == nil) + #expect(row.previousCost == nil) + #expect(row.previousSessionReferences == nil) + #expect(row.tokenComparison == .unavailable) + #expect(row.costComparison == .unavailable) + #expect(row.sessionReferenceComparison == .unavailable) + } + } + + @Test + func `ranking ties are deterministic`() { + let intervals = self.intervals(days: 7) + let current = [ + self.fragment(day: intervals.current.start, model: "model-z", input: 10, output: 0), + self.fragment(day: intervals.current.start, model: "model-a", input: 10, output: 0), + ] + let snapshot = self.build(current: current, previous: [], intervals: intervals, legacyTotal: 20) + #expect(snapshot.rows.map(\.id) == ["model-a", "model-z"]) + } + + @Test + func `dual run diagnostics flag total and identity mismatches`() { + let intervals = self.intervals(days: 7) + let current = [self.fragment(day: intervals.current.start, model: "model-a", input: 10, output: 0)] + let snapshot = CodexModelsAnalyticsBuilder().build(CodexModelsAnalyticsRequest( + source: CodexModelsAnalyticsSource(current: current, previous: []), + scopeID: nil, + periods: CodexModelsAnalyticsPeriods(current: intervals.current, previous: intervals.previous), + revision: CodexModelsAnalyticsRevision(generatedAt: intervals.current.end, indexRevision: "fixture"), + legacy: CodexModelsLegacyBaseline(totalTokens: 11, modelIDs: ["model-b"]))) + + #expect(snapshot.diagnostics.mismatches == ["total_tokens", "model_identities"]) + #expect(snapshot.diagnostics.mismatchDimensions == [.totalTokens, .modelIdentities]) + } + + @Test + func `dual run diagnostics compare current and previous per model raw values`() { + let intervals = self.intervals(days: 7) + let current = [self.fragment( + day: intervals.current.start, + model: "model-a", + input: 10, + output: 0, + session: "current", + costNanos: 100_000_000)] + let previous = [self.fragment( + day: intervals.previous.start, + model: "model-a", + input: 5, + output: 0, + session: "previous", + costNanos: 200_000_000)] + let snapshot = CodexModelsAnalyticsBuilder().build(CodexModelsAnalyticsRequest( + source: CodexModelsAnalyticsSource(current: current, previous: previous), + scopeID: nil, + periods: CodexModelsAnalyticsPeriods(current: intervals.current, previous: intervals.previous), + revision: CodexModelsAnalyticsRevision(generatedAt: intervals.current.end, indexRevision: "fixture"), + legacy: CodexModelsLegacyBaseline( + totalTokens: 10, + modelIDs: ["model-a"], + currentModels: [CodexModelsLegacyModelBaseline( + modelID: "model-a", + totalTokens: 11, + knownCost: Decimal(string: "0.3"), + pricedTokens: 10, + unpricedTokens: 0, + sessionReferences: 1)], + previousModels: [CodexModelsLegacyModelBaseline( + modelID: "model-a", + totalTokens: 5, + knownCost: Decimal(string: "0.2"), + pricedTokens: 0, + unpricedTokens: 5, + sessionReferences: 2)]))) + + #expect(snapshot.diagnostics.mismatchDimensions == [ + .modelTokens, + .modelKnownCost, + .modelPricingCoverage, + .modelSessionReferences, + ]) + } + + @Test + func `per model parity canonicalizes aliases and preserves unavailable pricing`() { + let intervals = self.intervals(days: 7) + let current = [ + self.fragment( + day: intervals.current.start, + model: "GPT-5", + input: 6, + output: 0, + session: "same", + costNanos: nil), + self.fragment( + day: intervals.current.start, + model: "gpt-5", + input: 4, + output: 0, + session: "same", + costNanos: nil), + ] + let previous = [self.fragment( + day: intervals.previous.start, + model: "OpenAI/GPT-5", + input: 5, + output: 0, + session: "earlier", + costNanos: nil)] + let snapshot = CodexModelsAnalyticsBuilder().build(CodexModelsAnalyticsRequest( + source: CodexModelsAnalyticsSource(current: current, previous: previous), + scopeID: nil, + periods: CodexModelsAnalyticsPeriods(current: intervals.current, previous: intervals.previous), + revision: CodexModelsAnalyticsRevision(generatedAt: intervals.current.end, indexRevision: "fixture"), + legacy: CodexModelsLegacyBaseline( + totalTokens: 10, + modelIDs: ["gpt-5"], + currentModels: [CodexModelsLegacyModelBaseline( + modelID: "gpt-5", + totalTokens: 10, + pricedTokens: 0, + unpricedTokens: 10, + sessionReferences: 1)], + previousModels: [CodexModelsLegacyModelBaseline( + modelID: "gpt-5", + totalTokens: 5, + pricedTokens: 0, + unpricedTokens: 5, + sessionReferences: 1)]))) + + #expect(snapshot.diagnostics.isMatched) + #expect(snapshot.rows[0].sessionReferences == 1) + #expect(snapshot.rows[0].cost.pricedTokens == 0) + #expect(snapshot.rows[0].cost.unpricedTokens == 10) + } + + @Test + func `per model parity skips periods whose source coverage is incomplete`() { + let intervals = self.intervals(days: 7) + let current = [self.fragment(day: intervals.current.start, model: "model-a", input: 10, output: 0)] + let snapshot = CodexModelsAnalyticsBuilder().build(CodexModelsAnalyticsRequest( + source: CodexModelsAnalyticsSource(current: current, previous: []), + scopeID: nil, + periods: CodexModelsAnalyticsPeriods(current: intervals.current, previous: intervals.previous), + revision: CodexModelsAnalyticsRevision(generatedAt: intervals.current.end, indexRevision: "fixture"), + legacy: CodexModelsLegacyBaseline( + totalTokens: 10, + modelIDs: ["model-a"], + currentModels: [CodexModelsLegacyModelBaseline( + modelID: "model-a", + totalTokens: 99, + knownCost: 99, + pricedTokens: 0, + unpricedTokens: 99, + sessionReferences: 99)]), + currentIsComplete: false, + previousIsComplete: true)) + + #expect(snapshot.diagnostics.isMatched) + } + + @Test + func `legacy event rows decode without new timestamp or pricing audit fields`() throws { + let data = Data(""" + { + "day": "2026-07-16", + "model": "model-a", + "turnID": "turn-1", + "eventIndex": 4, + "input": 10, + "cached": 3, + "output": 2 + } + """.utf8) + let row = try JSONDecoder().decode(CostUsageScanner.CodexUsageRow.self, from: data) + + #expect(row.rawModel == nil) + #expect(row.timestampUnixMs == nil) + #expect(row.knownCostNanos == nil) + #expect(row.unpricedTokens == nil) + #expect(row.input == 10) + } + + @Test + func `legacy analytics payload decodes without additive comparison and interval fields`() throws { + let intervals = self.intervals(days: 7) + let snapshot = self.build( + current: [self.fragment(day: intervals.current.start, model: "model-a", input: 10, output: 0)], + previous: [], + intervals: intervals, + legacyTotal: 10) + let encoded = try JSONEncoder().encode(snapshot) + var object = try #require(JSONSerialization.jsonObject(with: encoded) as? [String: Any]) + object.removeValue(forKey: "previousActiveModelCount") + object.removeValue(forKey: "currentIsComplete") + object.removeValue(forKey: "previousIsComplete") + object.removeValue(forKey: "newlyActiveModelCount") + object.removeValue(forKey: "previousSessionReferenceTotal") + object.removeValue(forKey: "sessionReferenceComparison") + object["rows"] = try (#require(object["rows"] as? [[String: Any]])).map { value in + var row = value + row.removeValue(forKey: "previousTotalTokens") + row.removeValue(forKey: "previousCost") + row.removeValue(forKey: "previousSessionReferences") + row.removeValue(forKey: "associatedSessionIDs") + return row + } + object["daily"] = try (#require(object["daily"] as? [[String: Any]])).map { value in + var bucket = value + bucket.removeValue(forKey: "interval") + bucket.removeValue(forKey: "sessionReferenceIDs") + return bucket + } + let legacyData = try JSONSerialization.data(withJSONObject: object) + let decoded = try JSONDecoder().decode(CodexModelsAnalyticsSnapshot.self, from: legacyData) + + #expect(decoded.previousActiveModelCount == nil) + #expect(decoded.currentIsComplete == nil) + #expect(decoded.previousIsComplete == nil) + #expect(decoded.newlyActiveModelCount == nil) + #expect(decoded.rows[0].previousTotalTokens == nil) + #expect(decoded.rows[0].associatedSessionIDs == nil) + #expect(decoded.daily[0].interval == nil) + #expect(decoded.daily[0].sessionReferenceIDs == decoded.daily[0].sessionIDs) + #expect(decoded.totalTokens == 10) + } + + @Test + func `feature flag defaults on and supports rollback`() throws { + let name = "CodexModelsAnalyticsTests.featureFlag.\(UUID().uuidString)" + let defaults = try #require(UserDefaults(suiteName: name)) + defer { defaults.removePersistentDomain(forName: name) } + + #expect(CodexModelsRollout.isEnabled(defaults: defaults)) + defaults.set(false, forKey: CodexModelsRollout.featureFlagKey) + #expect(!CodexModelsRollout.isEnabled(defaults: defaults)) + } + + private func build( + current: [CodexModelsUsageFragment], + previous: [CodexModelsUsageFragment], + intervals: (current: DateInterval, previous: DateInterval), + legacyTotal: Int64, + currentIsComplete: Bool = true, + previousIsComplete: Bool = true) -> CodexModelsAnalyticsSnapshot + { + CodexModelsAnalyticsBuilder().build(CodexModelsAnalyticsRequest( + source: CodexModelsAnalyticsSource(current: current, previous: previous), + scopeID: nil, + periods: CodexModelsAnalyticsPeriods(current: intervals.current, previous: intervals.previous), + revision: CodexModelsAnalyticsRevision(generatedAt: intervals.current.end, indexRevision: "fixture"), + legacy: CodexModelsLegacyBaseline( + totalTokens: legacyTotal, + modelIDs: Array(Set(current.map(\.rawModelID)))), + currentIsComplete: currentIsComplete, + previousIsComplete: previousIsComplete)) + } + + private func intervals(days: Int) -> (current: DateInterval, previous: DateInterval) { + let end = self.calendar.date(from: DateComponents(year: 2026, month: 7, day: 16))! + let currentStart = self.calendar.date(byAdding: .day, value: -days, to: end)! + let previousStart = self.calendar.date(byAdding: .day, value: -days, to: currentStart)! + return ( + DateInterval(start: currentStart, end: end), + DateInterval(start: previousStart, end: currentStart)) + } + + private func fragment( + day: Date, + timestamp: Date? = nil, + model: String, + input: Int64, + output: Int64, + session: String = "session", + costNanos: Int64? = 100_000_000) -> CodexModelsUsageFragment + { + CodexModelsUsageFragment( + workspaceID: "workspace", + sessionID: session, + day: day, + timestamp: timestamp, + rawModelID: model, + inputTokens: input, + cachedInputTokens: min(3, input), + outputTokens: output, + costNanos: costNanos) + } +} diff --git a/Tests/CodexBarTests/CodexModelsExportFormattingTests.swift b/Tests/CodexBarTests/CodexModelsExportFormattingTests.swift new file mode 100644 index 0000000000..60df959911 --- /dev/null +++ b/Tests/CodexBarTests/CodexModelsExportFormattingTests.swift @@ -0,0 +1,106 @@ +import Foundation +import Testing +@testable import CodexBar +@testable import CodexBarCore + +@Suite("Codex Models export and formatting") +struct CodexModelsExportFormattingTests { + private let calendar: Calendar = { + var calendar = Calendar(identifier: .gregorian) + calendar.locale = Locale(identifier: "en_US_POSIX") + calendar.timeZone = TimeZone(identifier: "America/Los_Angeles")! + return calendar + }() + + @Test + func `CSV export uses raw precision and explicit unknown cost`() { + let intervals = self.intervals(days: 7) + let current = [self.fragment( + day: intervals.current.start, + model: "model,quoted", + input: 7_100_000_000, + costNanos: nil)] + let snapshot = self.build(current: current, previous: [], intervals: intervals, legacyTotal: 7_100_000_000) + let csv = CodexModelsCSVExporter.export(snapshot: snapshot) + + #expect(csv.contains("7100000000")) + #expect(!csv.contains("7.1B")) + #expect(csv.contains(",0,7100000000,")) + #expect(csv.contains("\"model,quoted\"")) + #expect(csv.contains(",unavailable,")) + } + + @Test + func `CSV export keeps previous unavailable pricing blank and auditable`() throws { + let intervals = self.intervals(days: 7) + let snapshot = self.build( + current: [self.fragment( + day: intervals.current.start, + model: "model-a", + input: 10, + costNanos: 1_000_000_000)], + previous: [self.fragment( + day: intervals.previous.start, + model: "model-a", + input: 8, + costNanos: nil)], + intervals: intervals, + legacyTotal: 10) + + let lines = CodexModelsCSVExporter.export(snapshot: snapshot).split(separator: "\n") + let header = try #require(lines.first).split(separator: ",", omittingEmptySubsequences: false).map(String.init) + let values = try #require(lines.dropFirst().first) + .split(separator: ",", omittingEmptySubsequences: false) + .map(String.init) + let fields = Dictionary(uniqueKeysWithValues: zip(header, values)) + + #expect(fields["reasoning_tokens"]?.isEmpty == true) + #expect(fields["previous_known_cost"]?.isEmpty == true) + #expect(fields["previous_cost_status"] == "unavailable") + #expect(fields["previous_cost_coverage"] == "0") + #expect(fields["previous_priced_tokens"] == "0") + #expect(fields["previous_unpriced_tokens"] == "8") + } + + private func build( + current: [CodexModelsUsageFragment], + previous: [CodexModelsUsageFragment], + intervals: (current: DateInterval, previous: DateInterval), + legacyTotal: Int64) -> CodexModelsAnalyticsSnapshot + { + CodexModelsAnalyticsBuilder().build(CodexModelsAnalyticsRequest( + source: CodexModelsAnalyticsSource(current: current, previous: previous), + scopeID: nil, + periods: CodexModelsAnalyticsPeriods(current: intervals.current, previous: intervals.previous), + revision: CodexModelsAnalyticsRevision(generatedAt: intervals.current.end, indexRevision: "fixture"), + legacy: CodexModelsLegacyBaseline( + totalTokens: legacyTotal, + modelIDs: Array(Set(current.map(\.rawModelID)))))) + } + + private func intervals(days: Int) -> (current: DateInterval, previous: DateInterval) { + let end = self.calendar.date(from: DateComponents(year: 2026, month: 7, day: 16))! + let currentStart = self.calendar.date(byAdding: .day, value: -days, to: end)! + let previousStart = self.calendar.date(byAdding: .day, value: -days, to: currentStart)! + return ( + DateInterval(start: currentStart, end: end), + DateInterval(start: previousStart, end: currentStart)) + } + + private func fragment( + day: Date, + model: String, + input: Int64, + costNanos: Int64?) -> CodexModelsUsageFragment + { + CodexModelsUsageFragment( + workspaceID: "workspace", + sessionID: "session", + day: day, + rawModelID: model, + inputTokens: input, + cachedInputTokens: min(3, input), + outputTokens: 0, + costNanos: costNanos) + } +} diff --git a/Tests/CodexBarTests/CodexModelsPerformanceTests.swift b/Tests/CodexBarTests/CodexModelsPerformanceTests.swift new file mode 100644 index 0000000000..4aee7ba344 --- /dev/null +++ b/Tests/CodexBarTests/CodexModelsPerformanceTests.swift @@ -0,0 +1,156 @@ +import Foundation +import Testing +@testable import CodexBar +@testable import CodexBarCore + +@MainActor +@Suite(.serialized) +struct CodexModelsPerformanceTests { + @Test + func `workspace scale snapshot build stays within end to end budget`() throws { + let root = FileManager.default.temporaryDirectory + .appendingPathComponent("codexbar-workspace-performance-\(UUID().uuidString)", isDirectory: true) + defer { try? FileManager.default.removeItem(at: root) } + let sessionsRoot = root.appendingPathComponent("sessions", isDirectory: true) + let cacheRoot = root.appendingPathComponent("cache", isDirectory: true) + try FileManager.default.createDirectory(at: sessionsRoot, withIntermediateDirectories: true) + try FileManager.default.createDirectory(at: cacheRoot, withIntermediateDirectories: true) + + let options = CostUsageScanner.Options(codexSessionsRoot: sessionsRoot, cacheRoot: cacheRoot) + let end = Date(timeIntervalSince1970: 1_784_160_000) + let currentDay = end.addingTimeInterval(-24 * 60 * 60) + let previousDay = currentDay.addingTimeInterval(-30 * 24 * 60 * 60) + let currentDayKey = CostUsageScanner.CostUsageDayRange.dayKey(from: currentDay) + let previousDayKey = CostUsageScanner.CostUsageDayRange.dayKey(from: previousDay) + var cache = CostUsageCache() + cache.roots = CostUsageScanner.codexRootsFingerprint(options: options) + + for projectIndex in 0..<30 { + let project = root.appendingPathComponent("project-\(projectIndex)", isDirectory: true) + try FileManager.default.createDirectory( + at: project.appendingPathComponent(".git", isDirectory: true), + withIntermediateDirectories: true) + for sessionIndex in 0..<8 { + let sessionID = "project-\(projectIndex)-session-\(sessionIndex)" + cache.files[sessionsRoot.appendingPathComponent("\(sessionID).jsonl").path] = self.workspaceUsage( + sessionID: sessionID, + cwd: project.path, + currentDay: currentDay, + currentDayKey: currentDayKey, + previousDay: previousDay, + previousDayKey: previousDayKey, + seed: projectIndex * 8 + sessionIndex) + } + } + + func build() throws -> CodexLocalProjectUsageSnapshot { + try CodexLocalProjectUsageIndexer.buildSnapshotFromCostCache( + now: end, + historyDays: 30, + since: previousDay.addingTimeInterval(24 * 60 * 60), + until: end, + options: options, + cacheOverride: cache, + catalogOverride: .empty) + } + + _ = try build() + let clock = ContinuousClock() + var durations: [Duration] = [] + var snapshot: CodexLocalProjectUsageSnapshot? + for _ in 0..<3 { + let start = clock.now + snapshot = try build() + durations.append(start.duration(to: clock.now)) + } + + let measured = try #require(snapshot) + let median = durations.sorted()[1] + #expect(median < .seconds(1.5)) + #expect(measured.projects.count == 30) + #expect(measured.sessions.count == 240) + #expect((measured.total.totalTokens ?? 0) > 0) + #expect((measured.modelsAnalytics?.allWorkspaces.rows.count ?? 0) == 4) + #expect((measured.modelsAnalytics?.workspaces.count ?? 0) == 30) + #expect(measured.modelsAnalytics?.workspaces.values.allSatisfy { $0.rows.count == 4 } == true) + #expect(measured.modelsAnalytics?.allWorkspaces.comparison(.tokens) != .unavailable) + } + + // swiftlint:disable:next function_parameter_count + private func workspaceUsage( + sessionID: String, + cwd: String, + currentDay: Date, + currentDayKey: String, + previousDay: Date, + previousDayKey: String, + seed: Int) -> CostUsageFileUsage + { + let models = (0..<4).map { "model-\($0)" } + var days: [String: [String: [Int]]] = [:] + var rows: [CostUsageScanner.CodexUsageRow] = [] + for (index, model) in models.enumerated() { + let input = 100 + seed + index + let cached = input / 4 + let output = input / 5 + days[currentDayKey, default: [:]][model] = [input, cached, output] + days[previousDayKey, default: [:]][model] = [input - 1, cached, output] + rows.append(CostUsageScanner.CodexUsageRow( + day: currentDayKey, + model: model, + turnID: "current-\(index)", + eventIndex: index * 2, + timestampUnixMs: Int64(currentDay.timeIntervalSince1970 * 1000) + Int64(index), + input: input, + cached: cached, + output: output, + knownCostNanos: Int64(input * 1000), + unpricedTokens: 0, + pricingModel: model, + pricingMode: "standard")) + rows.append(CostUsageScanner.CodexUsageRow( + day: previousDayKey, + model: model, + turnID: "previous-\(index)", + eventIndex: index * 2 + 1, + timestampUnixMs: Int64(previousDay.timeIntervalSince1970 * 1000) + Int64(index), + input: input - 1, + cached: cached, + output: output, + knownCostNanos: Int64((input - 1) * 1000), + unpricedTokens: 0, + pricingModel: model, + pricingMode: "standard")) + } + return CostUsageFileUsage( + mtimeUnixMs: 1, + size: 1, + days: days, + parsedBytes: 1, + lastModel: models.last, + lastTotals: nil, + lastCountedTotals: nil, + lastRawTotalsBaseline: nil, + hasDivergentTotals: nil, + lastCodexTurnID: nil, + sessionId: sessionID, + forkedFromId: nil, + codexSession: CostUsageCodexSessionMetadata( + sessionId: sessionID, + forkedFromId: nil, + cwd: cwd, + title: nil, + startedAtUnixMs: Int64(previousDay.timeIntervalSince1970 * 1000), + latestActivityUnixMs: Int64(currentDay.timeIntervalSince1970 * 1000)), + codexCostNanos: nil, + codexPrioritySurchargeNanos: nil, + codexStandardCostNanos: nil, + codexPriorityCostNanos: nil, + codexStandardTokens: nil, + codexPriorityTokens: nil, + codexTurnIDs: nil, + codexRows: rows, + claudeRows: nil) + .refreshingCodexWorkspaceUsageFingerprint() + } +} diff --git a/Tests/CodexBarTests/CostUsageCacheTests.swift b/Tests/CodexBarTests/CostUsageCacheTests.swift index 5eee6fdc93..ed6fa68160 100644 --- a/Tests/CodexBarTests/CostUsageCacheTests.swift +++ b/Tests/CodexBarTests/CostUsageCacheTests.swift @@ -3,6 +3,33 @@ import Testing @testable import CodexBarCore struct CostUsageCacheTests { + @Test + func `legacy codex token cache decodes without reasoning while current rows round trip it`() throws { + let legacyTotals = try JSONDecoder().decode( + CostUsageCodexTotals.self, + from: Data(#"{"input":10,"cached":2,"output":4}"#.utf8)) + #expect(legacyTotals.reasoning == nil) + + let legacyRow = try JSONDecoder().decode( + CostUsageScanner.CodexUsageRow.self, + from: Data(#"{"day":"2026-07-17","model":"gpt-5.5","input":10,"cached":2,"output":4}"#.utf8)) + #expect(legacyRow.reasoning == nil) + + let currentRow = CostUsageScanner.CodexUsageRow( + day: "2026-07-17", + model: "gpt-5.5", + turnID: "turn", + eventIndex: 1, + input: 10, + cached: 2, + output: 4, + reasoning: 3) + let roundTripped = try JSONDecoder().decode( + CostUsageScanner.CodexUsageRow.self, + from: JSONEncoder().encode(currentRow)) + #expect(roundTripped.reasoning == 3) + } + @Test func `cache file URL uses provider artifact versions`() { let root = URL(fileURLWithPath: "/tmp/codexbar-cost-cache", isDirectory: true) @@ -11,7 +38,7 @@ struct CostUsageCacheTests { let claudeURL = CostUsageCacheIO.cacheFileURL(provider: .claude, cacheRoot: root) let vertexURL = CostUsageCacheIO.cacheFileURL(provider: .vertexai, cacheRoot: root) - #expect(codexURL.lastPathComponent == "codex-v10.json") + #expect(codexURL.lastPathComponent == "codex-v11.json") #expect(claudeURL.lastPathComponent == "claude-v5.json") #expect(vertexURL.lastPathComponent == "vertexai-v5.json") } diff --git a/Tests/CodexBarTests/CostUsageScannerBreakdownTests.swift b/Tests/CodexBarTests/CostUsageScannerBreakdownTests.swift index dd40efcbf7..547cf7fc0e 100644 --- a/Tests/CodexBarTests/CostUsageScannerBreakdownTests.swift +++ b/Tests/CodexBarTests/CostUsageScannerBreakdownTests.swift @@ -33,24 +33,30 @@ struct CostUsageScannerBreakdownTests { timestamp: String, model: String, total: Usage? = nil, - last: Usage? = nil) -> [String: Any] + last: Usage? = nil, + totalReasoning: Int? = nil, + lastReasoning: Int? = nil) -> [String: Any] { var info: [String: Any] = [ "model": model, ] if let total { - info["total_token_usage"] = [ + var usage: [String: Any] = [ "input_tokens": total.input, "cached_input_tokens": total.cached, "output_tokens": total.output, ] + usage["reasoning_output_tokens"] = totalReasoning + info["total_token_usage"] = usage } if let last { - info["last_token_usage"] = [ + var usage: [String: Any] = [ "input_tokens": last.input, "cached_input_tokens": last.cached, "output_tokens": last.output, ] + usage["reasoning_output_tokens"] = lastReasoning + info["last_token_usage"] = usage } return [ "type": "event_msg", @@ -1729,6 +1735,32 @@ struct CostUsageScannerBreakdownTests { #expect(parsed.days[dayKey]?[eventModel] == nil) } + @Test + func `codex foundation fallback preserves reasoning output tokens`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + let day = try env.makeLocalNoon(year: 2026, month: 5, day: 18) + let timestamp = env.isoString(for: day) + // Escaping the root type key bypasses the byte-fast parser. The literal nested marker + // keeps the line eligible for the Foundation fallback prefilter. + let line = #"{"\u0074ype":"event_msg","marker":{"type":"event_msg"},"timestamp":""# + + timestamp + + #"","payload":{"type":"token_count","info":{"model":"gpt-5.5","last_token_usage":{"# + + #""input_tokens":10,"cached_input_tokens":2,"output_tokens":7,"reasoning_output_tokens":4}}}}"# + let fileURL = try env.writeCodexSessionFile( + day: day, + filename: "foundation-reasoning.jsonl", + contents: line) + + let parsed = CostUsageScanner.parseCodexFile( + fileURL: fileURL, + range: CostUsageScanner.CostUsageDayRange(since: day, until: day)) + + #expect(parsed.rows.count == 1) + #expect(parsed.rows.first?.output == 7) + #expect(parsed.rows.first?.reasoning == 4) + } + @Test func `codex foundation fallback all blank context clears stale model`() throws { let env = try CostUsageTestEnvironment() @@ -1870,7 +1902,7 @@ struct CostUsageScannerBreakdownTests { #expect(first.data[0].totalTokens == 132) let newCacheURL = CostUsageCacheIO.cacheFileURL(provider: .codex, cacheRoot: env.cacheRoot) - #expect(newCacheURL.lastPathComponent == "codex-v10.json") + #expect(newCacheURL.lastPathComponent == "codex-v11.json") #expect(FileManager.default.fileExists(atPath: newCacheURL.path)) #expect(FileManager.default.fileExists(atPath: oldCacheURL.path)) @@ -2214,6 +2246,99 @@ struct CostUsageScannerBreakdownTests { #expect(parsed.hasInterleavedTotals) } + @Test + func `codex resolved fork subtracts inherited reasoning baseline`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + + let day = try env.makeLocalNoon(year: 2026, month: 6, day: 10) + let timestamp = env.isoString(for: day) + let model = "openai/gpt-5.5" + let fileURL = try env.writeCodexSessionFile( + day: day, + filename: "resolved-fork-reasoning.jsonl", + contents: env.jsonl([ + [ + "type": "session_meta", + "timestamp": timestamp, + "payload": [ + "id": "child-session", + "forked_from_id": "parent-session", + "timestamp": timestamp, + ], + ], + self.codexTurnContext(timestamp: timestamp, model: model), + self.codexTokenCount( + timestamp: env.isoString(for: day.addingTimeInterval(1)), + model: model, + total: (input: 110, cached: 0, output: 60), + totalReasoning: 24), + ])) + + let parsed = CostUsageScanner.parseCodexFile( + fileURL: fileURL, + range: CostUsageScanner.CostUsageDayRange(since: day, until: day), + inheritedTotalsResolver: { parentSessionID, _ in + #expect(parentSessionID == "parent-session") + return .resolved(.init(input: 100, cached: 0, output: 50, reasoning: 20)) + }) + + #expect(parsed.rows.count == 1) + #expect(parsed.rows.first?.input == 10) + #expect(parsed.rows.first?.output == 10) + #expect(parsed.rows.first?.reasoning == 4) + } + + @Test + func `codex interleaved containment carries reasoning without adding it to output`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + + let day = try env.makeLocalNoon(year: 2026, month: 6, day: 10) + let model = "openai/gpt-5.5" + let fileURL = try env.writeCodexSessionFile( + day: day, + filename: "interleaved-reasoning.jsonl", + contents: env.jsonl([ + self.codexTurnContext(timestamp: env.isoString(for: day), model: model), + self.codexTokenCount( + timestamp: env.isoString(for: day.addingTimeInterval(1)), + model: model, + total: (input: 0, cached: 0, output: 100), + totalReasoning: 60), + self.codexTokenCount( + timestamp: env.isoString(for: day.addingTimeInterval(2)), + model: model, + total: (input: 0, cached: 0, output: 50), + totalReasoning: 30), + self.codexTokenCount( + timestamp: env.isoString(for: day.addingTimeInterval(3)), + model: model, + total: (input: 0, cached: 0, output: 105), + totalReasoning: 63), + self.codexTokenCount( + timestamp: env.isoString(for: day.addingTimeInterval(4)), + model: model, + total: (input: 0, cached: 0, output: 55), + totalReasoning: 33), + self.codexTokenCount( + timestamp: env.isoString(for: day.addingTimeInterval(5)), + model: model, + total: (input: 0, cached: 0, output: 110), + totalReasoning: 66), + ])) + + let parsed = CostUsageScanner.parseCodexFile( + fileURL: fileURL, + range: CostUsageScanner.CostUsageDayRange(since: day, until: day)) + + #expect(parsed.rows.map(\.output) == [100, 5, 5]) + #expect(parsed.rows.compactMap(\.reasoning) == [60, 3, 3]) + #expect(parsed.rows.reduce(0) { $0 + $1.output } == 110) + #expect(parsed.rows.compactMap(\.reasoning).reduce(0, +) == 66) + #expect(parsed.hasInterleavedTotals) + } + @Test func `codex alternating repeated snapshots count zero`() throws { let env = try CostUsageTestEnvironment() diff --git a/Tests/CodexBarTests/CostUsageScannerTests.swift b/Tests/CodexBarTests/CostUsageScannerTests.swift index 076452b8fc..3776baa84b 100644 --- a/Tests/CodexBarTests/CostUsageScannerTests.swift +++ b/Tests/CodexBarTests/CostUsageScannerTests.swift @@ -2,6 +2,7 @@ import Foundation import Testing @testable import CodexBarCore +// swiftlint:disable:next type_body_length struct CostUsageScannerTests { @Test func `codex session metadata skips an oversized line without retaining it`() throws { @@ -452,6 +453,7 @@ struct CostUsageScannerTests { "input_tokens": 100, "cached_input_tokens": 20, "output_tokens": 10, + "reasoning_output_tokens": 4, ], "model": model, ], @@ -469,6 +471,8 @@ struct CostUsageScannerTests { #expect(first.lastTotals?.input == 100) #expect(first.lastTotals?.cached == 20) #expect(first.lastTotals?.output == 10) + #expect(first.lastTotals?.reasoning == 4) + #expect(first.rows.first?.reasoning == 4) let secondTokenCount: [String: Any] = [ "type": "event_msg", @@ -480,6 +484,7 @@ struct CostUsageScannerTests { "input_tokens": 160, "cached_input_tokens": 40, "output_tokens": 16, + "reasoning_output_tokens": 7, ], "model": model, ], @@ -500,6 +505,7 @@ struct CostUsageScannerTests { #expect(packed[0] == 60) #expect(packed[1] == 20) #expect(packed[2] == 6) + #expect(delta.rows.first?.reasoning == 3) } @Test diff --git a/Tests/CodexBarTests/SpendDashboardControllerTests.swift b/Tests/CodexBarTests/SpendDashboardControllerTests.swift index e1259d5ff9..69aa8b4414 100644 --- a/Tests/CodexBarTests/SpendDashboardControllerTests.swift +++ b/Tests/CodexBarTests/SpendDashboardControllerTests.swift @@ -838,6 +838,7 @@ struct SpendDashboardControllerTests { sessionCostUSD: nil, last30DaysTokens: 10, last30DaysCostUSD: cost, + currencyCode: "USD", daily: [entry], updatedAt: Date(timeIntervalSince1970: 1_784_179_200)) return SpendDashboardModel.ProviderInput( diff --git a/Tests/CodexBarTests/SpendDashboardTokenProvenanceTests.swift b/Tests/CodexBarTests/SpendDashboardTokenProvenanceTests.swift index 23902d4d79..6fc55887a4 100644 --- a/Tests/CodexBarTests/SpendDashboardTokenProvenanceTests.swift +++ b/Tests/CodexBarTests/SpendDashboardTokenProvenanceTests.swift @@ -102,6 +102,7 @@ struct SpendDashboardTokenProvenanceTests { @Test func `cached token account activation does not prove a forced refresh`() async throws { + let now = Date(timeIntervalSince1970: 1_784_179_200) let (settings, store) = Self.makeStore(provider: .mistral) settings.addTokenAccount(provider: .mistral, label: "Fixture", token: "fixture") let account = try #require(settings.effectiveSelectedTokenAccount(for: .mistral)) @@ -127,8 +128,13 @@ struct SpendDashboardTokenProvenanceTests { let controller = SpendDashboardController( userDefaults: settings.userDefaults, requestBuilder: { mode in - await SpendDashboardSource.makeRequest(settings: settings, store: store, mode: mode) - }) + await SpendDashboardSource.makeRequest( + settings: settings, + store: store, + mode: mode, + now: now) + }, + nowProvider: { now }) controller.update(configuration: SpendDashboardSource.configuration(settings: settings, store: store)) await Self.waitUntil { !controller.isRefreshing } #expect(controller.model.groups.first?.totalCost == 3) @@ -143,6 +149,7 @@ struct SpendDashboardTokenProvenanceTests { @Test func `forced successful empty publication removes prior spend without warning`() async { + let now = Date(timeIntervalSince1970: 1_784_203_200) let (settings, store) = Self.makeStore(provider: .bedrock) var loadCount = 0 store._test_tokenUsageSnapshotLoaderOverride = { _, _, _, _, _ in @@ -153,8 +160,13 @@ struct SpendDashboardTokenProvenanceTests { let controller = SpendDashboardController( userDefaults: settings.userDefaults, requestBuilder: { mode in - await SpendDashboardSource.makeRequest(settings: settings, store: store, mode: mode) - }) + await SpendDashboardSource.makeRequest( + settings: settings, + store: store, + mode: mode, + now: now) + }, + nowProvider: { now }) controller.update(configuration: SpendDashboardSource.configuration(settings: settings, store: store)) await Self.waitUntil { !controller.isRefreshing } #expect(controller.model.groups.first?.totalCost == 4) @@ -173,6 +185,7 @@ struct SpendDashboardTokenProvenanceTests { @Test func `first open accepts current empty publication without redundant refresh`() async { + let now = Date(timeIntervalSince1970: 1_784_179_200) let (settings, store) = Self.makeStore(provider: .bedrock) var loadCount = 0 store._test_tokenUsageSnapshotLoaderOverride = { _, _, _, _, _ in @@ -184,8 +197,13 @@ struct SpendDashboardTokenProvenanceTests { let controller = SpendDashboardController( userDefaults: settings.userDefaults, requestBuilder: { mode in - await SpendDashboardSource.makeRequest(settings: settings, store: store, mode: mode) - }) + await SpendDashboardSource.makeRequest( + settings: settings, + store: store, + mode: mode, + now: now) + }, + nowProvider: { now }) controller.update(configuration: SpendDashboardSource.configuration(settings: settings, store: store)) await Self.waitUntil { !controller.isRefreshing } @@ -320,6 +338,7 @@ struct SpendDashboardTokenProvenanceTests { sessionCostUSD: cost, last30DaysTokens: 10, last30DaysCostUSD: cost, + currencyCode: "USD", daily: [CostUsageDailyReport.Entry( date: "2026-07-16", inputTokens: 4, diff --git a/docs/codex-workspaces.md b/docs/codex-workspaces.md new file mode 100644 index 0000000000..20e71f93d8 --- /dev/null +++ b/docs/codex-workspaces.md @@ -0,0 +1,100 @@ +# Codex Workspaces local index + +Codex Workspaces attributes the existing local Codex cost scan to projects, +sessions, models, and days. This foundation is local-only library behavior; it +does not add a remote API, provider authentication flow, billing interface, or +public CLI JSON contract. + +## Data flow + +`CostUsageScanner` remains authoritative for JSONL parsing, cumulative-token +deltas, fork and subagent accounting, pricing, and incremental cursors. The +Workspaces index combines that scan cache with the read-only Codex thread +catalog: + +1. Scan local rollout JSONL into the Codex v11 cost cache. +2. Read catalog metadata without modifying the Codex catalog. +3. Canonicalize workspace attribution and scope it to the selected Codex home. +4. Publish the complete source state and derived snapshot in one SQLite + transaction. +5. Expose project, session, model, daily, source-status, progress, and CSV + library models to presentation consumers. + +The supported internal presentation boundary is: + +- `CostUsageFetcher.loadCachedCodexLocalProjectUsageSnapshot` +- `CostUsageFetcher.loadCodexLocalProjectUsageSnapshot` +- `CostUsageFetcher.clearCachedCodexLocalProjectUsageSnapshot` +- `CodexLocalProjectUsageSnapshot` +- `CodexLocalProjectUsageIndexProgress` + +## Persistence contracts + +The Codex cost cache is: + +```text +~/Library/Caches/CodexBar/cost-usage/codex-v11.json +``` + +The Workspaces sidecar is: + +```text +~/Library/Caches/CodexBar/local-usage/codex-workspaces-v1.sqlite +``` + +The sidecar currently uses SQLite schema version 5 and snapshot payload format +3. This is the first released Workspaces schema. A database with any other +`PRAGMA user_version` is rejected as incompatible and is never modified. + +### v10 to v11 + +`codex-v10.json` is not migrated in place and is not accepted as a v11 +incremental cursor. Parser and attribution semantics changed, so the first scan +after upgrading performs a one-time rebuild and writes a separate v11 artifact. +The v10 file remains untouched and recoverable. The v11 file is replaced +atomically only after a successful scan; later v11 refreshes can use the normal +incremental cursor. + +### Sidecar rollback + +Publication is transactional: a failed synchronization or snapshot write rolls +back and leaves the previous complete snapshot available. + +## Catalog completeness and last-good data + +Catalog access distinguishes complete, missing, locked, corrupt, and +incompatible states. A complete read replaces catalog metadata for that Codex +home scope. A later same-scope incomplete read reports partial or stale source +status while retaining the last-good catalog attribution and usage snapshot. +Sparse rollout updates do not erase retained titles, workspace paths, or other +catalog metadata. + +Scope identifiers and invalidation fingerprints are hashes; raw Codex-home +paths are not persisted as scope identifiers. Changed and deleted rollout files, +parser or pricing changes, history-window changes, and catalog changes +invalidate only the affected cached state. + +## Cost and display semantics + +Known model cost and unknown-cost coverage are represented separately. Unknown +pricing never becomes a synthetic zero-cost claim. Models analytics, parity +checks, performance telemetry, and CSV serialization operate on the same +persisted snapshot. + +Display-only projections are transient: + +- Hiding estimated cost does not rewrite the sidecar. +- Including or excluding cached input does not rescan or rewrite the sidecar. +- Hiding personal information removes persisted workspace paths, working + directories, workspace names, and session titles from snapshots before they + reach presentation code; it does not rewrite the local sidecar. +- Rankings, totals, charts, and breakdowns must use the same projection. + +## Privacy boundary + +The scanner, catalog reader, cache, sidecar, analytics, and CSV serializer run +locally. They do not upload rollout contents or catalog records. The stored +index can contain local workspace paths, session metadata, token totals, and +derived cost estimates, so runtime evidence must be sanitized before +publication. Remove paths, titles, session IDs, identities, tokens, keys, +network addresses, and database row contents from shared logs and screenshots. diff --git a/docs/codex.md b/docs/codex.md index f0a3dab926..bf75aea5d1 100644 --- a/docs/codex.md +++ b/docs/codex.md @@ -167,7 +167,7 @@ Example: - Native conversation rows reuse the corrected cached per-file totals and existing pricing tables. They are hidden when pi-compatible usage joins the aggregate because the native-only rows would not reconcile with the merged total. - Cache: - - Native + merged provider cache: `~/Library/Caches/CodexBar/cost-usage/codex-v10.json` + - Native + merged provider cache: `~/Library/Caches/CodexBar/cost-usage/codex-v11.json` - pi-compatible session cache: `~/Library/Caches/CodexBar/cost-usage/pi-sessions-v7.json` - Window: configurable 1-365 day rolling history, with a 60s minimum refresh interval.