From c72ce7169862ded578d760b97b9cb99cb0c45506 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 10 Jul 2026 10:29:41 +0100 Subject: [PATCH] fix: stabilize GPT-5.6 catalog pricing --- .../Generated/CodexParserHash.generated.swift | 2 +- .../CodexBarCore/PiSessionCostScanner.swift | 3 +- .../Vendored/CostUsage/CostUsagePricing.swift | 9 +- .../CostUsage/CostUsagePricingKey.swift | 22 +- .../CodexBarTests/CostUsagePricingTests.swift | 101 +++++++++ .../PiSessionCostScannerTests.swift | 195 +++++++++++++++++- 6 files changed, 320 insertions(+), 12 deletions(-) diff --git a/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift b/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift index e304ea6f74..800493ebeb 100644 --- a/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift +++ b/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift @@ -1,5 +1,5 @@ // Generated by Scripts/regenerate-codex-parser-hash.sh. Do not edit by hand. enum CodexParserHash { - static let value = "7985c9336eb88ef7" + static let value = "ed9cd73bd15c2398" } diff --git a/Sources/CodexBarCore/PiSessionCostScanner.swift b/Sources/CodexBarCore/PiSessionCostScanner.swift index ccd3361190..27b356892b 100644 --- a/Sources/CodexBarCore/PiSessionCostScanner.swift +++ b/Sources/CodexBarCore/PiSessionCostScanner.swift @@ -211,7 +211,8 @@ enum PiSessionCostScanner { pricingKey: CostUsagePricingKey.codex( modelsDevArtifact: modelsDevArtifact, formulaVersion: Self.costFormulaVersion, - parserHash: CodexParserHash.value)) + parserHash: CodexParserHash.value, + modelsDevProviderIDs: ["anthropic", "openai"])) } private static func requestedWindowExpandsCache( diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsagePricing.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsagePricing.swift index 08742e6067..b4d364d7f8 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsagePricing.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsagePricing.swift @@ -523,12 +523,17 @@ enum CostUsagePricing { modelsDevCacheRoot: URL? = nil) -> Double? { let key = self.normalizeCodexModel(model) - if let lookup = self.modelsDevLookup( + let modelsDevLookup = self.modelsDevLookup( providerID: self.codexModelsDevProviderID, model: model, catalog: modelsDevCatalog, cacheRoot: modelsDevCacheRoot) - { + ?? (model == key ? nil : self.modelsDevLookup( + providerID: self.codexModelsDevProviderID, + model: key, + catalog: modelsDevCatalog, + cacheRoot: modelsDevCacheRoot)) + if let lookup = modelsDevLookup { let bundled = self.codex[key] // A missing catalog context block means models.dev has no long-context opinion, so use // the bundled tuple. Once the block exists, preserve its omissions and normal fallback diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsagePricingKey.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsagePricingKey.swift index 6e109d7fd5..1e3e8d565e 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsagePricingKey.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsagePricingKey.swift @@ -9,7 +9,8 @@ enum CostUsagePricingKey { static func codex( modelsDevArtifact: ModelsDevCacheArtifact?, formulaVersion: Int, - parserHash: String? = nil) -> String + parserHash: String? = nil, + modelsDevProviderIDs: Set = ["openai"]) -> String { var parts = [ "costFormulaVersion=\(formulaVersion)", @@ -22,7 +23,10 @@ enum CostUsagePricingKey { let prefix: String if let modelsDevArtifact { prefix = "models-dev-v\(modelsDevArtifact.version)" - parts.append("modelsDevPricing:\n\(self.modelsDevPricingFingerprint(modelsDevArtifact.catalog))") + let modelsDevPricing = self.modelsDevPricingFingerprint( + modelsDevArtifact.catalog, + providerIDs: modelsDevProviderIDs) + parts.append("modelsDevPricing:\n\(modelsDevPricing)") } else { prefix = "builtin" parts.append("modelsDevPricing:none") @@ -30,27 +34,31 @@ enum CostUsagePricingKey { return "\(prefix)-\(self.sha256Hex(Data(parts.joined(separator: "\n").utf8)))" } - private static func modelsDevPricingFingerprint(_ catalog: ModelsDevCatalog) -> String { + private static func modelsDevPricingFingerprint( + _ catalog: ModelsDevCatalog, + providerIDs: Set) -> String + { var parts: [String] = [] - for providerID in catalog.providers.keys.sorted() { + let normalizedProviderIDs = Set(providerIDs.map(ModelsDevProvider.normalizeProviderID)) + for providerID in normalizedProviderIDs.sorted() { guard let provider = catalog.providers[providerID] else { continue } - parts.append("provider=\(providerID)|\(provider.id ?? "")") for modelKey in provider.models.keys.sorted() { - guard let model = provider.models[modelKey] else { continue } + guard let model = provider.models[modelKey], model.isPriceable else { continue } let cost = model.cost let contextOver200K = cost?.contextOver200K parts.append([ + "provider=\(providerID)", "model=\(modelKey)", model.id, self.optionalDoubleFingerprint(cost?.input), self.optionalDoubleFingerprint(cost?.output), self.optionalDoubleFingerprint(cost?.cacheRead), self.optionalDoubleFingerprint(cost?.cacheWrite), + contextOver200K == nil ? "contextOver200K=absent" : "contextOver200K=present", self.optionalDoubleFingerprint(contextOver200K?.input), self.optionalDoubleFingerprint(contextOver200K?.output), self.optionalDoubleFingerprint(contextOver200K?.cacheRead), self.optionalDoubleFingerprint(contextOver200K?.cacheWrite), - model.limit?.context.map(String.init) ?? "nil", ].joined(separator: "|")) } } diff --git a/Tests/CodexBarTests/CostUsagePricingTests.swift b/Tests/CodexBarTests/CostUsagePricingTests.swift index 7f9c3369e8..3298fb3bf6 100644 --- a/Tests/CodexBarTests/CostUsagePricingTests.swift +++ b/Tests/CodexBarTests/CostUsagePricingTests.swift @@ -116,6 +116,99 @@ struct CostUsagePricingTests { #expect(alias == sol) } + @Test + func `codex models dev falls back from gpt56 alias to canonical sol pricing`() throws { + let canonicalOnlyRoot = try Self.seedModelsDevCache(""" + { + "openai": { + "id": "openai", + "models": { + "gpt-5.6-sol": { + "id": "gpt-5.6-sol", + "cost": { "input": 7, "output": 31 } + } + } + } + } + """) + let aliasAndCanonicalRoot = try Self.seedModelsDevCache(""" + { + "openai": { + "id": "openai", + "models": { + "gpt-5.6": { + "id": "gpt-5.6", + "cost": { "input": 3, "output": 13 } + }, + "gpt-5.6-sol": { + "id": "gpt-5.6-sol", + "cost": { "input": 7, "output": 31 } + } + } + } + } + """) + + let canonicalFallback = CostUsagePricing.codexCostUSD( + model: "gpt-5.6", + inputTokens: 100, + cachedInputTokens: 0, + outputTokens: 0, + modelsDevCacheRoot: canonicalOnlyRoot) + let exactAlias = CostUsagePricing.codexCostUSD( + model: "gpt-5.6", + inputTokens: 100, + cachedInputTokens: 0, + outputTokens: 0, + modelsDevCacheRoot: aliasAndCanonicalRoot) + + #expect(canonicalFallback == 100.0 * 7e-6) + #expect(exactAlias == 100.0 * 3e-6) + } + + @Test + func `codex pricing key distinguishes an empty long context block from no block`() throws { + let withoutLongContext = try Self.modelsDevArtifact(""" + { + "openai": { + "id": "openai", + "models": { + "gpt-5.6-sol": { + "id": "gpt-5.6-sol", + "cost": { "input": 5, "output": 30 } + } + } + } + } + """) + let withEmptyLongContext = try Self.modelsDevArtifact(""" + { + "openai": { + "id": "openai", + "models": { + "gpt-5.6-sol": { + "id": "gpt-5.6-sol", + "cost": { + "input": 5, + "output": 30, + "context_over_200k": {} + } + } + } + } + } + """) + + let withoutKey = CostUsagePricingKey.codex( + modelsDevArtifact: withoutLongContext, + formulaVersion: 1) + let withEmptyKey = CostUsagePricingKey.codex( + modelsDevArtifact: withEmptyLongContext, + formulaVersion: 1) + + #expect(withoutKey != withEmptyKey) + } + @Test func `codex cost applies gpt56 long context rates`() throws { let root = try Self.cacheRoot() @@ -979,6 +1072,14 @@ extension CostUsagePricingTests { return root } + private static func modelsDevArtifact(_ json: String) throws -> ModelsDevCacheArtifact { + let catalog = try JSONDecoder().decode(ModelsDevCatalog.self, from: Data(json.utf8)) + return ModelsDevCacheArtifact( + version: ModelsDevCache.artifactVersion, + fetchedAt: Date(timeIntervalSince1970: 0), + catalog: catalog) + } + private static func cacheRoot() throws -> URL { let root = FileManager.default.temporaryDirectory .appendingPathComponent("codexbar-pricing-tests-\(UUID().uuidString)", isDirectory: true) diff --git a/Tests/CodexBarTests/PiSessionCostScannerTests.swift b/Tests/CodexBarTests/PiSessionCostScannerTests.swift index 2102bd84f6..e451cca1db 100644 --- a/Tests/CodexBarTests/PiSessionCostScannerTests.swift +++ b/Tests/CodexBarTests/PiSessionCostScannerTests.swift @@ -880,6 +880,78 @@ extension PiSessionCostScannerTests { #expect(abs((secondReport.data.first?.costUSD ?? 0) - 2.4) < 0.0000001) } + @Test + func `pi scanner reprices unchanged claude files when anthropic rates change`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + + let day = try env.makeLocalNoon(year: 2026, month: 7, day: 10) + let model = "claude-fable-5" + func assistant(at timestamp: Date) -> [String: Any] { + [ + "type": "message", + "timestamp": env.isoString(for: timestamp), + "message": [ + "role": "assistant", + "provider": "anthropic", + "model": model, + "timestamp": Int(timestamp.timeIntervalSince1970 * 1000), + "usage": [ + "input": 150_000, + "output": 0, + "totalTokens": 150_000, + ], + ], + ] + } + _ = try env.writePiSessionFile( + relativePath: "2026-07-10T10-00-00-000Z_anthropic-catalog-change.jsonl", + contents: env.jsonl([ + assistant(at: day.addingTimeInterval(-1)), + assistant(at: day), + ])) + + let firstCatalog = try Self.anthropicModelsDevCatalog(inputCostPerMillion: 4) + #expect(ModelsDevCache.save(catalog: firstCatalog, fetchedAt: day, cacheRoot: env.cacheRoot)) + let options = PiSessionCostScanner.Options( + piSessionsRoot: env.piSessionsRoot, + cacheRoot: env.cacheRoot, + refreshMinIntervalSeconds: 3600) + let firstReport = PiSessionCostScanner.loadDailyReport( + provider: .claude, + since: day, + until: day, + now: day, + options: options) + let firstCache = PiSessionCostCacheIO.load(cacheRoot: env.cacheRoot) + let firstPricingKey = try #require(firstCache.pricingKey) + #expect(firstReport.data.first?.totalTokens == 300_000) + #expect(abs((firstReport.data.first?.costUSD ?? 0) - 1.2) < 0.0000001) + + let secondCatalog = try Self.anthropicModelsDevCatalog(inputCostPerMillion: 8) + #expect(ModelsDevCache.save( + catalog: secondCatalog, + fetchedAt: day.addingTimeInterval(1), + cacheRoot: env.cacheRoot)) + #expect(PiSessionCostScanner.loadCachedDailyReport( + provider: .claude, + since: day, + until: day, + now: day.addingTimeInterval(1), + cacheRoot: env.cacheRoot) == nil) + + let secondReport = PiSessionCostScanner.loadDailyReport( + provider: .claude, + since: day, + until: day, + now: day.addingTimeInterval(2), + options: options) + let secondCache = PiSessionCostCacheIO.load(cacheRoot: env.cacheRoot) + #expect(secondCache.pricingKey != firstPricingKey) + #expect(secondReport.data.first?.totalTokens == 300_000) + #expect(abs((secondReport.data.first?.costUSD ?? 0) - 2.4) < 0.0000001) + } + @Test func `pi pricing key ignores catalog fetch time when rates are unchanged`() throws { let env = try CostUsageTestEnvironment() @@ -930,6 +1002,101 @@ extension PiSessionCostScannerTests { #expect(secondCache.lastScanUnixMs == firstCache.lastScanUnixMs) } + @Test + func `pi pricing key ignores unrelated providers and non pricing context metadata`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + + let day = try env.makeLocalNoon(year: 2026, month: 7, day: 10) + let firstCatalog = try Self.modelsDevCatalog(""" + { + "openai": { + "id": "openai", + "models": { + "gpt-5.6-sol": { + "id": "gpt-5.6-sol", + "limit": { "context": 1000000 }, + "cost": { "input": 4, "output": 30 } + } + } + }, + "google": { + "id": "google", + "models": { + "gemini-test": { + "id": "gemini-test", + "cost": { "input": 1, "output": 2 } + } + } + } + } + """) + #expect(ModelsDevCache.save(catalog: firstCatalog, fetchedAt: day, cacheRoot: env.cacheRoot)) + let assistant: [String: Any] = [ + "type": "message", + "timestamp": env.isoString(for: day), + "message": [ + "role": "assistant", + "provider": "openai-codex", + "model": "gpt-5.6-sol", + "timestamp": Int(day.timeIntervalSince1970 * 1000), + "usage": ["input": 100, "output": 0, "totalTokens": 100], + ], + ] + _ = try env.writePiSessionFile( + relativePath: "2026-07-10T10-00-00-000Z_catalog-metadata.jsonl", + contents: env.jsonl([assistant])) + let options = PiSessionCostScanner.Options( + piSessionsRoot: env.piSessionsRoot, + cacheRoot: env.cacheRoot, + refreshMinIntervalSeconds: 3600) + _ = PiSessionCostScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day, + options: options) + let firstCache = PiSessionCostCacheIO.load(cacheRoot: env.cacheRoot) + + let metadataOnlyChange = try Self.modelsDevCatalog(""" + { + "openai": { + "id": "openai", + "models": { + "gpt-5.6-sol": { + "id": "gpt-5.6-sol", + "limit": { "context": 2000000 }, + "cost": { "input": 4, "output": 30 } + } + } + }, + "google": { + "id": "google", + "models": { + "gemini-test": { + "id": "gemini-test", + "cost": { "input": 99, "output": 199 } + } + } + } + } + """) + #expect(ModelsDevCache.save( + catalog: metadataOnlyChange, + fetchedAt: day.addingTimeInterval(1), + cacheRoot: env.cacheRoot)) + _ = PiSessionCostScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day.addingTimeInterval(2), + options: options) + let secondCache = PiSessionCostCacheIO.load(cacheRoot: env.cacheRoot) + + #expect(secondCache.pricingKey == firstCache.pricingKey) + #expect(secondCache.lastScanUnixMs == firstCache.lastScanUnixMs) + } + @Test func `pi scanner reparses unchanged cached file when scan window expands`() throws { let env = try CostUsageTestEnvironment() @@ -1014,6 +1181,32 @@ extension PiSessionCostScannerTests { } } """ - return try JSONDecoder().decode(ModelsDevCatalog.self, from: Data(json.utf8)) + return try self.modelsDevCatalog(json) + } + + private static func anthropicModelsDevCatalog(inputCostPerMillion: Double) throws -> ModelsDevCatalog { + let json = """ + { + "anthropic": { + "id": "anthropic", + "models": { + "claude-fable-5": { + "id": "claude-fable-5", + "cost": { + "input": \(inputCostPerMillion), + "output": 15, + "cache_read": 0.3, + "cache_write": 3.75 + } + } + } + } + } + """ + return try self.modelsDevCatalog(json) + } + + private static func modelsDevCatalog(_ json: String) throws -> ModelsDevCatalog { + try JSONDecoder().decode(ModelsDevCatalog.self, from: Data(json.utf8)) } }