From fa99ed60110d1527db1b890b60da3f53b2cc453c Mon Sep 17 00:00:00 2001 From: Leo Date: Thu, 10 Sep 2026 14:19:47 -0400 Subject: [PATCH 1/2] fix(server): stop overpricing cached Claude tokens LiteLLM publishes canonical and provider-qualified rates for the same bare model name. Stripping prefixes before insert let a later entry overwrite the Anthropic cache discount and price cache reads as ordinary input. Keep each rate under its full key. Alias a bare name only when every qualified entry has the same effective rate. Upstream: pingdotgg/t3code#8806 Grok 4.6 High in Grok Build via Orca. --- apps/server/src/usage/usagePricing.test.ts | 55 +++++++++++++++++++ apps/server/src/usage/usagePricing.ts | 58 ++++++++++++++++---- apps/server/src/usage/usageScanCache.test.ts | 2 +- apps/server/src/usage/usageScanCache.ts | 2 +- 4 files changed, 105 insertions(+), 12 deletions(-) create mode 100644 apps/server/src/usage/usagePricing.test.ts diff --git a/apps/server/src/usage/usagePricing.test.ts b/apps/server/src/usage/usagePricing.test.ts new file mode 100644 index 0000000000000..2ea27375b1483 --- /dev/null +++ b/apps/server/src/usage/usagePricing.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it } from "@effect/vitest"; + +import { lookupRate, normalizeModelName, parseRateTable } from "./usagePricing.ts"; + +const rate = (input: number, cacheRead?: number) => ({ + input_cost_per_token: input, + output_cost_per_token: input * 5, + ...(cacheRead === undefined ? {} : { cache_read_input_token_cost: cacheRead }), +}); + +describe("usage pricing", () => { + it("keeps the existing model-name normalization contract", () => { + expect(normalizeModelName(" Anthropic/Claude-Opus-5 ")).toBe("claude-opus-5"); + }); + + it("keeps the canonical Fable rate separate from DeepInfra in either order", () => { + const canonical = ["claude-fable-5", rate(1e-5, 1e-6)] as const; + const deepInfra = ["deepinfra/anthropic/claude-fable-5", rate(1e-5)] as const; + + for (const entries of [ + [canonical, deepInfra], + [deepInfra, canonical], + ]) { + const table = parseRateTable(Object.fromEntries(entries)); + + expect(lookupRate(table, "claude-fable-5")?.cacheReadCostPerToken).toBe(1e-6); + expect(lookupRate(table, "deepinfra/anthropic/claude-fable-5")?.cacheReadCostPerToken).toBe( + 1e-5, + ); + expect(lookupRate(table, "other/claude-fable-5")).toBeNull(); + } + }); + + it("adds a bare alias when every qualified entry has the same rate", () => { + const table = parseRateTable({ + "provider-a/example-model": rate(1), + "provider-b/example-model": rate(1), + }); + + expect(lookupRate(table, "example-model")).toEqual( + lookupRate(table, "provider-a/example-model"), + ); + }); + + it("leaves an ambiguous bare name unpriced", () => { + const table = parseRateTable({ + "provider-a/example-model": rate(1), + "provider-b/example-model": rate(3), + }); + + expect(lookupRate(table, "provider-a/example-model")?.inputCostPerToken).toBe(1); + expect(lookupRate(table, "provider-b/example-model")?.inputCostPerToken).toBe(3); + expect(lookupRate(table, "example-model")).toBeNull(); + }); +}); diff --git a/apps/server/src/usage/usagePricing.ts b/apps/server/src/usage/usagePricing.ts index f0e59a8743994..3d7f5fd294859 100644 --- a/apps/server/src/usage/usagePricing.ts +++ b/apps/server/src/usage/usagePricing.ts @@ -44,6 +44,9 @@ function finiteNumber(value: unknown): number | null { * Entries without both an input and an output rate are dropped: a half-priced * model would silently under-report cost, which is worse than reporting the * model as unpriced. + * + * Entries keep their full normalized key; a bare name is aliased only when no + * canonical entry exists and every qualified entry has the same rate. */ export function parseRateTable(document: unknown): RateTable { const table = new Map(); @@ -56,7 +59,9 @@ export function parseRateTable(document: unknown): RateTable { const output = finiteNumber(entry.output_cost_per_token); if (input === null || output === null) continue; - table.set(normalizeModelName(name), { + const key = normalizeRateKey(name); + if (key.length === 0) continue; + table.set(key, { inputCostPerToken: input, outputCostPerToken: output, // Anthropic bills cache reads at a discount and cache writes at a @@ -66,20 +71,52 @@ export function parseRateTable(document: unknown): RateTable { cacheCreationCostPerToken: finiteNumber(entry.cache_creation_input_token_cost) ?? input, }); } + + // `null` marks a bare name claimed at conflicting rates: no alias for it. + const aliasCandidates = new Map(); + for (const [key, rate] of table) { + const alias = bareModelName(key); + if (alias.length === 0 || alias === key || table.has(alias)) continue; + const held = aliasCandidates.get(alias); + if (held === undefined) { + aliasCandidates.set(alias, rate); + } else if (held !== null && !sameRate(held, rate)) { + aliasCandidates.set(alias, null); + } + } + for (const [alias, rate] of aliasCandidates) { + if (rate !== null) table.set(alias, rate); + } + return table; } +function sameRate(a: ModelRate, b: ModelRate): boolean { + return ( + a.inputCostPerToken === b.inputCostPerToken && + a.outputCostPerToken === b.outputCostPerToken && + a.cacheReadCostPerToken === b.cacheReadCostPerToken && + a.cacheCreationCostPerToken === b.cacheCreationCostPerToken + ); +} + +function normalizeRateKey(model: string): string { + return model.trim().toLowerCase(); +} + /** * Canonicalises a model name for lookup. * - * Strips a `provider/` prefix (LiteLLM publishes both `claude-opus-5` and - * `anthropic/claude-opus-5`) and lowercases, since transcripts are inconsistent - * about casing. + * Strips a `provider/` prefix and lowercases, since transcripts are + * inconsistent about casing. */ export function normalizeModelName(model: string): string { - const trimmed = model.trim().toLowerCase(); - const slash = trimmed.lastIndexOf("/"); - return slash === -1 ? trimmed : trimmed.slice(slash + 1); + return bareModelName(normalizeRateKey(model)); +} + +function bareModelName(key: string): string { + const slash = key.lastIndexOf("/"); + return slash === -1 ? key : key.slice(slash + 1); } /** @@ -99,9 +136,10 @@ const UNPRICEABLE_MODELS = new Set([ ]); export function lookupRate(table: RateTable, model: string): ModelRate | null { - const normalized = normalizeModelName(model); - if (normalized.length === 0 || UNPRICEABLE_MODELS.has(normalized)) return null; - return table.get(normalized) ?? null; + const key = normalizeRateKey(model); + const bareName = bareModelName(key); + if (bareName.length === 0 || UNPRICEABLE_MODELS.has(bareName)) return null; + return table.get(key) ?? null; } export interface PricedUsage { diff --git a/apps/server/src/usage/usageScanCache.test.ts b/apps/server/src/usage/usageScanCache.test.ts index 64673e96c0908..645fa68f83b4a 100644 --- a/apps/server/src/usage/usageScanCache.test.ts +++ b/apps/server/src/usage/usageScanCache.test.ts @@ -79,7 +79,7 @@ describe("scan cache round trip", () => { it("rejects the whole cache when an intern table holds a non-string", () => { // models: [1] would pass the undefined guard, put a number in a record's - // model, and crash normalizeModelName at aggregate time. + // model, and crash lookupRate at aggregate time. const encoded = encodeScanCache(cacheWith([["/a.jsonl", 100, [record()]]])); const poisoned = { ...encoded, models: [1] }; diff --git a/apps/server/src/usage/usageScanCache.ts b/apps/server/src/usage/usageScanCache.ts index cc15ee9cee622..4ba20ce5d633b 100644 --- a/apps/server/src/usage/usageScanCache.ts +++ b/apps/server/src/usage/usageScanCache.ts @@ -124,7 +124,7 @@ export function decodeScanCache(document: unknown): ScanCache { // The intern tables must be all strings: a numeric entry would pass the // undefined guard below, land in a record's model, and crash the aggregate - // at normalizeModelName. A corrupt table rejects the whole cache. + // at lookupRate. A corrupt table rejects the whole cache. if (!root.models.every((value) => typeof value === "string")) return cache; if (!root.sessions.every((value) => typeof value === "string")) return cache; const models = root.models as readonly string[]; From 93fa69d9c06166f80c4923a1ce8d29ff24e947f8 Mon Sep 17 00:00:00 2001 From: Leo Date: Thu, 10 Sep 2026 14:34:22 -0400 Subject: [PATCH 2/2] fix(server): fall back to canonical rates for qualified usage names Exact-key lookup left transcripts such as anthropic/claude-fable-5 unpriced when LiteLLM only published the bare canonical rate. Keep provider-qualified rates when they exist, and use the bare canonical rate only when the exact key is missing. Grok 4.6 High in Grok Build via Orca. --- apps/server/src/usage/usagePricing.test.ts | 11 ++++++++++- apps/server/src/usage/usagePricing.ts | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/apps/server/src/usage/usagePricing.test.ts b/apps/server/src/usage/usagePricing.test.ts index 2ea27375b1483..951523ef6a270 100644 --- a/apps/server/src/usage/usagePricing.test.ts +++ b/apps/server/src/usage/usagePricing.test.ts @@ -27,7 +27,7 @@ describe("usage pricing", () => { expect(lookupRate(table, "deepinfra/anthropic/claude-fable-5")?.cacheReadCostPerToken).toBe( 1e-5, ); - expect(lookupRate(table, "other/claude-fable-5")).toBeNull(); + expect(lookupRate(table, "other/claude-fable-5")?.cacheReadCostPerToken).toBe(1e-6); } }); @@ -51,5 +51,14 @@ describe("usage pricing", () => { expect(lookupRate(table, "provider-a/example-model")?.inputCostPerToken).toBe(1); expect(lookupRate(table, "provider-b/example-model")?.inputCostPerToken).toBe(3); expect(lookupRate(table, "example-model")).toBeNull(); + expect(lookupRate(table, "other/example-model")).toBeNull(); + }); + + it("falls back to a canonical bare rate when the transcript is provider-qualified", () => { + const table = parseRateTable({ + "claude-fable-5": rate(1e-5, 1e-6), + }); + + expect(lookupRate(table, "anthropic/claude-fable-5")?.cacheReadCostPerToken).toBe(1e-6); }); }); diff --git a/apps/server/src/usage/usagePricing.ts b/apps/server/src/usage/usagePricing.ts index 3d7f5fd294859..347eb62b6030b 100644 --- a/apps/server/src/usage/usagePricing.ts +++ b/apps/server/src/usage/usagePricing.ts @@ -139,7 +139,7 @@ export function lookupRate(table: RateTable, model: string): ModelRate | null { const key = normalizeRateKey(model); const bareName = bareModelName(key); if (bareName.length === 0 || UNPRICEABLE_MODELS.has(bareName)) return null; - return table.get(key) ?? null; + return table.get(key) ?? table.get(bareName) ?? null; } export interface PricedUsage {