diff --git a/src/lib/providers/googleAiStudio.ts b/src/lib/providers/googleAiStudio.ts index 45607237e..a8eb93b4c 100644 --- a/src/lib/providers/googleAiStudio.ts +++ b/src/lib/providers/googleAiStudio.ts @@ -825,6 +825,7 @@ export class GoogleAIStudioProvider extends BaseProvider { let lastStepText = ""; let totalInputTokens = 0; let totalOutputTokens = 0; + let totalCacheReadTokens = 0; let step = 0; let completedWithFinalAnswer = false; const failedTools = new Map< @@ -866,6 +867,7 @@ export class GoogleAIStudioProvider extends BaseProvider { ); totalInputTokens += chunkResult.inputTokens; totalOutputTokens += chunkResult.outputTokens; + totalCacheReadTokens += chunkResult.cacheReadTokens ?? 0; const stepText = extractTextFromParts( chunkResult.rawResponseParts, @@ -1002,13 +1004,27 @@ export class GoogleAIStudioProvider extends BaseProvider { hitStepLimitWithoutFinalAnswer ? "max_steps" : "stop", ); + // Gemini promptTokenCount is OVERLAPPING: it already includes + // cachedContentTokenCount. Subtract once here so calculateCost + // bills the cached portion at the cheaper cacheRead rate without + // double-counting. Total billable tokens are conserved. + const adjustedInputTokens = Math.max( + 0, + totalInputTokens - totalCacheReadTokens, + ); analyticsResolve({ provider: this.providerName, model: modelName, tokenUsage: { - input: totalInputTokens, + input: adjustedInputTokens, output: totalOutputTokens, - total: totalInputTokens + totalOutputTokens, + total: + adjustedInputTokens + + totalCacheReadTokens + + totalOutputTokens, + ...(totalCacheReadTokens > 0 + ? { cacheReadTokens: totalCacheReadTokens } + : {}), }, requestDuration: responseTime, timestamp: new Date().toISOString(), @@ -1189,6 +1205,7 @@ export class GoogleAIStudioProvider extends BaseProvider { let lastStepText = ""; let totalInputTokens = 0; let totalOutputTokens = 0; + let totalCacheReadTokens = 0; const allToolCalls: Array<{ toolName: string; args: Record; @@ -1229,6 +1246,7 @@ export class GoogleAIStudioProvider extends BaseProvider { const chunkResult = await collectStreamChunks(stream); totalInputTokens += chunkResult.inputTokens; totalOutputTokens += chunkResult.outputTokens; + totalCacheReadTokens += chunkResult.cacheReadTokens ?? 0; const stepText = extractTextFromParts( chunkResult.rawResponseParts, @@ -1351,14 +1369,25 @@ export class GoogleAIStudioProvider extends BaseProvider { // analytics / evaluation / tracing stay attached. The native AI // Studio generate path bypasses BaseProvider.generate(), so // skipping enhanceResult would silently drop those features. + // Gemini promptTokenCount is OVERLAPPING (already includes + // cachedContentTokenCount). Subtract once so the cached portion is + // billed at the cheaper cacheRead rate without double-counting. + const adjustedInputTokens = Math.max( + 0, + totalInputTokens - totalCacheReadTokens, + ); const baseResult: EnhancedGenerateResult = { content: finalText, provider: this.providerName, model: modelName, usage: { - input: totalInputTokens, + input: adjustedInputTokens, output: totalOutputTokens, - total: totalInputTokens + totalOutputTokens, + total: + adjustedInputTokens + totalCacheReadTokens + totalOutputTokens, + ...(totalCacheReadTokens > 0 + ? { cacheReadTokens: totalCacheReadTokens } + : {}), }, responseTime, toolsUsed: allToolCalls.map((tc) => tc.toolName), diff --git a/src/lib/providers/googleNativeGemini3.ts b/src/lib/providers/googleNativeGemini3.ts index 103b047ca..4a7bef4d1 100644 --- a/src/lib/providers/googleNativeGemini3.ts +++ b/src/lib/providers/googleNativeGemini3.ts @@ -695,6 +695,7 @@ export async function collectStreamChunks( const stepFunctionCalls: NativeFunctionCall[] = []; let inputTokens = 0; let outputTokens = 0; + let cacheReadTokens = 0; for await (const chunk of stream) { // Extract raw parts from candidates FIRST @@ -717,15 +718,31 @@ export async function collectStreamChunks( // Accumulate usage metadata from chunks const usage = chunkRecord.usageMetadata as - | { promptTokenCount?: number; candidatesTokenCount?: number } + | { + promptTokenCount?: number; + candidatesTokenCount?: number; + cachedContentTokenCount?: number; + } | undefined; if (usage) { inputTokens = Math.max(inputTokens, usage.promptTokenCount || 0); outputTokens = Math.max(outputTokens, usage.candidatesTokenCount || 0); + // cachedContentTokenCount is OVERLAPPING (a subset already inside + // promptTokenCount). Surface it so the call site subtracts once. + cacheReadTokens = Math.max( + cacheReadTokens, + usage.cachedContentTokenCount || 0, + ); } } - return { rawResponseParts, stepFunctionCalls, inputTokens, outputTokens }; + return { + rawResponseParts, + stepFunctionCalls, + inputTokens, + outputTokens, + cacheReadTokens, + }; } /** @@ -829,6 +846,7 @@ export async function collectStreamChunksIncremental( const stepFunctionCalls: NativeFunctionCall[] = []; let inputTokens = 0; let outputTokens = 0; + let cacheReadTokens = 0; for await (const chunk of stream) { const chunkRecord = chunk as Record; @@ -853,15 +871,31 @@ export async function collectStreamChunksIncremental( } const usage = chunkRecord.usageMetadata as - | { promptTokenCount?: number; candidatesTokenCount?: number } + | { + promptTokenCount?: number; + candidatesTokenCount?: number; + cachedContentTokenCount?: number; + } | undefined; if (usage) { inputTokens = Math.max(inputTokens, usage.promptTokenCount || 0); outputTokens = Math.max(outputTokens, usage.candidatesTokenCount || 0); + // cachedContentTokenCount is OVERLAPPING (a subset already inside + // promptTokenCount). Surface it so the call site subtracts once. + cacheReadTokens = Math.max( + cacheReadTokens, + usage.cachedContentTokenCount || 0, + ); } } - return { rawResponseParts, stepFunctionCalls, inputTokens, outputTokens }; + return { + rawResponseParts, + stepFunctionCalls, + inputTokens, + outputTokens, + cacheReadTokens, + }; } /** diff --git a/src/lib/providers/googleVertex.ts b/src/lib/providers/googleVertex.ts index 79b868225..318bb8892 100644 --- a/src/lib/providers/googleVertex.ts +++ b/src/lib/providers/googleVertex.ts @@ -1806,6 +1806,7 @@ export class GoogleVertexProvider extends BaseProvider { // promptTokenCount is typically in the final chunk, candidatesTokenCount accumulates let totalInputTokens = 0; let totalOutputTokens = 0; + let totalCacheReadTokens = 0; // Track text parts as they arrive from the SDK so the returned async // iterable yields multiple chunks instead of a single buffered chunk. @@ -1941,6 +1942,7 @@ export class GoogleVertexProvider extends BaseProvider { | { promptTokenCount?: number; candidatesTokenCount?: number; + cachedContentTokenCount?: number; totalTokenCount?: number; } | undefined; @@ -1956,6 +1958,13 @@ export class GoogleVertexProvider extends BaseProvider { usageMetadata.promptTokenCount, usageMetadata.candidatesTokenCount ?? 0, ); + // cachedContentTokenCount is OVERLAPPING (a subset already inside + // promptTokenCount). Clamp to the prompt count so a later uncached + // step resets it to 0 instead of leaving a stale cached value. + totalCacheReadTokens = Math.min( + usageMetadata.cachedContentTokenCount ?? 0, + usageMetadata.promptTokenCount, + ); } // Take the latest candidatesTokenCount (accumulates through chunks) if ( @@ -2503,6 +2512,13 @@ export class GoogleVertexProvider extends BaseProvider { (te) => te.name !== "final_result", ); + // Gemini promptTokenCount is OVERLAPPING (already includes + // cachedContentTokenCount). Subtract once so the cached portion is billed at + // the cheaper cacheRead rate without double-counting; total is conserved. + const adjustedInputTokens = Math.max( + 0, + totalInputTokens - totalCacheReadTokens, + ); const result: StreamResult = { stream: createTextStream(), provider: this.providerName, @@ -2511,9 +2527,12 @@ export class GoogleVertexProvider extends BaseProvider { stopReason, rawFinishReason: lastFinishReason, usage: { - input: totalInputTokens, + input: adjustedInputTokens, output: totalOutputTokens, - total: totalInputTokens + totalOutputTokens, + total: adjustedInputTokens + totalCacheReadTokens + totalOutputTokens, + ...(totalCacheReadTokens > 0 && { + cacheReadTokens: totalCacheReadTokens, + }), }, toolCalls: externalToolCalls.map((tc) => ({ toolName: tc.toolName, @@ -2977,6 +2996,7 @@ export class GoogleVertexProvider extends BaseProvider { // promptTokenCount is typically in the final chunk, candidatesTokenCount accumulates let totalInputTokens = 0; let totalOutputTokens = 0; + let totalCacheReadTokens = 0; // Abort scaffolding (mirrors executeNativeAnthropicStream). The native // Gemini SDK cancels via config.abortSignal, so drive an internal @@ -3101,6 +3121,7 @@ export class GoogleVertexProvider extends BaseProvider { | { promptTokenCount?: number; candidatesTokenCount?: number; + cachedContentTokenCount?: number; totalTokenCount?: number; } | undefined; @@ -3116,6 +3137,13 @@ export class GoogleVertexProvider extends BaseProvider { usageMetadata.promptTokenCount, usageMetadata.candidatesTokenCount ?? 0, ); + // cachedContentTokenCount is OVERLAPPING (a subset already inside + // promptTokenCount). Clamp to the prompt count so a later uncached + // step resets it to 0 instead of leaving a stale cached value. + totalCacheReadTokens = Math.min( + usageMetadata.cachedContentTokenCount ?? 0, + usageMetadata.promptTokenCount, + ); } // Take the latest candidatesTokenCount (accumulates through chunks) if ( @@ -3643,6 +3671,13 @@ export class GoogleVertexProvider extends BaseProvider { ); // Build EnhancedGenerateResult + // Gemini promptTokenCount is OVERLAPPING (already includes + // cachedContentTokenCount). Subtract once so the cached portion is billed at + // the cheaper cacheRead rate without double-counting; total is conserved. + const adjustedInputTokens = Math.max( + 0, + totalInputTokens - totalCacheReadTokens, + ); const result: EnhancedGenerateResult = { content: finalText, provider: this.providerName, @@ -3652,9 +3687,12 @@ export class GoogleVertexProvider extends BaseProvider { rawFinishReason: lastFinishReason, stepsUsed: step, usage: { - input: totalInputTokens, + input: adjustedInputTokens, output: totalOutputTokens, - total: totalInputTokens + totalOutputTokens, + total: adjustedInputTokens + totalCacheReadTokens + totalOutputTokens, + ...(totalCacheReadTokens > 0 && { + cacheReadTokens: totalCacheReadTokens, + }), }, responseTime, toolsUsed: externalToolCalls.map((tc) => tc.toolName), diff --git a/src/lib/types/common.ts b/src/lib/types/common.ts index ef88629c9..e31118db8 100644 --- a/src/lib/types/common.ts +++ b/src/lib/types/common.ts @@ -357,6 +357,10 @@ export type RawUsageObject = { cacheCreationTokens?: number; cacheReadTokens?: number; + // OpenAI/DeepSeek/NIM/OpenAI-compatible nested cache field (overlapping: + // cached_tokens is a SUBSET already included in prompt_tokens) + prompt_tokens_details?: { cached_tokens?: number }; + // OpenAI o1/Anthropic reasoning tokens reasoningTokens?: number; reasoning?: number; diff --git a/src/lib/types/providers.ts b/src/lib/types/providers.ts index 0f0f43752..843b46ef3 100644 --- a/src/lib/types/providers.ts +++ b/src/lib/types/providers.ts @@ -1861,6 +1861,14 @@ export type CollectedChunkResult = { stepFunctionCalls: NativeFunctionCall[]; inputTokens: number; outputTokens: number; + /** + * Gemini cached-content tokens (overlapping: included in promptTokenCount). + * Surfaced so the call site can subtract from input and bill at cacheRead + * rate. Subtraction happens at the call site, not in the collector. + */ + cacheReadTokens?: number; + /** Cache creation tokens (symmetry; Gemini does not emit this). */ + cacheCreationTokens?: number; }; /** Push-based text channel for incremental streaming. */ diff --git a/src/lib/utils/pricing.ts b/src/lib/utils/pricing.ts index 2edcb9bdd..7e3c14821 100644 --- a/src/lib/utils/pricing.ts +++ b/src/lib/utils/pricing.ts @@ -100,8 +100,18 @@ const PRICING: Record< cacheRead: 1.5 / 1_000_000, cacheCreation: 18.75 / 1_000_000, }, - "claude-3-sonnet": { input: 3.0 / 1_000_000, output: 15.0 / 1_000_000 }, - "claude-3-haiku": { input: 0.25 / 1_000_000, output: 1.25 / 1_000_000 }, + "claude-3-sonnet": { + input: 3.0 / 1_000_000, + output: 15.0 / 1_000_000, + cacheRead: 0.3 / 1_000_000, + cacheCreation: 3.75 / 1_000_000, + }, + "claude-3-haiku": { + input: 0.25 / 1_000_000, + output: 1.25 / 1_000_000, + cacheRead: 0.025 / 1_000_000, + cacheCreation: 0.3125 / 1_000_000, + }, }, // Google Vertex AI — Claude models on Vertex (same pricing, @ date suffix) vertex: { @@ -151,26 +161,95 @@ const PRICING: Record< // OpenAI — updated March 2026 openai: { // GPT-5.x family - "gpt-5.4": { input: 2.5 / 1_000_000, output: 15.0 / 1_000_000 }, - "gpt-5.2": { input: 1.75 / 1_000_000, output: 14.0 / 1_000_000 }, - "gpt-5.1": { input: 0.625 / 1_000_000, output: 5.0 / 1_000_000 }, - "gpt-5.1-codex": { input: 1.25 / 1_000_000, output: 10.0 / 1_000_000 }, - "gpt-5": { input: 1.25 / 1_000_000, output: 10.0 / 1_000_000 }, - "gpt-5-mini": { input: 0.25 / 1_000_000, output: 2.0 / 1_000_000 }, - "gpt-5-nano": { input: 0.05 / 1_000_000, output: 0.4 / 1_000_000 }, + // cacheRead = 0.25x input (cached input tokens; no separate cacheCreation). + "gpt-5.4": { + input: 2.5 / 1_000_000, + output: 15.0 / 1_000_000, + cacheRead: 0.625 / 1_000_000, + }, + "gpt-5.2": { + input: 1.75 / 1_000_000, + output: 14.0 / 1_000_000, + cacheRead: 0.4375 / 1_000_000, + }, + "gpt-5.1": { + input: 0.625 / 1_000_000, + output: 5.0 / 1_000_000, + cacheRead: 0.15625 / 1_000_000, + }, + "gpt-5.1-codex": { + input: 1.25 / 1_000_000, + output: 10.0 / 1_000_000, + cacheRead: 0.3125 / 1_000_000, + }, + "gpt-5": { + input: 1.25 / 1_000_000, + output: 10.0 / 1_000_000, + cacheRead: 0.3125 / 1_000_000, + }, + "gpt-5-mini": { + input: 0.25 / 1_000_000, + output: 2.0 / 1_000_000, + cacheRead: 0.0625 / 1_000_000, + }, + "gpt-5-nano": { + input: 0.05 / 1_000_000, + output: 0.4 / 1_000_000, + cacheRead: 0.0125 / 1_000_000, + }, // GPT-4.1 family - "gpt-4.1": { input: 2.0 / 1_000_000, output: 8.0 / 1_000_000 }, - "gpt-4.1-mini": { input: 0.4 / 1_000_000, output: 1.6 / 1_000_000 }, - "gpt-4.1-nano": { input: 0.1 / 1_000_000, output: 0.4 / 1_000_000 }, + "gpt-4.1": { + input: 2.0 / 1_000_000, + output: 8.0 / 1_000_000, + cacheRead: 0.5 / 1_000_000, + }, + "gpt-4.1-mini": { + input: 0.4 / 1_000_000, + output: 1.6 / 1_000_000, + cacheRead: 0.1 / 1_000_000, + }, + "gpt-4.1-nano": { + input: 0.1 / 1_000_000, + output: 0.4 / 1_000_000, + cacheRead: 0.025 / 1_000_000, + }, // GPT-4o family - "gpt-4o": { input: 2.5 / 1_000_000, output: 10.0 / 1_000_000 }, - "gpt-4o-mini": { input: 0.15 / 1_000_000, output: 0.6 / 1_000_000 }, + "gpt-4o": { + input: 2.5 / 1_000_000, + output: 10.0 / 1_000_000, + cacheRead: 0.625 / 1_000_000, + }, + "gpt-4o-mini": { + input: 0.15 / 1_000_000, + output: 0.6 / 1_000_000, + cacheRead: 0.0375 / 1_000_000, + }, // o-series reasoning - o3: { input: 2.0 / 1_000_000, output: 8.0 / 1_000_000 }, - "o3-mini": { input: 1.1 / 1_000_000, output: 4.4 / 1_000_000 }, - "o4-mini": { input: 1.1 / 1_000_000, output: 4.4 / 1_000_000 }, - o1: { input: 15.0 / 1_000_000, output: 60.0 / 1_000_000 }, - "o1-mini": { input: 0.55 / 1_000_000, output: 2.2 / 1_000_000 }, + o3: { + input: 2.0 / 1_000_000, + output: 8.0 / 1_000_000, + cacheRead: 0.5 / 1_000_000, + }, + "o3-mini": { + input: 1.1 / 1_000_000, + output: 4.4 / 1_000_000, + cacheRead: 0.275 / 1_000_000, + }, + "o4-mini": { + input: 1.1 / 1_000_000, + output: 4.4 / 1_000_000, + cacheRead: 0.275 / 1_000_000, + }, + o1: { + input: 15.0 / 1_000_000, + output: 60.0 / 1_000_000, + cacheRead: 3.75 / 1_000_000, + }, + "o1-mini": { + input: 0.55 / 1_000_000, + output: 2.2 / 1_000_000, + cacheRead: 0.1375 / 1_000_000, + }, // Legacy "gpt-4-turbo": { input: 10.0 / 1_000_000, output: 30.0 / 1_000_000 }, "gpt-4": { input: 30.0 / 1_000_000, output: 60.0 / 1_000_000 }, @@ -179,52 +258,83 @@ const PRICING: Record< // Google (Gemini) — updated March 2026 google: { // Gemini 3.1 family (all require -preview suffix) + // cacheRead = 0.25x input (cached content tokens; explicit-cache storage is + // billed separately by TTL/time, so no per-token cacheCreation rate here). "gemini-3.1-pro-preview": { input: 2.0 / 1_000_000, output: 12.0 / 1_000_000, + cacheRead: 0.5 / 1_000_000, }, "gemini-3.1-flash-lite-preview": { input: 0.25 / 1_000_000, output: 1.5 / 1_000_000, + cacheRead: 0.0625 / 1_000_000, }, "gemini-3.1-flash-image-preview": { input: 0.5 / 1_000_000, output: 3.0 / 1_000_000, + cacheRead: 0.125 / 1_000_000, }, "gemini-3.1-pro-preview-customtools": { input: 2.0 / 1_000_000, output: 12.0 / 1_000_000, + cacheRead: 0.5 / 1_000_000, }, // Gemini 3 family "gemini-3-flash-preview": { input: 0.5 / 1_000_000, output: 3.0 / 1_000_000, + cacheRead: 0.125 / 1_000_000, }, "gemini-3-pro-image-preview": { input: 2.0 / 1_000_000, output: 12.0 / 1_000_000, + cacheRead: 0.5 / 1_000_000, }, /** @deprecated SHUT DOWN March 9, 2026. Migrate to gemini-3.1-pro-preview. */ "gemini-3-pro-preview": { input: 2.0 / 1_000_000, output: 12.0 / 1_000_000, + cacheRead: 0.5 / 1_000_000, }, // Gemini 2.5 family - "gemini-2.5-flash": { input: 0.3 / 1_000_000, output: 2.5 / 1_000_000 }, - "gemini-2.5-pro": { input: 1.25 / 1_000_000, output: 10.0 / 1_000_000 }, + "gemini-2.5-flash": { + input: 0.3 / 1_000_000, + output: 2.5 / 1_000_000, + cacheRead: 0.075 / 1_000_000, + }, + "gemini-2.5-pro": { + input: 1.25 / 1_000_000, + output: 10.0 / 1_000_000, + cacheRead: 0.3125 / 1_000_000, + }, "gemini-2.5-flash-lite": { input: 0.1 / 1_000_000, output: 0.4 / 1_000_000, + cacheRead: 0.025 / 1_000_000, }, // Gemini 2.0 family (deprecated June 2026) - "gemini-2.0-flash": { input: 0.15 / 1_000_000, output: 0.6 / 1_000_000 }, + "gemini-2.0-flash": { + input: 0.15 / 1_000_000, + output: 0.6 / 1_000_000, + cacheRead: 0.0375 / 1_000_000, + }, "gemini-2.0-flash-lite": { input: 0.075 / 1_000_000, output: 0.3 / 1_000_000, + cacheRead: 0.01875 / 1_000_000, }, // Gemini 1.5 family - "gemini-1.5-pro": { input: 1.25 / 1_000_000, output: 5.0 / 1_000_000 }, - "gemini-1.5-flash": { input: 0.075 / 1_000_000, output: 0.3 / 1_000_000 }, + "gemini-1.5-pro": { + input: 1.25 / 1_000_000, + output: 5.0 / 1_000_000, + cacheRead: 0.3125 / 1_000_000, + }, + "gemini-1.5-flash": { + input: 0.075 / 1_000_000, + output: 0.3 / 1_000_000, + cacheRead: 0.01875 / 1_000_000, + }, }, // Mistral AI mistral: { @@ -636,8 +746,16 @@ export function calculateCost( let cost = 0; cost += (usage.input || 0) * rates.input; cost += (usage.output || 0) * rates.output; - if (usage.cacheReadTokens && rates.cacheRead) { - cost += usage.cacheReadTokens * rates.cacheRead; + if (usage.cacheReadTokens) { + // Price cache reads at the discounted cacheRead rate when one exists; + // otherwise fall back to the full input rate. The overlapping extractor + // (tokenUtils) moves cached tokens out of `input` into `cacheReadTokens` + // for OpenAI/Gemini-family providers, so without this fallback a provider + // that reports cached tokens but has no cacheRead rate would bill those + // tokens at $0 (silent undercharge). Falling back to the input rate keeps + // total cost identical to pre-split billing for unpriced providers, while + // priced providers still get the cheaper cacheRead rate. + cost += usage.cacheReadTokens * (rates.cacheRead ?? rates.input); } if (usage.cacheCreationTokens && rates.cacheCreation) { cost += usage.cacheCreationTokens * rates.cacheCreation; diff --git a/src/lib/utils/tokenUtils.ts b/src/lib/utils/tokenUtils.ts index 4828ada3d..bf2401d25 100644 --- a/src/lib/utils/tokenUtils.ts +++ b/src/lib/utils/tokenUtils.ts @@ -132,6 +132,26 @@ export function extractCacheReadTokens( return undefined; } +/** + * Extract cache read token count from the OVERLAPPING-convention nested path + * used by OpenAI / DeepSeek / NIM / OpenAI-compatible providers: + * `usage.prompt_tokens_details.cached_tokens`. + * + * Unlike {@link extractCacheReadTokens} (non-overlapping Anthropic/Vertex + * convention where cache tokens are reported SEPARATELY from input), the value + * returned here is a SUBSET already included in `prompt_tokens`. Callers that + * use this value MUST subtract it from `input` to avoid double-counting. + */ +export function extractCachedInputTokensOverlapping( + usage: RawUsageObject, +): number | undefined { + const cached = usage.prompt_tokens_details?.cached_tokens; + if (typeof cached === "number" && cached > 0) { + return cached; + } + return undefined; +} + /** * Calculate cache savings percentage * @@ -196,14 +216,29 @@ export function extractTokenUsage( result.usage && typeof result.usage === "object" ? result.usage : result; // Extract base token counts - const input = extractInputTokens(usage); + let input = extractInputTokens(usage); const output = extractOutputTokens(usage); const total = extractTotalTokens(usage, input, output); // Extract optional token fields const reasoning = extractReasoningTokens(usage); const cacheCreationTokens = extractCacheCreationTokens(usage); - const cacheReadTokens = extractCacheReadTokens(usage); + let cacheReadTokens = extractCacheReadTokens(usage); + + // Overlapping-convention fallback (OpenAI/DeepSeek/NIM/OpenAI-compatible): + // when no non-overlapping cache-read field was present, look for the nested + // `prompt_tokens_details.cached_tokens`. That value is a SUBSET already + // included in `input`, so we must subtract it from `input` to avoid + // double-counting (and to let calculateCost apply the cheaper cacheRead rate + // to the cached portion). Only do this when cached <= input so a malformed + // response can never produce negative input or inflate the total. + if (cacheReadTokens === undefined) { + const overlappingCached = extractCachedInputTokensOverlapping(usage); + if (overlappingCached !== undefined && overlappingCached <= input) { + cacheReadTokens = overlappingCached; + input = Math.max(0, input - overlappingCached); + } + } // Calculate cache savings if enabled const cacheSavingsPercent = calculateCacheSavings diff --git a/test/continuous-test-suite-cache-breakpoints.ts b/test/continuous-test-suite-cache-breakpoints.ts index c3f032bbc..72ab6d608 100644 --- a/test/continuous-test-suite-cache-breakpoints.ts +++ b/test/continuous-test-suite-cache-breakpoints.ts @@ -14,6 +14,11 @@ import "dotenv/config"; */ import { applyVertexAnthropicCacheBreakpoints } from "../src/lib/utils/anthropicCacheBreakpoints.js"; +import { + extractTokenUsage, + extractCachedInputTokensOverlapping, +} from "../src/lib/utils/tokenUtils.js"; +import { calculateCost } from "../src/lib/utils/pricing.js"; import { defineSuite, logSection } from "./helpers/harness.js"; const { recordTest, runSuite } = defineSuite("Anthropic Cache Breakpoints"); @@ -247,10 +252,162 @@ function testEdgeCases(): void { } } +function testOverlappingCacheExtraction(): void { + logSection( + "Overlapping cache extraction (OpenAI/DeepSeek prompt_tokens_details)", + ); + try { + // Helper reads ONLY the nested overlapping path. + recordTest( + "extractCachedInputTokensOverlapping reads prompt_tokens_details.cached_tokens", + extractCachedInputTokensOverlapping({ + promptTokens: 1000, + prompt_tokens_details: { cached_tokens: 300 }, + }) === 300, + ); + recordTest( + "extractCachedInputTokensOverlapping returns undefined when absent", + extractCachedInputTokensOverlapping({ promptTokens: 1000 }) === undefined, + ); + // An explicit cached_tokens: 0 must behave exactly like absent — no + // cacheReadTokens surfaced and input left untouched (the > 0 guard). + recordTest( + "cached_tokens: 0 treated same as absent (no cacheReadTokens, input intact)", + extractCachedInputTokensOverlapping({ + promptTokens: 1000, + prompt_tokens_details: { cached_tokens: 0 }, + }) === undefined && + (() => { + const u = extractTokenUsage({ + promptTokens: 1000, + completionTokens: 200, + prompt_tokens_details: { cached_tokens: 0 }, + }); + return u.cacheReadTokens === undefined && u.input === 1000; + })(), + ); + + // extractTokenUsage subtracts cached from input (OVERLAPPING convention) + // so the cached portion is not double-counted. + const usage = extractTokenUsage({ + promptTokens: 1000, + completionTokens: 200, + prompt_tokens_details: { cached_tokens: 300 }, + }); + recordTest( + "input reduced by cached_tokens (1000 - 300 = 700)", + usage.input === 700, + ); + recordTest( + "cacheReadTokens populated (300)", + usage.cacheReadTokens === 300, + ); + recordTest("output unchanged (200)", usage.output === 200); + recordTest( + "total conserved (input+cacheRead+output = original prompt+completion)", + usage.input + (usage.cacheReadTokens ?? 0) + usage.output === 1200, + ); + + // Malformed: cached > prompt → no subtraction (guard against negative input). + const malformed = extractTokenUsage({ + promptTokens: 100, + completionTokens: 50, + prompt_tokens_details: { cached_tokens: 500 }, + }); + recordTest( + "cached > input → input left untouched (no negative, no inflation)", + malformed.input === 100 && malformed.cacheReadTokens === undefined, + ); + + // Non-overlapping (Anthropic-style) field is NOT subtracted from input. + const nonOverlap = extractTokenUsage({ + input: 1000, + output: 200, + cacheReadInputTokens: 400, + }); + recordTest( + "non-overlapping cacheReadInputTokens does NOT reduce input", + nonOverlap.input === 1000 && nonOverlap.cacheReadTokens === 400, + ); + + // Cost: cached portion billed at cheaper cacheRead rate (DeepSeek). + const fullCost = calculateCost("deepseek", "deepseek-chat", { + input: 1000, + output: 0, + total: 1000, + }); + const cachedCost = calculateCost("deepseek", "deepseek-chat", { + input: 700, + output: 0, + total: 1000, + cacheReadTokens: 300, + }); + recordTest( + "cached split is strictly cheaper than billing all input at full rate", + cachedCost < fullCost && cachedCost > 0, + ); + + // OpenAI + Gemini now expose cacheRead rates. + const openaiCached = calculateCost("openai", "gpt-4o", { + input: 700, + output: 0, + total: 1000, + cacheReadTokens: 300, + }); + recordTest("openai gpt-4o cacheRead rate applied", openaiCached > 0); + const geminiCached = calculateCost("google", "gemini-2.5-flash", { + input: 700, + output: 0, + total: 1000, + cacheReadTokens: 300, + }); + recordTest("google gemini cacheRead rate applied", geminiCached > 0); + // Vertex Gemini resolves google cacheRead via the existing fallback. + const vertexGeminiCached = calculateCost("vertex", "gemini-2.5-pro", { + input: 700, + output: 0, + total: 1000, + cacheReadTokens: 300, + }); + recordTest( + "vertex gemini cacheRead via google fallback", + vertexGeminiCached > 0, + ); + + // Option-C safety net: a provider with cacheReadTokens populated but NO + // cacheRead rate must bill the cached portion at the input rate (never $0), + // so the split cost equals billing the whole prompt at the input rate — + // identical to pre-split billing, no silent undercharge. + const noRateSplit = calculateCost("groq", "llama-3.3-70b-versatile", { + input: 700, + output: 0, + total: 1000, + cacheReadTokens: 300, + }); + const noRateFull = calculateCost("groq", "llama-3.3-70b-versatile", { + input: 1000, + output: 0, + total: 1000, + }); + recordTest( + "no-cacheRead-rate provider bills cached tokens at input rate (no undercharge)", + noRateSplit === noRateFull && noRateSplit > 0, + ); + } catch (error) { + recordTest( + "overlapping cache extraction", + false, + false, + error instanceof Error ? error.message : String(error), + ); + } +} + await runSuite(async () => { testSystemAndRollingHistory(); testNoSystemMarksLastTool(); testPurity(); testHistoryBreakpointCap(); testEdgeCases(); + testOverlappingCacheExtraction(); });