diff --git a/apps/web/src/lib/ai-gateway/processUsage.messages.ts b/apps/web/src/lib/ai-gateway/processUsage.messages.ts index 231868741c..2b0007f2a1 100644 --- a/apps/web/src/lib/ai-gateway/processUsage.messages.ts +++ b/apps/web/src/lib/ai-gateway/processUsage.messages.ts @@ -16,6 +16,7 @@ import { computeVercelCostMicrodollars, drainSseStream, extractVercelIsByok, + extractVercelUpstreamId, } from '@/lib/ai-gateway/processUsage.shared'; import { isErrorFinishReason } from '@/lib/ai-gateway/finishReason'; import type Anthropic from '@anthropic-ai/sdk'; @@ -222,7 +223,7 @@ export async function parseMessagesMicrodollarUsageFromStream( responseContent, inference_provider, finish_reason, - upstream_id: null, + upstream_id: extractVercelUpstreamId(providerMetadata), latency: null, moderation_latency: null, generation_time: null, @@ -260,7 +261,7 @@ export function parseMessagesMicrodollarUsageFromString( model: getVercelResolvedModel(providerMetadata) ?? responseJson?.model ?? null, responseContent, inference_provider, - upstream_id: null, + upstream_id: extractVercelUpstreamId(providerMetadata), finish_reason, latency: null, moderation_latency: null, diff --git a/apps/web/src/lib/ai-gateway/processUsage.responses.ts b/apps/web/src/lib/ai-gateway/processUsage.responses.ts index b5fefef3ef..1d3635ff13 100644 --- a/apps/web/src/lib/ai-gateway/processUsage.responses.ts +++ b/apps/web/src/lib/ai-gateway/processUsage.responses.ts @@ -17,6 +17,7 @@ import { computeVercelCostMicrodollars, drainSseStream, extractVercelIsByok, + extractVercelUpstreamId, } from '@/lib/ai-gateway/processUsage.shared'; import { isErrorFinishReason } from '@/lib/ai-gateway/finishReason'; @@ -210,7 +211,7 @@ export async function parseResponsesMicrodollarUsageFromStream( responseContent, inference_provider, finish_reason, - upstream_id: null, + upstream_id: extractVercelUpstreamId(providerMetadata), latency: null, moderation_latency: null, generation_time: null, @@ -240,7 +241,7 @@ export function parseResponsesMicrodollarUsageFromString( model: responseJson?.model ?? null, responseContent: responseJson?.output ? extractResponseContent(responseJson.output) : '', inference_provider, - upstream_id: null, + upstream_id: extractVercelUpstreamId(providerMetadata), finish_reason: responseJson?.status ?? null, latency: null, moderation_latency: null, diff --git a/apps/web/src/lib/ai-gateway/processUsage.shared.ts b/apps/web/src/lib/ai-gateway/processUsage.shared.ts index f21734b5e4..fb18d73a59 100644 --- a/apps/web/src/lib/ai-gateway/processUsage.shared.ts +++ b/apps/web/src/lib/ai-gateway/processUsage.shared.ts @@ -5,6 +5,7 @@ import { errorExceptInTest } from '@/lib/utils.server'; import { OPENROUTER_BYOK_COST_MULTIPLIER } from '@/lib/ai-gateway/processUsage.constants'; import type { NotYetCostedUsageStats, + VercelProviderAttempt, VercelProviderMetaData, } from '@/lib/ai-gateway/processUsage.types'; @@ -62,6 +63,22 @@ export function computeVercelCostMicrodollars( return toMicrodollars(isNaN(marketCost_USD) ? 0 : marketCost_USD); } +/** + * Finds the provider attempt that served the request in the Vercel AI Gateway + * routing metadata: the successful provider attempt within the successful model + * attempt, falling back to the last attempt of each when none reports success. + */ +function findVercelServingProviderAttempt( + vercelGateway: NonNullable | undefined | null +): VercelProviderAttempt | undefined { + const modelAttempts = vercelGateway?.routing?.modelAttempts; + if (!modelAttempts) return undefined; + const servingModel = modelAttempts.find(m => m.success) ?? modelAttempts.at(-1); + const providerAttempts = servingModel?.providerAttempts; + if (!providerAttempts) return undefined; + return providerAttempts.find(p => p.success) ?? providerAttempts.at(-1); +} + /** * Extracts whether the Vercel AI Gateway served the request with BYOK credentials. * @@ -73,18 +90,27 @@ export function computeVercelCostMicrodollars( export function extractVercelIsByok( vercelGateway: NonNullable | undefined | null ): boolean | null { - const modelAttempts = vercelGateway?.routing?.modelAttempts; - if (!modelAttempts) return null; - const successfulModel = modelAttempts.find(m => m.success) ?? modelAttempts.at(-1); - const providerAttempts = successfulModel?.providerAttempts; - if (!providerAttempts) return null; - const successfulProvider = providerAttempts.find(p => p.success) ?? providerAttempts.at(-1); - const credentialType = successfulProvider?.credentialType; + const credentialType = findVercelServingProviderAttempt(vercelGateway)?.credentialType; if (credentialType === 'byok') return true; if (credentialType === 'system') return false; return null; } +/** + * Extracts the upstream provider's own request/response id from the Vercel AI + * Gateway routing metadata, so it can be stored as + * `microdollar_usage_metadata.upstream_id` without a `/generation` lookup. + * + * Mirrors the attempt selection of `extractVercelIsByok`. Returns `null` when + * the gateway did not report an id. + */ +export function extractVercelUpstreamId( + vercelProviderMetadata: VercelProviderMetaData | undefined | null +): string | null { + const attempt = findVercelServingProviderAttempt(vercelProviderMetadata?.gateway); + return attempt?.providerRequestId ?? attempt?.providerResponseId ?? null; +} + export function isResponseInterruptedError(error: unknown): boolean { if (typeof error !== 'object' || error === null || !('name' in error)) return false; return error.name === 'ResponseAborted' || error.name === 'TimeoutError'; diff --git a/apps/web/src/lib/ai-gateway/processUsage.test.ts b/apps/web/src/lib/ai-gateway/processUsage.test.ts index 6e3a203dd2..c14ceecd99 100644 --- a/apps/web/src/lib/ai-gateway/processUsage.test.ts +++ b/apps/web/src/lib/ai-gateway/processUsage.test.ts @@ -296,6 +296,96 @@ describe('parseMicrodollarUsageFromStream approval tests', () => { expect(result.hasError).toBe(true); expect(result.status_code).toBe(502); }); + + test('records the Vercel upstream provider request id as upstream_id', async () => { + const chunk = `data: {"id":"gen_01KYMFY57BAFKZGK45197SCRGD","object":"chat.completion.chunk","created":1785246720,"model":"moonshotai/kimi-k3-fast","choices":[{"index":0,"delta":{"provider_metadata":{"fireworks":{},"gateway":{"routing":{"originalModelId":"moonshotai/kimi-k3-fast","resolvedProvider":"fireworks","fallbacksAvailable":[],"canonicalSlug":"moonshotai/kimi-k3-fast","finalProvider":"fireworks","speed":"fast","modelAttemptCount":1,"modelAttempts":[{"canonicalSlug":"moonshotai/kimi-k3-fast","success":true,"providerAttemptCount":1,"providerAttempts":[{"provider":"fireworks","credentialType":"system","success":true,"startTime":1785246717210,"endTime":1785246721497,"providerRequestId":"chatcmpl-ccebc94c526d4f8797cfe00023478a9a","statusCode":200,"providerResponseId":"chatcmpl-ccebc94c526d4f8797cfe00023478a9a"}]}],"totalProviderAttemptCount":1},"cost":"0.0474822","marketCost":"0.0474822","generationId":"gen_01KYMFY57BAFKZGK45197SCRGD"}}},"logprobs":null,"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":91904,"completion_tokens":134,"total_tokens":92038,"cost":0.0474822,"is_byok":false,"prompt_tokens_details":{"cached_tokens":91136},"cost_details":{"upstream_inference_cost":null},"completion_tokens_details":{"reasoning_tokens":0}},"generationId":"gen_01KYMFY57BAFKZGK45197SCRGD"}\n\ndata: [DONE]\n\n`; + + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(chunk)); + controller.close(); + }, + }); + + const result = await parseMicrodollarUsageFromStream( + stream, + 'fake-user-id', + undefined, + 'vercel', + 200 + ); + + expect(result.upstream_id).toBe('chatcmpl-ccebc94c526d4f8797cfe00023478a9a'); + expect(result.inference_provider).toBe('fireworks'); + expect(result.cost_mUsd).toBe(47482); + }); + + test('leaves upstream_id null when the Vercel gateway reports no provider request id', async () => { + const chunk = `data: {"id":"gen-1","object":"chat.completion.chunk","created":1,"model":"moonshotai/kimi-k3-fast","choices":[{"index":0,"delta":{"provider_metadata":{"gateway":{"routing":{"finalProvider":"fireworks","modelAttempts":[{"success":true,"providerAttempts":[{"provider":"fireworks","credentialType":"system","success":true}]}]},"marketCost":"0.001"}}},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}\n\ndata: [DONE]\n\n`; + + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(chunk)); + controller.close(); + }, + }); + + const result = await parseMicrodollarUsageFromStream( + stream, + 'fake-user-id', + undefined, + 'vercel', + 200 + ); + + expect(result.upstream_id).toBeNull(); + }); + + test('records the Vercel upstream provider request id for non-streamed responses', () => { + const response = JSON.stringify({ + id: 'gen-2', + object: 'chat.completion', + created: 1, + model: 'moonshotai/kimi-k3-fast', + choices: [ + { + index: 0, + finish_reason: 'stop', + message: { + role: 'assistant', + content: 'hi', + provider_metadata: { + gateway: { + routing: { + finalProvider: 'fireworks', + modelAttempts: [ + { + success: true, + providerAttempts: [ + { + provider: 'fireworks', + credentialType: 'system', + success: true, + providerResponseId: 'chatcmpl-non-streamed', + }, + ], + }, + ], + }, + marketCost: '0.001', + }, + }, + }, + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }); + + const result = parseMicrodollarUsageFromString(response, 'fake-user-id', 200); + + expect(result.upstream_id).toBe('chatcmpl-non-streamed'); + expect(result.inference_provider).toBe('fireworks'); + }); }); const sampleReqDir = join(process.cwd(), 'src/tests/req_sample'); diff --git a/apps/web/src/lib/ai-gateway/processUsage.ts b/apps/web/src/lib/ai-gateway/processUsage.ts index d52fca831a..1fddf95167 100644 --- a/apps/web/src/lib/ai-gateway/processUsage.ts +++ b/apps/web/src/lib/ai-gateway/processUsage.ts @@ -76,6 +76,7 @@ import { computeOpenRouterCostFields, drainSseStream, extractVercelIsByok, + extractVercelUpstreamId, isResponseInterruptedError, } from '@/lib/ai-gateway/processUsage.shared'; import { @@ -1119,7 +1120,7 @@ export async function parseMicrodollarUsageFromStream( responseContent, inference_provider, finish_reason, - upstream_id: null, + upstream_id: extractVercelUpstreamId(vercelProviderMetadata), latency: null, moderation_latency: null, generation_time: null, @@ -1152,6 +1153,7 @@ export function parseMicrodollarUsageFromString( } const choice = responseJson?.choices?.[0]; const finish_reason = choice?.finish_reason ?? null; + const vercelProviderMetadata = choice?.message?.provider_metadata ?? null; const coreProps = { kiloUserId, messageId: responseJson?.id ?? null, @@ -1159,10 +1161,8 @@ export function parseMicrodollarUsageFromString( model: responseJson?.model ?? null, responseContent: choice?.message.content ?? '', inference_provider: - responseJson?.provider ?? - choice?.message?.provider_metadata?.gateway?.routing?.finalProvider ?? - null, - upstream_id: null, + responseJson?.provider ?? vercelProviderMetadata?.gateway?.routing?.finalProvider ?? null, + upstream_id: extractVercelUpstreamId(vercelProviderMetadata), finish_reason, latency: null, moderation_latency: null, @@ -1172,11 +1172,7 @@ export function parseMicrodollarUsageFromString( status_code: statusCode, }; - const costs = processOpenRouterUsage( - responseJson?.usage, - coreProps, - choice?.message?.provider_metadata ?? null - ); + const costs = processOpenRouterUsage(responseJson?.usage, coreProps, vercelProviderMetadata); return { ...coreProps, ...costs }; } @@ -1252,6 +1248,7 @@ export async function processTokenData( } genStats.model = usageStats.model; // openrouter bug? + genStats.upstream_id ??= usageStats.upstream_id; // keep the id the response already reported genStats.hasError = usageStats.hasError; // retain by choice genStats.status_code = usageStats.status_code; // retain by choice genStats.streamed ??= usageContext.isStreaming; diff --git a/apps/web/src/lib/ai-gateway/processUsage.types.ts b/apps/web/src/lib/ai-gateway/processUsage.types.ts index 902baca5ee..f61220d793 100644 --- a/apps/web/src/lib/ai-gateway/processUsage.types.ts +++ b/apps/web/src/lib/ai-gateway/processUsage.types.ts @@ -23,6 +23,10 @@ export type VercelProviderAttempt = { provider?: string; credentialType?: string; success?: boolean; + /** The upstream provider's own request id, e.g. Fireworks' `chatcmpl-…`. */ + providerRequestId?: string; + /** The upstream provider's own response id; usually equal to `providerRequestId`. */ + providerResponseId?: string; }; export type VercelModelAttempt = { diff --git a/apps/web/src/tests/sample/vercel-messages.log.resp.sse.approved.json b/apps/web/src/tests/sample/vercel-messages.log.resp.sse.approved.json index 7326191747..116ccdb4ce 100644 --- a/apps/web/src/tests/sample/vercel-messages.log.resp.sse.approved.json +++ b/apps/web/src/tests/sample/vercel-messages.log.resp.sse.approved.json @@ -5,7 +5,7 @@ "responseContent": "Hi there! 👋 How are you doing? Is there something I can help you with today?", "inference_provider": "bedrock", "finish_reason": "end_turn", - "upstream_id": null, + "upstream_id": "305d0d3d-1a2c-47fe-a99f-96f30082adbd", "latency": null, "moderation_latency": null, "generation_time": null, diff --git a/apps/web/src/tests/sample/vercel-responses.log.resp.json.approved.json b/apps/web/src/tests/sample/vercel-responses.log.resp.json.approved.json index 3208289a7c..ec7784ae83 100644 --- a/apps/web/src/tests/sample/vercel-responses.log.resp.json.approved.json +++ b/apps/web/src/tests/sample/vercel-responses.log.resp.json.approved.json @@ -4,7 +4,7 @@ "model": "gpt-5.4-2026-03-05", "responseContent": "hi", "inference_provider": "openai", - "upstream_id": null, + "upstream_id": "req_869e7f2d33124834a2d0a7fb0a271c1a", "finish_reason": "completed", "latency": null, "moderation_latency": null, diff --git a/apps/web/src/tests/sample/vercel-responses.log.resp.sse.approved.json b/apps/web/src/tests/sample/vercel-responses.log.resp.sse.approved.json index a9a01fcafd..0162e44a86 100644 --- a/apps/web/src/tests/sample/vercel-responses.log.resp.sse.approved.json +++ b/apps/web/src/tests/sample/vercel-responses.log.resp.sse.approved.json @@ -5,7 +5,7 @@ "responseContent": "hi", "inference_provider": "openai", "finish_reason": "completed", - "upstream_id": null, + "upstream_id": "req_869e7f2d33124834a2d0a7fb0a271c1a", "latency": null, "moderation_latency": null, "generation_time": null,