From d3611b2353a432fb62ed5d9be26edb02ab55456a Mon Sep 17 00:00:00 2001 From: Luca Steeb Date: Mon, 28 Jul 2025 01:58:50 +0200 Subject: [PATCH 1/6] feat(chat): simplify Google streaming data parsing Refactored Google provider parsing to streamline logic by removing excessive JSON handling. Improved clarity and reduced redundant code. --- apps/gateway/src/chat/chat.ts | 474 ++++++++++++---------------------- 1 file changed, 169 insertions(+), 305 deletions(-) diff --git a/apps/gateway/src/chat/chat.ts b/apps/gateway/src/chat/chat.ts index 433c94595f..899887d469 100644 --- a/apps/gateway/src/chat/chat.ts +++ b/apps/gateway/src/chat/chat.ts @@ -1,17 +1,17 @@ import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi"; import { + type ApiKey, db, - shortid, type InferSelectModel, type Project, + shortid, type tables, - type ApiKey, } from "@llmgateway/db"; import { getCheapestFromAvailableProviders, + getModelStreamingSupport, getProviderEndpoint, getProviderHeaders, - getModelStreamingSupport, type Model, models, prepareRequestBody, @@ -39,8 +39,8 @@ import { import { calculateCosts } from "../lib/costs"; import { insertLog } from "../lib/logs"; import { - hasProviderEnvironmentToken, getProviderEnvVar, + hasProviderEnvironmentToken, } from "../lib/provider"; import type { ServerTypes } from "../vars"; @@ -2044,334 +2044,198 @@ chat.openapi(completions, async (c) => { continue; } - // For Google providers, handle raw JSON objects across newlines - if ( - usedProvider === "google-vertex" || - usedProvider === "google-ai-studio" - ) { - // Google sends raw JSON objects across multiple newlines, not SSE format - // We need to parse complete JSON objects from the accumulated buffer - let processedData = false; - - // Helper function to try parsing JSON from different positions - const tryParseJSON = (str: string, startIndex: number) => { - for (let i = startIndex; i < str.length; i++) { - try { - const substr = str.substring(startIndex, i + 1); - return { data: JSON.parse(substr), endIndex: i }; - } catch { - continue; - } - } - return null; - }; - - // Try to find and parse complete JSON objects - while (buffer.length > 0) { - // Find the start of a JSON object - const jsonStartIndex = buffer.indexOf("{"); - - if (jsonStartIndex === -1) { - // No JSON start found, clear buffer - buffer = ""; - break; - } - - // Try to parse JSON starting from this position - const parseResult = tryParseJSON(buffer, jsonStartIndex); - - if (parseResult) { - // Successfully parsed a JSON object - const data = parseResult.data; - buffer = buffer.substring(parseResult.endIndex + 1); - processedData = true; - - // Transform streaming responses to OpenAI format - const transformedData = transformStreamingChunkToOpenAIFormat( - usedProvider, - usedModel, - data, - ); - - await writeSSEAndCache({ - data: JSON.stringify(transformedData), - id: String(eventId++), - }); - - // Extract content for logging using helper function - const contentChunk = extractContentFromProvider( - data, - usedProvider, - ); - if (contentChunk) { - fullContent += contentChunk; - } + const lines = buffer.split("\n"); + // Keep the last line in buffer if it's incomplete (doesn't end with newline) + if (buffer.length > 0 && !buffer.endsWith("\n") && lines.length > 1) { + buffer = lines.pop() || ""; + } else { + buffer = ""; + } - // Extract reasoning content for logging using helper function - const reasoningContentChunk = - extractReasoningContentFromProvider(data, usedProvider); - if (reasoningContentChunk) { - fullReasoningContent += reasoningContentChunk; + for (const line of lines) { + if (line.startsWith("data: ")) { + if (line === "data: [DONE]") { + // Calculate final usage if we don't have complete data + let finalPromptTokens = promptTokens; + let finalCompletionTokens = completionTokens; + let finalTotalTokens = totalTokens; + + // Estimate missing tokens if needed using helper function + if (finalPromptTokens === null) { + finalPromptTokens = Math.round( + messages.reduce( + (acc, m) => acc + (m.content?.length || 0), + 0, + ) / 4, + ); } - // Check for finish reason - if (data.candidates && data.candidates[0]?.finishReason) { - finishReason = data.candidates[0].finishReason; - - // Send final chunk when we get a finish reason - if (finishReason) { - await writeSSEAndCache({ - event: "done", - data: "[DONE]", - id: String(eventId++), - }); - } + if (finalCompletionTokens === null) { + finalCompletionTokens = + estimateTokensFromContent(fullContent); } - // Extract token usage using helper function - const usage = extractTokenUsage(data, usedProvider); - if (usage.promptTokens !== null) { - promptTokens = usage.promptTokens; - } - if (usage.completionTokens !== null) { - completionTokens = usage.completionTokens; - } - if (usage.totalTokens !== null) { - totalTokens = usage.totalTokens; - } - if (usage.reasoningTokens !== null) { - reasoningTokens = usage.reasoningTokens; + if (finalTotalTokens === null) { + finalTotalTokens = + (finalPromptTokens || 0) + (finalCompletionTokens || 0); } - // For Google AI Studio, if candidatesTokenCount is not provided, - // we'll calculate it later from the fullContent + // Send final usage chunk before [DONE] if we have any usage data if ( - (usedProvider === "google-ai-studio" || - usedProvider === "google-vertex") && - !usage.completionTokens && - fullContent + finalPromptTokens !== null || + finalCompletionTokens !== null || + finalTotalTokens !== null ) { - completionTokens = null; // Mark as missing so we calculate later - } - } else { - // No complete JSON object found, try to find next JSON start - const nextStart = buffer.indexOf("{", jsonStartIndex + 1); - if (nextStart !== -1) { - // Skip to next potential JSON start - buffer = buffer.substring(nextStart); - } else { - // No more JSON starts found, break and wait for more data - break; - } - } - } + const finalUsageChunk = { + id: `chatcmpl-${Date.now()}`, + object: "chat.completion.chunk", + created: Math.floor(Date.now() / 1000), + model: usedModel, + choices: [ + { + index: 0, + delta: {}, + finish_reason: null, + }, + ], + usage: { + prompt_tokens: finalPromptTokens || 0, + completion_tokens: finalCompletionTokens || 0, + total_tokens: finalTotalTokens || 0, + }, + }; - // If we processed data but buffer is empty or very small, keep a bit for next iteration - if (processedData && buffer.length < 50) { - // Keep small remainder in buffer in case it's part of next JSON - } - } else { - // For non-Google providers, use the original line-by-line processing - const lines = buffer.split("\n"); - // Keep the last line in buffer if it's incomplete (doesn't end with newline) - if ( - buffer.length > 0 && - !buffer.endsWith("\n") && - lines.length > 1 - ) { - buffer = lines.pop() || ""; - } else { - buffer = ""; - } + await writeSSEAndCache({ + data: JSON.stringify(finalUsageChunk), + id: String(eventId++), + }); + } - for (const line of lines) { - if (line.startsWith("data: ")) { - if (line === "data: [DONE]") { - // Calculate final usage if we don't have complete data - let finalPromptTokens = promptTokens; - let finalCompletionTokens = completionTokens; - let finalTotalTokens = totalTokens; - - // Estimate missing tokens if needed using helper function - if (finalPromptTokens === null) { - finalPromptTokens = Math.round( - messages.reduce( - (acc, m) => acc + (m.content?.length || 0), - 0, - ) / 4, - ); + await writeSSEAndCache({ + event: "done", + data: "[DONE]", + id: String(eventId++), + }); + } else { + try { + const data = JSON.parse(line.substring(6)); + + // Transform streaming responses to OpenAI format for all providers + const transformedData = transformStreamingChunkToOpenAIFormat( + usedProvider, + usedModel, + data, + ); + + // For Anthropic, if we have partial usage data, complete it + if (usedProvider === "anthropic" && transformedData.usage) { + const usage = transformedData.usage; + if ( + usage.output_tokens !== undefined && + usage.prompt_tokens === undefined + ) { + // Estimate prompt tokens if not provided + const estimatedPromptTokens = Math.round( + messages.reduce( + (acc, m) => acc + (m.content?.length || 0), + 0, + ) / 4, + ); + transformedData.usage = { + prompt_tokens: estimatedPromptTokens, + completion_tokens: usage.output_tokens, + total_tokens: + estimatedPromptTokens + usage.output_tokens, + }; + } } - if (finalCompletionTokens === null) { - finalCompletionTokens = - estimateTokensFromContent(fullContent); - } + await writeSSEAndCache({ + data: JSON.stringify(transformedData), + id: String(eventId++), + }); - if (finalTotalTokens === null) { - finalTotalTokens = - (finalPromptTokens || 0) + (finalCompletionTokens || 0); + // Extract content for logging using helper function + const contentChunk = extractContentFromProvider( + data, + usedProvider, + ); + if (contentChunk) { + fullContent += contentChunk; } - // Send final usage chunk before [DONE] if we have any usage data - if ( - finalPromptTokens !== null || - finalCompletionTokens !== null || - finalTotalTokens !== null - ) { - const finalUsageChunk = { - id: `chatcmpl-${Date.now()}`, - object: "chat.completion.chunk", - created: Math.floor(Date.now() / 1000), - model: usedModel, - choices: [ - { - index: 0, - delta: {}, - finish_reason: null, - }, - ], - usage: { - prompt_tokens: finalPromptTokens || 0, - completion_tokens: finalCompletionTokens || 0, - total_tokens: finalTotalTokens || 0, - }, - }; - - await writeSSEAndCache({ - data: JSON.stringify(finalUsageChunk), - id: String(eventId++), - }); + // Extract reasoning content for logging using helper function + const reasoningContentChunk = + extractReasoningContentFromProvider(data, usedProvider); + if (reasoningContentChunk) { + fullReasoningContent += reasoningContentChunk; } - await writeSSEAndCache({ - event: "done", - data: "[DONE]", - id: String(eventId++), - }); - } else { - try { - const data = JSON.parse(line.substring(6)); - - // Transform streaming responses to OpenAI format for all providers - const transformedData = - transformStreamingChunkToOpenAIFormat( - usedProvider, - usedModel, - data, - ); - - // For Anthropic, if we have partial usage data, complete it - if (usedProvider === "anthropic" && transformedData.usage) { - const usage = transformedData.usage; + // Handle provider-specific finish reason extraction + switch (usedProvider) { + case "anthropic": if ( - usage.output_tokens !== undefined && - usage.prompt_tokens === undefined + data.type === "message_delta" && + data.delta?.stop_reason + ) { + finishReason = data.delta.stop_reason; + } else if ( + data.type === "message_stop" || + data.stop_reason ) { - // Estimate prompt tokens if not provided - const estimatedPromptTokens = Math.round( - messages.reduce( - (acc, m) => acc + (m.content?.length || 0), - 0, - ) / 4, - ); - transformedData.usage = { - prompt_tokens: estimatedPromptTokens, - completion_tokens: usage.output_tokens, - total_tokens: - estimatedPromptTokens + usage.output_tokens, - }; + finishReason = data.stop_reason || "end_turn"; + } else if (data.delta?.stop_reason) { + finishReason = data.delta.stop_reason; } - } - - await writeSSEAndCache({ - data: JSON.stringify(transformedData), - id: String(eventId++), - }); - - // Extract content for logging using helper function - const contentChunk = extractContentFromProvider( - data, - usedProvider, - ); - if (contentChunk) { - fullContent += contentChunk; - } + break; + default: // OpenAI format + if (data.choices && data.choices[0]?.finish_reason) { + finishReason = data.choices[0].finish_reason; + } + break; + } - // Extract reasoning content for logging using helper function - const reasoningContentChunk = - extractReasoningContentFromProvider(data, usedProvider); - if (reasoningContentChunk) { - fullReasoningContent += reasoningContentChunk; - } + // Extract token usage using helper function + const usage = extractTokenUsage(data, usedProvider); + if (usage.promptTokens !== null) { + promptTokens = usage.promptTokens; + } + if (usage.completionTokens !== null) { + completionTokens = usage.completionTokens; + } + if (usage.totalTokens !== null) { + totalTokens = usage.totalTokens; + } + if (usage.reasoningTokens !== null) { + reasoningTokens = usage.reasoningTokens; + } + if (usage.cachedTokens !== null) { + cachedTokens = usage.cachedTokens; + } - // Handle provider-specific finish reason extraction - switch (usedProvider) { - case "anthropic": - if ( - data.type === "message_delta" && - data.delta?.stop_reason - ) { - finishReason = data.delta.stop_reason; - } else if ( - data.type === "message_stop" || - data.stop_reason - ) { - finishReason = data.stop_reason || "end_turn"; - } else if (data.delta?.stop_reason) { - finishReason = data.delta.stop_reason; - } - break; - default: // OpenAI format - if (data.choices && data.choices[0]?.finish_reason) { - finishReason = data.choices[0].finish_reason; - } - break; + // Estimate tokens if not provided and we have a finish reason + if (finishReason && (!promptTokens || !completionTokens)) { + if (!promptTokens) { + promptTokens = Math.round( + messages.reduce( + (acc, m) => acc + (m.content?.length || 0), + 0, + ) / 4, + ); } - // Extract token usage using helper function - const usage = extractTokenUsage(data, usedProvider); - if (usage.promptTokens !== null) { - promptTokens = usage.promptTokens; - } - if (usage.completionTokens !== null) { - completionTokens = usage.completionTokens; - } - if (usage.totalTokens !== null) { - totalTokens = usage.totalTokens; - } - if (usage.reasoningTokens !== null) { - reasoningTokens = usage.reasoningTokens; - } - if (usage.cachedTokens !== null) { - cachedTokens = usage.cachedTokens; + if (!completionTokens) { + completionTokens = estimateTokensFromContent(fullContent); } - // Estimate tokens if not provided and we have a finish reason - if (finishReason && (!promptTokens || !completionTokens)) { - if (!promptTokens) { - promptTokens = Math.round( - messages.reduce( - (acc, m) => acc + (m.content?.length || 0), - 0, - ) / 4, - ); - } - - if (!completionTokens) { - completionTokens = - estimateTokensFromContent(fullContent); - } - - totalTokens = - (promptTokens || 0) + (completionTokens || 0); - } - } catch (e) { - console.warn("Failed to parse streaming JSON:", { - error: e instanceof Error ? e.message : String(e), - lineContent: line.substring(0, 100), // First 100 chars for debugging - provider: usedProvider, - }); + totalTokens = (promptTokens || 0) + (completionTokens || 0); } + } catch (e) { + console.warn("Failed to parse streaming JSON:", { + error: e instanceof Error ? e.message : String(e), + lineContent: line.substring(0, 100), // First 100 chars for debugging + provider: usedProvider, + }); } } } @@ -2554,7 +2418,6 @@ chat.openapi(completions, async (c) => { tools, toolChoice: tool_choice, }); - // Save streaming cache if enabled and not canceled if (cachingEnabled && streamingCacheKey && !canceled && finishReason) { try { @@ -2569,6 +2432,7 @@ chat.openapi(completions, async (c) => { completed: true, }, }; + await setStreamingCache( streamingCacheKey, streamingCacheData, From 2ecddb3932b2fc8f32dae2931989dc5301b423af Mon Sep 17 00:00:00 2001 From: Luca Steeb Date: Mon, 28 Jul 2025 02:22:13 +0200 Subject: [PATCH 2/6] fix(chat): improve error handling for JSON parsing Enhanced error handling by adding detailed logs for failed streaming JSON parsing, including line content and provider info. Refactored nested logic for improved clarity and robustness. --- apps/gateway/src/chat/chat.ts | 215 +++++++++++++++++----------------- 1 file changed, 108 insertions(+), 107 deletions(-) diff --git a/apps/gateway/src/chat/chat.ts b/apps/gateway/src/chat/chat.ts index 899887d469..c9624400c0 100644 --- a/apps/gateway/src/chat/chat.ts +++ b/apps/gateway/src/chat/chat.ts @@ -2117,125 +2117,126 @@ chat.openapi(completions, async (c) => { id: String(eventId++), }); } else { + let data; try { - const data = JSON.parse(line.substring(6)); - - // Transform streaming responses to OpenAI format for all providers - const transformedData = transformStreamingChunkToOpenAIFormat( - usedProvider, - usedModel, - data, - ); - - // For Anthropic, if we have partial usage data, complete it - if (usedProvider === "anthropic" && transformedData.usage) { - const usage = transformedData.usage; - if ( - usage.output_tokens !== undefined && - usage.prompt_tokens === undefined - ) { - // Estimate prompt tokens if not provided - const estimatedPromptTokens = Math.round( - messages.reduce( - (acc, m) => acc + (m.content?.length || 0), - 0, - ) / 4, - ); - transformedData.usage = { - prompt_tokens: estimatedPromptTokens, - completion_tokens: usage.output_tokens, - total_tokens: - estimatedPromptTokens + usage.output_tokens, - }; - } - } - - await writeSSEAndCache({ - data: JSON.stringify(transformedData), - id: String(eventId++), + data = JSON.parse(line.substring(6)); + } catch (e) { + console.warn("Failed to parse streaming JSON:", { + error: e instanceof Error ? e.message : String(e), + lineContent: line, // First 100 chars for debugging + provider: usedProvider, }); + continue; + } - // Extract content for logging using helper function - const contentChunk = extractContentFromProvider( - data, - usedProvider, - ); - if (contentChunk) { - fullContent += contentChunk; + // Transform streaming responses to OpenAI format for all providers + const transformedData = transformStreamingChunkToOpenAIFormat( + usedProvider, + usedModel, + data, + ); + + // For Anthropic, if we have partial usage data, complete it + if (usedProvider === "anthropic" && transformedData.usage) { + const usage = transformedData.usage; + if ( + usage.output_tokens !== undefined && + usage.prompt_tokens === undefined + ) { + // Estimate prompt tokens if not provided + const estimatedPromptTokens = Math.round( + messages.reduce( + (acc, m) => acc + (m.content?.length || 0), + 0, + ) / 4, + ); + transformedData.usage = { + prompt_tokens: estimatedPromptTokens, + completion_tokens: usage.output_tokens, + total_tokens: estimatedPromptTokens + usage.output_tokens, + }; } + } - // Extract reasoning content for logging using helper function - const reasoningContentChunk = - extractReasoningContentFromProvider(data, usedProvider); - if (reasoningContentChunk) { - fullReasoningContent += reasoningContentChunk; - } + await writeSSEAndCache({ + data: JSON.stringify(transformedData), + id: String(eventId++), + }); - // Handle provider-specific finish reason extraction - switch (usedProvider) { - case "anthropic": - if ( - data.type === "message_delta" && - data.delta?.stop_reason - ) { - finishReason = data.delta.stop_reason; - } else if ( - data.type === "message_stop" || - data.stop_reason - ) { - finishReason = data.stop_reason || "end_turn"; - } else if (data.delta?.stop_reason) { - finishReason = data.delta.stop_reason; - } - break; - default: // OpenAI format - if (data.choices && data.choices[0]?.finish_reason) { - finishReason = data.choices[0].finish_reason; - } - break; - } + // Extract content for logging using helper function + const contentChunk = extractContentFromProvider( + data, + usedProvider, + ); + if (contentChunk) { + fullContent += contentChunk; + } - // Extract token usage using helper function - const usage = extractTokenUsage(data, usedProvider); - if (usage.promptTokens !== null) { - promptTokens = usage.promptTokens; - } - if (usage.completionTokens !== null) { - completionTokens = usage.completionTokens; - } - if (usage.totalTokens !== null) { - totalTokens = usage.totalTokens; - } - if (usage.reasoningTokens !== null) { - reasoningTokens = usage.reasoningTokens; - } - if (usage.cachedTokens !== null) { - cachedTokens = usage.cachedTokens; - } + // Extract reasoning content for logging using helper function + const reasoningContentChunk = + extractReasoningContentFromProvider(data, usedProvider); + if (reasoningContentChunk) { + fullReasoningContent += reasoningContentChunk; + } - // Estimate tokens if not provided and we have a finish reason - if (finishReason && (!promptTokens || !completionTokens)) { - if (!promptTokens) { - promptTokens = Math.round( - messages.reduce( - (acc, m) => acc + (m.content?.length || 0), - 0, - ) / 4, - ); + // Handle provider-specific finish reason extraction + switch (usedProvider) { + case "anthropic": + if ( + data.type === "message_delta" && + data.delta?.stop_reason + ) { + finishReason = data.delta.stop_reason; + } else if ( + data.type === "message_stop" || + data.stop_reason + ) { + finishReason = data.stop_reason || "end_turn"; + } else if (data.delta?.stop_reason) { + finishReason = data.delta.stop_reason; } - - if (!completionTokens) { - completionTokens = estimateTokensFromContent(fullContent); + break; + default: // OpenAI format + if (data.choices && data.choices[0]?.finish_reason) { + finishReason = data.choices[0].finish_reason; } + break; + } + + // Extract token usage using helper function + const usage = extractTokenUsage(data, usedProvider); + if (usage.promptTokens !== null) { + promptTokens = usage.promptTokens; + } + if (usage.completionTokens !== null) { + completionTokens = usage.completionTokens; + } + if (usage.totalTokens !== null) { + totalTokens = usage.totalTokens; + } + if (usage.reasoningTokens !== null) { + reasoningTokens = usage.reasoningTokens; + } + if (usage.cachedTokens !== null) { + cachedTokens = usage.cachedTokens; + } - totalTokens = (promptTokens || 0) + (completionTokens || 0); + // Estimate tokens if not provided and we have a finish reason + if (finishReason && (!promptTokens || !completionTokens)) { + if (!promptTokens) { + promptTokens = Math.round( + messages.reduce( + (acc, m) => acc + (m.content?.length || 0), + 0, + ) / 4, + ); } - } catch (e) { - console.warn("Failed to parse streaming JSON:", { - error: e instanceof Error ? e.message : String(e), - lineContent: line.substring(0, 100), // First 100 chars for debugging - provider: usedProvider, - }); + + if (!completionTokens) { + completionTokens = estimateTokensFromContent(fullContent); + } + + totalTokens = (promptTokens || 0) + (completionTokens || 0); } } } From b1fd080be345197770e1589aebcfbf0095bb668b Mon Sep 17 00:00:00 2001 From: Luca Steeb Date: Mon, 28 Jul 2025 02:41:52 +0200 Subject: [PATCH 3/6] fix(chat): handle finishReason for Google providers Added logic to extract and assign finishReason from responses of Google Vertex and Google AI Studio providers. Improved consistency in provider-specific handling of response finish reasons. --- apps/gateway/src/chat/chat.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/apps/gateway/src/chat/chat.ts b/apps/gateway/src/chat/chat.ts index c9624400c0..de5454277a 100644 --- a/apps/gateway/src/chat/chat.ts +++ b/apps/gateway/src/chat/chat.ts @@ -2181,6 +2181,12 @@ chat.openapi(completions, async (c) => { // Handle provider-specific finish reason extraction switch (usedProvider) { + case "google-vertex": + case "google-ai-studio": + if (data.candidates?.[0]?.finishReason) { + finishReason = data.candidates[0].finishReason; + } + break; case "anthropic": if ( data.type === "message_delta" && From 2701497a064a41dc8a6ff6769d860e52c74ac6e0 Mon Sep 17 00:00:00 2001 From: Luca Steeb Date: Mon, 28 Jul 2025 13:14:01 +0200 Subject: [PATCH 4/6] fix(chat): estimate missing token counts in usage metadata Added fallback logic to estimate missing completion token counts using content length when data is incomplete. Improved token usage calculation for providers, ensuring accurate results even if metadata is partially available. Enhanced error handling for encoding failures with descriptive logs. --- apps/gateway/src/chat/chat.ts | 64 +++++++++++++++++++++++++++++++---- 1 file changed, 58 insertions(+), 6 deletions(-) diff --git a/apps/gateway/src/chat/chat.ts b/apps/gateway/src/chat/chat.ts index de5454277a..19ead62e44 100644 --- a/apps/gateway/src/chat/chat.ts +++ b/apps/gateway/src/chat/chat.ts @@ -161,10 +161,27 @@ function parseProviderResponse(usedProvider: Provider, json: any) { promptTokens = json.usageMetadata?.promptTokenCount || null; completionTokens = json.usageMetadata?.candidatesTokenCount || null; reasoningTokens = json.usageMetadata?.thoughtsTokenCount || null; - totalTokens = - promptTokens !== null && completionTokens !== null - ? promptTokens + completionTokens - : json.usageMetadata?.totalTokenCount || null; + totalTokens = json.usageMetadata?.totalTokenCount || null; + + // If candidatesTokenCount is missing, estimate it from the content + if (completionTokens === null && content) { + try { + completionTokens = encode(content).length; + } catch (error) { + // Fallback to simple estimation if encoding fails + console.error(`Failed to encode completion text: ${error}`); + completionTokens = Math.max(1, Math.round(content.length / 4)); + } + } + + // Calculate totalTokens if not provided but we have prompt and completion tokens + if ( + totalTokens === null && + promptTokens !== null && + completionTokens !== null + ) { + totalTokens = promptTokens + completionTokens + (reasoningTokens || 0); + } break; case "mistral": content = json.choices?.[0]?.message?.content || null; @@ -319,7 +336,11 @@ function extractReasoningContentFromProvider( /** * Extracts token usage information from streaming data based on provider format */ -function extractTokenUsage(data: any, provider: Provider) { +function extractTokenUsage( + data: any, + provider: Provider, + fullContent?: string, +) { let promptTokens = null; let completionTokens = null; let totalTokens = null; @@ -334,6 +355,33 @@ function extractTokenUsage(data: any, provider: Provider) { completionTokens = data.usageMetadata.candidatesTokenCount || null; totalTokens = data.usageMetadata.totalTokenCount || null; reasoningTokens = data.usageMetadata.thoughtsTokenCount || null; + + // If candidatesTokenCount is missing and we have content, estimate it + if (completionTokens === null && fullContent) { + try { + completionTokens = encode(fullContent).length; + console.log( + `Google streaming: estimated ${completionTokens} completion tokens from content: "${fullContent}"`, + ); + } catch (error) { + // Fallback to simple estimation if encoding fails + console.error( + `Failed to encode completion text in streaming: ${error}`, + ); + completionTokens = Math.max(1, Math.round(fullContent.length / 4)); + console.log( + `Google streaming: fallback estimated ${completionTokens} completion tokens from content: "${fullContent}"`, + ); + } + } else if (completionTokens === null) { + console.log( + `Google streaming: candidatesTokenCount is null and no fullContent provided`, + ); + } else { + console.log( + `Google streaming: using provided candidatesTokenCount: ${completionTokens}`, + ); + } } break; case "anthropic": @@ -2210,7 +2258,11 @@ chat.openapi(completions, async (c) => { } // Extract token usage using helper function - const usage = extractTokenUsage(data, usedProvider); + const usage = extractTokenUsage( + data, + usedProvider, + fullContent, + ); if (usage.promptTokens !== null) { promptTokens = usage.promptTokens; } From 19bf1cd6f39a78cb3d9dba066042746f36c689fd Mon Sep 17 00:00:00 2001 From: Luca Steeb Date: Mon, 28 Jul 2025 13:48:41 +0200 Subject: [PATCH 5/6] fix(chat): add token usage metadata for Google providers Added logic to extract and include token usage metadata (prompt, completion, and total tokens) for Google Vertex and Google AI Studio providers when available. Improved streaming data consistency by integrating usage details into streaming chunks. --- apps/gateway/src/chat/chat.ts | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/apps/gateway/src/chat/chat.ts b/apps/gateway/src/chat/chat.ts index 19ead62e44..467122f96e 100644 --- a/apps/gateway/src/chat/chat.ts +++ b/apps/gateway/src/chat/chat.ts @@ -2206,6 +2206,31 @@ chat.openapi(completions, async (c) => { } } + // For Google providers, add usage information when available + if ( + usedProvider === "google-vertex" || + usedProvider === "google-ai-studio" + ) { + const usage = extractTokenUsage( + data, + usedProvider, + fullContent, + ); + + // If we have usage data from Google, add it to the streaming chunk + if ( + usage.promptTokens !== null || + usage.completionTokens !== null || + usage.totalTokens !== null + ) { + transformedData.usage = { + prompt_tokens: usage.promptTokens, + completion_tokens: usage.completionTokens, + total_tokens: usage.totalTokens, + }; + } + } + await writeSSEAndCache({ data: JSON.stringify(transformedData), id: String(eventId++), From a58d7788a406f1be0aef7d4f72ad57c8104389a0 Mon Sep 17 00:00:00 2001 From: Luca Steeb Date: Mon, 28 Jul 2025 14:00:22 +0200 Subject: [PATCH 6/6] fix(chat): remove redundant logs in token estimation Removed excessive logging in token estimation logic for Google streaming. Improved clarity and reduced unnecessary console clutter. --- apps/gateway/src/chat/chat.ts | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/apps/gateway/src/chat/chat.ts b/apps/gateway/src/chat/chat.ts index 467122f96e..2b6b2847a6 100644 --- a/apps/gateway/src/chat/chat.ts +++ b/apps/gateway/src/chat/chat.ts @@ -360,27 +360,13 @@ function extractTokenUsage( if (completionTokens === null && fullContent) { try { completionTokens = encode(fullContent).length; - console.log( - `Google streaming: estimated ${completionTokens} completion tokens from content: "${fullContent}"`, - ); } catch (error) { // Fallback to simple estimation if encoding fails console.error( `Failed to encode completion text in streaming: ${error}`, ); completionTokens = Math.max(1, Math.round(fullContent.length / 4)); - console.log( - `Google streaming: fallback estimated ${completionTokens} completion tokens from content: "${fullContent}"`, - ); } - } else if (completionTokens === null) { - console.log( - `Google streaming: candidatesTokenCount is null and no fullContent provided`, - ); - } else { - console.log( - `Google streaming: using provided candidatesTokenCount: ${completionTokens}`, - ); } } break;