diff --git a/apps/gateway/src/chat/chat.ts b/apps/gateway/src/chat/chat.ts index 03f620a1d9..005b2244bc 100644 --- a/apps/gateway/src/chat/chat.ts +++ b/apps/gateway/src/chat/chat.ts @@ -2647,6 +2647,7 @@ chat.openapi(completions, async (c) => { { prompt: messages.map((m) => m.content).join("\n"), completion: fullContent, + toolResults: streamingToolCalls || undefined, }, ); @@ -3033,6 +3034,7 @@ chat.openapi(completions, async (c) => { { prompt: messages.map((m) => m.content).join("\n"), completion: content, + toolResults: toolResults, }, ); diff --git a/apps/gateway/src/lib/costs.spec.ts b/apps/gateway/src/lib/costs.spec.ts index c4b365e8b0..4730ea4111 100644 --- a/apps/gateway/src/lib/costs.spec.ts +++ b/apps/gateway/src/lib/costs.spec.ts @@ -171,4 +171,83 @@ describe("calculateCosts", () => { expect(result.discount).toBeUndefined(); // Should not include discount field when discount is 1 }); + + it("should calculate input costs even when output tokens are zero", () => { + const result = calculateCosts("gpt-4", "openai", 100, 0, null); + + expect(result.inputCost).toBeCloseTo(0.001); // 100 * 0.00001 + expect(result.outputCost).toBeCloseTo(0); // 0 * 0.00003 + expect(result.totalCost).toBeCloseTo(0.001); // 0.001 + 0 + expect(result.promptTokens).toBe(100); + expect(result.completionTokens).toBe(0); + expect(result.estimatedCost).toBe(false); + }); + + it("should calculate input costs when completion tokens are null but prompt tokens exist", () => { + const result = calculateCosts("gpt-4", "openai", 100, null, null); + + expect(result.inputCost).toBeCloseTo(0.001); // 100 * 0.00001 + expect(result.outputCost).toBeCloseTo(0); // 0 * 0.00003 (completion tokens set to 0) + expect(result.totalCost).toBeCloseTo(0.001); // 0.001 + 0 + expect(result.promptTokens).toBe(100); + expect(result.completionTokens).toBe(0); // Should default to 0 + expect(result.estimatedCost).toBe(false); + }); + + it("should include tool results in completion token estimation", () => { + const result = calculateCosts("gpt-4", "openai", null, null, null, { + prompt: "What's the weather like?", + completion: "", // Empty completion + toolResults: [ + { + id: "call_1", + type: "function", + function: { + name: "get_weather", + arguments: '{"location": "San Francisco"}', + }, + }, + { + id: "call_2", + type: "function", + function: { + name: "get_temperature", + arguments: '{"location": "New York", "units": "celsius"}', + }, + }, + ], + }); + + // Should calculate tokens for tool calls even with empty completion + expect(result.promptTokens).toBeGreaterThan(0); + expect(result.completionTokens).toBeGreaterThan(0); // Should include tool call tokens + expect(result.inputCost).toBeGreaterThan(0); + expect(result.outputCost).toBeGreaterThan(0); // Should have cost from tool calls + expect(result.totalCost).toBeGreaterThan(0); + expect(result.estimatedCost).toBe(true); + }); + + it("should handle tool results with missing function data gracefully", () => { + const result = calculateCosts("gpt-4", "openai", null, null, null, { + prompt: "What's the weather like?", + completion: "Here's the weather:", + toolResults: [ + { id: "call_1", type: "function" } as any, // Missing function data + { id: "call_2", type: "function", function: {} as any }, // Missing name and arguments + { + id: "call_3", + type: "function", + function: { + name: "get_weather", + arguments: '{"location": "Paris"}', + }, + }, + ], + }); + + // Should still work with partial tool result data + expect(result.promptTokens).toBeGreaterThan(0); + expect(result.completionTokens).toBeGreaterThan(0); + expect(result.estimatedCost).toBe(true); + }); }); diff --git a/apps/gateway/src/lib/costs.ts b/apps/gateway/src/lib/costs.ts index 8fbb6612f3..864db788fb 100644 --- a/apps/gateway/src/lib/costs.ts +++ b/apps/gateway/src/lib/costs.ts @@ -1,7 +1,12 @@ import { encode, encodeChat } from "gpt-tokenizer"; import { logger } from "@llmgateway/logger"; -import { type Model, type ModelDefinition, models } from "@llmgateway/models"; +import { + type Model, + type ModelDefinition, + models, + type ToolCall, +} from "@llmgateway/models"; // Define ChatMessage type to match what gpt-tokenizer expects interface ChatMessage { @@ -27,6 +32,7 @@ export function calculateCosts( messages?: ChatMessage[]; prompt?: string; completion?: string; + toolResults?: ToolCall[]; }, ) { // Find the model info - try both base model name and provider model name @@ -89,20 +95,39 @@ export function calculateCosts( } // Calculate completion tokens - if (!completionTokens && fullOutput && fullOutput.completion) { - try { - calculatedCompletionTokens = encode( - JSON.stringify(fullOutput.completion), - ).length; - } catch (error) { - // If encoding fails, leave as null - logger.error(`Failed to encode completion text: ${error}`); + if (!completionTokens && fullOutput) { + let completionText = ""; + + // Include main completion content + if (fullOutput.completion) { + completionText += fullOutput.completion; + } + + // Include tool results if available + if (fullOutput.toolResults && Array.isArray(fullOutput.toolResults)) { + for (const toolResult of fullOutput.toolResults) { + if (toolResult.function?.name) { + completionText += toolResult.function.name; + } + if (toolResult.function?.arguments) { + completionText += JSON.stringify(toolResult.function.arguments); + } + } + } + + if (completionText) { + try { + calculatedCompletionTokens = encode(completionText).length; + } catch (error) { + // If encoding fails, leave as null + logger.error(`Failed to encode completion text: ${error}`); + } } } } - // If we still don't have token counts, return null costs - if (!calculatedPromptTokens || !calculatedCompletionTokens) { + // If we don't have prompt tokens, we can't calculate any costs + if (!calculatedPromptTokens) { return { inputCost: null, outputCost: null, @@ -117,6 +142,11 @@ export function calculateCosts( }; } + // Set completion tokens to 0 if not available (but still calculate input costs) + if (!calculatedCompletionTokens) { + calculatedCompletionTokens = 0; + } + // Find the provider-specific pricing const providerInfo = modelInfo.providers.find( (p) => p.providerId === provider,