feat(routing): cache weight via length-based estimate

apps/api/src/routes/logs.ts

@@ -125,6 +125,7 @@ const logSchema = z.object({
 						throughput: z.number().optional(),
 						price: z.number().optional(),
 						priority: z.number().optional(),
+						cacheSupported: z.boolean().optional(),
 						failed: z.boolean().optional(),
 						status_code: z.number().optional(),
 						error_type: z.string().optional(),

apps/code/src/lib/api/v1.d.ts

@@ -992,6 +992,7 @@ export interface paths {
                                         throughput?: number;
                                         price?: number;
                                         priority?: number;
+                                        cacheSupported?: boolean;
                                         failed?: boolean;
                                         status_code?: number;
                                         error_type?: string;
@@ -1236,6 +1237,7 @@ export interface paths {
                                         throughput?: number;
                                         price?: number;
                                         priority?: number;
+                                        cacheSupported?: boolean;
                                         failed?: boolean;
                                         status_code?: number;
                                         error_type?: string;

apps/docs/content/features/routing.mdx

@@ -49,6 +49,10 @@ The algorithm calculates a weighted score for each available provider and select
 
 For non-streaming requests, the latency weight (10%) is redistributed proportionally to the other factors since time-to-first-token is less relevant when waiting for the complete response.
 
+**Cache Support for Large Prompts**:
+
+When the estimated prompt is at least 5,000 tokens, an additional 20% weight is added to the score for whether each provider supports prompt caching (advertised via a cached input price). Providers that support caching score better than ones that do not, since caching can substantially reduce the cost of large or repeated prompts. Below the 5k threshold, this weight is dropped entirely — caching has little impact on small prompts, so cache support is ignored. The selected provider's cache support is exposed as `cacheSupported` on the routing metadata.
+
 **Exponential Uptime Penalty**:
 
 Providers with uptime below 95% receive an additional exponential penalty that increases rapidly as uptime drops:

apps/gateway/package.json

@@ -40,7 +40,6 @@
 		"@opentelemetry/sdk-node": "0.205.0",
 		"decimal.js": "10.5.0",
 		"dotenv": "16.5.0",
-		"gpt-tokenizer": "3.0.1",
 		"hono": "^4.12.7",
 		"hono-openapi": "^1.1.1",
 		"ioredis": "5.7.0",

apps/gateway/src/chat/chat.ts

@@ -1,5 +1,4 @@
 import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
-import { encode } from "gpt-tokenizer";
 import { HTTPException } from "hono/http-exception";
 import { streamSSE } from "hono/streaming";
 
@@ -245,6 +244,7 @@ function collapseProvidersToBestRegionPerProvider(
 	options: {
 		metricsMap: Map<string, ProviderMetrics>;
 		isStreaming: boolean;
+		promptTokens?: number;
 	},
 ): ProviderModelMapping[] {
 	const providersById = new Map<string, ProviderModelMapping[]>();
@@ -1087,6 +1087,18 @@ chat.openapi(completions, async (c) => {
 		}
 	}
 
+	// Estimate prompt tokens once so all routing decisions can reuse the
+	// same value (e.g. cache-support weighting kicks in for large prompts).
+	// Uses a cheap chars/4 heuristic — accuracy is intentionally traded
+	// for throughput on the gateway hot path.
+	let routingPromptTokens = 0;
+	if (messages && messages.length > 0) {
+		routingPromptTokens = encodeChatMessages(messages);
+	}
+	if (tools && tools.length > 0) {
+		routingPromptTokens += Math.round(JSON.stringify(tools).length / 4);
+	}
+
 	// Extract and validate source from x-source header with HTTP-Referer fallback
 	let source = validateSource(
 		c.req.header("x-source"),
@@ -1498,34 +1510,9 @@ chat.openapi(completions, async (c) => {
 		(usedProvider === "llmgateway" && usedModel === "auto") ||
 		usedModel === "auto"
 	) {
-		// Estimate prompt/input tokens first so auto-routing can react to large prompts
-		let estimatedInputTokens = 0;
-
-		// Estimate prompt tokens from messages
-		if (messages && messages.length > 0) {
-			try {
-				estimatedInputTokens = encodeChatMessages(messages);
-			} catch {
-				// Fallback to simple estimation if encoding fails
-				const messageTokens = messages.reduce(
-					(acc, m) => acc + (m.content?.length ?? 0),
-					0,
-				);
-				estimatedInputTokens = Math.max(1, Math.round(messageTokens / 4));
-			}
-		}
-
-		// Add tool definitions to context estimation
-		if (tools && tools.length > 0) {
-			try {
-				const toolsString = JSON.stringify(tools);
-				const toolTokens = Math.round(toolsString.length / 4);
-				estimatedInputTokens += toolTokens;
-			} catch {
-				// Fallback estimation for tools
-				estimatedInputTokens += tools.length * 100; // Rough estimate per tool
-			}
-		}
+		// Reuse the prompt-token estimate computed earlier so auto-routing can
+		// react to large prompts when picking a model.
+		const estimatedInputTokens = routingPromptTokens;
 
 		// Estimate the full context needed based on the request
 		let requiredContextSize = estimatedInputTokens;
@@ -1744,13 +1731,18 @@ chat.openapi(completions, async (c) => {
 					{
 						metricsMap,
 						isStreaming: stream,
+						promptTokens: routingPromptTokens,
 					},
 				);
 
 			const cheapestResult = getCheapestFromAvailableProviders(
 				providerAgnosticSelectedProviders,
 				selectedModel,
-				{ metricsMap, isStreaming: stream },
+				{
+					metricsMap,
+					isStreaming: stream,
+					promptTokens: routingPromptTokens,
+				},
 			);
 
 			if (cheapestResult) {
@@ -1910,6 +1902,7 @@ chat.openapi(completions, async (c) => {
 						{
 							metricsMap,
 							isStreaming: stream,
+							promptTokens: routingPromptTokens,
 						},
 					);
 
@@ -2094,6 +2087,7 @@ chat.openapi(completions, async (c) => {
 							{
 								metricsMap: allMetricsMap,
 								isStreaming: stream,
+								promptTokens: routingPromptTokens,
 							},
 						);
 
@@ -2231,7 +2225,11 @@ chat.openapi(completions, async (c) => {
 							collapseProvidersToBestRegionPerProvider(
 								availableModelProviders,
 								modelWithPricing,
-								{ metricsMap: allMetricsMap, isStreaming: stream },
+								{
+									metricsMap: allMetricsMap,
+									isStreaming: stream,
+									promptTokens: routingPromptTokens,
+								},
 							);
 
 						// Filter to only providers with better uptime than the original
@@ -2256,7 +2254,11 @@ chat.openapi(completions, async (c) => {
 							const cheapestResult = getCheapestFromAvailableProviders(
 								betterUptimeProviders,
 								modelWithPricing,
-								{ metricsMap: allMetricsMap, isStreaming: stream },
+								{
+									metricsMap: allMetricsMap,
+									isStreaming: stream,
+									promptTokens: routingPromptTokens,
+								},
 							);
 
 							// Get price info for the original requested provider to include in scores
@@ -2433,13 +2435,21 @@ chat.openapi(completions, async (c) => {
 					collapseProvidersToBestRegionPerProvider(
 						routingCandidates,
 						modelWithPricing,
-						{ metricsMap, isStreaming: stream },
+						{
+							metricsMap,
+							isStreaming: stream,
+							promptTokens: routingPromptTokens,
+						},
 					);
 
 				const cheapestResult = getCheapestFromAvailableProviders(
 					providerAgnosticCandidates,
 					modelWithPricing,
-					{ metricsMap, isStreaming: stream },
+					{
+						metricsMap,
+						isStreaming: stream,
+						promptTokens: routingPromptTokens,
+					},
 				);
 
 				if (cheapestResult) {
@@ -2606,6 +2616,7 @@ chat.openapi(completions, async (c) => {
 						{
 							metricsMap,
 							isStreaming: stream,
+							promptTokens: routingPromptTokens,
 						},
 					);
 
@@ -6866,27 +6877,8 @@ chat.openapi(completions, async (c) => {
 								imageTokens = 258 + Math.ceil(imageByteSize / 750);
 							}
 
-							// Skip expensive token encoding for image responses - use simple estimation
-							// Token encoding on large base64 content causes CPU spikes
-							if (imageByteSize > 0) {
-								const textTokens = estimateTokensFromContent(fullContent);
-								calculatedCompletionTokens = textTokens + imageTokens;
-							} else {
-								try {
-									const textTokens = fullContent
-										? encode(JSON.stringify(fullContent)).length
-										: 0;
-									calculatedCompletionTokens = textTokens + imageTokens;
-								} catch (error) {
-									// Fallback to simple estimation if encoding fails
-									logger.error(
-										"Failed to encode completion text in streaming",
-										error instanceof Error ? error : new Error(String(error)),
-									);
-									const textTokens = estimateTokensFromContent(fullContent);
-									calculatedCompletionTokens = textTokens + imageTokens;
-								}
-							}
+							const textTokens = estimateTokensFromContent(fullContent);
+							calculatedCompletionTokens = textTokens + imageTokens;
 						}
 
 						calculatedTotalTokens =
@@ -6896,17 +6888,8 @@ chat.openapi(completions, async (c) => {
 					// Estimate reasoning tokens if not provided but reasoning content exists
 					let calculatedReasoningTokens = reasoningTokens;
 					if (!reasoningTokens && fullReasoningContent) {
-						try {
-							calculatedReasoningTokens = encode(fullReasoningContent).length;
-						} catch (error) {
-							// Fallback to simple estimation if encoding fails
-							logger.error(
-								"Failed to encode reasoning text in streaming",
-								error instanceof Error ? error : new Error(String(error)),
-							);
-							calculatedReasoningTokens =
-								estimateTokensFromContent(fullReasoningContent);
-						}
+						calculatedReasoningTokens =
+							estimateTokensFromContent(fullReasoningContent);
 					}
 
 					if (
@@ -8962,16 +8945,7 @@ chat.openapi(completions, async (c) => {
 	// Estimate reasoning tokens if not provided but reasoning content exists
 	let calculatedReasoningTokens = reasoningTokens;
 	if (!reasoningTokens && reasoningContent) {
-		try {
-			calculatedReasoningTokens = encode(reasoningContent).length;
-		} catch (error) {
-			// Fallback to simple estimation if encoding fails
-			logger.error(
-				"Failed to encode reasoning text",
-				error instanceof Error ? error : new Error(String(error)),
-			);
-			calculatedReasoningTokens = estimateTokensFromContent(reasoningContent);
-		}
+		calculatedReasoningTokens = estimateTokensFromContent(reasoningContent);
 	}
 	const costs = await calculateCosts(
 		usedModel,

apps/gateway/src/chat/tools/estimate-tokens-from-content.ts

@@ -1,9 +1,9 @@
+import { estimateTokensFromText } from "@llmgateway/shared";
+
 /**
- * Estimates tokens from content length using simple division
+ * Estimates tokens from content length using a chars/4 heuristic. Backed by
+ * the shared text-only estimator.
  */
 export function estimateTokensFromContent(content: string): number {
-	if (!content) {
-		return 0;
-	}
-	return Math.max(1, Math.round(content.length / 4));
+	return estimateTokensFromText(content);
 }

apps/gateway/src/chat/tools/estimate-tokens.ts

@@ -1,13 +1,12 @@
-import { encode } from "gpt-tokenizer";
-
-import { logger } from "@llmgateway/logger";
-
+import { estimateTokensFromContent } from "./estimate-tokens-from-content.js";
 import { encodeChatMessages } from "./tokenizer.js";
 
 import type { Provider } from "@llmgateway/models";
 
 /**
- * Estimates token counts when not provided by the API using gpt-tokenizer
+ * Estimates token counts when not provided by the API. Uses a cheap
+ * length-based heuristic rather than running a tokenizer on the
+ * gateway hot path.
  */
 export function estimateTokens(
 	usedProvider: Provider,
@@ -19,25 +18,13 @@ export function estimateTokens(
 	let calculatedPromptTokens = promptTokens;
 	let calculatedCompletionTokens = completionTokens;
 
-	// Always estimate missing tokens for any provider
 	if (!promptTokens || !completionTokens) {
-		// Estimate prompt tokens using encodeChat for better accuracy
 		if (!promptTokens && messages && messages.length > 0) {
 			calculatedPromptTokens = encodeChatMessages(messages);
 		}
 
-		// Estimate completion tokens using encode for better accuracy
 		if (!completionTokens && content) {
-			try {
-				calculatedCompletionTokens = encode(JSON.stringify(content)).length;
-			} catch (error) {
-				// Fallback to simple estimation if encoding fails
-				logger.error(
-					"Failed to encode completion text",
-					error instanceof Error ? error : new Error(String(error)),
-				);
-				calculatedCompletionTokens = content.length / 4;
-			}
+			calculatedCompletionTokens = estimateTokensFromContent(content);
 		}
 	}
 

apps/gateway/src/chat/tools/tokenizer.ts

@@ -1,12 +1,10 @@
-import { encodeChat } from "gpt-tokenizer";
-
-import { logger } from "@llmgateway/logger";
-
-import { DEFAULT_TOKENIZER_MODEL } from "./types.js";
+import { estimateChatMessageTokens } from "@llmgateway/shared";
 
 /**
  * Converts a message content value (string, array of content parts, null, or
- * undefined) to a plain string suitable for the gpt-tokenizer library.
+ * undefined) to a plain string. Used by call sites that need a flat string
+ * (e.g. for cost estimation) — not for token counting; see
+ * `encodeChatMessages` for that.
  */
 export function messageContentToString(
 	content: string | unknown[] | null | undefined,
@@ -21,36 +19,12 @@ export function messageContentToString(
 }
 
 /**
- * Encodes an array of chat messages and returns the token count. Handles
- * messages whose content may be a string, an array of content parts, null, or
- * undefined – all of which are valid shapes in the OpenAI chat format but
- * would otherwise crash gpt-tokenizer.
+ * Rough length-based prompt-token estimate for a chat message array.
+ *
+ * Backed by the shared `estimateChatMessageTokens` helper, which only counts
+ * text and ignores multimodal parts (image_url, file, etc.). Image input
+ * billing is handled separately in costs.ts.
  */
 export function encodeChatMessages(messages: any[]): number {
-	try {
-		const chatMessages = messages.map((m) => ({
-			role: m.role as "user" | "assistant" | "system" | undefined,
-			content: messageContentToString(m.content),
-			...(m.name !== null && m.name !== undefined && { name: m.name }),
-		}));
-		return encodeChat(chatMessages, DEFAULT_TOKENIZER_MODEL).length;
-	} catch (error) {
-		logger.error("Failed to encode chat messages", {
-			error: error instanceof Error ? error.message : String(error),
-			messageCount: messages.length,
-			messageRoles: messages.map((m) => m.role),
-			messageContentTypes: messages.map((m) => typeof m.content),
-		});
-		// Fallback: rough 4-chars-per-token estimate
-		return Math.max(
-			1,
-			Math.round(
-				messages.reduce(
-					(acc: number, m: any) =>
-						acc + messageContentToString(m.content).length,
-					0,
-				) / 4,
-			),
-		);
-	}
+	return estimateChatMessageTokens(messages);
 }