From 378eb19812e740abee46249e0f777cb5ebae9009 Mon Sep 17 00:00:00 2001 From: Luca Steeb Date: Sat, 4 Apr 2026 21:17:53 +0700 Subject: [PATCH 1/6] fix: prefer api keys in hybrid mode --- apps/gateway/src/api.spec.ts | 75 +++++++++++++++++++++++++++++++++++ apps/gateway/src/chat/chat.ts | 37 ----------------- 2 files changed, 75 insertions(+), 37 deletions(-) diff --git a/apps/gateway/src/api.spec.ts b/apps/gateway/src/api.spec.ts index a74da50405..4891eaf700 100644 --- a/apps/gateway/src/api.spec.ts +++ b/apps/gateway/src/api.spec.ts @@ -3860,6 +3860,81 @@ describe("api", () => { expect(Number(cachedLog?.promptTokens)).toBeGreaterThan(0); }); + test("/v1/chat/completions hybrid prefers provider key over regional env token", async () => { + await harness.setProjectMode("hybrid"); + + await db.insert(tables.apiKey).values({ + id: "token-id", + token: "real-token", + projectId: "project-id", + description: "Test API Key", + createdBy: "user-id", + }); + + await db.insert(tables.providerKey).values({ + id: "provider-key-id", + token: "sk-db-key", + provider: "alibaba", + organizationId: "org-id", + baseUrl: mockServerUrl, + }); + + const previousAlibabaRegionalKey = + process.env.LLM_ALIBABA_API_KEY__US_VIRGINIA; + const originalFetch = globalThis.fetch; + let sawAlibabaRequest = false; + const fetchSpy = vi + .spyOn(globalThis, "fetch") + .mockImplementation(async (input, init) => { + const url = + typeof input === "string" + ? input + : input instanceof URL + ? input.toString() + : input.url; + + if (url.startsWith(mockServerUrl)) { + sawAlibabaRequest = true; + const headers = new Headers(init?.headers); + expect(headers.get("authorization")).toBe("Bearer sk-db-key"); + } + + return await originalFetch(input as RequestInfo | URL, init); + }); + + try { + process.env.LLM_ALIBABA_API_KEY__US_VIRGINIA = "sk-env-key"; + + const res = await app.request("/v1/chat/completions", { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: "Bearer real-token", + }, + body: JSON.stringify({ + model: "alibaba/qwen-plus:us-virginia", + messages: [ + { + role: "user", + content: "Hello from hybrid regional routing!", + }, + ], + }), + }); + + expect(res.status).toBe(200); + expect(sawAlibabaRequest).toBe(true); + } finally { + fetchSpy.mockRestore(); + if (previousAlibabaRegionalKey === undefined) { + delete process.env.LLM_ALIBABA_API_KEY__US_VIRGINIA; + } else { + process.env.LLM_ALIBABA_API_KEY__US_VIRGINIA = + previousAlibabaRegionalKey; + } + } + }); + // test for model with multiple providers (llama-3.3-70b-instruct) test.skip("/v1/chat/completions with model that has multiple providers", async () => { await db.insert(tables.apiKey).values({ diff --git a/apps/gateway/src/chat/chat.ts b/apps/gateway/src/chat/chat.ts index ca8bfacd51..7e8fdeba8d 100644 --- a/apps/gateway/src/chat/chat.ts +++ b/apps/gateway/src/chat/chat.ts @@ -4593,25 +4593,6 @@ chat.openapi(completions, async (c) => { ) { usedRegion ??= resolveRegionFromProviderKey(providerKey); } - // Override with region-specific env var if the DB key doesn't match the requested region. - // When we do override, route health attribution to the regional env credential. - // providerKey stays set so endpoint/options/baseUrl construction keeps the BYOK context; - // only trackedKeyHealthId is cleared so reportTrackedKey* doesn't blame the unused DB key. - if (usedRegion) { - const regionEnvVarName = getRegionSpecificEnvVarName( - usedProvider, - usedRegion, - ); - if (regionEnvVarName) { - const regionToken = process.env[regionEnvVarName]; - if (regionToken && regionToken !== usedToken) { - usedToken = regionToken; - envVarName = regionEnvVarName; - configIndex = 0; - trackedKeyHealthId = undefined; - } - } - } } else if (project.mode === "credits") { // Check regular credits, dev plan credits, and chat plan credits. assertDevPlanPremiumCapNotExceeded( @@ -4726,24 +4707,6 @@ chat.openapi(completions, async (c) => { ) { usedRegion ??= resolveRegionFromProviderKey(providerKey); } - // Override with region-specific env var if the DB key doesn't match the requested region. - // Route health attribution to the env credential while keeping providerKey for - // endpoint/options resolution (BYOK base URLs and provider options). - if (usedRegion) { - const regionEnvVarName = getRegionSpecificEnvVarName( - usedProvider, - usedRegion, - ); - if (regionEnvVarName) { - const regionToken = process.env[regionEnvVarName]; - if (regionToken && regionToken !== usedToken) { - usedToken = regionToken; - envVarName = regionEnvVarName; - configIndex = 0; - trackedKeyHealthId = undefined; - } - } - } } else { // No API key available, fall back to credits // Check regular credits, dev plan credits, and chat plan credits. From c3c488061f988457300d003addfc6ffddd25c5cc Mon Sep 17 00:00:00 2001 From: Luca Steeb Date: Sat, 4 Apr 2026 21:43:46 +0700 Subject: [PATCH 2/6] fix: prioritize keyed hybrid routes --- apps/gateway/src/api.spec.ts | 91 +++++++++++++ apps/gateway/src/chat/chat.ts | 248 +++++++++++++++++++++++----------- 2 files changed, 261 insertions(+), 78 deletions(-) diff --git a/apps/gateway/src/api.spec.ts b/apps/gateway/src/api.spec.ts index 4891eaf700..6daaf52ad8 100644 --- a/apps/gateway/src/api.spec.ts +++ b/apps/gateway/src/api.spec.ts @@ -3935,6 +3935,97 @@ describe("api", () => { } }); + test("/v1/chat/completions hybrid prefers keyed provider over credits-backed provider for gemini-2.5-flash-lite", async () => { + await harness.setProjectMode("hybrid"); + await harness.setRoutingMetrics( + "gemini-2.5-flash-lite", + "google-ai-studio", + { + uptime: 90, + latency: 1200, + throughput: 5, + }, + ); + await harness.setRoutingMetrics("gemini-2.5-flash-lite", "google-vertex", { + uptime: 100, + latency: 10, + throughput: 500, + }); + + await db.insert(tables.apiKey).values({ + id: "token-id", + token: "real-token", + projectId: "project-id", + description: "Test API Key", + createdBy: "user-id", + }); + + await db.insert(tables.providerKey).values({ + id: "provider-key-id", + token: "studio-db-key", + provider: "google-ai-studio", + organizationId: "org-id", + baseUrl: mockServerUrl, + }); + + const previousVertexKey = process.env.LLM_GOOGLE_VERTEX_API_KEY; + const previousGoogleCloudProject = process.env.LLM_GOOGLE_CLOUD_PROJECT; + const previousVertexBaseUrl = process.env.LLM_GOOGLE_VERTEX_BASE_URL; + const requestId = "chat-hybrid-keyed-provider-request-id"; + + try { + process.env.LLM_GOOGLE_VERTEX_API_KEY = "vertex-env-key"; + process.env.LLM_GOOGLE_CLOUD_PROJECT = "vertex-project"; + process.env.LLM_GOOGLE_VERTEX_BASE_URL = mockServerUrl; + + const res = await app.request("/v1/chat/completions", { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: "Bearer real-token", + "x-request-id": requestId, + }, + body: JSON.stringify({ + model: "gemini-2.5-flash-lite", + messages: [ + { + role: "user", + content: "Hello from hybrid provider routing!", + }, + ], + }), + }); + + expect(res.status).toBe(200); + + const json = await res.json(); + expect(json.metadata.used_provider).toBe("google-ai-studio"); + expect(json.choices[0].message.content).toContain( + "mock Google AI response", + ); + + const logs = await waitForLogs(1); + const completedLog = logs.find((log) => log.requestId === requestId); + expect(completedLog?.usedProvider).toBe("google-ai-studio"); + } finally { + if (previousVertexKey === undefined) { + delete process.env.LLM_GOOGLE_VERTEX_API_KEY; + } else { + process.env.LLM_GOOGLE_VERTEX_API_KEY = previousVertexKey; + } + if (previousGoogleCloudProject === undefined) { + delete process.env.LLM_GOOGLE_CLOUD_PROJECT; + } else { + process.env.LLM_GOOGLE_CLOUD_PROJECT = previousGoogleCloudProject; + } + if (previousVertexBaseUrl === undefined) { + delete process.env.LLM_GOOGLE_VERTEX_BASE_URL; + } else { + process.env.LLM_GOOGLE_VERTEX_BASE_URL = previousVertexBaseUrl; + } + } + }); + // test for model with multiple providers (llama-3.3-70b-instruct) test.skip("/v1/chat/completions with model that has multiple providers", async () => { await db.insert(tables.apiKey).values({ diff --git a/apps/gateway/src/chat/chat.ts b/apps/gateway/src/chat/chat.ts index 7e8fdeba8d..2a884b147a 100644 --- a/apps/gateway/src/chat/chat.ts +++ b/apps/gateway/src/chat/chat.ts @@ -402,6 +402,108 @@ function applyPinnedDefaultRegions( }); } +function getEnvironmentBackedProviders(providerIds?: string[]): string[] { + const candidateProviders = providerIds + ? providers.filter((provider) => providerIds.includes(provider.id)) + : providers; + + return candidateProviders + .filter((provider) => provider.id !== "llmgateway") + .filter((provider) => hasProviderEnvironmentToken(provider.id as Provider)) + .map((provider) => provider.id); +} + +function getAvailableProvidersForProjectMode( + projectMode: string, + providerKeys: Array<{ provider: string }>, + providerIds?: string[], +): { + availableProviders: string[]; + providersWithKeys: Set; +} { + const providersWithKeys = new Set(providerKeys.map((key) => key.provider)); + + if (projectMode === "api-keys") { + return { + availableProviders: Array.from(providersWithKeys), + providersWithKeys, + }; + } + + const envProviders = getEnvironmentBackedProviders(providerIds); + + if (projectMode === "credits") { + return { + availableProviders: envProviders, + providersWithKeys, + }; + } + + return { + availableProviders: Array.from( + new Set([...providersWithKeys, ...envProviders]), + ), + providersWithKeys, + }; +} + +function preferProvidersWithKeys( + projectMode: string, + candidates: ProviderModelMapping[], + providersWithKeys: Set, +): ProviderModelMapping[] { + if (projectMode !== "hybrid") { + return candidates; + } + + const keyedCandidates = candidates.filter((candidate) => + providersWithKeys.has(candidate.providerId), + ); + + return keyedCandidates.length > 0 ? keyedCandidates : candidates; +} + +function getRoutingCandidatesForProjectMode( + projectMode: string, + candidates: ProviderModelMapping[], + rateLimitedProviderIds: Set, + providersWithKeys: Set, +): ProviderModelMapping[] { + const nonRateLimitedCandidates = candidates.filter( + (candidate) => !rateLimitedProviderIds.has(candidate.providerId), + ); + + if (projectMode !== "hybrid") { + return nonRateLimitedCandidates.length > 0 + ? nonRateLimitedCandidates + : candidates; + } + + const keyedCandidates = candidates.filter((candidate) => + providersWithKeys.has(candidate.providerId), + ); + + if (keyedCandidates.length === 0) { + return nonRateLimitedCandidates.length > 0 + ? nonRateLimitedCandidates + : candidates; + } + + const nonRateLimitedKeyedCandidates = keyedCandidates.filter( + (candidate) => !rateLimitedProviderIds.has(candidate.providerId), + ); + + if (nonRateLimitedKeyedCandidates.length > 0) { + return nonRateLimitedKeyedCandidates; + } + + if (nonRateLimitedCandidates.length > 0) { + return nonRateLimitedCandidates; + } + + return keyedCandidates; +} + function preferConcreteRegionalMappings( providers: ProviderModelMapping[], ): ProviderModelMapping[] { @@ -2886,40 +2988,20 @@ chat.openapi(completions, async (c) => { } // Get available providers based on project mode - let availableProviders: string[] = []; + const providerKeys = await findActiveProviderKeys(project.organizationId); + const supportedProviderIds = providers + .filter((provider) => provider.id !== "llmgateway") + .map((provider) => provider.id); + const { availableProviders, providersWithKeys } = + getAvailableProvidersForProjectMode( + project.mode, + providerKeys, + supportedProviderIds, + ); // Region locks from DB provider keys, so auto-routing honors an org's // configured region (e.g. aws_bedrock_region: "eu") instead of being // collapsed to the pinned default by applyPinnedDefaultRegions. - let autoProviderLockedRegions = new Map(); - - if (project.mode === "api-keys") { - const providerKeys = await findActiveProviderKeys(project.organizationId); - availableProviders = providerKeys.map((key) => key.provider); - autoProviderLockedRegions = buildProviderLockedRegions(providerKeys); - } else if (project.mode === "credits" || project.mode === "hybrid") { - const providerKeys = await findActiveProviderKeys(project.organizationId); - const databaseProviders = providerKeys.map((key) => key.provider); - autoProviderLockedRegions = buildProviderLockedRegions(providerKeys); - - // Check which providers have environment tokens available - const envProviders: string[] = []; - const supportedProviders = providers - .filter((p) => p.id !== "llmgateway") - .map((p) => p.id); - for (const provider of supportedProviders) { - if (hasProviderEnvironmentToken(provider as Provider)) { - envProviders.push(provider); - } - } - - if (project.mode === "credits") { - availableProviders = envProviders; - } else { - availableProviders = [ - ...new Set([...databaseProviders, ...envProviders]), - ]; - } - } + const autoProviderLockedRegions = buildProviderLockedRegions(providerKeys); // Find the cheapest model that meets our context size requirements // Only consider hardcoded models for auto selection @@ -3025,7 +3107,6 @@ chat.openapi(completions, async (c) => { (!candidateAllowedProviders || candidateAllowedProviders.includes(provider.providerId)), ); - const cachedFilteredProviders = isDevPlanRestricted ? availableModelProviders.filter(providerSupportsCachedInput) : availableModelProviders; @@ -3041,7 +3122,6 @@ chat.openapi(completions, async (c) => { anyPostComplianceCandidate = true; } } - // Filter by context size requirement, reasoning capability, and deprecation status const suitableProviders = complianceFilteredProviders.filter( (provider) => { @@ -3107,7 +3187,6 @@ chat.openapi(completions, async (c) => { return false; } } - // Check JSON output capability if json_object or json_schema response format is requested if ( response_format?.type === "json_object" || @@ -3159,10 +3238,15 @@ chat.openapi(completions, async (c) => { return contextSizeMet; }, ); + const preferredSuitableProviders = preferProvidersWithKeys( + project.mode, + suitableProviders, + providersWithKeys, + ); - if (suitableProviders.length > 0) { + if (preferredSuitableProviders.length > 0) { // Find the cheapest among the suitable providers for this model - for (const provider of suitableProviders) { + for (const provider of preferredSuitableProviders) { const { price } = await getDiscountedProviderSelectionPrice( provider, modelDef.id, @@ -3176,7 +3260,7 @@ chat.openapi(completions, async (c) => { if (totalPrice < lowestPrice) { lowestPrice = totalPrice; selectedModel = modelDef; - selectedProviders = suitableProviders; + selectedProviders = preferredSuitableProviders; } } } @@ -3576,14 +3660,12 @@ chat.openapi(completions, async (c) => { project.organizationId, providerIds, ); - - const availableProviders = - project.mode === "api-keys" - ? providerKeys.map((key) => key.provider) - : providers - .filter((p) => p.id !== "llmgateway" && p.id !== usedProvider) - .filter((p) => hasProviderEnvironmentToken(p.id as Provider)) - .map((p) => p.id); + const { availableProviders, providersWithKeys } = + getAvailableProvidersForProjectMode( + project.mode, + providerKeys, + providerIds, + ); const availableModelProviders = preferConcreteRegionalMappings( applyPinnedDefaultRegions(iamFilteredModelProviders, { @@ -3642,8 +3724,13 @@ chat.openapi(completions, async (c) => { baseModelId, availableModelProviders, ); + const preferredCandidatesForRouting = preferProvidersWithKeys( + project.mode, + candidatesForRouting, + providersWithKeys, + ); - if (candidatesForRouting.length > 0) { + if (preferredCandidatesForRouting.length > 0) { const rawModelForFallback = models.find((m) => m.id === baseModelId); const modelWithPricing = rawModelForFallback ? { @@ -3778,14 +3865,12 @@ chat.openapi(completions, async (c) => { project.organizationId, providerIds, ); - - const availableProviders = - project.mode === "api-keys" - ? providerKeys.map((key) => key.provider) - : providers - .filter((p) => p.id !== "llmgateway" && p.id !== usedProvider) - .filter((p) => hasProviderEnvironmentToken(p.id as Provider)) - .map((p) => p.id); + const { availableProviders, providersWithKeys } = + getAvailableProvidersForProjectMode( + project.mode, + providerKeys, + providerIds, + ); // Filter model providers to only those available (excluding the low-uptime one) // If web search is requested, also filter to providers that support it @@ -3818,14 +3903,18 @@ chat.openapi(completions, async (c) => { provider.region === usedRegion ), ); - const uptimeFallbackCandidates = await pickNonRateLimitedCandidates( project.organizationId, baseModelId, availableModelProviders, ); + const preferredUptimeFallbackCandidates = preferProvidersWithKeys( + project.mode, + uptimeFallbackCandidates, + providersWithKeys, + ); - if (uptimeFallbackCandidates.length > 0) { + if (preferredUptimeFallbackCandidates.length > 0) { const rawModelForFallback = models.find((m) => m.id === baseModelId); const modelWithPricing = rawModelForFallback ? { @@ -3838,18 +3927,20 @@ chat.openapi(completions, async (c) => { if (modelWithPricing) { // Fetch metrics for all available providers - const metricsCombinations = uptimeFallbackCandidates.map((p) => ({ - modelId: modelWithPricing.id, - providerId: p.providerId, - region: p.region, - })); + const metricsCombinations = preferredUptimeFallbackCandidates.map( + (p) => ({ + modelId: modelWithPricing.id, + providerId: p.providerId, + region: p.region, + }), + ); const allMetricsMap = await getProviderMetricsForRouting( metricsCombinations, routingCfg, ); const providerAgnosticCandidates = await collapseProvidersToBestRegionPerProvider( - uptimeFallbackCandidates, + preferredUptimeFallbackCandidates, modelWithPricing, { metricsMap: allMetricsMap, @@ -3967,14 +4058,12 @@ chat.openapi(completions, async (c) => { project.organizationId, providerIds, ); - - const availableProviders = - project.mode === "api-keys" - ? providerKeys.map((key) => key.provider) - : providers - .filter((p) => p.id !== "llmgateway") - .filter((p) => hasProviderEnvironmentToken(p.id as Provider)) - .map((p) => p.id); + const { availableProviders, providersWithKeys } = + getAvailableProvidersForProjectMode( + project.mode, + providerKeys, + providerIds, + ); // Build a map of provider → locked region from DB provider keys. // When a user sets a region in their provider key (e.g. alibaba_region: "cn-beijing"), @@ -4065,23 +4154,26 @@ chat.openapi(completions, async (c) => { contentFilterRoutingExcludedProviders = contentFilterRoutingDecision.excludedProviders; contentFilterRoutingApplied = contentFilterRoutingDecision.rerouted; + const preferredRoutingProviders = preferProvidersWithKeys( + project.mode, + contentFilterPreferredProviders, + providersWithKeys, + ); // Filter out rate-limited providers during routing const rateLimitedProviderIds = await filterRateLimitedProviders( project.organizationId, - contentFilterPreferredProviders.map((p) => ({ + preferredRoutingProviders.map((p) => ({ providerId: p.providerId, model: (modelInfo as ModelDefinition).id, })), ); - const nonRateLimitedProviders = contentFilterPreferredProviders.filter( - (p) => !rateLimitedProviderIds.has(p.providerId), + const routingCandidates = getRoutingCandidatesForProjectMode( + project.mode, + preferredRoutingProviders, + rateLimitedProviderIds, + providersWithKeys, ); - // Fail-open: if all are rate-limited, use them all anyway - const routingCandidates = - nonRateLimitedProviders.length > 0 - ? nonRateLimitedProviders - : contentFilterPreferredProviders; const rawModelWithPricing = models.find( (m) => m.id === usedInternalModel, From 88cce012acd537e3cc1751adb32ff81d9380e3e5 Mon Sep 17 00:00:00 2001 From: Luca Steeb Date: Sun, 28 Jun 2026 18:51:11 +0100 Subject: [PATCH 3/6] test: cover hybrid routing --- apps/gateway/src/chat/chat.ts | 108 +------------- .../tools/hybrid-provider-routing.spec.ts | 139 ++++++++++++++++++ .../src/chat/tools/hybrid-provider-routing.ts | 110 ++++++++++++++ 3 files changed, 254 insertions(+), 103 deletions(-) create mode 100644 apps/gateway/src/chat/tools/hybrid-provider-routing.spec.ts create mode 100644 apps/gateway/src/chat/tools/hybrid-provider-routing.ts diff --git a/apps/gateway/src/chat/chat.ts b/apps/gateway/src/chat/chat.ts index 2a884b147a..102477647c 100644 --- a/apps/gateway/src/chat/chat.ts +++ b/apps/gateway/src/chat/chat.ts @@ -135,7 +135,6 @@ import { type BaseMessage, getModelStreamingSupport, hasMaxTokens, - hasProviderEnvironmentToken, hasRegionSpecificEnvKey, type ModelDefinition, models, @@ -197,6 +196,11 @@ import { } from "./tools/get-provider-env.js"; import { hasMeaningfulAssistantOutput } from "./tools/has-meaningful-assistant-output.js"; import { healJsonResponse } from "./tools/heal-json-response.js"; +import { + getAvailableProvidersForProjectMode, + getRoutingCandidatesForProjectMode, + preferProvidersWithKeys, +} from "./tools/hybrid-provider-routing.js"; import { isModelTrulyFree } from "./tools/is-model-truly-free.js"; import { mapFinishReasonToOpenai } from "./tools/map-finish-reason-to-openai.js"; import { @@ -402,108 +406,6 @@ function applyPinnedDefaultRegions( }); } -function getEnvironmentBackedProviders(providerIds?: string[]): string[] { - const candidateProviders = providerIds - ? providers.filter((provider) => providerIds.includes(provider.id)) - : providers; - - return candidateProviders - .filter((provider) => provider.id !== "llmgateway") - .filter((provider) => hasProviderEnvironmentToken(provider.id as Provider)) - .map((provider) => provider.id); -} - -function getAvailableProvidersForProjectMode( - projectMode: string, - providerKeys: Array<{ provider: string }>, - providerIds?: string[], -): { - availableProviders: string[]; - providersWithKeys: Set; -} { - const providersWithKeys = new Set(providerKeys.map((key) => key.provider)); - - if (projectMode === "api-keys") { - return { - availableProviders: Array.from(providersWithKeys), - providersWithKeys, - }; - } - - const envProviders = getEnvironmentBackedProviders(providerIds); - - if (projectMode === "credits") { - return { - availableProviders: envProviders, - providersWithKeys, - }; - } - - return { - availableProviders: Array.from( - new Set([...providersWithKeys, ...envProviders]), - ), - providersWithKeys, - }; -} - -function preferProvidersWithKeys( - projectMode: string, - candidates: ProviderModelMapping[], - providersWithKeys: Set, -): ProviderModelMapping[] { - if (projectMode !== "hybrid") { - return candidates; - } - - const keyedCandidates = candidates.filter((candidate) => - providersWithKeys.has(candidate.providerId), - ); - - return keyedCandidates.length > 0 ? keyedCandidates : candidates; -} - -function getRoutingCandidatesForProjectMode( - projectMode: string, - candidates: ProviderModelMapping[], - rateLimitedProviderIds: Set, - providersWithKeys: Set, -): ProviderModelMapping[] { - const nonRateLimitedCandidates = candidates.filter( - (candidate) => !rateLimitedProviderIds.has(candidate.providerId), - ); - - if (projectMode !== "hybrid") { - return nonRateLimitedCandidates.length > 0 - ? nonRateLimitedCandidates - : candidates; - } - - const keyedCandidates = candidates.filter((candidate) => - providersWithKeys.has(candidate.providerId), - ); - - if (keyedCandidates.length === 0) { - return nonRateLimitedCandidates.length > 0 - ? nonRateLimitedCandidates - : candidates; - } - - const nonRateLimitedKeyedCandidates = keyedCandidates.filter( - (candidate) => !rateLimitedProviderIds.has(candidate.providerId), - ); - - if (nonRateLimitedKeyedCandidates.length > 0) { - return nonRateLimitedKeyedCandidates; - } - - if (nonRateLimitedCandidates.length > 0) { - return nonRateLimitedCandidates; - } - - return keyedCandidates; -} - function preferConcreteRegionalMappings( providers: ProviderModelMapping[], ): ProviderModelMapping[] { diff --git a/apps/gateway/src/chat/tools/hybrid-provider-routing.spec.ts b/apps/gateway/src/chat/tools/hybrid-provider-routing.spec.ts new file mode 100644 index 0000000000..2c329740db --- /dev/null +++ b/apps/gateway/src/chat/tools/hybrid-provider-routing.spec.ts @@ -0,0 +1,139 @@ +import { afterEach, describe, expect, it } from "vitest"; + +import { + getAvailableProvidersForProjectMode, + getRoutingCandidatesForProjectMode, + preferProvidersWithKeys, +} from "./hybrid-provider-routing.js"; + +import type { ProviderModelMapping } from "@llmgateway/models"; + +describe("hybrid-provider-routing", () => { + const originalOpenAIKey = process.env.LLM_OPENAI_API_KEY; + const originalGoogleVertexKey = process.env.LLM_GOOGLE_VERTEX_API_KEY; + + afterEach(() => { + if (originalOpenAIKey === undefined) { + delete process.env.LLM_OPENAI_API_KEY; + } else { + process.env.LLM_OPENAI_API_KEY = originalOpenAIKey; + } + + if (originalGoogleVertexKey === undefined) { + delete process.env.LLM_GOOGLE_VERTEX_API_KEY; + } else { + process.env.LLM_GOOGLE_VERTEX_API_KEY = originalGoogleVertexKey; + } + }); + + it("returns only provider keys in api-keys mode", () => { + process.env.LLM_OPENAI_API_KEY = "sk-openai"; + + const result = getAvailableProvidersForProjectMode( + "api-keys", + [{ provider: "google-ai-studio" }, { provider: "alibaba" }], + ["google-ai-studio", "google-vertex", "alibaba"], + ); + + expect(result.availableProviders).toEqual(["google-ai-studio", "alibaba"]); + expect([...result.providersWithKeys]).toEqual([ + "google-ai-studio", + "alibaba", + ]); + }); + + it("prefers keyed providers over credits-backed providers in hybrid mode", () => { + process.env.LLM_GOOGLE_VERTEX_API_KEY = "vertex-env-key"; + + const result = getAvailableProvidersForProjectMode( + "hybrid", + [{ provider: "google-ai-studio" }], + ["google-ai-studio", "google-vertex"], + ); + + expect(result.availableProviders).toEqual([ + "google-ai-studio", + "google-vertex", + ]); + + const candidates: ProviderModelMapping[] = [ + { + providerId: "google-vertex", + externalId: "gemini-2.5-flash-lite", + streaming: true, + }, + { + providerId: "google-ai-studio", + externalId: "gemini-2.5-flash-lite", + streaming: true, + }, + ]; + + expect( + preferProvidersWithKeys("hybrid", candidates, result.providersWithKeys), + ).toMatchObject([ + { + providerId: "google-ai-studio", + externalId: "gemini-2.5-flash-lite", + }, + ]); + }); + + it("falls back to non-rate-limited credits providers when every keyed candidate is rate limited", () => { + const candidates: ProviderModelMapping[] = [ + { + providerId: "google-ai-studio", + externalId: "gemini-2.5-flash-lite", + streaming: true, + }, + { + providerId: "google-vertex", + externalId: "gemini-2.5-flash-lite", + streaming: true, + }, + ]; + + expect( + getRoutingCandidatesForProjectMode( + "hybrid", + candidates, + new Set(["google-ai-studio"]), + new Set(["google-ai-studio"]), + ), + ).toMatchObject([ + { + providerId: "google-vertex", + externalId: "gemini-2.5-flash-lite", + }, + ]); + }); + + it("keeps keyed candidates if every provider is rate limited", () => { + const candidates: ProviderModelMapping[] = [ + { + providerId: "google-ai-studio", + externalId: "gemini-2.5-flash-lite", + streaming: true, + }, + { + providerId: "google-vertex", + externalId: "gemini-2.5-flash-lite", + streaming: true, + }, + ]; + + expect( + getRoutingCandidatesForProjectMode( + "hybrid", + candidates, + new Set(["google-ai-studio", "google-vertex"]), + new Set(["google-ai-studio"]), + ), + ).toMatchObject([ + { + providerId: "google-ai-studio", + externalId: "gemini-2.5-flash-lite", + }, + ]); + }); +}); diff --git a/apps/gateway/src/chat/tools/hybrid-provider-routing.ts b/apps/gateway/src/chat/tools/hybrid-provider-routing.ts new file mode 100644 index 0000000000..9668f8ba29 --- /dev/null +++ b/apps/gateway/src/chat/tools/hybrid-provider-routing.ts @@ -0,0 +1,110 @@ +import { + hasProviderEnvironmentToken, + type Provider, + type ProviderModelMapping, + providers, +} from "@llmgateway/models"; + +export function getEnvironmentBackedProviders( + providerIds?: string[], +): string[] { + const candidateProviders = providerIds + ? providers.filter((provider) => providerIds.includes(provider.id)) + : providers; + + return candidateProviders + .filter((provider) => provider.id !== "llmgateway") + .filter((provider) => hasProviderEnvironmentToken(provider.id as Provider)) + .map((provider) => provider.id); +} + +export function getAvailableProvidersForProjectMode( + projectMode: string, + providerKeys: Array<{ provider: string }>, + providerIds?: string[], +): { + availableProviders: string[]; + providersWithKeys: Set; +} { + const providersWithKeys = new Set(providerKeys.map((key) => key.provider)); + + if (projectMode === "api-keys") { + return { + availableProviders: Array.from(providersWithKeys), + providersWithKeys, + }; + } + + const envProviders = getEnvironmentBackedProviders(providerIds); + + if (projectMode === "credits") { + return { + availableProviders: envProviders, + providersWithKeys, + }; + } + + return { + availableProviders: Array.from( + new Set([...providersWithKeys, ...envProviders]), + ), + providersWithKeys, + }; +} + +export function preferProvidersWithKeys( + projectMode: string, + candidates: ProviderModelMapping[], + providersWithKeys: Set, +): ProviderModelMapping[] { + if (projectMode !== "hybrid") { + return candidates; + } + + const keyedCandidates = candidates.filter((candidate) => + providersWithKeys.has(candidate.providerId), + ); + + return keyedCandidates.length > 0 ? keyedCandidates : candidates; +} + +export function getRoutingCandidatesForProjectMode( + projectMode: string, + candidates: ProviderModelMapping[], + rateLimitedProviderIds: Set, + providersWithKeys: Set, +): ProviderModelMapping[] { + const nonRateLimitedCandidates = candidates.filter( + (candidate) => !rateLimitedProviderIds.has(candidate.providerId), + ); + + if (projectMode !== "hybrid") { + return nonRateLimitedCandidates.length > 0 + ? nonRateLimitedCandidates + : candidates; + } + + const keyedCandidates = candidates.filter((candidate) => + providersWithKeys.has(candidate.providerId), + ); + + if (keyedCandidates.length === 0) { + return nonRateLimitedCandidates.length > 0 + ? nonRateLimitedCandidates + : candidates; + } + + const nonRateLimitedKeyedCandidates = keyedCandidates.filter( + (candidate) => !rateLimitedProviderIds.has(candidate.providerId), + ); + + if (nonRateLimitedKeyedCandidates.length > 0) { + return nonRateLimitedKeyedCandidates; + } + + if (nonRateLimitedCandidates.length > 0) { + return nonRateLimitedCandidates; + } + + return keyedCandidates; +} From 9c76b46b08a0b778d5a60c4df104660c2e57f661 Mon Sep 17 00:00:00 2001 From: Luca Steeb Date: Thu, 9 Jul 2026 20:32:42 +0100 Subject: [PATCH 4/6] fix: overflow to credits when keyed provider hits cap The main routing path pre-filtered candidates to keyed providers before handing them to getRoutingCandidatesForProjectMode, which made its credits-overflow branch unreachable: when every keyed candidate was rate limited, routing fail-opened to the rate-limited keyed provider and the consume step proceeded past the org's provider cap with only a warning. Pass the full candidate list and peek rate limits across all of it so hybrid mode overflows to credits-backed providers once keyed candidates are exhausted, matching the helper's unit-tested behavior. Co-Authored-By: Claude Opus 4.8 (1M context) --- apps/gateway/src/api.spec.ts | 93 +++++++++++++++++++++++++++++++++++ apps/gateway/src/chat/chat.ts | 15 +++--- 2 files changed, 99 insertions(+), 9 deletions(-) diff --git a/apps/gateway/src/api.spec.ts b/apps/gateway/src/api.spec.ts index 7034be43d8..ab56bb2b8a 100644 --- a/apps/gateway/src/api.spec.ts +++ b/apps/gateway/src/api.spec.ts @@ -4026,6 +4026,99 @@ describe("api", () => { } }); + test("/v1/chat/completions hybrid overflows to credits provider when keyed provider is rate limited", async () => { + await harness.setProjectMode("hybrid"); + await harness.setRoutingMetrics( + "gemini-2.5-flash-lite", + "google-ai-studio", + { + uptime: 100, + latency: 100, + throughput: 100, + }, + ); + await harness.setRoutingMetrics("gemini-2.5-flash-lite", "google-vertex", { + uptime: 100, + latency: 100, + throughput: 100, + }); + + await db.insert(tables.apiKey).values({ + id: "token-id", + token: "real-token", + projectId: "project-id", + description: "Test API Key", + createdBy: "user-id", + }); + + await db.insert(tables.providerKey).values({ + id: "provider-key-id", + token: "studio-db-key", + provider: "google-ai-studio", + organizationId: "org-id", + baseUrl: mockServerUrl, + }); + + // Org-level RPM cap on the keyed provider: the first request consumes the + // only slot, so the second must overflow to the credits-backed provider. + await db.insert(tables.rateLimit).values({ + id: "rate-limit-studio", + organizationId: "org-id", + provider: "google-ai-studio", + model: "gemini-2.5-flash-lite", + maxRpm: 1, + }); + + const previousVertexKey = process.env.LLM_GOOGLE_VERTEX_API_KEY; + const previousGoogleCloudProject = process.env.LLM_GOOGLE_CLOUD_PROJECT; + const previousVertexBaseUrl = process.env.LLM_GOOGLE_VERTEX_BASE_URL; + + try { + process.env.LLM_GOOGLE_VERTEX_API_KEY = "vertex-test-token"; + process.env.LLM_GOOGLE_CLOUD_PROJECT = "vertex-project"; + process.env.LLM_GOOGLE_VERTEX_BASE_URL = mockServerUrl; + + const makeRequest = (content: string) => + app.request("/v1/chat/completions", { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: "Bearer real-token", + }, + body: JSON.stringify({ + model: "gemini-2.5-flash-lite", + messages: [{ role: "user", content }], + }), + }); + + const firstRes = await makeRequest("Hybrid rate limit request one"); + expect(firstRes.status).toBe(200); + const firstJson = await firstRes.json(); + expect(firstJson.metadata.used_provider).toBe("google-ai-studio"); + + const secondRes = await makeRequest("Hybrid rate limit request two"); + expect(secondRes.status).toBe(200); + const secondJson = await secondRes.json(); + expect(secondJson.metadata.used_provider).toBe("google-vertex"); + } finally { + if (previousVertexKey === undefined) { + delete process.env.LLM_GOOGLE_VERTEX_API_KEY; + } else { + process.env.LLM_GOOGLE_VERTEX_API_KEY = previousVertexKey; + } + if (previousGoogleCloudProject === undefined) { + delete process.env.LLM_GOOGLE_CLOUD_PROJECT; + } else { + process.env.LLM_GOOGLE_CLOUD_PROJECT = previousGoogleCloudProject; + } + if (previousVertexBaseUrl === undefined) { + delete process.env.LLM_GOOGLE_VERTEX_BASE_URL; + } else { + process.env.LLM_GOOGLE_VERTEX_BASE_URL = previousVertexBaseUrl; + } + } + }); + // Non-streaming responses are cached in OpenAI format, so the stored // finish_reason is normalized (e.g. "stop"). The cache-hit log must classify // it using the OpenAI mapping, not the upstream provider's native format — diff --git a/apps/gateway/src/chat/chat.ts b/apps/gateway/src/chat/chat.ts index b928d38677..87d9d631c0 100644 --- a/apps/gateway/src/chat/chat.ts +++ b/apps/gateway/src/chat/chat.ts @@ -4066,23 +4066,20 @@ chat.openapi(completions, async (c) => { contentFilterRoutingExcludedProviders = contentFilterRoutingDecision.excludedProviders; contentFilterRoutingApplied = contentFilterRoutingDecision.rerouted; - const preferredRoutingProviders = preferProvidersWithKeys( - project.mode, - contentFilterPreferredProviders, - providersWithKeys, - ); - - // Filter out rate-limited providers during routing + // Filter out rate-limited providers during routing. Rate limits must be + // peeked across the full candidate list (not just keyed providers) so + // hybrid mode can overflow to credits-backed providers when every keyed + // candidate is rate limited. const rateLimitedProviderIds = await filterRateLimitedProviders( project.organizationId, - preferredRoutingProviders.map((p) => ({ + contentFilterPreferredProviders.map((p) => ({ providerId: p.providerId, model: (modelInfo as ModelDefinition).id, })), ); const routingCandidates = getRoutingCandidatesForProjectMode( project.mode, - preferredRoutingProviders, + contentFilterPreferredProviders, rateLimitedProviderIds, providersWithKeys, ); From 1989d111c8b18733012aadd68301bbbddea6b831 Mon Sep 17 00:00:00 2001 From: Luca Steeb Date: Thu, 9 Jul 2026 21:01:06 +0100 Subject: [PATCH 5/6] fix: honor keyed preference in fallback selection Address PR review feedback: - Pinned rate-limit fallback computed a keyed-preferred candidate list but only used it as a length guard; metrics and cheapest-provider selection still ran over the full list, so hybrid re-routes could pick a cheaper credits-backed provider over the org's keyed provider. Use the preferred list for selection. - Low-uptime fallback applied keyed preference before the better-uptime filter, so when no keyed alternative beat the degraded provider's uptime the request stayed degraded even though a healthy credits-backed provider existed. Prefer keyed providers only among better-uptime candidates. - Type the helpers' projectMode as the closed "api-keys"|"credits"|"hybrid" union and make candidate filters generic instead of ProviderModelMapping[]. Co-Authored-By: Claude Opus 4.8 (1M context) --- apps/gateway/src/chat/chat.ts | 48 ++++++++++--------- .../src/chat/tools/hybrid-provider-routing.ts | 23 +++++---- 2 files changed, 39 insertions(+), 32 deletions(-) diff --git a/apps/gateway/src/chat/chat.ts b/apps/gateway/src/chat/chat.ts index 87d9d631c0..ba9e4e1773 100644 --- a/apps/gateway/src/chat/chat.ts +++ b/apps/gateway/src/chat/chat.ts @@ -3654,18 +3654,20 @@ chat.openapi(completions, async (c) => { : undefined; if (modelWithPricing) { - const metricsCombinations = candidatesForRouting.map((p) => ({ - modelId: modelWithPricing.id, - providerId: p.providerId, - region: p.region, - })); + const metricsCombinations = preferredCandidatesForRouting.map( + (p) => ({ + modelId: modelWithPricing.id, + providerId: p.providerId, + region: p.region, + }), + ); const allMetricsMap = await getProviderMetricsForRouting( metricsCombinations, routingCfg, ); const cheapestResult = await getCheapestFromAvailableProviders( - candidatesForRouting, + preferredCandidatesForRouting, modelWithPricing, { metricsMap: allMetricsMap, @@ -3820,13 +3822,8 @@ chat.openapi(completions, async (c) => { baseModelId, availableModelProviders, ); - const preferredUptimeFallbackCandidates = preferProvidersWithKeys( - project.mode, - uptimeFallbackCandidates, - providersWithKeys, - ); - if (preferredUptimeFallbackCandidates.length > 0) { + if (uptimeFallbackCandidates.length > 0) { const rawModelForFallback = models.find((m) => m.id === baseModelId); const modelWithPricing = rawModelForFallback ? { @@ -3839,20 +3836,18 @@ chat.openapi(completions, async (c) => { if (modelWithPricing) { // Fetch metrics for all available providers - const metricsCombinations = preferredUptimeFallbackCandidates.map( - (p) => ({ - modelId: modelWithPricing.id, - providerId: p.providerId, - region: p.region, - }), - ); + const metricsCombinations = uptimeFallbackCandidates.map((p) => ({ + modelId: modelWithPricing.id, + providerId: p.providerId, + region: p.region, + })); const allMetricsMap = await getProviderMetricsForRouting( metricsCombinations, routingCfg, ); const providerAgnosticCandidates = await collapseProvidersToBestRegionPerProvider( - preferredUptimeFallbackCandidates, + uptimeFallbackCandidates, modelWithPricing, { metricsMap: allMetricsMap, @@ -3878,12 +3873,21 @@ chat.openapi(completions, async (c) => { ); }, ); + // Prefer keyed providers only among the better-uptime candidates: + // escaping the degraded provider takes priority, so a healthy + // credits-backed provider still wins over staying degraded when no + // keyed candidate has better uptime. + const preferredBetterUptimeProviders = preferProvidersWithKeys( + project.mode, + betterUptimeProviders, + providersWithKeys, + ); // Only proceed with fallback if there are providers with better uptime // Otherwise stick with the original provider - if (betterUptimeProviders.length > 0) { + if (preferredBetterUptimeProviders.length > 0) { const cheapestResult = await getCheapestFromAvailableProviders( - betterUptimeProviders, + preferredBetterUptimeProviders, modelWithPricing, { metricsMap: allMetricsMap, diff --git a/apps/gateway/src/chat/tools/hybrid-provider-routing.ts b/apps/gateway/src/chat/tools/hybrid-provider-routing.ts index 9668f8ba29..c23f6444b9 100644 --- a/apps/gateway/src/chat/tools/hybrid-provider-routing.ts +++ b/apps/gateway/src/chat/tools/hybrid-provider-routing.ts @@ -1,10 +1,11 @@ import { hasProviderEnvironmentToken, type Provider, - type ProviderModelMapping, providers, } from "@llmgateway/models"; +export type ProjectMode = "api-keys" | "credits" | "hybrid"; + export function getEnvironmentBackedProviders( providerIds?: string[], ): string[] { @@ -19,7 +20,7 @@ export function getEnvironmentBackedProviders( } export function getAvailableProvidersForProjectMode( - projectMode: string, + projectMode: ProjectMode, providerKeys: Array<{ provider: string }>, providerIds?: string[], ): { @@ -52,11 +53,11 @@ export function getAvailableProvidersForProjectMode( }; } -export function preferProvidersWithKeys( - projectMode: string, - candidates: ProviderModelMapping[], +export function preferProvidersWithKeys( + projectMode: ProjectMode, + candidates: T[], providersWithKeys: Set, -): ProviderModelMapping[] { +): T[] { if (projectMode !== "hybrid") { return candidates; } @@ -68,12 +69,14 @@ export function preferProvidersWithKeys( return keyedCandidates.length > 0 ? keyedCandidates : candidates; } -export function getRoutingCandidatesForProjectMode( - projectMode: string, - candidates: ProviderModelMapping[], +export function getRoutingCandidatesForProjectMode< + T extends { providerId: string }, +>( + projectMode: ProjectMode, + candidates: T[], rateLimitedProviderIds: Set, providersWithKeys: Set, -): ProviderModelMapping[] { +): T[] { const nonRateLimitedCandidates = candidates.filter( (candidate) => !rateLimitedProviderIds.has(candidate.providerId), ); From 584f74fecfaae911abc42a426b3b7bf852cf98f6 Mon Sep 17 00:00:00 2001 From: Luca Steeb Date: Thu, 9 Jul 2026 21:20:42 +0100 Subject: [PATCH 6/6] fix: keep demoted credits providers as retry targets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hybrid keyed-provider preference narrowed routingMetadata.providerScores to keyed candidates, so the retry loop (selectNextProvider) could no longer escape to a credits-backed provider when a BYOK key failed and its provider had no env credential — the request died even though a healthy credits provider existed (pre-preference behavior recovered fine). Append the candidates demoted by the keyed preference to providerScores with a worst-rank score and a hybrid_demoted flag on all three routing paths (main, rate-limit fallback, low-uptime fallback): they stay behind every keyed candidate but remain reachable as last-resort retry targets. The flag also makes keyed-preference routing decisions queryable in logs for post-deploy monitoring. Also log a structured warning when a BYOK key's configured region differs from the request region, since such requests now surface upstream auth errors instead of silently switching to the platform's regional env credential. Co-Authored-By: Claude Opus 4.8 (1M context) --- apps/gateway/src/api.spec.ts | 81 +++++++++++++ apps/gateway/src/chat/chat.ts | 112 +++++++++++++++++- .../get-cheapest-from-available-providers.ts | 3 + 3 files changed, 194 insertions(+), 2 deletions(-) diff --git a/apps/gateway/src/api.spec.ts b/apps/gateway/src/api.spec.ts index ab56bb2b8a..2046327a44 100644 --- a/apps/gateway/src/api.spec.ts +++ b/apps/gateway/src/api.spec.ts @@ -4026,6 +4026,87 @@ describe("api", () => { } }); + test("/v1/chat/completions hybrid escapes to credits provider when keyed provider fails", async () => { + await harness.setProjectMode("hybrid"); + await harness.setRoutingMetrics( + "gemini-2.5-flash-lite", + "google-ai-studio", + { + uptime: 100, + latency: 100, + throughput: 100, + }, + ); + await harness.setRoutingMetrics("gemini-2.5-flash-lite", "google-vertex", { + uptime: 100, + latency: 100, + throughput: 100, + }); + + await db.insert(tables.apiKey).values({ + id: "token-id", + token: "real-token", + projectId: "project-id", + description: "Test API Key", + createdBy: "user-id", + }); + + // Keyed provider whose upstream is unreachable: routing prefers it, the + // request fails with a network error, and the retry loop must escape to + // the credits-backed provider via its demoted score entry. + await db.insert(tables.providerKey).values({ + id: "provider-key-id", + token: "studio-db-key", + provider: "google-ai-studio", + organizationId: "org-id", + baseUrl: "http://127.0.0.1:9", + }); + + const previousVertexKey = process.env.LLM_GOOGLE_VERTEX_API_KEY; + const previousGoogleCloudProject = process.env.LLM_GOOGLE_CLOUD_PROJECT; + const previousVertexBaseUrl = process.env.LLM_GOOGLE_VERTEX_BASE_URL; + + try { + process.env.LLM_GOOGLE_VERTEX_API_KEY = "vertex-test-token"; + process.env.LLM_GOOGLE_CLOUD_PROJECT = "vertex-project"; + process.env.LLM_GOOGLE_VERTEX_BASE_URL = mockServerUrl; + + const res = await app.request("/v1/chat/completions", { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: "Bearer real-token", + }, + body: JSON.stringify({ + model: "gemini-2.5-flash-lite", + messages: [ + { role: "user", content: "Hybrid dead key escape request" }, + ], + }), + }); + + expect(res.status).toBe(200); + const json = await res.json(); + expect(json.metadata.used_provider).toBe("google-vertex"); + } finally { + if (previousVertexKey === undefined) { + delete process.env.LLM_GOOGLE_VERTEX_API_KEY; + } else { + process.env.LLM_GOOGLE_VERTEX_API_KEY = previousVertexKey; + } + if (previousGoogleCloudProject === undefined) { + delete process.env.LLM_GOOGLE_CLOUD_PROJECT; + } else { + process.env.LLM_GOOGLE_CLOUD_PROJECT = previousGoogleCloudProject; + } + if (previousVertexBaseUrl === undefined) { + delete process.env.LLM_GOOGLE_VERTEX_BASE_URL; + } else { + process.env.LLM_GOOGLE_VERTEX_BASE_URL = previousVertexBaseUrl; + } + } + }); + test("/v1/chat/completions hybrid overflows to credits provider when keyed provider is rate limited", async () => { await harness.setProjectMode("hybrid"); await harness.setRoutingMetrics( diff --git a/apps/gateway/src/chat/chat.ts b/apps/gateway/src/chat/chat.ts index ba9e4e1773..0aae2f1200 100644 --- a/apps/gateway/src/chat/chat.ts +++ b/apps/gateway/src/chat/chat.ts @@ -1907,6 +1907,48 @@ chat.openapi(completions, async (c) => { project.organizationId, ); + // Candidates demoted by hybrid keyed-provider preference stay in the scores + // as last-resort retry targets: their worst-rank score keeps them behind + // every keyed candidate, but the retry loop can still escape to them when a + // BYOK key fails and the provider has no env credential to fall back to. + const appendHybridDemotedProviderScores = async ( + metadata: RoutingMetadata, + demotedCandidates: ProviderModelMapping[], + modelId: string, + ) => { + const seenProviderIds = new Set( + metadata.providerScores.map((score) => score.providerId), + ); + const maxScore = Math.max( + 0, + ...metadata.providerScores.map((score) => score.score), + ); + let offset = 0; + for (const candidate of demotedCandidates) { + if (seenProviderIds.has(candidate.providerId)) { + continue; + } + seenProviderIds.add(candidate.providerId); + const { price, discount } = await getDiscountedProviderSelectionPrice( + candidate, + modelId, + { + organizationId: project.organizationId, + providerDiscountResolver, + }, + ); + metadata.providerScores.push({ + providerId: candidate.providerId, + region: candidate.region, + score: maxScore + 1000 + offset, + price: price.toNumber(), + discount: discount.toNumber(), + hybrid_demoted: true, + }); + offset++; + } + }; + const buildFinalResponseMetadata = (discount?: number | null) => toResponseMetadataExtras({ logId: finalLogId, @@ -3722,6 +3764,16 @@ chat.openapi(completions, async (c) => { xNoFallbackHeaderSet, ), }; + const preferredCandidateSet = new Set( + preferredCandidatesForRouting, + ); + await appendHybridDemotedProviderScores( + routingMetadata, + candidatesForRouting.filter( + (candidate) => !preferredCandidateSet.has(candidate), + ), + modelWithPricing.id, + ); } } } @@ -3947,6 +3999,16 @@ chat.openapi(completions, async (c) => { xNoFallbackHeaderSet, ), }; + const preferredBetterUptimeSet = new Set( + preferredBetterUptimeProviders, + ); + await appendHybridDemotedProviderScores( + routingMetadata, + betterUptimeProviders.filter( + (candidate) => !preferredBetterUptimeSet.has(candidate), + ), + modelWithPricing.id, + ); } } } @@ -4240,6 +4302,18 @@ chat.openapi(completions, async (c) => { } } } + { + const routingCandidateSet = new Set(routingCandidates); + await appendHybridDemotedProviderScores( + routingMetadata, + contentFilterPreferredProviders.filter( + (candidate) => + !routingCandidateSet.has(candidate) && + !rateLimitedProviderIds.has(candidate.providerId), + ), + modelWithPricing.id, + ); + } } else { usedProvider = routingCandidates[0].providerId; usedInternalModel = modelInfo.id; @@ -4596,7 +4670,24 @@ chat.openapi(completions, async (c) => { usedProvider, ) ) { - usedRegion ??= resolveRegionFromProviderKey(providerKey); + const keyConfiguredRegion = resolveRegionFromProviderKey(providerKey); + usedRegion ??= keyConfiguredRegion; + // The BYOK key is always used for the requested region (no silent env + // fallback), so a mismatch surfaces as an upstream auth error; log it + // for support diagnosis. + if ( + usedRegion && + keyConfiguredRegion && + keyConfiguredRegion !== usedRegion + ) { + logger.warn("BYOK provider key region differs from request region", { + organizationId: project.organizationId, + provider: usedProvider, + providerKeyId: providerKey.id, + keyRegion: keyConfiguredRegion, + requestedRegion: usedRegion, + }); + } } } else if (project.mode === "credits") { // Check regular credits, dev plan credits, and chat plan credits. @@ -4710,7 +4801,24 @@ chat.openapi(completions, async (c) => { usedProvider, ) ) { - usedRegion ??= resolveRegionFromProviderKey(providerKey); + const keyConfiguredRegion = resolveRegionFromProviderKey(providerKey); + usedRegion ??= keyConfiguredRegion; + // The BYOK key is always used for the requested region (no silent env + // fallback), so a mismatch surfaces as an upstream auth error; log it + // for support diagnosis. + if ( + usedRegion && + keyConfiguredRegion && + keyConfiguredRegion !== usedRegion + ) { + logger.warn("BYOK provider key region differs from request region", { + organizationId: project.organizationId, + provider: usedProvider, + providerKeyId: providerKey.id, + keyRegion: keyConfiguredRegion, + requestedRegion: usedRegion, + }); + } } } else { // No API key available, fall back to credits diff --git a/packages/actions/src/get-cheapest-from-available-providers.ts b/packages/actions/src/get-cheapest-from-available-providers.ts index 4946c96c0e..67c1869f28 100644 --- a/packages/actions/src/get-cheapest-from-available-providers.ts +++ b/packages/actions/src/get-cheapest-from-available-providers.ts @@ -102,6 +102,9 @@ export interface RoutingMetadata { contentFilterProvider?: boolean; // Set when the provider was excluded because the gateway content filter matched excludedByContentFilter?: boolean; + // Set when hybrid keyed-provider preference demoted this credits-backed + // candidate; kept in the scores as a last-resort retry target + hybrid_demoted?: boolean; }>; // Optional fields for low-uptime fallback routing originalProvider?: string;