diff --git a/apps/gateway/src/chat/chat.ts b/apps/gateway/src/chat/chat.ts index d8e7cfeaca..5d3d9d62f3 100644 --- a/apps/gateway/src/chat/chat.ts +++ b/apps/gateway/src/chat/chat.ts @@ -57,7 +57,6 @@ import { getProviderSelectionPrice, googleProviderSupportsAudioFormat, prepareRequestBody, - resolveMetricsModelId, type RoutingMetadata, } from "@llmgateway/actions"; import { @@ -509,12 +508,7 @@ function addContentFilterRoutingMetadata( : [ ...excludedProviders.map((provider) => { const metrics = metricsMap.get( - metricsKey( - resolveMetricsModelId(modelId, provider.modelName), - provider.providerId, - provider.region, - provider.modelName, - ), + metricsKey(modelId, provider.providerId, provider.region), ); return { @@ -1857,10 +1851,9 @@ chat.openapi(completions, async (c) => { if (selectedModel && selectedProviders.length > 0) { // Fetch uptime/latency metrics from last 5 minutes for provider selection const metricsCombinations = selectedProviders.map((p) => ({ - modelId: resolveMetricsModelId(selectedModel.id, p.modelName), + modelId: selectedModel.id, providerId: p.providerId, region: p.region, - modelName: p.modelName, })); const metricsMap = await getProviderMetricsForCombinations(metricsCombinations); @@ -2068,10 +2061,9 @@ chat.openapi(completions, async (c) => { if (eligibleMappings.length > 1) { const metricsCombinations = eligibleMappings.map((provider) => ({ - modelId: resolveMetricsModelId(modelInfo.id, provider.modelName), + modelId: modelInfo.id, providerId: provider.providerId, region: provider.region, - modelName: provider.modelName, })); const metricsMap = await getProviderMetricsForCombinations(metricsCombinations); @@ -2256,10 +2248,9 @@ chat.openapi(completions, async (c) => { if (modelWithPricing) { const metricsCombinations = candidatesForRouting.map((p) => ({ - modelId: resolveMetricsModelId(modelWithPricing.id, p.modelName), + modelId: modelWithPricing.id, providerId: p.providerId, region: p.region, - modelName: p.modelName, })); const allMetricsMap = await getProviderMetricsForCombinations(metricsCombinations); @@ -2328,20 +2319,18 @@ chat.openapi(completions, async (c) => { // Find the base model ID for metrics lookup // Since custom providers are excluded above, modelInfo always has 'id' const baseModelId = (modelInfo as ModelDefinition).id; - const metricsModelId = resolveMetricsModelId(baseModelId, usedModel); // Fetch uptime metrics for the requested provider const metricsMap = await getProviderMetricsForCombinations([ { - modelId: metricsModelId, + modelId: baseModelId, providerId: usedProvider, region: usedRegion, - modelName: usedModel, }, ]); const metrics = metricsMap.get( - metricsKey(metricsModelId, usedProvider, usedRegion, usedModel), + metricsKey(baseModelId, usedProvider, usedRegion), ); // If we have metrics and uptime is below 90%, route to an alternative @@ -2412,10 +2401,9 @@ chat.openapi(completions, async (c) => { if (modelWithPricing) { // Fetch metrics for all available providers const metricsCombinations = uptimeFallbackCandidates.map((p) => ({ - modelId: resolveMetricsModelId(modelWithPricing.id, p.modelName), + modelId: modelWithPricing.id, providerId: p.providerId, region: p.region, - modelName: p.modelName, })); const allMetricsMap = await getProviderMetricsForCombinations(metricsCombinations); @@ -2435,12 +2423,7 @@ chat.openapi(completions, async (c) => { const betterUptimeProviders = providerAgnosticCandidates.filter( (p) => { const providerMetrics = allMetricsMap.get( - metricsKey( - resolveMetricsModelId(modelWithPricing.id, p.modelName), - p.providerId, - p.region, - p.modelName, - ), + metricsKey(modelWithPricing.id, p.providerId, p.region), ); // If no metrics, assume the provider is healthy (100% uptime) // If has metrics, only include if uptime is better than original @@ -2631,13 +2614,9 @@ chat.openapi(completions, async (c) => { ...routingCandidates, ...contentFilterRoutingExcludedProviders, ].map((provider) => ({ - modelId: resolveMetricsModelId( - modelWithPricing.id, - provider.modelName, - ), + modelId: modelWithPricing.id, providerId: provider.providerId, region: provider.region, - modelName: provider.modelName, })); const metricsMap = await getProviderMetricsForCombinations(metricsCombinations); @@ -2808,10 +2787,9 @@ chat.openapi(completions, async (c) => { ...routingMetadataProviders, ...contentFilterRoutingExcludedProviders, ].map((provider) => ({ - modelId: resolveMetricsModelId(baseModelId, provider.modelName), + modelId: baseModelId, providerId: provider.providerId, region: provider.region, - modelName: provider.modelName, })); metricsMap = await getProviderMetricsForCombinations(metricsCombinations); } @@ -2837,12 +2815,7 @@ chat.openapi(completions, async (c) => { weightedScores?.metadata.providerScores ?? routingMetadataProviders.map((p) => { const metrics = metricsMap.get( - metricsKey( - resolveMetricsModelId(baseModelId, p.modelName), - p.providerId, - p.region, - p.modelName, - ), + metricsKey(baseModelId, p.providerId, p.region), ); return { providerId: p.providerId, diff --git a/apps/gateway/src/chat/tools/parse-model-input.ts b/apps/gateway/src/chat/tools/parse-model-input.ts index 3acfec34e7..8eb2dc5101 100644 --- a/apps/gateway/src/chat/tools/parse-model-input.ts +++ b/apps/gateway/src/chat/tools/parse-model-input.ts @@ -94,19 +94,10 @@ export function parseModelInput(modelInput: string): ParseModelInputResult { }); } - // Use the provider-specific model name if available - // For models with multiple mappings for the same provider (routing models), - // keep the base model ID so routing can select the right variant later - const providerMappings = modelDef.providers.filter( + const providerMapping = modelDef.providers.find( (p) => p.providerId === requestedProvider, ); - if (providerMappings.length > 1) { - requestedModel = modelDef.id as Model; - } else if (providerMappings.length === 1) { - requestedModel = providerMappings[0].modelName; - } else { - requestedModel = modelName as Model; - } + requestedModel = (providerMapping?.modelName ?? modelName) as Model; } } else if (models.find((m) => m.id === modelInput)) { requestedModel = modelInput as Model; diff --git a/apps/gateway/src/videos/videos.ts b/apps/gateway/src/videos/videos.ts index a3af416314..3b1edf9449 100644 --- a/apps/gateway/src/videos/videos.ts +++ b/apps/gateway/src/videos/videos.ts @@ -24,7 +24,6 @@ import { getProviderHeaders, getProviderSelectionPrice, processImageUrl, - resolveMetricsModelId, type RoutingMetadata, type VideoPricingContext, } from "@llmgateway/actions"; @@ -1453,10 +1452,9 @@ async function resolveVideoExecution( if (configuredEligibleMappings.length > 1) { const metricsCombinations = configuredEligibleMappings.map((provider) => ({ - modelId: resolveMetricsModelId(modelInfo.id, provider.modelName), + modelId: modelInfo.id, providerId: provider.providerId, region: provider.region, - modelName: provider.modelName, })); const metricsMap = await getProviderMetricsForCombinations(metricsCombinations); @@ -1468,10 +1466,9 @@ async function resolveVideoExecution( : undefined; const requestedKey = requestedMapping ? metricsKey( - resolveMetricsModelId(modelInfo.id, requestedMapping.modelName), + modelInfo.id, requestedMapping.providerId, requestedMapping.region, - requestedMapping.modelName, ) : undefined; @@ -1491,12 +1488,7 @@ async function resolveVideoExecution( } const providerMetrics = metricsMap.get( - metricsKey( - resolveMetricsModelId(modelInfo.id, provider.modelName), - provider.providerId, - provider.region, - provider.modelName, - ), + metricsKey(modelInfo.id, provider.providerId, provider.region), ); return ( !providerMetrics || diff --git a/packages/actions/src/get-cheapest-from-available-providers.ts b/packages/actions/src/get-cheapest-from-available-providers.ts index bb859f5501..6a7bc22b1d 100644 --- a/packages/actions/src/get-cheapest-from-available-providers.ts +++ b/packages/actions/src/get-cheapest-from-available-providers.ts @@ -3,30 +3,11 @@ import { Decimal } from "decimal.js"; import { type ProviderMetrics, metricsKey } from "@llmgateway/db"; import { getProviderDefinition, - models, type AvailableModelProvider, type ModelWithPricing, type ProviderModelMapping, } from "@llmgateway/models"; -/** - * Resolve the model id to use when looking up routing metrics for a candidate. - * - * For virtual models like `grok-4-1-fast`, the worker writes metrics to the - * concrete variant's mapping row (e.g. `grok-4-1-fast-non-reasoning`) because - * the request flows through the concrete model. The candidate's `modelName` - * matches that concrete model's id, so we use it. For non-virtual models the - * candidate's `modelName` is a provider-specific name with no matching catalog - * entry, and we fall back to the parent model id. - */ -export function resolveMetricsModelId( - parentModelId: string, - candidateModelName: string, -): string { - const concrete = models.find((m) => m.id === candidateModelName); - return concrete?.id ?? parentModelId; -} - interface ProviderScore { provider: T; score: Decimal; @@ -399,10 +380,9 @@ export function getCheapestFromAvailableProviders< const priority = providerDef?.priority ?? 1; const metrics = metricsMap?.get( metricsKey( - resolveMetricsModelId(modelWithPricing.id, provider.modelName), + modelWithPricing.id, provider.providerId, provider.region, - provider.modelName, ), ); @@ -443,10 +423,9 @@ export function getCheapestFromAvailableProviders< const price = getProviderSelectionPrice(providerInfo, videoPricing); const mKey = metricsKey( - resolveMetricsModelId(modelWithPricing.id, provider.modelName), + modelWithPricing.id, provider.providerId, provider.region, - provider.modelName, ); const metrics = metricsMap.get(mKey); diff --git a/packages/actions/src/models.spec.ts b/packages/actions/src/models.spec.ts index e6d7b2bed0..be92d5b3e5 100644 --- a/packages/actions/src/models.spec.ts +++ b/packages/actions/src/models.spec.ts @@ -12,7 +12,6 @@ import { import { getCheapestFromAvailableProviders, getProviderSelectionPrice, - resolveMetricsModelId, } from "./get-cheapest-from-available-providers.js"; import { getCheapestModelForProvider } from "./get-cheapest-model-for-provider.js"; import { prepareRequestBody } from "./prepare-request-body.js"; @@ -693,12 +692,7 @@ describe("getCheapestFromAvailableProviders", () => { { metricsMap: new Map([ [ - metricsKey( - "veo-3.1-generate-preview", - "avalanche", - undefined, - "veo3", - ), + metricsKey("veo-3.1-generate-preview", "avalanche", undefined), { modelId: "veo-3.1-generate-preview", providerId: "avalanche", @@ -713,7 +707,6 @@ describe("getCheapestFromAvailableProviders", () => { "veo-3.1-generate-preview", "google-vertex", undefined, - "veo-3.1-generate-001", ), { modelId: "veo-3.1-generate-preview", @@ -789,12 +782,7 @@ describe("getCheapestFromAvailableProviders", () => { { metricsMap: new Map([ [ - metricsKey( - "veo-3.1-generate-preview", - "avalanche", - undefined, - "veo3", - ), + metricsKey("veo-3.1-generate-preview", "avalanche", undefined), { modelId: "veo-3.1-generate-preview", providerId: "avalanche", @@ -809,7 +797,6 @@ describe("getCheapestFromAvailableProviders", () => { "veo-3.1-generate-preview", "google-vertex", undefined, - "veo-3.1-generate-001", ), { modelId: "veo-3.1-generate-preview", @@ -856,320 +843,6 @@ describe("getCheapestFromAvailableProviders", () => { expect(result).toBe(null); }); - describe("resolveMetricsModelId", () => { - it("returns the candidate's modelName when it matches a concrete catalog model", () => { - expect( - resolveMetricsModelId("grok-4-1-fast", "grok-4-1-fast-non-reasoning"), - ).toBe("grok-4-1-fast-non-reasoning"); - expect( - resolveMetricsModelId("grok-4-1-fast", "grok-4-1-fast-reasoning"), - ).toBe("grok-4-1-fast-reasoning"); - }); - - it("falls back to the parent model id when the modelName is provider-specific", () => { - expect( - resolveMetricsModelId("gpt-4o-mini", "gpt-4o-mini-2024-07-18"), - ).toBe("gpt-4o-mini"); - }); - - it("falls back to the parent model id when the modelName is unknown", () => { - expect( - resolveMetricsModelId("custom-parent", "totally-unknown-model"), - ).toBe("custom-parent"); - }); - }); - - describe("virtual model variant routing", () => { - const virtualModel: Parameters< - typeof getCheapestFromAvailableProviders - >[1] = { - id: "virtual-test", - providers: [ - { - providerId: "openai", - modelName: "virtual-test-non-reasoning", - inputPrice: "1e-6", - outputPrice: "2e-6", - }, - { - providerId: "openai", - modelName: "virtual-test-reasoning", - inputPrice: "10e-6", - outputPrice: "20e-6", - }, - ], - }; - - it("scores the reasoning variant by its own pricing, not the first variant in the array", () => { - const result = getCheapestFromAvailableProviders( - [{ providerId: "openai", modelName: "virtual-test-reasoning" }], - virtualModel, - ); - - expect(result?.provider.modelName).toBe("virtual-test-reasoning"); - expect(result?.metadata.providerScores[0]?.price).toBeCloseTo(15 / 1e6); - }); - - it("scores the non-reasoning variant by its own pricing", () => { - const result = getCheapestFromAvailableProviders( - [{ providerId: "openai", modelName: "virtual-test-non-reasoning" }], - virtualModel, - ); - - expect(result?.provider.modelName).toBe("virtual-test-non-reasoning"); - expect(result?.metadata.providerScores[0]?.price).toBeCloseTo(1.5 / 1e6); - }); - - it("filters out a reasoning variant whose stability is unstable while keeping the non-reasoning sibling", () => { - const modelWithUnstableReasoning: Parameters< - typeof getCheapestFromAvailableProviders - >[1] = { - id: "virtual-stability-test", - providers: [ - { - providerId: "openai", - modelName: "virtual-stability-non-reasoning", - inputPrice: "1e-6", - outputPrice: "2e-6", - }, - { - providerId: "openai", - modelName: "virtual-stability-reasoning", - inputPrice: "10e-6", - outputPrice: "20e-6", - stability: "unstable", - }, - ], - }; - - const reasoningResult = getCheapestFromAvailableProviders( - [ - { - providerId: "openai", - modelName: "virtual-stability-reasoning", - }, - ], - modelWithUnstableReasoning, - ); - expect(reasoningResult).toBe(null); - - const nonReasoningResult = getCheapestFromAvailableProviders( - [ - { - providerId: "openai", - modelName: "virtual-stability-non-reasoning", - }, - ], - modelWithUnstableReasoning, - ); - expect(nonReasoningResult?.provider.modelName).toBe( - "virtual-stability-non-reasoning", - ); - }); - - it("preserves the legacy providerId+region match when the candidate does not name a specific variant", () => { - const result = getCheapestFromAvailableProviders( - [{ providerId: "openai", modelName: "unrelated-name" }], - virtualModel, - ); - - expect(result?.provider.modelName).toBe("unrelated-name"); - expect(result?.metadata.providerScores[0]?.price).toBeCloseTo(1.5 / 1e6); - }); - - it("scores the reasoning variant under the price-only-no-metrics path", () => { - const result = getCheapestFromAvailableProviders( - [{ providerId: "openai", modelName: "virtual-test-reasoning" }], - virtualModel, - { metricsMap: new Map() }, - ); - - expect(result?.metadata.selectionReason).toBe("price-only-no-metrics"); - expect(result?.provider.modelName).toBe("virtual-test-reasoning"); - expect(result?.metadata.providerScores[0]?.price).toBeCloseTo(15 / 1e6); - }); - - it("routes the catalog grok-4-1-fast reasoning variant with reasoning-variant cache support", () => { - const grokModel = models.find((model) => model.id === "grok-4-1-fast"); - expect(grokModel).toBeDefined(); - if (!grokModel) { - throw new Error("Missing grok-4-1-fast fixture"); - } - - const reasoningProvider = grokModel.providers.find( - (p) => p.modelName === "grok-4-1-fast-reasoning", - ); - expect(reasoningProvider).toBeDefined(); - if (!reasoningProvider) { - throw new Error("Missing reasoning variant"); - } - - const result = getCheapestFromAvailableProviders( - [reasoningProvider], - grokModel, - { metricsMap: new Map(), promptTokens: 200_000 }, - ); - - expect(result?.provider.modelName).toBe("grok-4-1-fast-reasoning"); - }); - - it("scores the weighted-score path using variant-specific metrics", () => { - const reasoningProvider = { - providerId: "openai" as const, - modelName: "virtual-test-reasoning", - }; - const nonReasoningProvider = { - providerId: "openai" as const, - modelName: "virtual-test-non-reasoning", - }; - - // Variant-keyed metrics: non-reasoning is healthy, reasoning is degraded. - // Without modelName-aware lookup these would clobber each other under the - // same `modelId:providerId:region` legacy key. - const metricsMap = new Map([ - [ - metricsKey( - "virtual-test", - "openai", - undefined, - "virtual-test-non-reasoning", - ), - { - modelId: "virtual-test", - providerId: "openai", - modelName: "virtual-test-non-reasoning", - uptime: 99.9, - averageLatency: 100, - throughput: 200, - totalRequests: 100, - }, - ], - [ - metricsKey( - "virtual-test", - "openai", - undefined, - "virtual-test-reasoning", - ), - { - modelId: "virtual-test", - providerId: "openai", - modelName: "virtual-test-reasoning", - uptime: 50, - averageLatency: 1000, - throughput: 10, - totalRequests: 100, - }, - ], - ]); - - const reasoningResult = getCheapestFromAvailableProviders( - [reasoningProvider], - virtualModel, - { metricsMap }, - ); - expect(reasoningResult?.provider.modelName).toBe( - "virtual-test-reasoning", - ); - expect(reasoningResult?.metadata.providerScores[0]?.uptime).toBe(50); - expect(reasoningResult?.metadata.providerScores[0]?.latency).toBe(1000); - expect(reasoningResult?.metadata.providerScores[0]?.throughput).toBe(10); - - const nonReasoningResult = getCheapestFromAvailableProviders( - [nonReasoningProvider], - virtualModel, - { metricsMap }, - ); - expect(nonReasoningResult?.provider.modelName).toBe( - "virtual-test-non-reasoning", - ); - expect(nonReasoningResult?.metadata.providerScores[0]?.uptime).toBe(99.9); - expect(nonReasoningResult?.metadata.providerScores[0]?.latency).toBe(100); - expect(nonReasoningResult?.metadata.providerScores[0]?.throughput).toBe( - 200, - ); - }); - - it("scores the random-exploration metadata using variant-specific metrics", () => { - const reasoningProvider = { - providerId: "openai" as const, - modelName: "virtual-test-reasoning", - }; - - const metricsMap = new Map([ - [ - metricsKey( - "virtual-test", - "openai", - undefined, - "virtual-test-non-reasoning", - ), - { - modelId: "virtual-test", - providerId: "openai", - modelName: "virtual-test-non-reasoning", - uptime: 99.9, - averageLatency: 100, - throughput: 200, - totalRequests: 100, - }, - ], - [ - metricsKey( - "virtual-test", - "openai", - undefined, - "virtual-test-reasoning", - ), - { - modelId: "virtual-test", - providerId: "openai", - modelName: "virtual-test-reasoning", - uptime: 75, - averageLatency: 800, - throughput: 25, - totalRequests: 100, - }, - ], - ]); - - const randomSpy = vi.spyOn(Math, "random").mockReturnValue(0); - const originalArgv = process.argv; - const originalNodeEnv = process.env.NODE_ENV; - const originalVitest = process.env.VITEST; - delete process.env.NODE_ENV; - delete process.env.VITEST; - process.argv = ["node", "/tmp/not-a-test-run.mjs"]; - - try { - const result = getCheapestFromAvailableProviders( - [reasoningProvider], - virtualModel, - { metricsMap }, - ); - expect(result?.metadata.selectionReason).toBe("random-exploration"); - expect(result?.provider.modelName).toBe("virtual-test-reasoning"); - expect(result?.metadata.providerScores[0]?.uptime).toBe(75); - expect(result?.metadata.providerScores[0]?.latency).toBe(800); - expect(result?.metadata.providerScores[0]?.throughput).toBe(25); - expect(result?.metadata.providerScores[0]?.price).toBeCloseTo(15 / 1e6); - } finally { - randomSpy.mockRestore(); - process.argv = originalArgv; - if (originalNodeEnv !== undefined) { - process.env.NODE_ENV = originalNodeEnv; - } else { - delete process.env.NODE_ENV; - } - if (originalVitest !== undefined) { - process.env.VITEST = originalVitest; - } else { - delete process.env.VITEST; - } - } - }); - }); - it("should use the default exploration rate when EXPLORATION_RATE is empty", () => { const originalExplorationRate = process.env.EXPLORATION_RATE; process.env.EXPLORATION_RATE = ""; @@ -1315,7 +988,7 @@ describe("getCheapestFromAvailableProviders", () => { const equalMetrics = new Map([ [ - metricsKey("cache-test-model", "openai", undefined, "cache-test"), + metricsKey("cache-test-model", "openai", undefined), { modelId: "cache-test-model", providerId: "openai", @@ -1326,7 +999,7 @@ describe("getCheapestFromAvailableProviders", () => { }, ], [ - metricsKey("cache-test-model", "deepseek", undefined, "cache-test"), + metricsKey("cache-test-model", "deepseek", undefined), { modelId: "cache-test-model", providerId: "deepseek", diff --git a/packages/db/src/provider-metrics.spec.ts b/packages/db/src/provider-metrics.spec.ts index d0019d0365..ec42edbf4d 100644 --- a/packages/db/src/provider-metrics.spec.ts +++ b/packages/db/src/provider-metrics.spec.ts @@ -94,9 +94,7 @@ describe("provider-metrics", () => { const metrics = await getProviderMetrics(); expect(metrics.size).toBe(1); - const metric = metrics.get( - metricsKey("gpt-4", "openai", undefined, "gpt-4"), - ); + const metric = metrics.get(metricsKey("gpt-4", "openai", undefined)); expect(metric).toBeDefined(); expect(metric?.modelId).toBe("gpt-4"); expect(metric?.providerId).toBe("openai"); @@ -131,20 +129,13 @@ describe("provider-metrics", () => { expect(metrics.size).toBe(2); - const gptMetric = metrics.get( - metricsKey("gpt-4", "openai", undefined, "gpt-4"), - ); + const gptMetric = metrics.get(metricsKey("gpt-4", "openai", undefined)); expect(gptMetric?.uptime).toBe(80); expect(gptMetric?.averageLatency).toBe(1000); expect(gptMetric?.totalRequests).toBe(100); const claudeMetric = metrics.get( - metricsKey( - "claude-3-5-sonnet", - "anthropic", - undefined, - "claude-3-5-sonnet-20241022", - ), + metricsKey("claude-3-5-sonnet", "anthropic", undefined), ); expect(claudeMetric?.uptime).toBe(95); expect(claudeMetric?.averageLatency).toBe(2000); @@ -167,18 +158,9 @@ describe("provider-metrics", () => { const metrics = await getProviderMetrics(); expect(metrics.size).toBe(1); expect( - metrics.has( - metricsKey( - "claude-3-5-sonnet", - "anthropic", - undefined, - "claude-3-5-sonnet-20241022", - ), - ), + metrics.has(metricsKey("claude-3-5-sonnet", "anthropic", undefined)), ).toBe(true); - expect( - metrics.has(metricsKey("gpt-4", "openai", undefined, "gpt-4")), - ).toBe(false); + expect(metrics.has(metricsKey("gpt-4", "openai", undefined))).toBe(false); }); it("should skip mappings with zero total requests", async () => { @@ -208,9 +190,7 @@ describe("provider-metrics", () => { .where(eq(modelProviderMapping.id, "mapping-1")); const metrics = await getProviderMetrics(); - const metric = metrics.get( - metricsKey("gpt-4", "openai", undefined, "gpt-4"), - ); + const metric = metrics.get(metricsKey("gpt-4", "openai", undefined)); expect(metric?.uptime).toBe(100); }); @@ -226,9 +206,7 @@ describe("provider-metrics", () => { .where(eq(modelProviderMapping.id, "mapping-1")); const metrics = await getProviderMetrics(); - const metric = metrics.get( - metricsKey("gpt-4", "openai", undefined, "gpt-4"), - ); + const metric = metrics.get(metricsKey("gpt-4", "openai", undefined)); expect(metric?.uptime).toBe(0); }); @@ -244,9 +222,7 @@ describe("provider-metrics", () => { .where(eq(modelProviderMapping.id, "mapping-1")); const metrics = await getProviderMetrics(); - const metric = metrics.get( - metricsKey("gpt-4", "openai", undefined, "gpt-4"), - ); + const metric = metrics.get(metricsKey("gpt-4", "openai", undefined)); expect(metric).toBeDefined(); expect(metric?.uptime).toBe(95); expect(metric?.averageLatency).toBeUndefined(); @@ -266,9 +242,7 @@ describe("provider-metrics", () => { .where(eq(modelProviderMapping.id, "mapping-1")); const metrics = await getProviderMetrics(); - const metric = metrics.get( - metricsKey("gpt-4", "openai", undefined, "gpt-4"), - ); + const metric = metrics.get(metricsKey("gpt-4", "openai", undefined)); expect(metric).toBeDefined(); expect(metric?.uptime).toBeUndefined(); expect(metric?.averageLatency).toBe(500); @@ -305,22 +279,13 @@ describe("provider-metrics", () => { .where(eq(modelProviderMapping.id, "mapping-2")); const metrics = await getProviderMetricsForCombinations([ - { modelId: "gpt-4", providerId: "openai", modelName: "gpt-4" }, + { modelId: "gpt-4", providerId: "openai" }, ]); expect(metrics.size).toBe(1); + expect(metrics.has(metricsKey("gpt-4", "openai", undefined))).toBe(true); expect( - metrics.has(metricsKey("gpt-4", "openai", undefined, "gpt-4")), - ).toBe(true); - expect( - metrics.has( - metricsKey( - "claude-3-5-sonnet", - "anthropic", - undefined, - "claude-3-5-sonnet-20241022", - ), - ), + metrics.has(metricsKey("claude-3-5-sonnet", "anthropic", undefined)), ).toBe(false); }); @@ -346,28 +311,20 @@ describe("provider-metrics", () => { .where(eq(modelProviderMapping.id, "mapping-2")); const metrics = await getProviderMetricsForCombinations([ - { modelId: "gpt-4", providerId: "openai", modelName: "gpt-4" }, + { modelId: "gpt-4", providerId: "openai" }, { modelId: "claude-3-5-sonnet", providerId: "anthropic", - modelName: "claude-3-5-sonnet-20241022", }, ]); expect(metrics.size).toBe(2); expect( - metrics.get(metricsKey("gpt-4", "openai", undefined, "gpt-4")) - ?.totalRequests, + metrics.get(metricsKey("gpt-4", "openai", undefined))?.totalRequests, ).toBe(100); expect( - metrics.get( - metricsKey( - "claude-3-5-sonnet", - "anthropic", - undefined, - "claude-3-5-sonnet-20241022", - ), - )?.totalRequests, + metrics.get(metricsKey("claude-3-5-sonnet", "anthropic", undefined)) + ?.totalRequests, ).toBe(200); }); @@ -384,27 +341,17 @@ describe("provider-metrics", () => { .where(eq(modelProviderMapping.id, "mapping-1")); const metrics = await getProviderMetricsForCombinations([ - { modelId: "gpt-4", providerId: "openai", modelName: "gpt-4" }, + { modelId: "gpt-4", providerId: "openai" }, { modelId: "claude-3-5-sonnet", providerId: "anthropic", - modelName: "claude-3-5-sonnet-20241022", }, ]); expect(metrics.size).toBe(1); + expect(metrics.has(metricsKey("gpt-4", "openai", undefined))).toBe(true); expect( - metrics.has(metricsKey("gpt-4", "openai", undefined, "gpt-4")), - ).toBe(true); - expect( - metrics.has( - metricsKey( - "claude-3-5-sonnet", - "anthropic", - undefined, - "claude-3-5-sonnet-20241022", - ), - ), + metrics.has(metricsKey("claude-3-5-sonnet", "anthropic", undefined)), ).toBe(false); }); @@ -420,7 +367,7 @@ describe("provider-metrics", () => { .where(eq(modelProviderMapping.id, "mapping-1")); const metrics = await getProviderMetricsForCombinations([ - { modelId: "gpt-4", providerId: "openai", modelName: "gpt-4" }, + { modelId: "gpt-4", providerId: "openai" }, ]); expect(metrics.size).toBe(0); @@ -438,13 +385,11 @@ describe("provider-metrics", () => { .where(eq(modelProviderMapping.id, "mapping-1")); const metrics = await getProviderMetricsForCombinations([ - { modelId: "gpt-4", providerId: "openai", modelName: "gpt-4" }, + { modelId: "gpt-4", providerId: "openai" }, ]); expect(metrics.size).toBe(1); - const metric = metrics.get( - metricsKey("gpt-4", "openai", undefined, "gpt-4"), - ); + const metric = metrics.get(metricsKey("gpt-4", "openai", undefined)); expect(metric?.modelId).toBe("gpt-4"); expect(metric?.providerId).toBe("openai"); expect(metric?.uptime).toBe(75); @@ -452,74 +397,5 @@ describe("provider-metrics", () => { expect(metric?.throughput).toBe(25); expect(metric?.totalRequests).toBe(100); }); - - it("disambiguates virtual model variants that share modelId/providerId/region", async () => { - await db.insert(model).values({ - id: "virtual-grok", - name: "Virtual Grok", - family: "grok", - status: "active", - }); - - await db.insert(modelProviderMapping).values([ - { - id: "virtual-grok-non-reasoning", - modelId: "virtual-grok", - providerId: "openai", - modelName: "virtual-grok-non-reasoning", - status: "active", - routingUptime: 99, - routingLatency: 100, - routingThroughput: 200, - routingTotalRequests: 100, - }, - { - id: "virtual-grok-reasoning", - modelId: "virtual-grok", - providerId: "openai", - modelName: "virtual-grok-reasoning", - status: "active", - routingUptime: 50, - routingLatency: 1000, - routingThroughput: 10, - routingTotalRequests: 100, - }, - ]); - - const metrics = await getProviderMetricsForCombinations([ - { - modelId: "virtual-grok", - providerId: "openai", - modelName: "virtual-grok-non-reasoning", - }, - { - modelId: "virtual-grok", - providerId: "openai", - modelName: "virtual-grok-reasoning", - }, - ]); - - const nonReasoning = metrics.get( - metricsKey( - "virtual-grok", - "openai", - undefined, - "virtual-grok-non-reasoning", - ), - ); - const reasoning = metrics.get( - metricsKey( - "virtual-grok", - "openai", - undefined, - "virtual-grok-reasoning", - ), - ); - - expect(nonReasoning?.uptime).toBe(99); - expect(nonReasoning?.averageLatency).toBe(100); - expect(reasoning?.uptime).toBe(50); - expect(reasoning?.averageLatency).toBe(1000); - }); }); }); diff --git a/packages/db/src/provider-metrics.ts b/packages/db/src/provider-metrics.ts index 2bf8758925..20a56f45fd 100644 --- a/packages/db/src/provider-metrics.ts +++ b/packages/db/src/provider-metrics.ts @@ -8,7 +8,6 @@ import { modelProviderMapping } from "./schema.js"; export interface ProviderMetrics { providerId: string; modelId: string; - modelName?: string; region?: string; uptime?: number; // Percentage (0-100, undefined = no data) averageLatency?: number; // Milliseconds (undefined = no data) @@ -17,18 +16,14 @@ export interface ProviderMetrics { } /** - * Build a metrics map key from modelId, providerId, optional region, and - * optional provider modelName. Including modelName disambiguates virtual - * model variants (e.g. reasoning vs non-reasoning) that share the same - * (modelId, providerId, region) tuple in the routing tables. + * Build a metrics map key from modelId, providerId, and optional region. */ export function metricsKey( modelId: string, providerId: string, region?: string | null, - modelName?: string | null, ): string { - return `${modelId}:${providerId}:${region ?? ""}:${modelName ?? ""}`; + return `${modelId}:${providerId}:${region ?? ""}`; } const modelProviderMappingTableName = getTableName(modelProviderMapping); @@ -36,7 +31,6 @@ const modelProviderMappingTableName = getTableName(modelProviderMapping); interface ProviderMetricsRow { modelId: string; providerId: string; - modelName: string; region: string | null; routingUptime: number | null; routingLatency: number | null; @@ -53,7 +47,6 @@ async function fetchAllProviderMetricsRows(): Promise { .select({ modelId: modelProviderMapping.modelId, providerId: modelProviderMapping.providerId, - modelName: modelProviderMapping.modelName, region: modelProviderMapping.region, routingUptime: modelProviderMapping.routingUptime, routingLatency: modelProviderMapping.routingLatency, @@ -77,7 +70,6 @@ function rowToMetrics(row: ProviderMetricsRow): ProviderMetrics | undefined { return { providerId: row.providerId, modelId: row.modelId, - modelName: row.modelName, region: row.region ?? undefined, uptime: row.routingUptime ?? undefined, averageLatency: row.routingLatency ?? undefined, @@ -91,10 +83,7 @@ function rowToMetrics(row: ProviderMetricsRow): ProviderMetrics | undefined { * Metrics are computed by the worker with time-tier weighting * (last 1 min = 10x, last 5 min = 3x, last hour = 1x). * - * Keys include modelName so virtual model variants that share the same - * (modelId, providerId, region) tuple do not overwrite each other. - * - * @returns Map of metrics keyed by `metricsKey(modelId, providerId, region, modelName)` + * @returns Map of metrics keyed by `metricsKey(modelId, providerId, region)` */ export async function getProviderMetrics(): Promise< Map @@ -107,7 +96,7 @@ export async function getProviderMetrics(): Promise< continue; } metricsMap.set( - metricsKey(row.modelId, row.providerId, row.region, row.modelName), + metricsKey(row.modelId, row.providerId, row.region), metrics, ); } @@ -119,70 +108,40 @@ export async function getProviderMetrics(): Promise< * Uses the same cached "all active mappings" query as getProviderMetrics so * every request hits a single SWR mirror that survives Postgres outages. * - * Pass `modelName` for virtual-model variants so the routing math reads the - * variant's own metrics rather than letting siblings overwrite each other in - * the result map. - * - * @param combinations - Array of {modelId, providerId, region?, modelName?} to fetch metrics for - * @returns Map of metrics keyed by `metricsKey(modelId, providerId, region, modelName?)` + * @param combinations - Array of {modelId, providerId, region?} to fetch metrics for + * @returns Map of metrics keyed by `metricsKey(modelId, providerId, region)` */ export async function getProviderMetricsForCombinations( combinations: Array<{ modelId: string; providerId: string; region?: string; - modelName?: string; }>, ): Promise> { if (combinations.length === 0) { return new Map(); } - const wantedWithModelName = new Set(); - const wantedLegacy = new Set(); + const wanted = new Set(); for (const combo of combinations) { - if (combo.modelName) { - wantedWithModelName.add( - metricsKey( - combo.modelId, - combo.providerId, - combo.region ?? null, - combo.modelName, - ), - ); - } else { - wantedLegacy.add( - metricsKey(combo.modelId, combo.providerId, combo.region ?? null), - ); - } + wanted.add( + metricsKey(combo.modelId, combo.providerId, combo.region ?? null), + ); } const rows = await fetchAllProviderMetricsRows(); const metricsMap = new Map(); for (const row of rows) { - const variantKey = metricsKey( - row.modelId, - row.providerId, - row.region, - row.modelName, - ); - const legacyKey = metricsKey(row.modelId, row.providerId, row.region); - const matchedVariant = wantedWithModelName.has(variantKey); - const matchedLegacy = wantedLegacy.has(legacyKey); - if (!matchedVariant && !matchedLegacy) { + const key = metricsKey(row.modelId, row.providerId, row.region); + if (!wanted.has(key)) { continue; } const metrics = rowToMetrics(row); if (!metrics) { continue; } - if (matchedVariant) { - metricsMap.set(variantKey, metrics); - } - if (matchedLegacy && !metricsMap.has(legacyKey)) { - metricsMap.set(legacyKey, metrics); - } + metricsMap.set(key, metrics); } return metricsMap; diff --git a/packages/models/src/models/xai.ts b/packages/models/src/models/xai.ts index 476a3267b0..c39e9724e7 100644 --- a/packages/models/src/models/xai.ts +++ b/packages/models/src/models/xai.ts @@ -486,202 +486,6 @@ export const xaiModels = [ }, ], }, - { - id: "grok-4-fast", - name: "Grok 4 Fast", - description: - "Grok 4 Fast - routes to reasoning or non-reasoning variant based on reasoning_effort parameter.", - family: "xai", - releasedAt: new Date("2025-07-09"), - providers: [ - // Non-reasoning variant (default - listed first) - { - providerId: "xai", - contentFilterPrice: 0.05, - modelName: "grok-4-fast-non-reasoning", - deactivatedAt: new Date("2026-05-15"), - inputPrice: "0.2e-6", - outputPrice: "0.5e-6", - pricingTiers: [ - { - name: "Up to 128K", - upToTokens: 128000, - inputPrice: "0.2e-6", - outputPrice: "0.5e-6", - cachedInputPrice: "0.05e-6", - }, - { - name: "Over 128K", - upToTokens: Infinity, - inputPrice: "0.4e-6", - outputPrice: "1.0e-6", - cachedInputPrice: "0", - }, - ], - cachedInputPrice: "0.05e-6", - requestPrice: "0", - imageInputPrice: undefined, - contextSize: 2_000_000, - maxOutput: 30000, - streaming: true, - vision: true, - tools: true, - jsonOutput: true, - supportedParameters: xaiSupportedParamsNoFreqPresence, - }, - // Reasoning variant (used when reasoning_effort is set) - { - providerId: "xai", - contentFilterPrice: 0.05, - modelName: "grok-4-fast-reasoning", - deactivatedAt: new Date("2026-05-15"), - inputPrice: "0.2e-6", - outputPrice: "0.5e-6", - pricingTiers: [ - { - name: "Up to 128K", - upToTokens: 128000, - inputPrice: "0.2e-6", - outputPrice: "0.5e-6", - cachedInputPrice: "0.05e-6", - }, - { - name: "Over 128K", - upToTokens: Infinity, - inputPrice: "0.4e-6", - outputPrice: "1.0e-6", - cachedInputPrice: "0", - }, - ], - cachedInputPrice: "0.05e-6", - requestPrice: "0", - imageInputPrice: undefined, - contextSize: 2_000_000, - maxOutput: 30000, - streaming: true, - vision: true, - reasoning: true, - tools: true, - jsonOutput: true, - supportedParameters: xaiSupportedParamsNoFreqPresence, - }, - ], - }, - { - id: "grok-4-1-fast", - name: "Grok 4.1 Fast", - description: - "Grok 4.1 Fast - routes to reasoning or non-reasoning variant based on reasoning_effort parameter.", - family: "xai", - releasedAt: new Date("2025-11-19"), - providers: [ - // Non-reasoning variant (default - listed first) - { - providerId: "xai", - contentFilterPrice: 0.05, - modelName: "grok-4-1-fast-non-reasoning", - deactivatedAt: new Date("2026-05-15"), - inputPrice: "0.2e-6", - outputPrice: "0.5e-6", - pricingTiers: [ - { - name: "Up to 128K", - upToTokens: 128000, - inputPrice: "0.2e-6", - outputPrice: "0.5e-6", - cachedInputPrice: "0.05e-6", - }, - { - name: "Over 128K", - upToTokens: Infinity, - inputPrice: "0.4e-6", - outputPrice: "1.0e-6", - cachedInputPrice: "0", - }, - ], - cachedInputPrice: "0.05e-6", - requestPrice: "0", - imageInputPrice: undefined, - contextSize: 2_000_000, - maxOutput: 30000, - streaming: true, - vision: true, - tools: true, - jsonOutput: true, - supportedParameters: xaiSupportedParamsNoFreqPresence, - }, - { - providerId: "azure-ai-foundry", - modelName: "grok-4-1-fast-non-reasoning", - inputPrice: "0.2e-6", - outputPrice: "0.5e-6", - requestPrice: "0", - imageInputPrice: undefined, - discount: "0.2", - contextSize: 2_000_000, - maxOutput: 30000, - streaming: true, - vision: true, - tools: true, - jsonOutput: true, - supportedParameters: xaiSupportedParamsNoFreqPresence, - }, - // Reasoning variant (used when reasoning_effort is set) - { - providerId: "xai", - contentFilterPrice: 0.05, - modelName: "grok-4-1-fast-reasoning", - deactivatedAt: new Date("2026-05-15"), - inputPrice: "0.2e-6", - outputPrice: "0.5e-6", - pricingTiers: [ - { - name: "Up to 128K", - upToTokens: 128000, - inputPrice: "0.2e-6", - outputPrice: "0.5e-6", - cachedInputPrice: "0.05e-6", - }, - { - name: "Over 128K", - upToTokens: Infinity, - inputPrice: "0.4e-6", - outputPrice: "1.0e-6", - cachedInputPrice: "0", - }, - ], - cachedInputPrice: "0.05e-6", - requestPrice: "0", - imageInputPrice: undefined, - contextSize: 2_000_000, - maxOutput: 30000, - streaming: true, - vision: true, - reasoning: true, - tools: true, - jsonOutput: true, - supportedParameters: xaiSupportedParamsNoFreqPresence, - }, - { - providerId: "azure-ai-foundry", - modelName: "grok-4-1-fast-reasoning", - inputPrice: "0.2e-6", - outputPrice: "0.5e-6", - requestPrice: "0", - imageInputPrice: undefined, - discount: "0.2", - contextSize: 2_000_000, - maxOutput: 30000, - streaming: true, - vision: true, - reasoning: true, - reasoningOutput: "omit", - tools: true, - jsonOutput: true, - supportedParameters: xaiSupportedParamsNoFreqPresence, - }, - ], - }, { id: "grok-4-20-multi-agent-beta-0309", name: "Grok 4.20 Multi-Agent Beta (0309)",