Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 43 additions & 1 deletion apps/gateway/src/chat/chat.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1186,6 +1186,25 @@ chat.openapi(completions, async (c) => {
allMetricsMap,
);

// Get price info for the original requested provider to include in scores
const originalProviderInfo = modelInfo.providers.find(
(p) => p.providerId === requestedProvider,
);
const originalProviderPrice = originalProviderInfo
? (originalProviderInfo.inputPrice ?? 0) +
(originalProviderInfo.outputPrice ?? 0)
: 0;

// Create score entry for the original requested provider
const originalProviderScore = {
providerId: requestedProvider,
score: -1, // Negative score indicates this provider was skipped due to low uptime
price: originalProviderPrice,
uptime: metrics.uptime,
latency: metrics.averageLatency,
throughput: metrics.throughput,
};

if (cheapestResult) {
usedProvider = cheapestResult.provider.providerId;
usedModel = cheapestResult.provider.modelName;
Expand All @@ -1194,6 +1213,11 @@ chat.openapi(completions, async (c) => {
selectionReason: "low-uptime-fallback",
originalProvider: requestedProvider,
originalProviderUptime: metrics.uptime,
// Add the original provider's score to the scores array
providerScores: [
originalProviderScore,
...cheapestResult.metadata.providerScores,
],
};
} else {
// Use first available provider as fallback
Expand All @@ -1205,7 +1229,7 @@ chat.openapi(completions, async (c) => {
),
selectedProvider: usedProvider,
selectionReason: "low-uptime-fallback",
providerScores: [],
providerScores: [originalProviderScore],
originalProvider: requestedProvider,
originalProviderUptime: metrics.uptime,
};
Expand Down Expand Up @@ -1320,6 +1344,21 @@ chat.openapi(completions, async (c) => {
(selectedProviderInfo.outputPrice ?? 0)
: 0;

// Fetch metrics for the selected provider to include in routing metadata
// This provides visibility into uptime/latency/throughput even for direct provider selection
const baseModelId = (modelInfo as ModelDefinition).id;
let providerMetrics:
| { uptime: number; averageLatency: number; throughput: number }
| undefined;

if (baseModelId && usedProvider !== "custom") {
const directMetricsMap = await getProviderMetricsForCombinations(
[{ modelId: baseModelId, providerId: usedProvider }],
5,
);
providerMetrics = directMetricsMap.get(`${baseModelId}:${usedProvider}`);
}

routingMetadata = {
availableProviders: [usedProvider],
selectedProvider: usedProvider,
Expand All @@ -1329,6 +1368,9 @@ chat.openapi(completions, async (c) => {
providerId: usedProvider,
score: 1,
price,
uptime: providerMetrics?.uptime,
latency: providerMetrics?.averageLatency,
throughput: providerMetrics?.throughput,
},
],
};
Expand Down
Loading