fix(gateway): respect RPM limits in low-uptime fallback

apps/gateway/src/chat/chat.ts

@@ -39,6 +39,7 @@ import {
 	filterRateLimitedProviders,
 	getExceededProviderRateLimitLabels,
 	peekProviderRateLimit,
+	pickNonRateLimitedCandidates,
 	providerRateLimitWindows,
 } from "@/lib/provider-rate-limit.js";
 import { getResponsesContext } from "@/lib/responses-context.js";
@@ -2161,24 +2162,12 @@ chat.openapi(completions, async (c) => {
 					return true;
 				});
 
-				// Also filter out rate-limited alternatives
-				const rateLimitedAlternatives = await filterRateLimitedProviders(
+				const candidatesForRouting = await pickNonRateLimitedCandidates(
 					project.organizationId,
-					availableModelProviders.map((p) => ({
-						providerId: p.providerId,
-						model: baseModelId,
-						providerModelName: p.modelName,
-					})),
-				);
-				const nonRateLimitedAlternatives = availableModelProviders.filter(
-					(p) => !rateLimitedAlternatives.has(p.providerId),
+					baseModelId,
+					availableModelProviders,
 				);
 
-				const candidatesForRouting =
-					nonRateLimitedAlternatives.length > 0
-						? nonRateLimitedAlternatives
-						: availableModelProviders;
-
 				if (candidatesForRouting.length > 0) {
 					const rawModelForFallback = models.find((m) => m.id === baseModelId);
 					const modelWithPricing = rawModelForFallback
@@ -2326,7 +2315,13 @@ chat.openapi(completions, async (c) => {
 						),
 				);
 
-				if (availableModelProviders.length > 0) {
+				const uptimeFallbackCandidates = await pickNonRateLimitedCandidates(
+					project.organizationId,
+					baseModelId,
+					availableModelProviders,
+				);
+
+				if (uptimeFallbackCandidates.length > 0) {
 					const rawModelForFallback = models.find((m) => m.id === baseModelId);
 					const modelWithPricing = rawModelForFallback
 						? {
@@ -2339,7 +2334,7 @@ chat.openapi(completions, async (c) => {
 
 					if (modelWithPricing) {
 						// Fetch metrics for all available providers
-						const metricsCombinations = availableModelProviders.map((p) => ({
+						const metricsCombinations = uptimeFallbackCandidates.map((p) => ({
 							modelId: resolveMetricsModelId(modelWithPricing.id, p.modelName),
 							providerId: p.providerId,
 							region: p.region,
@@ -2349,7 +2344,7 @@ chat.openapi(completions, async (c) => {
 							await getProviderMetricsForCombinations(metricsCombinations);
 						const providerAgnosticCandidates =
 							collapseProvidersToBestRegionPerProvider(
-								availableModelProviders,
+								uptimeFallbackCandidates,
 								modelWithPricing,
 								{
 									metricsMap: allMetricsMap,

apps/gateway/src/lib/provider-rate-limit.spec.ts

@@ -5,6 +5,7 @@ import {
 	filterRateLimitedProviders,
 	getExceededProviderRateLimitLabels,
 	peekProviderRateLimit,
+	pickNonRateLimitedCandidates,
 } from "./provider-rate-limit.js";
 
 vi.mock("@llmgateway/cache", () => ({
@@ -245,6 +246,94 @@ describe("filterRateLimitedProviders", () => {
 	});
 });
 
+describe("pickNonRateLimitedCandidates", () => {
+	beforeEach(() => {
+		vi.resetAllMocks();
+	});
+
+	const cappedRpm = {
+		maxRpm: 10,
+		maxRpd: 0,
+		rpmSource: "global_provider",
+		rpdSource: "none",
+		rpmRateLimitId: "rl-rpm",
+	} as const;
+
+	const openRpm = {
+		maxRpm: 100,
+		maxRpd: 0,
+		rpmSource: "global_provider",
+		rpdSource: "none",
+		rpmRateLimitId: "rl-rpm",
+	} as const;
+
+	it("drops a rate-limited candidate so the router falls back to the next one", async () => {
+		vi.mocked(mockDb.getEffectiveRateLimit)
+			.mockResolvedValueOnce(cappedRpm)
+			.mockResolvedValueOnce(openRpm);
+		vi.mocked(redis.zcard).mockResolvedValueOnce(10).mockResolvedValueOnce(20);
+		vi.mocked(redis.zrange).mockResolvedValueOnce([
+			"member",
+			Date.now().toString(),
+		]);
+
+		const result = await pickNonRateLimitedCandidates("org-1", "glm-4.7", [
+			{ providerId: "together-ai", modelName: "glm-4.7" },
+			{ providerId: "cerebras", modelName: "glm-4.7" },
+		]);
+
+		expect(result).toEqual([{ providerId: "cerebras", modelName: "glm-4.7" }]);
+	});
+
+	it("fails open when every candidate is rate-limited", async () => {
+		vi.mocked(mockDb.getEffectiveRateLimit)
+			.mockResolvedValueOnce(cappedRpm)
+			.mockResolvedValueOnce(cappedRpm);
+		vi.mocked(redis.zcard).mockResolvedValueOnce(10).mockResolvedValueOnce(10);
+		vi.mocked(redis.zrange)
+			.mockResolvedValueOnce(["m1", Date.now().toString()])
+			.mockResolvedValueOnce(["m2", Date.now().toString()]);
+
+		const candidates = [
+			{ providerId: "together-ai", modelName: "glm-4.7" },
+			{ providerId: "cerebras", modelName: "glm-4.7" },
+		];
+		const result = await pickNonRateLimitedCandidates(
+			"org-1",
+			"glm-4.7",
+			candidates,
+		);
+
+		expect(result).toEqual(candidates);
+	});
+
+	it("dedupes peeks across region-expanded variants of the same provider+model", async () => {
+		vi.mocked(mockDb.getEffectiveRateLimit).mockResolvedValueOnce(openRpm);
+		vi.mocked(redis.zcard).mockResolvedValueOnce(0);
+
+		const candidates = [
+			{ providerId: "alibaba", modelName: "glm-4.6", region: "singapore" },
+			{ providerId: "alibaba", modelName: "glm-4.6", region: "cn-beijing" },
+			{ providerId: "alibaba", modelName: "glm-4.6", region: "us-east-1" },
+		];
+		const result = await pickNonRateLimitedCandidates(
+			"org-1",
+			"glm-4.6",
+			candidates,
+		);
+
+		expect(vi.mocked(mockDb.getEffectiveRateLimit)).toHaveBeenCalledTimes(1);
+		expect(result).toEqual(candidates);
+	});
+
+	it("returns an empty list unchanged without calling Redis", async () => {
+		const result = await pickNonRateLimitedCandidates("org-1", "glm-4.7", []);
+
+		expect(result).toEqual([]);
+		expect(vi.mocked(mockDb.getEffectiveRateLimit)).not.toHaveBeenCalled();
+	});
+});
+
 describe("getExceededProviderRateLimitLabels", () => {
 	it("formats the blocked limit types for logging", () => {
 		expect(getExceededProviderRateLimitLabels(["rpm"])).toBe("RPM");

apps/gateway/src/lib/provider-rate-limit.ts

@@ -273,6 +273,44 @@ export async function filterRateLimitedProviders(
 	);
 }
 
+/**
+ * Pick fallback candidates that are not at their RPM/RPD cap.
+ * Dedupes peeks by providerId+modelName since region-expanded variants share
+ * the same rate-limit window. Falls open to the original candidates if every
+ * one is capped, so callers always get a non-empty list when input was non-empty.
+ */
+export async function pickNonRateLimitedCandidates<
+	T extends { providerId: string; modelName: string },
+>(organizationId: string, baseModelId: string, candidates: T[]): Promise<T[]> {
+	if (candidates.length === 0) {
+		return candidates;
+	}
+
+	const uniquePeekCandidates = Array.from(
+		new Map(
+			candidates.map((p) => [
+				`${p.providerId}:${p.modelName}`,
+				{
+					providerId: p.providerId,
+					model: baseModelId,
+					providerModelName: p.modelName,
+				},
+			]),
+		).values(),
+	);
+
+	const rateLimited = await filterRateLimitedProviders(
+		organizationId,
+		uniquePeekCandidates,
+	);
+
+	const nonRateLimited = candidates.filter(
+		(p) => !rateLimited.has(p.providerId),
+	);
+
+	return nonRateLimited.length > 0 ? nonRateLimited : candidates;
+}
+
 /**
  * Check configurable provider/model caps stored in the database.
  * Uses a Redis sliding window approach identical to free model rate limiting.