From 8b3d527cd40cc7b39bb5bb1a5041fd10d07c9a55 Mon Sep 17 00:00:00 2001 From: Herjarsa Date: Sun, 7 Jun 2026 13:09:12 +0200 Subject: [PATCH 1/2] fix(combo): add 429 to PROVIDER_FAILURE_ERROR_CODES to prevent infinite retry loop isProviderFailureCode(429) returns false because 429 was excluded from PROVIDER_FAILURE_ERROR_CODES, so recordProviderFailure() is never called on rate-limited responses. Without recordProviderFailure, the circuit breaker never opens and the combo keeps retrying all targets from the same failing provider infinitely. Per-error-type cooldowns (rate_limit: 60s, quota_exhausted: 1h) prevent cascading provider trips at scale (Issue #1846). Also syncs PROVIDER_BREAKER_FAILURE_STATUSES in chat.ts for consistency. Fixes: #3200 (infinite same-provider fallback loop) --- open-sse/services/accountFallback.ts | 8 +++++--- src/sse/handlers/chat.ts | 2 +- tests/unit/account-fallback-service.test.ts | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/open-sse/services/accountFallback.ts b/open-sse/services/accountFallback.ts index 140317964aa..2db460320d6 100644 --- a/open-sse/services/accountFallback.ts +++ b/open-sse/services/accountFallback.ts @@ -81,9 +81,11 @@ function toJsonRecord(value: unknown): JsonRecord { // Error codes that count toward provider-level failure threshold // 429 (rate limit) is intentionally excluded: rate limits are connection-scoped // and handled via Connection Cooldown, not provider-wide circuit breaker. -// Counting 429 toward provider failure causes cascading provider trips at scale -// when many connections hit rate limits simultaneously (Issue #1846). -const PROVIDER_FAILURE_ERROR_CODES = new Set([408, 500, 502, 503, 504]); +// 429 included so the circuit breaker opens on repeated rate limits, +// preventing infinite combo retries. Per-error-type cooldowns +// (rate_limit: 60s, quota_exhausted: 1h) prevent cascading provider +// trips at scale (Issue #1846). +const PROVIDER_FAILURE_ERROR_CODES = new Set([408, 429, 500, 502, 503, 504]); // Per-connection failure deduplication: prevents rapid-fire failures from the // same connection from counting multiple times toward the provider breaker. diff --git a/src/sse/handlers/chat.ts b/src/sse/handlers/chat.ts index 2e5efb681a9..6e8f0415ae9 100644 --- a/src/sse/handlers/chat.ts +++ b/src/sse/handlers/chat.ts @@ -173,7 +173,7 @@ function intersectAllowedConnectionIds(primary: unknown, secondary: unknown): st return first || second || null; } -const PROVIDER_BREAKER_FAILURE_STATUSES = new Set([408, 500, 502, 503, 504]); +const PROVIDER_BREAKER_FAILURE_STATUSES = new Set([408, 429, 500, 502, 503, 504]); /** * Handle chat completion request diff --git a/tests/unit/account-fallback-service.test.ts b/tests/unit/account-fallback-service.test.ts index 713a93440e1..753e40e3ed1 100644 --- a/tests/unit/account-fallback-service.test.ts +++ b/tests/unit/account-fallback-service.test.ts @@ -536,7 +536,7 @@ test("recordModelLockoutFailure uses provider profile cooldowns, backoff, and re // Provider-level failure circuit breaker tests test("isProviderFailureCode correctly identifies provider-wide transient error codes", () => { - assert.equal(isProviderFailureCode(429), false); + assert.equal(isProviderFailureCode(429), true); assert.equal(isProviderFailureCode(408), true); assert.equal(isProviderFailureCode(500), true); assert.equal(isProviderFailureCode(502), true); From aff2faa920b5febb0037ab115d8ec19089af2b39 Mon Sep 17 00:00:00 2001 From: herjarsa Date: Sun, 7 Jun 2026 10:55:34 -0300 Subject: [PATCH 2/2] fix(review): reconcile PROVIDER_FAILURE_ERROR_CODES comment The original comment block had the first 2 lines still saying '429 is intentionally excluded' while the new lines said '429 included', making the comment contradictory. Rewrite as a single consistent block explaining the reasoning: per-error-type cooldowns address the cascading concern from Issue #1846, while the inclusion prevents infinite combo retries (#3200). Co-authored-by: diegosouzapw --- open-sse/services/accountFallback.ts | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/open-sse/services/accountFallback.ts b/open-sse/services/accountFallback.ts index 2db460320d6..dcd33e42f7e 100644 --- a/open-sse/services/accountFallback.ts +++ b/open-sse/services/accountFallback.ts @@ -78,13 +78,11 @@ function toJsonRecord(value: unknown): JsonRecord { } // Provider-level failure tracking for circuit breaker behavior -// Error codes that count toward provider-level failure threshold -// 429 (rate limit) is intentionally excluded: rate limits are connection-scoped -// and handled via Connection Cooldown, not provider-wide circuit breaker. -// 429 included so the circuit breaker opens on repeated rate limits, -// preventing infinite combo retries. Per-error-type cooldowns -// (rate_limit: 60s, quota_exhausted: 1h) prevent cascading provider -// trips at scale (Issue #1846). +// Error codes that count toward provider-level failure threshold. +// 429 is included: per-error-type cooldowns (rate_limit: 60s, quota_exhausted: 1h) +// prevent cascading provider trips at scale (Issue #1846 concern addressed), +// while still allowing the circuit breaker to open on sustained 429s and +// prevent infinite combo retries (Issue #3200). const PROVIDER_FAILURE_ERROR_CODES = new Set([408, 429, 500, 502, 503, 504]); // Per-connection failure deduplication: prevents rapid-fire failures from the