From 74549cfd68e9fd7fb32c7dd194ea4464fbaff6a6 Mon Sep 17 00:00:00 2001 From: clousky Date: Tue, 21 Apr 2026 22:47:48 +0800 Subject: [PATCH 1/2] fix(fallback): use shared CircuitBreaker instead of undefined constants recordProviderFailure was referencing PROVIDER_FAILURE_WINDOW_MS, PROVIDER_FAILURE_THRESHOLD, and PROVIDER_COOLDOWN_MS which don't exist, causing runtime ReferenceError. Replace with direct delegation to the existing getCircuitBreaker utility which already handles failure counting, threshold detection, and state transitions. Remove redundant ProviderFailureEntry type, providerFailureState Map, and providerFailureLocks Set. Co-Authored-By: Claude Opus 4.6 --- open-sse/services/accountFallback.ts | 71 +++------------------------- 1 file changed, 7 insertions(+), 64 deletions(-) diff --git a/open-sse/services/accountFallback.ts b/open-sse/services/accountFallback.ts index fcaecd09f62d..c3f94d8100bc 100644 --- a/open-sse/services/accountFallback.ts +++ b/open-sse/services/accountFallback.ts @@ -49,24 +49,9 @@ type ModelFailureState = { }; // Provider-level failure tracking for circuit breaker behavior -type ProviderFailureEntry = { - failureCount: number; - lastFailureAt: number; - resetAfterMs: number; - cooldownUntil: number | null; -}; - // Error codes that count toward provider-level failure threshold const PROVIDER_FAILURE_ERROR_CODES = new Set([408, 500, 502, 503, 504]); -// Provider-level failure state map: providerId -> failure entry -const providerFailureState = new Map(); -// Guard against synchronous re-entrant calls within the same event-loop tick. -// NOT a true mutex — Node.js is single-threaded, so different SSE streams -// can interleave across ticks. This Set prevents a single call from recursively -// re-entering recordProviderFailure within the same synchronous call stack. -const providerFailureLocks = new Set(); - // T06 (sub2api PR #1037): Signals that indicate permanent account deactivation. // When a 401 body contains these strings, the account is permanently dead // and should NOT be retried after token refresh. @@ -529,6 +514,8 @@ export function getProviderCooldownRemainingMs(provider: string | null | undefin /** * Record a provider failure against the shared circuit breaker. + * Delegates to the existing CircuitBreaker utility which handles + * failure counting, threshold detection, and state transitions. */ export function recordProviderFailure( provider: string | null | undefined, @@ -536,57 +523,13 @@ export function recordProviderFailure( ): void { if (!provider) return; - // Guard against concurrent re-entrant calls within the same tick - if (providerFailureLocks.has(provider)) return; - providerFailureLocks.add(provider); - - try { - const now = Date.now(); - const entry = providerFailureState.get(provider); - - // Check if we're in cooldown period - if (entry && entry.cooldownUntil !== null && now < entry.cooldownUntil) { - return; // Already in cooldown, don't record - } + const breaker = getProviderBreaker(provider); + if (!breaker) return; - // Check if failure window has expired - if (entry && now - entry.lastFailureAt > entry.resetAfterMs) { - // Window expired, reset count - providerFailureState.set(provider, { - failureCount: 1, - lastFailureAt: now, - resetAfterMs: PROVIDER_FAILURE_WINDOW_MS, - cooldownUntil: null, - }); - return; - } + breaker._onFailure(); - // Increment failure count - const newCount = entry ? entry.failureCount + 1 : 1; - - if (newCount >= PROVIDER_FAILURE_THRESHOLD) { - // Threshold reached, enter cooldown - const cooldownUntil = now + PROVIDER_COOLDOWN_MS; - providerFailureState.set(provider, { - failureCount: newCount, - lastFailureAt: now, - resetAfterMs: PROVIDER_FAILURE_WINDOW_MS, - cooldownUntil, - }); - log?.warn?.( - `[ProviderFailure] ${provider}: ${newCount} failures in ${PROVIDER_FAILURE_WINDOW_MS / 1000}s — entering ${PROVIDER_COOLDOWN_MS / 1000}s cooldown` - ); - } else { - // Just increment counter - providerFailureState.set(provider, { - failureCount: newCount, - lastFailureAt: now, - resetAfterMs: PROVIDER_FAILURE_WINDOW_MS, - cooldownUntil: null, - }); - } - } finally { - providerFailureLocks.delete(provider); + if (!breaker.canExecute()) { + log?.warn?.(`[ProviderFailure] ${provider}: circuit breaker opened after repeated failures`); } } From 4d9ef29fc8fcd96f01b4dde70a3ff7ffce9e0493 Mon Sep 17 00:00:00 2001 From: clousky Date: Tue, 21 Apr 2026 23:51:24 +0800 Subject: [PATCH 2/2] fix(breaker): skip recording failures during cooldown to prevent indefinite lockout --- open-sse/services/accountFallback.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/open-sse/services/accountFallback.ts b/open-sse/services/accountFallback.ts index c3f94d8100bc..0714f979c9e5 100644 --- a/open-sse/services/accountFallback.ts +++ b/open-sse/services/accountFallback.ts @@ -516,6 +516,11 @@ export function getProviderCooldownRemainingMs(provider: string | null | undefin * Record a provider failure against the shared circuit breaker. * Delegates to the existing CircuitBreaker utility which handles * failure counting, threshold detection, and state transitions. + * + * IMPORTANT: If the breaker is already OPEN (in cooldown), we skip + * recording the failure to prevent resetting the cooldown timer. + * This matches the original behavior where failures during cooldown + * were ignored to avoid indefinite lockout. */ export function recordProviderFailure( provider: string | null | undefined, @@ -526,6 +531,9 @@ export function recordProviderFailure( const breaker = getProviderBreaker(provider); if (!breaker) return; + // Skip if already in cooldown to prevent timer reset (indefinite lockout bug) + if (!breaker.canExecute()) return; + breaker._onFailure(); if (!breaker.canExecute()) {