From 297d4d5a3b667487f5a2bf80539d5b4a5e16d52c Mon Sep 17 00:00:00 2001 From: ggdayup Date: Wed, 7 Oct 2026 10:45:16 +0800 Subject: [PATCH 1/5] fix(resilience): add payload-aware anti-avalanche failover guard Oversized payloads (>=50k tokens) or TPM 429s can trigger a cascade where cross-pool retries repeatedly extend key cooldowns and trip the long model-lockout. This guard suppresses cross-pool fallback for identified avalanche risks and bounds the transient key cooldown, without tripping the 1800s lockout, so a single heavy request cannot avalanche the whole pool. --- .../services/accountFallback/antiAvalanche.ts | 250 ++++++++++++++++++ src/sse/handlers/chat.ts | 32 +++ src/sse/services/auth.ts | 32 ++- ...anti-avalanche-failover-protection.test.ts | 239 +++++++++++++++++ 4 files changed, 546 insertions(+), 7 deletions(-) create mode 100644 open-sse/services/accountFallback/antiAvalanche.ts create mode 100644 tests/unit/anti-avalanche-failover-protection.test.ts diff --git a/open-sse/services/accountFallback/antiAvalanche.ts b/open-sse/services/accountFallback/antiAvalanche.ts new file mode 100644 index 000000000000..bf03e7ca39b4 --- /dev/null +++ b/open-sse/services/accountFallback/antiAvalanche.ts @@ -0,0 +1,250 @@ +/** + * open-sse/services/accountFallback/antiAvalanche.ts + * + * Payload-aware cascading failover protection (Anti-Avalanche Guard). + * + * Prevents domino exhaustion across multiple credentials when an upstream + * rate limit (HTTP 429) or payload rejection (HTTP 413 / context overflow) is + * triggered by the payload size, prompt token volume, or TPM/burst limits + * rather than transient per-key exhaustion. + * + * When a request fails with 429 and: + * 1. Upstream explicitly indicates token rate / TPM exhaustion (e.g. Gemini + * generate_content_free_tier_input_token_count, or OpenAI/Anthropic TPM limit), OR + * 2. The payload is abnormally large (e.g. > 50,000 tokens) and exceeds the + * allowed retry threshold (at most 1 retry for 50k-100k, 0 retries for >= 100k), + * + * the guard suppresses cascading fallback across the remaining active credentials + * in the pool. This prevents 1 oversized request from knocking down all N keys + * and triggering escalated 1800s model lockouts on healthy accounts. + */ + +import { classifyGeminiQuotaMetricFromText, isTpmExhausted } from "../geminiRateLimitTracker.ts"; +import { isContextOverflow400 } from "../combo/comboPredicates.ts"; + +export const DEFAULT_AVALANCHE_TOKEN_THRESHOLD = 50_000; +export const MASSIVE_PAYLOAD_TOKEN_THRESHOLD = 100_000; +export const DEFAULT_AVALANCHE_MAX_RETRIES = 1; +export const DEFAULT_AVALANCHE_BURST_COOLDOWN_MS = 60_000; + +const EXPLICIT_TPM_PATTERNS = [ + /free_tier_input_token_count/i, + /\bgenerate_content_.*_token_count\b/i, + /\b(?:tpm|tokens?\s*per\s*minute)\b/i, + /\btoken\s*rate\s*limit\b/i, + /\binput\s*tokens?\s*exceed(?:ed)?\b/i, + /\brate_limit_exceeded.*tokens?\b/i, + /\bexceeded.*token.*quota\b/i, + /\btoken_count\b/i, + /\btoken\s*limit\s*reached\b/i, +]; + +export type AvalancheFailureReason = + | "explicit_tpm_limit" + | "oversized_payload_rate_limit" + | "payload_too_large" + | "context_length_exceeded"; + +export interface AvalancheEvaluationContext { + status: number; + errorText: string | null | undefined; + provider?: string | null; + model?: string | null; + promptTokens?: number | null; + fallbackAttemptCount?: number; + settings?: Record | null; +} + +export interface AvalancheRiskResult { + isAvalancheRisk: boolean; + reason?: AvalancheFailureReason; + shouldSuppressFallback: boolean; + suggestedCooldownMs: number; + promptTokens: number; + explanation?: string; +} + +export interface AvalancheSettings { + enabled: boolean; + tokenThreshold: number; + massiveThreshold: number; + maxRetries: number; + burstCooldownMs: number; +} + +export function resolveAvalancheSettings( + settings?: Record | null +): AvalancheSettings { + const envEnabled = process.env.OMNIROUTE_AVALANCHE_PROTECTION_ENABLED; + const enabled = + envEnabled === "false" || envEnabled === "0" + ? false + : typeof settings?.avalancheProtectionEnabled === "boolean" + ? settings.avalancheProtectionEnabled + : true; + + const parsedThreshold = Number( + process.env.OMNIROUTE_AVALANCHE_TOKEN_THRESHOLD || + settings?.avalancheTokenThreshold || + DEFAULT_AVALANCHE_TOKEN_THRESHOLD + ); + const tokenThreshold = + Number.isFinite(parsedThreshold) && parsedThreshold > 0 + ? parsedThreshold + : DEFAULT_AVALANCHE_TOKEN_THRESHOLD; + + const parsedMaxRetries = Number( + process.env.OMNIROUTE_AVALANCHE_MAX_RETRIES || + settings?.avalancheMaxRetries || + DEFAULT_AVALANCHE_MAX_RETRIES + ); + const maxRetries = + Number.isFinite(parsedMaxRetries) && parsedMaxRetries >= 0 + ? parsedMaxRetries + : DEFAULT_AVALANCHE_MAX_RETRIES; + + const parsedCooldown = Number( + process.env.OMNIROUTE_AVALANCHE_BURST_COOLDOWN_MS || + settings?.avalancheBurstCooldownMs || + DEFAULT_AVALANCHE_BURST_COOLDOWN_MS + ); + const burstCooldownMs = + Number.isFinite(parsedCooldown) && parsedCooldown > 0 + ? parsedCooldown + : DEFAULT_AVALANCHE_BURST_COOLDOWN_MS; + + return { + enabled, + tokenThreshold, + massiveThreshold: MASSIVE_PAYLOAD_TOKEN_THRESHOLD, + maxRetries, + burstCooldownMs, + }; +} + +export function isExplicitTokenRateLimitError( + status: number, + errorText: string | null | undefined, + provider?: string | null, + model?: string | null +): boolean { + if (status !== 429) return false; + const text = String(errorText || ""); + + if (provider === "gemini") { + if (classifyGeminiQuotaMetricFromText(text) === "tpm") return true; + if (model && isTpmExhausted(model)) return true; + } + + return EXPLICIT_TPM_PATTERNS.some((pattern) => pattern.test(text)); +} + +export function evaluateAvalancheRisk(ctx: AvalancheEvaluationContext): AvalancheRiskResult { + const settings = resolveAvalancheSettings(ctx.settings); + const promptTokens = + typeof ctx.promptTokens === "number" && ctx.promptTokens >= 0 ? ctx.promptTokens : 0; + const attempt = + typeof ctx.fallbackAttemptCount === "number" && ctx.fallbackAttemptCount >= 0 + ? ctx.fallbackAttemptCount + : 0; + const provider = ctx.provider || "unknown"; + const model = ctx.model || "unknown"; + + if (!settings.enabled) { + return { + isAvalancheRisk: false, + shouldSuppressFallback: false, + suggestedCooldownMs: 0, + promptTokens, + }; + } + + // 1. HTTP 413: Payload Too Large + if (ctx.status === 413) { + return { + isAvalancheRisk: true, + reason: "payload_too_large", + shouldSuppressFallback: true, + suggestedCooldownMs: settings.burstCooldownMs, + promptTokens, + explanation: `[OmniRoute Anti-Avalanche] Request rejected with HTTP 413 (Payload Too Large) for ${provider}/${model}. Cascading fallback suppressed.`, + }; + } + + // 2. HTTP 400: Context length / overflow + if (ctx.status === 400 && isContextOverflow400(ctx.errorText)) { + return { + isAvalancheRisk: true, + reason: "context_length_exceeded", + shouldSuppressFallback: true, + suggestedCooldownMs: 0, + promptTokens, + explanation: `[OmniRoute Anti-Avalanche] Context length exceeded for ${provider}/${model}. Cascading fallback across same-model accounts suppressed.`, + }; + } + + // 3. HTTP 429: Rate limited + if (ctx.status === 429) { + // 3a. Explicit TPM / Token Limit in error body or metric + if (isExplicitTokenRateLimitError(ctx.status, ctx.errorText, ctx.provider, ctx.model)) { + return { + isAvalancheRisk: true, + reason: "explicit_tpm_limit", + shouldSuppressFallback: true, + suggestedCooldownMs: settings.burstCooldownMs, + promptTokens, + explanation: `[OmniRoute Anti-Avalanche] Token rate limit (TPM/token count) hit for ${provider}/${model}. Cascading fallback suppressed across remaining keys to protect pool.`, + }; + } + + // 3b. Massive payload (>= 100k tokens) + if (promptTokens >= settings.massiveThreshold) { + return { + isAvalancheRisk: true, + reason: "oversized_payload_rate_limit", + shouldSuppressFallback: true, + suggestedCooldownMs: settings.burstCooldownMs, + promptTokens, + explanation: `[OmniRoute Anti-Avalanche] Massive payload (~${promptTokens} tokens) hit rate limit on ${provider}/${model}. Cascading fallback suppressed to prevent key exhaustion.`, + }; + } + + // 3c. Oversized payload (>= 50k tokens) with bounded retry limit + if (promptTokens >= settings.tokenThreshold) { + const shouldSuppress = attempt >= settings.maxRetries; + return { + isAvalancheRisk: true, + reason: "oversized_payload_rate_limit", + shouldSuppressFallback: shouldSuppress, + suggestedCooldownMs: settings.burstCooldownMs, + promptTokens, + explanation: shouldSuppress + ? `[OmniRoute Anti-Avalanche] Large payload (~${promptTokens} tokens) hit rate limit on ${provider}/${model} across ${attempt + 1} attempt(s). Cascading fallback suppressed.` + : undefined, + }; + } + } + + return { + isAvalancheRisk: false, + shouldSuppressFallback: false, + suggestedCooldownMs: 0, + promptTokens, + }; +} + +export function formatAntiAvalancheMessage( + provider: string, + model: string, + promptTokens: number, + reason: string, + originalError?: string +): string { + const base = `[OmniRoute Anti-Avalanche] Request to ${provider}/${model} (~${promptTokens.toLocaleString()} tokens) triggered a payload-induced rate limit (${reason}). Cascading fallback across remaining credentials was suppressed to preserve pool availability.`; + const suggestion = + provider === "gemini" + ? ` Tip: Large context requests (>50k tokens) with deep agent history should be routed to 'agy/*' (Antigravity OAuth) which has higher burst token limits, or wait 60s for the TPM sliding window to reset.` + : ` Tip: Consider reducing context size or waiting for rate limit window to clear.`; + const upstream = originalError ? ` Upstream error: ${originalError}` : ""; + return `${base}${suggestion}${upstream}`; +} diff --git a/src/sse/handlers/chat.ts b/src/sse/handlers/chat.ts index 40db7681766a..5407d50378aa 100644 --- a/src/sse/handlers/chat.ts +++ b/src/sse/handlers/chat.ts @@ -24,6 +24,8 @@ import { recordModelLockoutFailure, isDailyQuotaExhausted, } from "@omniroute/open-sse/services/accountFallback.ts"; +import { evaluateAvalancheRisk } from "@omniroute/open-sse/services/accountFallback/antiAvalanche.ts"; +import { estimateTokens } from "@omniroute/open-sse/services/contextManager.ts"; import { getCombo, getComboForModel, getModelInfo } from "../services/model"; import { stripContextWindowSuffix } from "@omniroute/open-sse/services/model.ts"; import { resolveBareModelToConnectionDefault } from "@omniroute/open-sse/services/model.ts"; @@ -2525,6 +2527,20 @@ async function handleSingleModelChat( const is401 = result.status === 401; const skipConnectionDisable = shouldSkipConnDisable(result, is401, hasExtraKeys, provider); + // Anti-Avalanche Guard: detect payload-induced rate limits / TPM rejections to suppress domino failover + const avalancheRisk = + result.status === 429 || result.status === 413 || result.status === 400 + ? evaluateAvalancheRisk({ + status: result.status, + errorText: errorStr, + provider, + model: effectiveModel, + promptTokens: estimateTokens(requestBody?.messages ?? requestBody), + fallbackAttemptCount: excludedConnectionIds.size, + settings: runtimeOptions?.cachedSettings, + }) + : null; + const { shouldFallback, cooldownMs } = skipConnectionDisable ? { shouldFallback: false, cooldownMs: 0 } : await markAccountUnavailable( @@ -2542,9 +2558,25 @@ async function handleSingleModelChat( ), isCombo, headers: result.response.headers, + isAvalancheRisk: avalancheRisk?.isAvalancheRisk ?? false, + avalancheCooldownMs: avalancheRisk?.suggestedCooldownMs, + suppressFallback: avalancheRisk?.shouldSuppressFallback ?? false, }) ); + if (avalancheRisk?.shouldSuppressFallback) { + log.warn( + "ANTI_AVALANCHE", + `${provider}/${model} cascading failover suppressed (${avalancheRisk.reason}, ~${avalancheRisk.promptTokens} tokens, attempt ${excludedConnectionIds.size + 1}) to prevent key exhaustion` + ); + if (result.response?.headers) { + result.response.headers.set("x-omniroute-anti-avalanche", "suppressed"); + if (avalancheRisk.reason) { + result.response.headers.set("x-omniroute-avalanche-reason", avalancheRisk.reason); + } + } + } + // An explicit pin (combo step `connectionId` / `x-omniroute-connection`) is an // operator instruction, not a suggestion: the account cooldown above is still // recorded, but selection must NOT silently rotate to a sibling account of the diff --git a/src/sse/services/auth.ts b/src/sse/services/auth.ts index e29c2ba97f6e..3d78034834fd 100644 --- a/src/sse/services/auth.ts +++ b/src/sse/services/auth.ts @@ -2702,6 +2702,9 @@ export async function markAccountUnavailable( headers?: Headers | Record | null; correlationId?: string | null; streamOutputEmitted?: boolean; + isAvalancheRisk?: boolean; + avalancheCooldownMs?: number; + suppressFallback?: boolean; } = {} ) { const currentMutex = markMutexes.get(connectionId) || Promise.resolve(); @@ -3136,12 +3139,18 @@ export async function markAccountUnavailable( effectiveProviderProfile, { ...modelLockoutOptions, - exactCooldownMs: isModelScopedClaudeQuota - ? claudeQuotaScope.cooldownMs - : fallbackResult.usedUpstreamRetryHint === true - ? fallbackResult.cooldownMs - : (fallbackResult.quotaResetHintMs ?? null), - maxCooldownMs: mlSettings.maxCooldownMs, + exactCooldownMs: + options.isAvalancheRisk && typeof options.avalancheCooldownMs === "number" + ? options.avalancheCooldownMs + : isModelScopedClaudeQuota + ? claudeQuotaScope.cooldownMs + : fallbackResult.usedUpstreamRetryHint === true + ? fallbackResult.cooldownMs + : (fallbackResult.quotaResetHintMs ?? null), + maxCooldownMs: + options.isAvalancheRisk && typeof options.avalancheCooldownMs === "number" + ? options.avalancheCooldownMs + : mlSettings.maxCooldownMs, scope: usesExactAntigravityLock ? "exact" : undefined, // Only a transport header, google.rpc.RetryInfo, or cached Claude reset can bypass // maxCooldownMs. Prose and generic JSON hints remain exact but operator-capped. @@ -3168,7 +3177,10 @@ export async function markAccountUnavailable( cooldownMs: lockout.cooldownMs, reason, }); - return { shouldFallback: true, cooldownMs: lockout.cooldownMs }; + return { + shouldFallback: options.suppressFallback ? false : true, + cooldownMs: lockout.cooldownMs, + }; } const result = fallbackResult; if (isSharedWalletCredits402(provider, status, errorText)) { @@ -3178,6 +3190,12 @@ export async function markAccountUnavailable( } const { shouldFallback, cooldownMs: rawCooldownMs, newBackoffLevel, reason } = result; if (!shouldFallback) return { shouldFallback: false, cooldownMs: 0 }; + if (options.suppressFallback) { + return { + shouldFallback: false, + cooldownMs: options.avalancheCooldownMs ?? rawCooldownMs, + }; + } const providerErrorType = classifyProviderError(status, errorText, provider); if ( diff --git a/tests/unit/anti-avalanche-failover-protection.test.ts b/tests/unit/anti-avalanche-failover-protection.test.ts new file mode 100644 index 000000000000..984091a2053e --- /dev/null +++ b/tests/unit/anti-avalanche-failover-protection.test.ts @@ -0,0 +1,239 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { + evaluateAvalancheRisk, + formatAntiAvalancheMessage, + resolveAvalancheSettings, + DEFAULT_AVALANCHE_BURST_COOLDOWN_MS, +} from "../../open-sse/services/accountFallback/antiAvalanche.ts"; + +test("Anti-Avalanche: small payload without explicit token metric does not suppress fallback", () => { + const result = evaluateAvalancheRisk({ + status: 429, + errorText: "Rate limit exceeded. Please try again later.", + provider: "gemini", + model: "gemini-3.8-flash", + promptTokens: 1200, + fallbackAttemptCount: 0, + }); + + assert.equal(result.isAvalancheRisk, false); + assert.equal(result.shouldSuppressFallback, false); + assert.equal(result.suggestedCooldownMs, 0); +}); + +test("Anti-Avalanche: explicit Gemini TPM metric in error text triggers immediate suppression (0 retries)", () => { + const tpmError = + "Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_input_token_count, limit: 16000"; + const result = evaluateAvalancheRisk({ + status: 429, + errorText: tpmError, + provider: "gemini", + model: "gemini-3.8-flash", + promptTokens: 25000, + fallbackAttemptCount: 0, + }); + + assert.equal(result.isAvalancheRisk, true); + assert.equal(result.reason, "explicit_tpm_limit"); + assert.equal(result.shouldSuppressFallback, true); + assert.equal(result.suggestedCooldownMs, DEFAULT_AVALANCHE_BURST_COOLDOWN_MS); + assert.match(result.explanation || "", /Token rate limit/); +}); + +test("Anti-Avalanche: OpenAI-style tokens per minute 429 triggers immediate suppression", () => { + const openaiTpmError = + "Rate limit reached for model gpt-4o on tokens per minute (TPM): Limit 30000, Requested 45000."; + const result = evaluateAvalancheRisk({ + status: 429, + errorText: openaiTpmError, + provider: "openai", + model: "gpt-4o", + promptTokens: 45000, + fallbackAttemptCount: 0, + }); + + assert.equal(result.isAvalancheRisk, true); + assert.equal(result.reason, "explicit_tpm_limit"); + assert.equal(result.shouldSuppressFallback, true); +}); + +test("Anti-Avalanche: massive payload (>= 100k tokens, incident repro) immediately suppresses failover", () => { + // Incident condition: 115,618 tokens hitting 429 + const result = evaluateAvalancheRisk({ + status: 429, + errorText: "Resource has been exhausted (e.g. check quota).", + provider: "gemini", + model: "gemini-3.8-flash", + promptTokens: 115618, + fallbackAttemptCount: 0, + }); + + assert.equal(result.isAvalancheRisk, true); + assert.equal(result.reason, "oversized_payload_rate_limit"); + assert.equal(result.shouldSuppressFallback, true); + assert.equal(result.suggestedCooldownMs, 60000); + assert.match(result.explanation || "", /Massive payload/); +}); + +test("Anti-Avalanche: oversized payload (50k-100k tokens) allows 1 retry then suppresses consecutive failure", () => { + // Attempt 0: first key fails on 60k tokens -> allow 1 failover to test if key was just busy + const attempt0 = evaluateAvalancheRisk({ + status: 429, + errorText: "Too Many Requests", + provider: "gemini", + model: "gemini-3.8-flash", + promptTokens: 60000, + fallbackAttemptCount: 0, + }); + + assert.equal(attempt0.isAvalancheRisk, true); + assert.equal(attempt0.reason, "oversized_payload_rate_limit"); + assert.equal(attempt0.shouldSuppressFallback, false); + + // Attempt 1: second key ALSO fails on 60k tokens -> suppress failover to protect keys 3..8 + const attempt1 = evaluateAvalancheRisk({ + status: 429, + errorText: "Too Many Requests", + provider: "gemini", + model: "gemini-3.8-flash", + promptTokens: 60000, + fallbackAttemptCount: 1, + }); + + assert.equal(attempt1.isAvalancheRisk, true); + assert.equal(attempt1.reason, "oversized_payload_rate_limit"); + assert.equal(attempt1.shouldSuppressFallback, true); + assert.match(attempt1.explanation || "", /Cascading fallback suppressed/); +}); + +test("Anti-Avalanche: HTTP 413 Payload Too Large suppresses failover immediately", () => { + const result = evaluateAvalancheRisk({ + status: 413, + errorText: "Request entity too large", + provider: "anthropic", + model: "claude-3-5-sonnet", + promptTokens: 180000, + fallbackAttemptCount: 0, + }); + + assert.equal(result.isAvalancheRisk, true); + assert.equal(result.reason, "payload_too_large"); + assert.equal(result.shouldSuppressFallback, true); +}); + +test("Anti-Avalanche: HTTP 400 context length overflow suppresses failover immediately", () => { + const result = evaluateAvalancheRisk({ + status: 400, + errorText: + "InvalidRequestError: This model's maximum context length is 32768 tokens. However, your messages resulted in 35000 tokens.", + provider: "openai", + model: "gpt-4", + promptTokens: 35000, + fallbackAttemptCount: 0, + }); + + assert.equal(result.isAvalancheRisk, true); + assert.equal(result.reason, "context_length_exceeded"); + assert.equal(result.shouldSuppressFallback, true); +}); + +test("Anti-Avalanche: disabled via settings bypasses protection", () => { + const result = evaluateAvalancheRisk({ + status: 429, + errorText: "generate_content_free_tier_input_token_count", + provider: "gemini", + model: "gemini-3.8-flash", + promptTokens: 115618, + fallbackAttemptCount: 5, + settings: { avalancheProtectionEnabled: false }, + }); + + assert.equal(result.isAvalancheRisk, false); + assert.equal(result.shouldSuppressFallback, false); +}); + +test("Anti-Avalanche: formatAntiAvalancheMessage provides actionable guidance", () => { + const msg = formatAntiAvalancheMessage( + "gemini", + "gemini-3.8-flash", + 115618, + "oversized_payload_rate_limit", + "RESOURCE_EXHAUSTED" + ); + + assert.match(msg, /Anti-Avalanche/); + assert.match(msg, /115,618/); + assert.match(msg, /agy\/\*/); + assert.match(msg, /RESOURCE_EXHAUSTED/); +}); + +test("Anti-Avalanche: resolveAvalancheSettings respects custom thresholds", () => { + const custom = resolveAvalancheSettings({ + avalancheTokenThreshold: 30000, + avalancheMaxRetries: 2, + avalancheBurstCooldownMs: 45000, + }); + + assert.equal(custom.enabled, true); + assert.equal(custom.tokenThreshold, 30000); + assert.equal(custom.maxRetries, 2); + assert.equal(custom.burstCooldownMs, 45000); +}); + +test("Anti-Avalanche: markAccountUnavailable honors suppressFallback and bounds lockout cooldown", async () => { + const { markAccountUnavailable } = await import("../../src/sse/services/auth.ts"); + const providersDb = await import("../../src/lib/db/providers.ts"); + + const conn = await providersDb.createProviderConnection({ + provider: "gemini", + authType: "apikey", + name: "Anti-Avalanche Test Key", + apiKey: "test-gemini-key-avalanche", + }); + const connId = (conn as { id: string }).id; + + const result = await markAccountUnavailable( + connId, + 429, + "Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_input_token_count, limit: 16000", + "gemini", + "gemini-3.8-flash", + null, + { + isAvalancheRisk: true, + avalancheCooldownMs: 60000, + suppressFallback: true, + } + ); + + assert.equal(result.shouldFallback, false, "suppressFallback must return shouldFallback: false"); + assert.equal( + result.cooldownMs, + 60000, + "cooldown must be bounded to 60000ms instead of escalating to 1800s" + ); +}); + +test("Anti-Avalanche: normal 429 without suppressFallback still allows fallback", async () => { + const { markAccountUnavailable } = await import("../../src/sse/services/auth.ts"); + const providersDb = await import("../../src/lib/db/providers.ts"); + + const conn = await providersDb.createProviderConnection({ + provider: "gemini", + authType: "apikey", + name: "Normal Fallback Test Key", + apiKey: "test-gemini-key-normal", + }); + const connId = (conn as { id: string }).id; + + const result = await markAccountUnavailable( + connId, + 429, + "Rate limit exceeded (generic RPM)", + "gemini", + "gemini-3.8-flash" + ); + + assert.equal(result.shouldFallback, true, "normal 429 must allow fallback across accounts"); +}); From 0a4e91543f35f214334aa729efe67141a9a568e7 Mon Sep 17 00:00:00 2001 From: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com> Date: Thu, 22 Oct 2026 15:07:29 -0300 Subject: [PATCH 2/5] fix(resilience): keep 413 fallback, narrow TPM patterns, env-only knobs, guard avalanche headers - 413 keeps the #13350 retry-on-another-account contract (no longer suppressed) - drop the generic token_count / exceeded.*token.*quota TPM patterns - tuning is env-only (OMNIROUTE_AVALANCHE_*), documented in .env.example and ENVIRONMENT.md - best-effort suppression headers (immutable Response headers no longer throw) - add chat pipeline wiring tests and a changelog fragment --- .env.example | 11 ++ ...5775-payload-aware-anti-avalanche-guard.md | 1 + docs/reference/ENVIRONMENT.md | 4 + .../services/accountFallback/antiAvalanche.ts | 123 +++++++--------- src/sse/handlers/chat.ts | 15 +- tests/unit/anti-avalanche-chat-wiring.test.ts | 135 ++++++++++++++++++ ...anti-avalanche-failover-protection.test.ts | 120 ++++++++++++---- 7 files changed, 306 insertions(+), 103 deletions(-) create mode 100644 changelog.d/features/15775-payload-aware-anti-avalanche-guard.md create mode 100644 tests/unit/anti-avalanche-chat-wiring.test.ts diff --git a/.env.example b/.env.example index eb1fe64cf5c2..3f6e5b4c8d6c 100644 --- a/.env.example +++ b/.env.example @@ -2053,6 +2053,17 @@ CURSOR_USER_AGENT="Cursor/3.4" # Retry-After. Used by: open-sse/services/apiKeyRotator.ts (#14573). Default 60000. # OMNIROUTE_API_KEY_COOLDOWN_MS=60000 +# ── Anti-avalanche failover guard (payload-aware) ── +# Used by: open-sse/services/accountFallback/antiAvalanche.ts → src/sse/handlers/chat.ts. +# When a 429 carries an explicit TPM / token-rate-limit signal, or an oversized prompt +# (>= token threshold) is rejected, the guard stops the cascading failover across the +# remaining keys of the provider and bounds the model-lockout cooldown. 413 is NOT +# affected (it keeps the #13350 retry-on-another-account behaviour). +# OMNIROUTE_AVALANCHE_PROTECTION_ENABLED=true +# OMNIROUTE_AVALANCHE_TOKEN_THRESHOLD=50000 +# OMNIROUTE_AVALANCHE_MAX_RETRIES=1 +# OMNIROUTE_AVALANCHE_BURST_COOLDOWN_MS=60000 + # ── Provider-level circuit breaker thresholds and cooldowns ── # Used by: open-sse/config/constants.ts (PROVIDER_PROFILES → accountFallback). # These control the provider-level fuse (entire provider cooldown after repeated diff --git a/changelog.d/features/15775-payload-aware-anti-avalanche-guard.md b/changelog.d/features/15775-payload-aware-anti-avalanche-guard.md new file mode 100644 index 000000000000..01a8aa091a58 --- /dev/null +++ b/changelog.d/features/15775-payload-aware-anti-avalanche-guard.md @@ -0,0 +1 @@ +- **feat(resilience):** payload-aware anti-avalanche failover guard — an explicit TPM / token-rate-limit 429 (or a 429 on a 50k+ token prompt) no longer cascades over the provider's remaining keys and the model lockout is bounded to a short cooldown instead of escalating; HTTP 413 keeps its retry-on-another-account behaviour. Tunable via `OMNIROUTE_AVALANCHE_*` (`OMNIROUTE_AVALANCHE_PROTECTION_ENABLED=false` disables it) ([#15775](https://github.com/cheaper-inference/OmniRoute/pull/15775) — thanks @ggdayup) diff --git a/docs/reference/ENVIRONMENT.md b/docs/reference/ENVIRONMENT.md index f74331fc977a..9efd9ddce29e 100644 --- a/docs/reference/ENVIRONMENT.md +++ b/docs/reference/ENVIRONMENT.md @@ -945,6 +945,10 @@ Provider-level circuit breaker tuning. Defaults reflect the scaled values used s | `OMNIROUTE_PROVIDER_BREAKER_LOCAL_FAILURE_WINDOW_MS` | `300000` | `open-sse/config/constants.ts` | Provider-level breaker: rolling failure-count window (ms) for local providers. | | `OMNIROUTE_PROVIDER_BREAKER_LOCAL_COOLDOWN_MS` | `60000` | `open-sse/config/constants.ts` | Provider-level breaker: cooldown (ms) once the local provider threshold is reached. | | `OMNIROUTE_API_KEY_COOLDOWN_MS` | `60000` | `open-sse/services/apiKeyRotator.ts` | Per-key cooldown (ms) applied by multi-key API-key rotation when an upstream 429 carries no `Retry-After` (#14573). Non-positive or non-numeric values fall back to the default. | +| `OMNIROUTE_AVALANCHE_PROTECTION_ENABLED` | `true` | `open-sse/services/accountFallback/antiAvalanche.ts` | Payload-aware anti-avalanche guard: set `false`/`0` to disable. When on, an explicit TPM / token-rate-limit 429 or an oversized-prompt 429 (and a context-overflow 400) stops the cascading failover across the provider's remaining keys and bounds the model-lockout cooldown. HTTP 413 is not affected (it keeps the #13350 retry-on-another-account behaviour). | +| `OMNIROUTE_AVALANCHE_TOKEN_THRESHOLD` | `50000` | `open-sse/services/accountFallback/antiAvalanche.ts` | Estimated prompt tokens at which a 429 counts as a payload-induced rate limit (allowed `OMNIROUTE_AVALANCHE_MAX_RETRIES` failovers before suppression; 100000+ is suppressed immediately). Non-positive or non-numeric values fall back to the default. | +| `OMNIROUTE_AVALANCHE_MAX_RETRIES` | `1` | `open-sse/services/accountFallback/antiAvalanche.ts` | Failovers still allowed for a 429 on a prompt between the token threshold and 100000 tokens before the guard suppresses further failover. `0` suppresses immediately. | +| `OMNIROUTE_AVALANCHE_BURST_COOLDOWN_MS` | `60000` | `open-sse/services/accountFallback/antiAvalanche.ts` | Bounded model-lockout cooldown (ms) applied when the guard fires, instead of the escalating lockout. Non-positive or non-numeric values fall back to the default. | | `PIN_DROP_BACKOFF_LEVEL` | `2` | `open-sse/services/combo.ts` | Backoff depth at which a context-cache pin's provider is deemed durably unhealthy and the pin is dropped for failover. | | `PIN_DROP_GRACE_MS` | `20000` | `open-sse/services/combo.ts` | Anti-flap window (ms) tolerating brief transient cooldowns before dropping a context-cache pin. | diff --git a/open-sse/services/accountFallback/antiAvalanche.ts b/open-sse/services/accountFallback/antiAvalanche.ts index bf03e7ca39b4..20054299847e 100644 --- a/open-sse/services/accountFallback/antiAvalanche.ts +++ b/open-sse/services/accountFallback/antiAvalanche.ts @@ -4,10 +4,14 @@ * Payload-aware cascading failover protection (Anti-Avalanche Guard). * * Prevents domino exhaustion across multiple credentials when an upstream - * rate limit (HTTP 429) or payload rejection (HTTP 413 / context overflow) is + * rate limit (HTTP 429) or context-overflow rejection (HTTP 400) is * triggered by the payload size, prompt token volume, or TPM/burst limits * rather than transient per-key exhaustion. * + * HTTP 413 is deliberately NOT handled here: accountFallback.ts treats + * PAYLOAD_TOO_LARGE as a retryable fallback (#13350), so another account or + * provider may still accept the payload. + * * When a request fails with 429 and: * 1. Upstream explicitly indicates token rate / TPM exhaustion (e.g. Gemini * generate_content_free_tier_input_token_count, or OpenAI/Anthropic TPM limit), OR @@ -17,6 +21,8 @@ * the guard suppresses cascading fallback across the remaining active credentials * in the pool. This prevents 1 oversized request from knocking down all N keys * and triggering escalated 1800s model lockouts on healthy accounts. + * + * Tuning is env-only (OMNIROUTE_AVALANCHE_*, see docs/reference/ENVIRONMENT.md). */ import { classifyGeminiQuotaMetricFromText, isTpmExhausted } from "../geminiRateLimitTracker.ts"; @@ -34,16 +40,11 @@ const EXPLICIT_TPM_PATTERNS = [ /\btoken\s*rate\s*limit\b/i, /\binput\s*tokens?\s*exceed(?:ed)?\b/i, /\brate_limit_exceeded.*tokens?\b/i, - /\bexceeded.*token.*quota\b/i, - /\btoken_count\b/i, /\btoken\s*limit\s*reached\b/i, ]; export type AvalancheFailureReason = - | "explicit_tpm_limit" - | "oversized_payload_rate_limit" - | "payload_too_large" - | "context_length_exceeded"; + "explicit_tpm_limit" | "oversized_payload_rate_limit" | "context_length_exceeded"; export interface AvalancheEvaluationContext { status: number; @@ -52,7 +53,6 @@ export interface AvalancheEvaluationContext { model?: string | null; promptTokens?: number | null; fallbackAttemptCount?: number; - settings?: Record | null; } export interface AvalancheRiskResult { @@ -72,53 +72,33 @@ export interface AvalancheSettings { burstCooldownMs: number; } -export function resolveAvalancheSettings( - settings?: Record | null -): AvalancheSettings { +function readPositiveNumber(raw: string | undefined, fallback: number, allowZero = false): number { + if (raw === undefined || raw.trim() === "") return fallback; + const parsed = Number(raw); + if (!Number.isFinite(parsed)) return fallback; + return (allowZero ? parsed >= 0 : parsed > 0) ? parsed : fallback; +} + +export function resolveAvalancheSettings(): AvalancheSettings { const envEnabled = process.env.OMNIROUTE_AVALANCHE_PROTECTION_ENABLED; - const enabled = - envEnabled === "false" || envEnabled === "0" - ? false - : typeof settings?.avalancheProtectionEnabled === "boolean" - ? settings.avalancheProtectionEnabled - : true; - - const parsedThreshold = Number( - process.env.OMNIROUTE_AVALANCHE_TOKEN_THRESHOLD || - settings?.avalancheTokenThreshold || - DEFAULT_AVALANCHE_TOKEN_THRESHOLD - ); - const tokenThreshold = - Number.isFinite(parsedThreshold) && parsedThreshold > 0 - ? parsedThreshold - : DEFAULT_AVALANCHE_TOKEN_THRESHOLD; - - const parsedMaxRetries = Number( - process.env.OMNIROUTE_AVALANCHE_MAX_RETRIES || - settings?.avalancheMaxRetries || - DEFAULT_AVALANCHE_MAX_RETRIES - ); - const maxRetries = - Number.isFinite(parsedMaxRetries) && parsedMaxRetries >= 0 - ? parsedMaxRetries - : DEFAULT_AVALANCHE_MAX_RETRIES; - - const parsedCooldown = Number( - process.env.OMNIROUTE_AVALANCHE_BURST_COOLDOWN_MS || - settings?.avalancheBurstCooldownMs || - DEFAULT_AVALANCHE_BURST_COOLDOWN_MS - ); - const burstCooldownMs = - Number.isFinite(parsedCooldown) && parsedCooldown > 0 - ? parsedCooldown - : DEFAULT_AVALANCHE_BURST_COOLDOWN_MS; + const enabled = !(envEnabled === "false" || envEnabled === "0"); return { enabled, - tokenThreshold, + tokenThreshold: readPositiveNumber( + process.env.OMNIROUTE_AVALANCHE_TOKEN_THRESHOLD, + DEFAULT_AVALANCHE_TOKEN_THRESHOLD + ), massiveThreshold: MASSIVE_PAYLOAD_TOKEN_THRESHOLD, - maxRetries, - burstCooldownMs, + maxRetries: readPositiveNumber( + process.env.OMNIROUTE_AVALANCHE_MAX_RETRIES, + DEFAULT_AVALANCHE_MAX_RETRIES, + true + ), + burstCooldownMs: readPositiveNumber( + process.env.OMNIROUTE_AVALANCHE_BURST_COOLDOWN_MS, + DEFAULT_AVALANCHE_BURST_COOLDOWN_MS + ), }; } @@ -140,7 +120,7 @@ export function isExplicitTokenRateLimitError( } export function evaluateAvalancheRisk(ctx: AvalancheEvaluationContext): AvalancheRiskResult { - const settings = resolveAvalancheSettings(ctx.settings); + const settings = resolveAvalancheSettings(); const promptTokens = typeof ctx.promptTokens === "number" && ctx.promptTokens >= 0 ? ctx.promptTokens : 0; const attempt = @@ -159,19 +139,7 @@ export function evaluateAvalancheRisk(ctx: AvalancheEvaluationContext): Avalanch }; } - // 1. HTTP 413: Payload Too Large - if (ctx.status === 413) { - return { - isAvalancheRisk: true, - reason: "payload_too_large", - shouldSuppressFallback: true, - suggestedCooldownMs: settings.burstCooldownMs, - promptTokens, - explanation: `[OmniRoute Anti-Avalanche] Request rejected with HTTP 413 (Payload Too Large) for ${provider}/${model}. Cascading fallback suppressed.`, - }; - } - - // 2. HTTP 400: Context length / overflow + // 1. HTTP 400: Context length / overflow if (ctx.status === 400 && isContextOverflow400(ctx.errorText)) { return { isAvalancheRisk: true, @@ -183,9 +151,9 @@ export function evaluateAvalancheRisk(ctx: AvalancheEvaluationContext): Avalanch }; } - // 3. HTTP 429: Rate limited + // 2. HTTP 429: Rate limited if (ctx.status === 429) { - // 3a. Explicit TPM / Token Limit in error body or metric + // 2a. Explicit TPM / Token Limit in error body or metric if (isExplicitTokenRateLimitError(ctx.status, ctx.errorText, ctx.provider, ctx.model)) { return { isAvalancheRisk: true, @@ -197,7 +165,7 @@ export function evaluateAvalancheRisk(ctx: AvalancheEvaluationContext): Avalanch }; } - // 3b. Massive payload (>= 100k tokens) + // 2b. Massive payload (>= 100k tokens) if (promptTokens >= settings.massiveThreshold) { return { isAvalancheRisk: true, @@ -209,7 +177,7 @@ export function evaluateAvalancheRisk(ctx: AvalancheEvaluationContext): Avalanch }; } - // 3c. Oversized payload (>= 50k tokens) with bounded retry limit + // 2c. Oversized payload (>= 50k tokens) with bounded retry limit if (promptTokens >= settings.tokenThreshold) { const shouldSuppress = attempt >= settings.maxRetries; return { @@ -248,3 +216,22 @@ export function formatAntiAvalancheMessage( const upstream = originalError ? ` Upstream error: ${originalError}` : ""; return `${base}${suggestion}${upstream}`; } + +/** + * Tags a suppressed failover on the client-facing response. Best-effort: the headers of + * some Response objects (e.g. Response.error() or a response built by a fetch wrapper) are + * immutable and .set() throws, which must never break the failover/error path. + */ +export function applyAvalancheSuppressionHeaders( + response: Response | null | undefined, + reason?: AvalancheFailureReason +): void { + const headers = response?.headers; + if (!headers) return; + try { + headers.set("x-omniroute-anti-avalanche", "suppressed"); + if (reason) headers.set("x-omniroute-avalanche-reason", reason); + } catch { + // immutable headers: the suppression is still enforced, only the debug tag is dropped. + } +} diff --git a/src/sse/handlers/chat.ts b/src/sse/handlers/chat.ts index b75f1b3bbb5f..7c2844809d70 100644 --- a/src/sse/handlers/chat.ts +++ b/src/sse/handlers/chat.ts @@ -28,7 +28,10 @@ import { recordModelLockoutFailure, isDailyQuotaExhausted, } from "@omniroute/open-sse/services/accountFallback.ts"; -import { evaluateAvalancheRisk } from "@omniroute/open-sse/services/accountFallback/antiAvalanche.ts"; +import { + applyAvalancheSuppressionHeaders, + evaluateAvalancheRisk, +} from "@omniroute/open-sse/services/accountFallback/antiAvalanche.ts"; import { estimateTokens } from "@omniroute/open-sse/services/contextManager.ts"; import { getCombo, getComboForModel, getModelInfo } from "../services/model"; import { stripContextWindowSuffix } from "@omniroute/open-sse/services/model.ts"; @@ -2706,7 +2709,7 @@ async function handleSingleModelChat( // Anti-Avalanche Guard: detect payload-induced rate limits / TPM rejections to suppress domino failover const avalancheRisk = - result.status === 429 || result.status === 413 || result.status === 400 + result.status === 429 || result.status === 400 ? evaluateAvalancheRisk({ status: result.status, errorText: errorStr, @@ -2714,7 +2717,6 @@ async function handleSingleModelChat( model: effectiveModel, promptTokens: estimateTokens(requestBody?.messages ?? requestBody), fallbackAttemptCount: excludedConnectionIds.size, - settings: runtimeOptions?.cachedSettings, }) : null; @@ -2747,12 +2749,7 @@ async function handleSingleModelChat( "ANTI_AVALANCHE", `${provider}/${model} cascading failover suppressed (${avalancheRisk.reason}, ~${avalancheRisk.promptTokens} tokens, attempt ${excludedConnectionIds.size + 1}) to prevent key exhaustion` ); - if (result.response?.headers) { - result.response.headers.set("x-omniroute-anti-avalanche", "suppressed"); - if (avalancheRisk.reason) { - result.response.headers.set("x-omniroute-avalanche-reason", avalancheRisk.reason); - } - } + applyAvalancheSuppressionHeaders(result.response, avalancheRisk.reason); } // An explicit pin (combo step `connectionId` / `x-omniroute-connection`) is an diff --git a/tests/unit/anti-avalanche-chat-wiring.test.ts b/tests/unit/anti-avalanche-chat-wiring.test.ts new file mode 100644 index 000000000000..009f5ad766e2 --- /dev/null +++ b/tests/unit/anti-avalanche-chat-wiring.test.ts @@ -0,0 +1,135 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +import { createChatPipelineHarness } from "../integration/_chatPipelineHarness.ts"; + +const harness = await createChatPipelineHarness("anti-avalanche-chat-wiring"); +const { clearAllModelLockouts } = await import("../../open-sse/services/accountFallback.ts"); +const { + BaseExecutor, + buildOpenAIResponse, + buildRequest, + handleChat, + resetStorage, + seedConnection, +} = harness; + +const originalFetch = globalThis.fetch; +const originalRetryConfig = { + maxAttempts: BaseExecutor.RETRY_CONFIG.maxAttempts, + delayMs: BaseExecutor.RETRY_CONFIG.delayMs, +}; + +const TPM_BODY = + "Rate limit reached for gpt-4o-mini on tokens per minute (TPM): Limit 30000, Requested 45000."; + +function chatBody(text: string) { + return { + model: "openai/gpt-4o-mini", + stream: false, + messages: [{ role: "user", content: text }], + }; +} + +// Same-key transport retries are not the avalanche: count distinct upstream credentials instead. +function bearerOf(init?: RequestInit) { + return new Headers(init?.headers).get("authorization") ?? ""; +} + +function upstreamError(status: number, message: string) { + return new Response(JSON.stringify({ error: { message } }), { + status, + headers: { "Content-Type": "application/json" }, + }); +} + +async function seedThreeOpenAIKeys() { + for (let index = 1; index <= 3; index += 1) { + await seedConnection("openai", { name: `openai-key-${index}`, priority: index }); + } +} + +test.beforeEach(async () => { + BaseExecutor.RETRY_CONFIG.maxAttempts = 1; + BaseExecutor.RETRY_CONFIG.delayMs = 0; + delete process.env.OMNIROUTE_AVALANCHE_PROTECTION_ENABLED; + await resetStorage(); + clearAllModelLockouts(); +}); + +test.afterEach(async () => { + BaseExecutor.RETRY_CONFIG.maxAttempts = originalRetryConfig.maxAttempts; + BaseExecutor.RETRY_CONFIG.delayMs = originalRetryConfig.delayMs; + globalThis.fetch = originalFetch; + delete process.env.OMNIROUTE_AVALANCHE_PROTECTION_ENABLED; + clearAllModelLockouts(); + await resetStorage(); +}); + +test.after(async () => { + await harness.cleanup(); +}); + +test("an explicit TPM 429 stops at the first key and tags the response as suppressed", async () => { + await seedThreeOpenAIKeys(); + const keysUsed = new Set(); + globalThis.fetch = async (_url, init) => { + keysUsed.add(bearerOf(init)); + return upstreamError(429, TPM_BODY); + }; + + const response = await handleChat(buildRequest({ body: chatBody("a very large agent history") })); + + assert.equal(response.status, 429); + assert.equal(keysUsed.size, 1, "the TPM rejection must not cascade over the remaining keys"); + assert.equal(response.headers.get("x-omniroute-anti-avalanche"), "suppressed"); + assert.equal(response.headers.get("x-omniroute-avalanche-reason"), "explicit_tpm_limit"); +}); + +test("an ordinary small 429 still fails over to the next key", async () => { + await seedThreeOpenAIKeys(); + const keysUsed = new Set(); + globalThis.fetch = async (_url, init) => { + keysUsed.add(bearerOf(init)); + if (keysUsed.size === 1) return upstreamError(429, "Too many requests, slow down"); + return buildOpenAIResponse("served by a sibling key"); + }; + + const response = await handleChat(buildRequest({ body: chatBody("hello") })); + + assert.equal(response.status, 200); + assert.equal(keysUsed.size, 2); + assert.equal(response.headers.get("x-omniroute-anti-avalanche"), null); +}); + +test("HTTP 413 keeps the #13350 fallback to another key", async () => { + await seedThreeOpenAIKeys(); + const keysUsed = new Set(); + globalThis.fetch = async (_url, init) => { + keysUsed.add(bearerOf(init)); + if (keysUsed.size === 1) return upstreamError(413, "Request entity too large"); + return buildOpenAIResponse("served by a sibling key"); + }; + + const response = await handleChat(buildRequest({ body: chatBody("hello") })); + + assert.equal(response.status, 200); + assert.equal(keysUsed.size, 2); + assert.equal(response.headers.get("x-omniroute-anti-avalanche"), null); +}); + +test("OMNIROUTE_AVALANCHE_PROTECTION_ENABLED=false restores the cascading failover", async () => { + process.env.OMNIROUTE_AVALANCHE_PROTECTION_ENABLED = "false"; + await seedThreeOpenAIKeys(); + const keysUsed = new Set(); + globalThis.fetch = async (_url, init) => { + keysUsed.add(bearerOf(init)); + if (keysUsed.size < 3) return upstreamError(429, TPM_BODY); + return buildOpenAIResponse("served by the third key"); + }; + + const response = await handleChat(buildRequest({ body: chatBody("a very large agent history") })); + + assert.equal(response.status, 200); + assert.equal(keysUsed.size, 3); +}); diff --git a/tests/unit/anti-avalanche-failover-protection.test.ts b/tests/unit/anti-avalanche-failover-protection.test.ts index 984091a2053e..974e866fc5d8 100644 --- a/tests/unit/anti-avalanche-failover-protection.test.ts +++ b/tests/unit/anti-avalanche-failover-protection.test.ts @@ -1,6 +1,16 @@ import test from "node:test"; import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +// The markAccountUnavailable tests below create provider connections: keep them off the real +// home database. +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-anti-avalanche-")); +const previousDataDir = process.env.DATA_DIR; +process.env.DATA_DIR = TEST_DATA_DIR; import { + applyAvalancheSuppressionHeaders, evaluateAvalancheRisk, formatAntiAvalancheMessage, resolveAvalancheSettings, @@ -107,7 +117,9 @@ test("Anti-Avalanche: oversized payload (50k-100k tokens) allows 1 retry then su assert.match(attempt1.explanation || "", /Cascading fallback suppressed/); }); -test("Anti-Avalanche: HTTP 413 Payload Too Large suppresses failover immediately", () => { +test("Anti-Avalanche: HTTP 413 Payload Too Large keeps the #13350 fallback contract (not suppressed)", () => { + // accountFallback.ts classifies 413 PAYLOAD_TOO_LARGE as a retryable fallback (#13350): + // another account/provider may accept the payload, so the guard must not suppress it. const result = evaluateAvalancheRisk({ status: 413, errorText: "Request entity too large", @@ -117,9 +129,27 @@ test("Anti-Avalanche: HTTP 413 Payload Too Large suppresses failover immediately fallbackAttemptCount: 0, }); - assert.equal(result.isAvalancheRisk, true); - assert.equal(result.reason, "payload_too_large"); - assert.equal(result.shouldSuppressFallback, true); + assert.equal(result.isAvalancheRisk, false); + assert.equal(result.shouldSuppressFallback, false); + assert.equal(result.suggestedCooldownMs, 0); +}); + +test("Anti-Avalanche: generic token_count / token-quota wording on a 429 is not an explicit TPM limit", () => { + for (const errorText of [ + "Invalid request: token_count field is required", + "You exceeded your current token quota, please check your plan and billing details.", + ]) { + const result = evaluateAvalancheRisk({ + status: 429, + errorText, + provider: "openai", + model: "gpt-4o", + promptTokens: 1200, + fallbackAttemptCount: 0, + }); + assert.equal(result.isAvalancheRisk, false, errorText); + assert.equal(result.shouldSuppressFallback, false, errorText); + } }); test("Anti-Avalanche: HTTP 400 context length overflow suppresses failover immediately", () => { @@ -138,19 +168,25 @@ test("Anti-Avalanche: HTTP 400 context length overflow suppresses failover immed assert.equal(result.shouldSuppressFallback, true); }); -test("Anti-Avalanche: disabled via settings bypasses protection", () => { - const result = evaluateAvalancheRisk({ - status: 429, - errorText: "generate_content_free_tier_input_token_count", - provider: "gemini", - model: "gemini-3.8-flash", - promptTokens: 115618, - fallbackAttemptCount: 5, - settings: { avalancheProtectionEnabled: false }, - }); +test("Anti-Avalanche: disabled via OMNIROUTE_AVALANCHE_PROTECTION_ENABLED bypasses protection", () => { + const previous = process.env.OMNIROUTE_AVALANCHE_PROTECTION_ENABLED; + process.env.OMNIROUTE_AVALANCHE_PROTECTION_ENABLED = "false"; + try { + const result = evaluateAvalancheRisk({ + status: 429, + errorText: "generate_content_free_tier_input_token_count", + provider: "gemini", + model: "gemini-3.8-flash", + promptTokens: 115618, + fallbackAttemptCount: 5, + }); - assert.equal(result.isAvalancheRisk, false); - assert.equal(result.shouldSuppressFallback, false); + assert.equal(result.isAvalancheRisk, false); + assert.equal(result.shouldSuppressFallback, false); + } finally { + if (previous === undefined) delete process.env.OMNIROUTE_AVALANCHE_PROTECTION_ENABLED; + else process.env.OMNIROUTE_AVALANCHE_PROTECTION_ENABLED = previous; + } }); test("Anti-Avalanche: formatAntiAvalancheMessage provides actionable guidance", () => { @@ -168,17 +204,41 @@ test("Anti-Avalanche: formatAntiAvalancheMessage provides actionable guidance", assert.match(msg, /RESOURCE_EXHAUSTED/); }); -test("Anti-Avalanche: resolveAvalancheSettings respects custom thresholds", () => { - const custom = resolveAvalancheSettings({ - avalancheTokenThreshold: 30000, - avalancheMaxRetries: 2, - avalancheBurstCooldownMs: 45000, - }); +test("Anti-Avalanche: resolveAvalancheSettings respects env thresholds", () => { + const keys = [ + "OMNIROUTE_AVALANCHE_TOKEN_THRESHOLD", + "OMNIROUTE_AVALANCHE_MAX_RETRIES", + "OMNIROUTE_AVALANCHE_BURST_COOLDOWN_MS", + ] as const; + const previous = keys.map((key) => process.env[key]); + process.env.OMNIROUTE_AVALANCHE_TOKEN_THRESHOLD = "30000"; + process.env.OMNIROUTE_AVALANCHE_MAX_RETRIES = "2"; + process.env.OMNIROUTE_AVALANCHE_BURST_COOLDOWN_MS = "45000"; + try { + const custom = resolveAvalancheSettings(); + assert.equal(custom.enabled, true); + assert.equal(custom.tokenThreshold, 30000); + assert.equal(custom.maxRetries, 2); + assert.equal(custom.burstCooldownMs, 45000); + } finally { + keys.forEach((key, index) => { + const value = previous[index]; + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + }); + } +}); + +test("Anti-Avalanche: suppression headers are best-effort on an immutable response", () => { + const mutable = new Response("x", { status: 429 }); + applyAvalancheSuppressionHeaders(mutable, "explicit_tpm_limit"); + assert.equal(mutable.headers.get("x-omniroute-anti-avalanche"), "suppressed"); + assert.equal(mutable.headers.get("x-omniroute-avalanche-reason"), "explicit_tpm_limit"); - assert.equal(custom.enabled, true); - assert.equal(custom.tokenThreshold, 30000); - assert.equal(custom.maxRetries, 2); - assert.equal(custom.burstCooldownMs, 45000); + // Response.error() has immutable headers: .set() throws a TypeError. + const immutable = Response.error(); + assert.doesNotThrow(() => applyAvalancheSuppressionHeaders(immutable, "explicit_tpm_limit")); + assert.doesNotThrow(() => applyAvalancheSuppressionHeaders(null, undefined)); }); test("Anti-Avalanche: markAccountUnavailable honors suppressFallback and bounds lockout cooldown", async () => { @@ -237,3 +297,11 @@ test("Anti-Avalanche: normal 429 without suppressFallback still allows fallback" assert.equal(result.shouldFallback, true, "normal 429 must allow fallback across accounts"); }); + +test.after(async () => { + const core = await import("../../src/lib/db/core.ts"); + core.resetDbInstance(); + if (previousDataDir === undefined) delete process.env.DATA_DIR; + else process.env.DATA_DIR = previousDataDir; + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); +}); From e8db1ed9cea4132fd53e094ad328371f5d90d7f0 Mon Sep 17 00:00:00 2001 From: ggdayup Date: Wed, 28 Oct 2026 20:09:51 -0300 Subject: [PATCH 3/5] refactor(resilience): split evaluateAvalancheRisk under the cognitive-complexity threshold Extract the HTTP 429 branch and the no-risk result into helpers so the new function no longer adds a cognitive-complexity ratchet violation (17 > 15). Behaviour is unchanged; the 18 anti-avalanche tests stay green. Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com> --- .../services/accountFallback/antiAvalanche.ts | 129 ++++++++++-------- 1 file changed, 69 insertions(+), 60 deletions(-) diff --git a/open-sse/services/accountFallback/antiAvalanche.ts b/open-sse/services/accountFallback/antiAvalanche.ts index 20054299847e..6e6aa222eb94 100644 --- a/open-sse/services/accountFallback/antiAvalanche.ts +++ b/open-sse/services/accountFallback/antiAvalanche.ts @@ -119,26 +119,76 @@ export function isExplicitTokenRateLimitError( return EXPLICIT_TPM_PATTERNS.some((pattern) => pattern.test(text)); } -export function evaluateAvalancheRisk(ctx: AvalancheEvaluationContext): AvalancheRiskResult { - const settings = resolveAvalancheSettings(); - const promptTokens = - typeof ctx.promptTokens === "number" && ctx.promptTokens >= 0 ? ctx.promptTokens : 0; - const attempt = - typeof ctx.fallbackAttemptCount === "number" && ctx.fallbackAttemptCount >= 0 - ? ctx.fallbackAttemptCount - : 0; - const provider = ctx.provider || "unknown"; - const model = ctx.model || "unknown"; - - if (!settings.enabled) { +function nonNegativeOrZero(value: number | null | undefined): number { + return typeof value === "number" && value >= 0 ? value : 0; +} + +function noAvalancheRisk(promptTokens: number): AvalancheRiskResult { + return { + isAvalancheRisk: false, + shouldSuppressFallback: false, + suggestedCooldownMs: 0, + promptTokens, + }; +} + +/** HTTP 429 branch: explicit TPM limit, massive payload, or oversized payload past the retry budget. */ +function evaluateRateLimitAvalanche( + ctx: AvalancheEvaluationContext, + settings: AvalancheSettings, + promptTokens: number, + label: string +): AvalancheRiskResult | null { + // 2a. Explicit TPM / Token Limit in error body or metric + if (isExplicitTokenRateLimitError(ctx.status, ctx.errorText, ctx.provider, ctx.model)) { return { - isAvalancheRisk: false, - shouldSuppressFallback: false, - suggestedCooldownMs: 0, + isAvalancheRisk: true, + reason: "explicit_tpm_limit", + shouldSuppressFallback: true, + suggestedCooldownMs: settings.burstCooldownMs, promptTokens, + explanation: `[OmniRoute Anti-Avalanche] Token rate limit (TPM/token count) hit for ${label}. Cascading fallback suppressed across remaining keys to protect pool.`, }; } + // 2b. Massive payload (>= 100k tokens) + if (promptTokens >= settings.massiveThreshold) { + return { + isAvalancheRisk: true, + reason: "oversized_payload_rate_limit", + shouldSuppressFallback: true, + suggestedCooldownMs: settings.burstCooldownMs, + promptTokens, + explanation: `[OmniRoute Anti-Avalanche] Massive payload (~${promptTokens} tokens) hit rate limit on ${label}. Cascading fallback suppressed to prevent key exhaustion.`, + }; + } + + // 2c. Oversized payload (>= 50k tokens) with bounded retry limit + if (promptTokens >= settings.tokenThreshold) { + const attempt = nonNegativeOrZero(ctx.fallbackAttemptCount); + const shouldSuppress = attempt >= settings.maxRetries; + return { + isAvalancheRisk: true, + reason: "oversized_payload_rate_limit", + shouldSuppressFallback: shouldSuppress, + suggestedCooldownMs: settings.burstCooldownMs, + promptTokens, + explanation: shouldSuppress + ? `[OmniRoute Anti-Avalanche] Large payload (~${promptTokens} tokens) hit rate limit on ${label} across ${attempt + 1} attempt(s). Cascading fallback suppressed.` + : undefined, + }; + } + + return null; +} + +export function evaluateAvalancheRisk(ctx: AvalancheEvaluationContext): AvalancheRiskResult { + const settings = resolveAvalancheSettings(); + const promptTokens = nonNegativeOrZero(ctx.promptTokens); + if (!settings.enabled) return noAvalancheRisk(promptTokens); + + const label = `${ctx.provider || "unknown"}/${ctx.model || "unknown"}`; + // 1. HTTP 400: Context length / overflow if (ctx.status === 400 && isContextOverflow400(ctx.errorText)) { return { @@ -147,58 +197,17 @@ export function evaluateAvalancheRisk(ctx: AvalancheEvaluationContext): Avalanch shouldSuppressFallback: true, suggestedCooldownMs: 0, promptTokens, - explanation: `[OmniRoute Anti-Avalanche] Context length exceeded for ${provider}/${model}. Cascading fallback across same-model accounts suppressed.`, + explanation: `[OmniRoute Anti-Avalanche] Context length exceeded for ${label}. Cascading fallback across same-model accounts suppressed.`, }; } // 2. HTTP 429: Rate limited if (ctx.status === 429) { - // 2a. Explicit TPM / Token Limit in error body or metric - if (isExplicitTokenRateLimitError(ctx.status, ctx.errorText, ctx.provider, ctx.model)) { - return { - isAvalancheRisk: true, - reason: "explicit_tpm_limit", - shouldSuppressFallback: true, - suggestedCooldownMs: settings.burstCooldownMs, - promptTokens, - explanation: `[OmniRoute Anti-Avalanche] Token rate limit (TPM/token count) hit for ${provider}/${model}. Cascading fallback suppressed across remaining keys to protect pool.`, - }; - } - - // 2b. Massive payload (>= 100k tokens) - if (promptTokens >= settings.massiveThreshold) { - return { - isAvalancheRisk: true, - reason: "oversized_payload_rate_limit", - shouldSuppressFallback: true, - suggestedCooldownMs: settings.burstCooldownMs, - promptTokens, - explanation: `[OmniRoute Anti-Avalanche] Massive payload (~${promptTokens} tokens) hit rate limit on ${provider}/${model}. Cascading fallback suppressed to prevent key exhaustion.`, - }; - } - - // 2c. Oversized payload (>= 50k tokens) with bounded retry limit - if (promptTokens >= settings.tokenThreshold) { - const shouldSuppress = attempt >= settings.maxRetries; - return { - isAvalancheRisk: true, - reason: "oversized_payload_rate_limit", - shouldSuppressFallback: shouldSuppress, - suggestedCooldownMs: settings.burstCooldownMs, - promptTokens, - explanation: shouldSuppress - ? `[OmniRoute Anti-Avalanche] Large payload (~${promptTokens} tokens) hit rate limit on ${provider}/${model} across ${attempt + 1} attempt(s). Cascading fallback suppressed.` - : undefined, - }; - } + const rateLimitRisk = evaluateRateLimitAvalanche(ctx, settings, promptTokens, label); + if (rateLimitRisk) return rateLimitRisk; } - return { - isAvalancheRisk: false, - shouldSuppressFallback: false, - suggestedCooldownMs: 0, - promptTokens, - }; + return noAvalancheRisk(promptTokens); } export function formatAntiAvalancheMessage( From d0062f6e0c44762695ce68a6c93c279a674ef124 Mon Sep 17 00:00:00 2001 From: ggdayup Date: Mon, 2 Nov 2026 17:57:50 -0300 Subject: [PATCH 4/5] test(resilience): register anti-avalanche suites in the mutation tap list check:mutation-test-coverage --strict flagged both new test files as covering accountFallback.ts and auth.ts without being listed in stryker tap.testFiles. Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com> --- stryker.conf.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/stryker.conf.json b/stryker.conf.json index 7b1145274400..75d298c3eb11 100644 --- a/stryker.conf.json +++ b/stryker.conf.json @@ -148,6 +148,8 @@ "tests/unit/anthropic-thinking-signature-recovery.test.ts", "tests/unit/anthropic-request-not-allowed-cooldown-escalation.test.ts", "tests/unit/anthropic-request-not-allowed-not-a-ban.test.ts", + "tests/unit/anti-avalanche-chat-wiring.test.ts", + "tests/unit/anti-avalanche-failover-protection.test.ts", "tests/unit/grok-request-refusal-not-a-ban.test.ts", "tests/unit/agy-family-not-connection-cooldown.test.ts", "tests/unit/antigravity-family-cooldown-reader.test.ts", From 5560dc8f65bc4c2e606b9d29958829a2a0209179 Mon Sep 17 00:00:00 2001 From: ggdayup Date: Mon, 2 Nov 2026 18:39:31 -0300 Subject: [PATCH 5/5] fix(resilience): make the anti-avalanche guard opt-in and TPM-only The guard is off unless OMNIROUTE_AVALANCHE_PROTECTION_ENABLED is exactly true. When enabled, only a 429 that names a tokens-per-minute limit suppresses failover, so a quota-exhausted 429 with a large prompt fails over as usual; the now-dead token threshold and retry settings are removed. The burst cooldown is a floor that never shortens an upstream reset, and a suppressed connection-scoped 429 now records its lock. Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com> --- .env.example | 15 +- ...5775-payload-aware-anti-avalanche-guard.md | 2 +- docs/reference/ENVIRONMENT.md | 6 +- .../services/accountFallback/antiAvalanche.ts | 161 ++++----- src/sse/handlers/chat.ts | 6 +- src/sse/services/auth.ts | 51 +-- tests/unit/anti-avalanche-chat-wiring.test.ts | 57 +++- ...anti-avalanche-failover-protection.test.ts | 312 +++++++++++++----- 8 files changed, 389 insertions(+), 221 deletions(-) diff --git a/.env.example b/.env.example index 534e74de96e0..1e1a72997394 100644 --- a/.env.example +++ b/.env.example @@ -2143,15 +2143,14 @@ CURSOR_USER_AGENT="Cursor/3.4" # Retry-After. Used by: open-sse/services/apiKeyRotator.ts (#14573). Default 60000. # OMNIROUTE_API_KEY_COOLDOWN_MS=60000 -# ── Anti-avalanche failover guard (payload-aware) ── +# ── Anti-avalanche failover guard (opt-in, off by default) ── # Used by: open-sse/services/accountFallback/antiAvalanche.ts → src/sse/handlers/chat.ts. -# When a 429 carries an explicit TPM / token-rate-limit signal, or an oversized prompt -# (>= token threshold) is rejected, the guard stops the cascading failover across the -# remaining keys of the provider and bounds the model-lockout cooldown. 413 is NOT -# affected (it keeps the #13350 retry-on-another-account behaviour). -# OMNIROUTE_AVALANCHE_PROTECTION_ENABLED=true -# OMNIROUTE_AVALANCHE_TOKEN_THRESHOLD=50000 -# OMNIROUTE_AVALANCHE_MAX_RETRIES=1 +# Only an explicit "true" enables it. When on, a 429 that names a tokens-per-minute limit +# (or a context-overflow 400) stops the cascading failover across the remaining keys of the +# provider. A quota-exhausted 429 keeps failing over however large the prompt is, 413 keeps +# the #13350 retry-on-another-account behaviour, and the burst cooldown never shortens a +# longer upstream reset. +# OMNIROUTE_AVALANCHE_PROTECTION_ENABLED=false # OMNIROUTE_AVALANCHE_BURST_COOLDOWN_MS=60000 # ── Provider-level circuit breaker thresholds and cooldowns ── diff --git a/changelog.d/features/15775-payload-aware-anti-avalanche-guard.md b/changelog.d/features/15775-payload-aware-anti-avalanche-guard.md index 01a8aa091a58..8d391abcffd7 100644 --- a/changelog.d/features/15775-payload-aware-anti-avalanche-guard.md +++ b/changelog.d/features/15775-payload-aware-anti-avalanche-guard.md @@ -1 +1 @@ -- **feat(resilience):** payload-aware anti-avalanche failover guard — an explicit TPM / token-rate-limit 429 (or a 429 on a 50k+ token prompt) no longer cascades over the provider's remaining keys and the model lockout is bounded to a short cooldown instead of escalating; HTTP 413 keeps its retry-on-another-account behaviour. Tunable via `OMNIROUTE_AVALANCHE_*` (`OMNIROUTE_AVALANCHE_PROTECTION_ENABLED=false` disables it) ([#15775](https://github.com/cheaper-inference/OmniRoute/pull/15775) — thanks @ggdayup) +- **feat(resilience):** opt-in anti-avalanche failover guard (`OMNIROUTE_AVALANCHE_PROTECTION_ENABLED=true`, off by default) — when enabled, a 429 that names a tokens-per-minute limit no longer cascades over the provider's remaining keys and the model lockout uses a short cooldown instead of escalating, while a longer upstream reset always wins; quota-exhausted 429s keep failing over however large the prompt is and HTTP 413 keeps its retry-on-another-account behaviour ([#15775](https://github.com/cheaper-inference/OmniRoute/pull/15775) — thanks @ggdayup) diff --git a/docs/reference/ENVIRONMENT.md b/docs/reference/ENVIRONMENT.md index 430f8ab59419..1b26f448cc1a 100644 --- a/docs/reference/ENVIRONMENT.md +++ b/docs/reference/ENVIRONMENT.md @@ -986,10 +986,8 @@ Provider-level circuit breaker tuning. Defaults reflect the scaled values used s | `OMNIROUTE_PROVIDER_BREAKER_LOCAL_FAILURE_WINDOW_MS` | `300000` | `open-sse/config/constants.ts` | Provider-level breaker: rolling failure-count window (ms) for local providers. | | `OMNIROUTE_PROVIDER_BREAKER_LOCAL_COOLDOWN_MS` | `60000` | `open-sse/config/constants.ts` | Provider-level breaker: cooldown (ms) once the local provider threshold is reached. | | `OMNIROUTE_API_KEY_COOLDOWN_MS` | `60000` | `open-sse/services/apiKeyRotator.ts` | Per-key cooldown (ms) applied by multi-key API-key rotation when an upstream 429 carries no `Retry-After` (#14573). Non-positive or non-numeric values fall back to the default. | -| `OMNIROUTE_AVALANCHE_PROTECTION_ENABLED` | `true` | `open-sse/services/accountFallback/antiAvalanche.ts` | Payload-aware anti-avalanche guard: set `false`/`0` to disable. When on, an explicit TPM / token-rate-limit 429 or an oversized-prompt 429 (and a context-overflow 400) stops the cascading failover across the provider's remaining keys and bounds the model-lockout cooldown. HTTP 413 is not affected (it keeps the #13350 retry-on-another-account behaviour). | -| `OMNIROUTE_AVALANCHE_TOKEN_THRESHOLD` | `50000` | `open-sse/services/accountFallback/antiAvalanche.ts` | Estimated prompt tokens at which a 429 counts as a payload-induced rate limit (allowed `OMNIROUTE_AVALANCHE_MAX_RETRIES` failovers before suppression; 100000+ is suppressed immediately). Non-positive or non-numeric values fall back to the default. | -| `OMNIROUTE_AVALANCHE_MAX_RETRIES` | `1` | `open-sse/services/accountFallback/antiAvalanche.ts` | Failovers still allowed for a 429 on a prompt between the token threshold and 100000 tokens before the guard suppresses further failover. `0` suppresses immediately. | -| `OMNIROUTE_AVALANCHE_BURST_COOLDOWN_MS` | `60000` | `open-sse/services/accountFallback/antiAvalanche.ts` | Bounded model-lockout cooldown (ms) applied when the guard fires, instead of the escalating lockout. Non-positive or non-numeric values fall back to the default. | +| `OMNIROUTE_AVALANCHE_PROTECTION_ENABLED` | `false` | `open-sse/services/accountFallback/antiAvalanche.ts` | Opt-in anti-avalanche guard: only an explicit `true` enables it. When on, a 429 that names a tokens-per-minute limit (and a context-overflow 400) stops the cascading failover across the provider's remaining keys. A quota-exhausted 429 keeps failing over however large the prompt is, and HTTP 413 is not affected (it keeps the #13350 retry-on-another-account behaviour). | +| `OMNIROUTE_AVALANCHE_BURST_COOLDOWN_MS` | `60000` | `open-sse/services/accountFallback/antiAvalanche.ts` | Model-lockout cooldown (ms) applied when the guard fires, instead of the escalating lockout. A longer upstream reset always wins. Non-positive or non-numeric values fall back to the default. | | `PIN_DROP_BACKOFF_LEVEL` | `2` | `open-sse/services/combo.ts` | Backoff depth at which a context-cache pin's provider is deemed durably unhealthy and the pin is dropped for failover. | | `PIN_DROP_GRACE_MS` | `20000` | `open-sse/services/combo.ts` | Anti-flap window (ms) tolerating brief transient cooldowns before dropping a context-cache pin. | diff --git a/open-sse/services/accountFallback/antiAvalanche.ts b/open-sse/services/accountFallback/antiAvalanche.ts index 6e6aa222eb94..f6a3940a9a4e 100644 --- a/open-sse/services/accountFallback/antiAvalanche.ts +++ b/open-sse/services/accountFallback/antiAvalanche.ts @@ -1,36 +1,31 @@ /** * open-sse/services/accountFallback/antiAvalanche.ts * - * Payload-aware cascading failover protection (Anti-Avalanche Guard). + * Anti-Avalanche Guard: opt-in cascading-failover protection + * (OMNIROUTE_AVALANCHE_PROTECTION_ENABLED=true, off by default). * * Prevents domino exhaustion across multiple credentials when an upstream - * rate limit (HTTP 429) or context-overflow rejection (HTTP 400) is - * triggered by the payload size, prompt token volume, or TPM/burst limits - * rather than transient per-key exhaustion. + * rejection comes from a limit every sibling key would hit too: + * - an HTTP 429 that names a tokens-per-minute limit (e.g. Gemini + * generate_content_free_tier_input_token_count, or an OpenAI/Anthropic TPM + * limit). This stops 1 oversized request from knocking down all N keys and + * triggering escalated 1800s model lockouts on healthy accounts; + * - an HTTP 400 context-overflow rejection. + * + * Prompt size alone never suppresses failover: a quota-exhausted 429 with a large + * prompt says nothing about the sibling accounts, so it fails over as usual. * * HTTP 413 is deliberately NOT handled here: accountFallback.ts treats * PAYLOAD_TOO_LARGE as a retryable fallback (#13350), so another account or * provider may still accept the payload. * - * When a request fails with 429 and: - * 1. Upstream explicitly indicates token rate / TPM exhaustion (e.g. Gemini - * generate_content_free_tier_input_token_count, or OpenAI/Anthropic TPM limit), OR - * 2. The payload is abnormally large (e.g. > 50,000 tokens) and exceeds the - * allowed retry threshold (at most 1 retry for 50k-100k, 0 retries for >= 100k), - * - * the guard suppresses cascading fallback across the remaining active credentials - * in the pool. This prevents 1 oversized request from knocking down all N keys - * and triggering escalated 1800s model lockouts on healthy accounts. - * * Tuning is env-only (OMNIROUTE_AVALANCHE_*, see docs/reference/ENVIRONMENT.md). */ -import { classifyGeminiQuotaMetricFromText, isTpmExhausted } from "../geminiRateLimitTracker.ts"; +import { classifyGeminiQuotaMetricFromText } from "../geminiRateLimitTracker.ts"; import { isContextOverflow400 } from "../combo/comboPredicates.ts"; +import { isSubscriptionQuotaText } from "../quotaTextCooldowns.ts"; -export const DEFAULT_AVALANCHE_TOKEN_THRESHOLD = 50_000; -export const MASSIVE_PAYLOAD_TOKEN_THRESHOLD = 100_000; -export const DEFAULT_AVALANCHE_MAX_RETRIES = 1; export const DEFAULT_AVALANCHE_BURST_COOLDOWN_MS = 60_000; const EXPLICIT_TPM_PATTERNS = [ @@ -38,13 +33,19 @@ const EXPLICIT_TPM_PATTERNS = [ /\bgenerate_content_.*_token_count\b/i, /\b(?:tpm|tokens?\s*per\s*minute)\b/i, /\btoken\s*rate\s*limit\b/i, - /\binput\s*tokens?\s*exceed(?:ed)?\b/i, /\brate_limit_exceeded.*tokens?\b/i, - /\btoken\s*limit\s*reached\b/i, ]; -export type AvalancheFailureReason = - "explicit_tpm_limit" | "oversized_payload_rate_limit" | "context_length_exceeded"; +// A window longer than a minute is a quota: sibling accounts keep their own, so it must fail over +// even when the body also carries a token wording (OpenAI "tokens per day", Gemini "...PerDay"). +const LONG_WINDOW_QUOTA_PATTERNS = [ + /\b(?:tpd|rpd)\b/i, + /per[\s_-]*(?:hour|day|week|month)/i, + /\b(?:hourly|daily|weekly|monthly)\b/i, + /\binsufficient_quota\b/i, +]; + +export type AvalancheFailureReason = "explicit_tpm_limit" | "context_length_exceeded"; export interface AvalancheEvaluationContext { status: number; @@ -52,7 +53,6 @@ export interface AvalancheEvaluationContext { provider?: string | null; model?: string | null; promptTokens?: number | null; - fallbackAttemptCount?: number; } export interface AvalancheRiskResult { @@ -66,35 +66,19 @@ export interface AvalancheRiskResult { export interface AvalancheSettings { enabled: boolean; - tokenThreshold: number; - massiveThreshold: number; - maxRetries: number; burstCooldownMs: number; } -function readPositiveNumber(raw: string | undefined, fallback: number, allowZero = false): number { +function readPositiveNumber(raw: string | undefined, fallback: number): number { if (raw === undefined || raw.trim() === "") return fallback; const parsed = Number(raw); - if (!Number.isFinite(parsed)) return fallback; - return (allowZero ? parsed >= 0 : parsed > 0) ? parsed : fallback; + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; } export function resolveAvalancheSettings(): AvalancheSettings { - const envEnabled = process.env.OMNIROUTE_AVALANCHE_PROTECTION_ENABLED; - const enabled = !(envEnabled === "false" || envEnabled === "0"); - return { - enabled, - tokenThreshold: readPositiveNumber( - process.env.OMNIROUTE_AVALANCHE_TOKEN_THRESHOLD, - DEFAULT_AVALANCHE_TOKEN_THRESHOLD - ), - massiveThreshold: MASSIVE_PAYLOAD_TOKEN_THRESHOLD, - maxRetries: readPositiveNumber( - process.env.OMNIROUTE_AVALANCHE_MAX_RETRIES, - DEFAULT_AVALANCHE_MAX_RETRIES, - true - ), + // Opt-in: it changes failover semantics, so only an explicit "true" enables it. + enabled: process.env.OMNIROUTE_AVALANCHE_PROTECTION_ENABLED === "true", burstCooldownMs: readPositiveNumber( process.env.OMNIROUTE_AVALANCHE_BURST_COOLDOWN_MS, DEFAULT_AVALANCHE_BURST_COOLDOWN_MS @@ -102,18 +86,28 @@ export function resolveAvalancheSettings(): AvalancheSettings { }; } +/** + * True only when the 429 itself names a tokens-per-minute limit. Local usage counters do not + * count: they are per model across every key, so they cannot describe this key's rejection. + */ export function isExplicitTokenRateLimitError( status: number, errorText: string | null | undefined, - provider?: string | null, - model?: string | null + provider?: string | null ): boolean { if (status !== 429) return false; const text = String(errorText || ""); + if ( + isSubscriptionQuotaText(text.toLowerCase(), provider) || + LONG_WINDOW_QUOTA_PATTERNS.some((pattern) => pattern.test(text)) + ) { + return false; + } + if (provider === "gemini") { - if (classifyGeminiQuotaMetricFromText(text) === "tpm") return true; - if (model && isTpmExhausted(model)) return true; + const metric = classifyGeminiQuotaMetricFromText(text); + if (metric) return metric === "tpm"; } return EXPLICIT_TPM_PATTERNS.some((pattern) => pattern.test(text)); @@ -132,56 +126,6 @@ function noAvalancheRisk(promptTokens: number): AvalancheRiskResult { }; } -/** HTTP 429 branch: explicit TPM limit, massive payload, or oversized payload past the retry budget. */ -function evaluateRateLimitAvalanche( - ctx: AvalancheEvaluationContext, - settings: AvalancheSettings, - promptTokens: number, - label: string -): AvalancheRiskResult | null { - // 2a. Explicit TPM / Token Limit in error body or metric - if (isExplicitTokenRateLimitError(ctx.status, ctx.errorText, ctx.provider, ctx.model)) { - return { - isAvalancheRisk: true, - reason: "explicit_tpm_limit", - shouldSuppressFallback: true, - suggestedCooldownMs: settings.burstCooldownMs, - promptTokens, - explanation: `[OmniRoute Anti-Avalanche] Token rate limit (TPM/token count) hit for ${label}. Cascading fallback suppressed across remaining keys to protect pool.`, - }; - } - - // 2b. Massive payload (>= 100k tokens) - if (promptTokens >= settings.massiveThreshold) { - return { - isAvalancheRisk: true, - reason: "oversized_payload_rate_limit", - shouldSuppressFallback: true, - suggestedCooldownMs: settings.burstCooldownMs, - promptTokens, - explanation: `[OmniRoute Anti-Avalanche] Massive payload (~${promptTokens} tokens) hit rate limit on ${label}. Cascading fallback suppressed to prevent key exhaustion.`, - }; - } - - // 2c. Oversized payload (>= 50k tokens) with bounded retry limit - if (promptTokens >= settings.tokenThreshold) { - const attempt = nonNegativeOrZero(ctx.fallbackAttemptCount); - const shouldSuppress = attempt >= settings.maxRetries; - return { - isAvalancheRisk: true, - reason: "oversized_payload_rate_limit", - shouldSuppressFallback: shouldSuppress, - suggestedCooldownMs: settings.burstCooldownMs, - promptTokens, - explanation: shouldSuppress - ? `[OmniRoute Anti-Avalanche] Large payload (~${promptTokens} tokens) hit rate limit on ${label} across ${attempt + 1} attempt(s). Cascading fallback suppressed.` - : undefined, - }; - } - - return null; -} - export function evaluateAvalancheRisk(ctx: AvalancheEvaluationContext): AvalancheRiskResult { const settings = resolveAvalancheSettings(); const promptTokens = nonNegativeOrZero(ctx.promptTokens); @@ -201,15 +145,30 @@ export function evaluateAvalancheRisk(ctx: AvalancheEvaluationContext): Avalanch }; } - // 2. HTTP 429: Rate limited - if (ctx.status === 429) { - const rateLimitRisk = evaluateRateLimitAvalanche(ctx, settings, promptTokens, label); - if (rateLimitRisk) return rateLimitRisk; + // 2. HTTP 429 naming a tokens-per-minute limit + if (isExplicitTokenRateLimitError(ctx.status, ctx.errorText, ctx.provider)) { + return { + isAvalancheRisk: true, + reason: "explicit_tpm_limit", + shouldSuppressFallback: true, + suggestedCooldownMs: settings.burstCooldownMs, + promptTokens, + explanation: `[OmniRoute Anti-Avalanche] Token rate limit (TPM/token count) hit for ${label}. Cascading fallback suppressed across remaining keys to protect pool.`, + }; } return noAvalancheRisk(promptTokens); } +/** The guard's burst cooldown is a floor: it never shortens an upstream reset. */ +export function resolveAvalancheCooldownMs( + avalancheCooldownMs: number | null | undefined, + upstreamResetMs: number | null | undefined +): number | null { + if (typeof avalancheCooldownMs !== "number") return upstreamResetMs ?? null; + return Math.max(avalancheCooldownMs, upstreamResetMs ?? 0); +} + export function formatAntiAvalancheMessage( provider: string, model: string, diff --git a/src/sse/handlers/chat.ts b/src/sse/handlers/chat.ts index f4b83eacdbc9..bdf3e319792f 100644 --- a/src/sse/handlers/chat.ts +++ b/src/sse/handlers/chat.ts @@ -31,6 +31,7 @@ import { import { applyAvalancheSuppressionHeaders, evaluateAvalancheRisk, + resolveAvalancheSettings, } from "@omniroute/open-sse/services/accountFallback/antiAvalanche.ts"; import { estimateTokens } from "@omniroute/open-sse/services/contextManager.ts"; import { getCombo, getComboForModel, getModelInfo } from "../services/model"; @@ -2732,16 +2733,15 @@ async function handleSingleModelChat( const skipConnectionDisable = isRefusalReplay || shouldSkipConnDisable(result, is401, hasExtraKeys, provider); - // Anti-Avalanche Guard: detect payload-induced rate limits / TPM rejections to suppress domino failover + // Anti-Avalanche Guard (opt-in): detect TPM / context-overflow rejections to suppress domino failover const avalancheRisk = - result.status === 429 || result.status === 400 + (result.status === 429 || result.status === 400) && resolveAvalancheSettings().enabled ? evaluateAvalancheRisk({ status: result.status, errorText: errorStr, provider, model: effectiveModel, promptTokens: estimateTokens(requestBody?.messages ?? requestBody), - fallbackAttemptCount: excludedConnectionIds.size, }) : null; diff --git a/src/sse/services/auth.ts b/src/sse/services/auth.ts index 3fee7109808f..2279e4334892 100644 --- a/src/sse/services/auth.ts +++ b/src/sse/services/auth.ts @@ -106,6 +106,7 @@ import { } from "@omniroute/open-sse/services/accountFallback.ts"; import { copilotRejectionFlag, lockCopilotModelNotSupported } from "./copilotModelNotSupportedLock"; import { isSharedWalletCredits402 } from "@omniroute/open-sse/services/accountFallback/sharedWalletCredits.ts"; +import { resolveAvalancheCooldownMs } from "@omniroute/open-sse/services/accountFallback/antiAvalanche.ts"; import { isModelScopedAccessDenial } from "@omniroute/open-sse/services/accountFallback/modelScopedAccessDenial.ts"; import { isModelUnsupported401 } from "@omniroute/open-sse/services/accountFallback/perModelFailureScope.ts"; import { postOutputFailureReachesLockout } from "@omniroute/open-sse/services/accountFallback/postOutputFailureStreak.ts"; @@ -2332,6 +2333,16 @@ function isStreamContentStall(errorText: string): boolean { return /stream content stall/i.test(String(errorText || "")); } +/** The reset an upstream announced for this failure, as opposed to a locally computed backoff. */ +function resolveUpstreamResetMs(fallback: { + usedUpstreamRetryHint?: boolean; + cooldownMs?: number; + quotaResetHintMs?: number | null; +}): number | null { + if (fallback.usedUpstreamRetryHint === true) return fallback.cooldownMs ?? null; + return fallback.quotaResetHintMs ?? null; +} + export async function markAccountUnavailable( connectionId: string, status: number, @@ -2804,6 +2815,12 @@ export async function markAccountUnavailable( status === 429 ? ANTIGRAVITY_FAMILY_INFERRED_BASE_COOLDOWN_MS : null; + const upstreamCooldownMs = isModelScopedClaudeQuota + ? claudeQuotaScope.cooldownMs + : resolveUpstreamResetMs(fallbackResult); + const exactCooldownMs = options.isAvalancheRisk + ? resolveAvalancheCooldownMs(options.avalancheCooldownMs, upstreamCooldownMs) + : upstreamCooldownMs; const lockout = recordModelLockoutFailure( provider, connectionId, @@ -2820,24 +2837,16 @@ export async function markAccountUnavailable( effectiveProviderProfile, { ...modelLockoutOptions, - exactCooldownMs: - options.isAvalancheRisk && typeof options.avalancheCooldownMs === "number" - ? options.avalancheCooldownMs - : isModelScopedClaudeQuota - ? claudeQuotaScope.cooldownMs - : fallbackResult.usedUpstreamRetryHint === true - ? fallbackResult.cooldownMs - : (fallbackResult.quotaResetHintMs ?? null), - maxCooldownMs: - options.isAvalancheRisk && typeof options.avalancheCooldownMs === "number" - ? options.avalancheCooldownMs - : mlSettings.maxCooldownMs, + exactCooldownMs, + maxCooldownMs: mlSettings.maxCooldownMs, scope: usesExactAntigravityLock ? "exact" : undefined, // Only a transport header, google.rpc.RetryInfo, or cached Claude reset can bypass - // maxCooldownMs. Prose and generic JSON hints remain exact but operator-capped. + // maxCooldownMs. Prose and generic JSON hints remain exact but operator-capped, and + // so does the avalanche burst cooldown when it outlasts the upstream reset. exactCooldownIsUpstreamReset: - retryHintBypassesMaxCooldownMs(fallbackResult.retryHintSource) || - isModelScopedClaudeQuota, + exactCooldownMs === upstreamCooldownMs && + (retryHintBypassesMaxCooldownMs(fallbackResult.retryHintSource) || + isModelScopedClaudeQuota), } ); // Update last error for observability (without changing terminal status) @@ -2872,10 +2881,14 @@ export async function markAccountUnavailable( const { shouldFallback, cooldownMs: rawCooldownMs, newBackoffLevel, reason } = result; if (!shouldFallback) return { shouldFallback: false, cooldownMs: 0 }; if (options.suppressFallback) { - return { - shouldFallback: false, - cooldownMs: options.avalancheCooldownMs ?? rawCooldownMs, - }; + const suppressedCooldownMs = + resolveAvalancheCooldownMs(options.avalancheCooldownMs, resolveUpstreamResetMs(result)) ?? + rawCooldownMs; + // This path skips the connection cooldown below, so the lock is what keeps the key out. + if (provider && suppressedCooldownMs > 0 && !disableCooling) { + lockModel(provider, connectionId, model, reason || "unknown", suppressedCooldownMs); + } + return { shouldFallback: false, cooldownMs: suppressedCooldownMs }; } const providerErrorType = classifyProviderError(status, errorText, provider); diff --git a/tests/unit/anti-avalanche-chat-wiring.test.ts b/tests/unit/anti-avalanche-chat-wiring.test.ts index 009f5ad766e2..02665b50dc6c 100644 --- a/tests/unit/anti-avalanche-chat-wiring.test.ts +++ b/tests/unit/anti-avalanche-chat-wiring.test.ts @@ -52,7 +52,8 @@ async function seedThreeOpenAIKeys() { test.beforeEach(async () => { BaseExecutor.RETRY_CONFIG.maxAttempts = 1; BaseExecutor.RETRY_CONFIG.delayMs = 0; - delete process.env.OMNIROUTE_AVALANCHE_PROTECTION_ENABLED; + // Opt-in guard: these tests exercise it enabled unless they say otherwise. + process.env.OMNIROUTE_AVALANCHE_PROTECTION_ENABLED = "true"; await resetStorage(); clearAllModelLockouts(); }); @@ -86,6 +87,27 @@ test("an explicit TPM 429 stops at the first key and tags the response as suppre assert.equal(response.headers.get("x-omniroute-avalanche-reason"), "explicit_tpm_limit"); }); +test("a suppressed TPM 429 keeps the key out for the whole upstream Retry-After", async () => { + await seedThreeOpenAIKeys(); + const keysUsed: string[] = []; + globalThis.fetch = async (_url, init) => { + keysUsed.push(bearerOf(init)); + if (keysUsed.length > 1) return buildOpenAIResponse("served by a sibling key"); + return new Response(JSON.stringify({ error: { message: TPM_BODY } }), { + status: 429, + headers: { "Content-Type": "application/json", "Retry-After": "3600" }, + }); + }; + + const first = await handleChat(buildRequest({ body: chatBody("a very large agent history") })); + assert.equal(first.status, 429); + assert.equal(keysUsed.length, 1); + + const second = await handleChat(buildRequest({ body: chatBody("the next request") })); + assert.equal(second.status, 200); + assert.notEqual(keysUsed[1], keysUsed[0], "the first key must still be cooling down"); +}); + test("an ordinary small 429 still fails over to the next key", async () => { await seedThreeOpenAIKeys(); const keysUsed = new Set(); @@ -102,6 +124,22 @@ test("an ordinary small 429 still fails over to the next key", async () => { assert.equal(response.headers.get("x-omniroute-anti-avalanche"), null); }); +test("a quota-exhausted 429 on a massive prompt still fails over to a sibling key", async () => { + await seedThreeOpenAIKeys(); + const keysUsed = new Set(); + globalThis.fetch = async (_url, init) => { + keysUsed.add(bearerOf(init)); + if (keysUsed.size === 1) return upstreamError(429, "You exceeded your current quota"); + return buildOpenAIResponse("served by a sibling key"); + }; + + const response = await handleChat(buildRequest({ body: chatBody("large ".repeat(73_500)) })); + + assert.equal(response.status, 200); + assert.equal(keysUsed.size, 2); + assert.equal(response.headers.get("x-omniroute-anti-avalanche"), null); +}); + test("HTTP 413 keeps the #13350 fallback to another key", async () => { await seedThreeOpenAIKeys(); const keysUsed = new Set(); @@ -118,6 +156,23 @@ test("HTTP 413 keeps the #13350 fallback to another key", async () => { assert.equal(response.headers.get("x-omniroute-anti-avalanche"), null); }); +test("the guard is off by default: an unset env keeps the normal failover over a TPM 429", async () => { + delete process.env.OMNIROUTE_AVALANCHE_PROTECTION_ENABLED; + await seedThreeOpenAIKeys(); + const keysUsed = new Set(); + globalThis.fetch = async (_url, init) => { + keysUsed.add(bearerOf(init)); + if (keysUsed.size < 3) return upstreamError(429, TPM_BODY); + return buildOpenAIResponse("served by the third key"); + }; + + const response = await handleChat(buildRequest({ body: chatBody("a very large agent history") })); + + assert.equal(response.status, 200); + assert.equal(keysUsed.size, 3); + assert.equal(response.headers.get("x-omniroute-anti-avalanche"), null); +}); + test("OMNIROUTE_AVALANCHE_PROTECTION_ENABLED=false restores the cascading failover", async () => { process.env.OMNIROUTE_AVALANCHE_PROTECTION_ENABLED = "false"; await seedThreeOpenAIKeys(); diff --git a/tests/unit/anti-avalanche-failover-protection.test.ts b/tests/unit/anti-avalanche-failover-protection.test.ts index 974e866fc5d8..9e677d232e28 100644 --- a/tests/unit/anti-avalanche-failover-protection.test.ts +++ b/tests/unit/anti-avalanche-failover-protection.test.ts @@ -16,6 +16,43 @@ import { resolveAvalancheSettings, DEFAULT_AVALANCHE_BURST_COOLDOWN_MS, } from "../../open-sse/services/accountFallback/antiAvalanche.ts"; +import { + incrementTokenUsage, + resetCounters, +} from "../../open-sse/services/geminiRateLimitTracker.ts"; + +// The guard is opt-in: every test below exercises the enabled guard unless it says otherwise. +const ENABLE_ENV = "OMNIROUTE_AVALANCHE_PROTECTION_ENABLED"; +const previousEnabled = process.env[ENABLE_ENV]; + +test.beforeEach(() => { + process.env[ENABLE_ENV] = "true"; +}); + +const OPENAI_TPM_ERROR = + "Rate limit reached for model gpt-4o on tokens per minute (TPM): Limit 30000, Requested 45000."; + +test("Anti-Avalanche: the guard is OFF unless the env var is exactly true", () => { + for (const value of [undefined, "", "false", "0", "1", "yes", "TRUE"]) { + if (value === undefined) delete process.env[ENABLE_ENV]; + else process.env[ENABLE_ENV] = value; + + assert.equal(resolveAvalancheSettings().enabled, false, String(value)); + const result = evaluateAvalancheRisk({ + status: 429, + errorText: OPENAI_TPM_ERROR, + provider: "openai", + model: "gpt-4o", + promptTokens: 150000, + }); + assert.equal(result.isAvalancheRisk, false, String(value)); + assert.equal(result.shouldSuppressFallback, false, String(value)); + assert.equal(result.suggestedCooldownMs, 0, String(value)); + } + + process.env[ENABLE_ENV] = "true"; + assert.equal(resolveAvalancheSettings().enabled, true); +}); test("Anti-Avalanche: small payload without explicit token metric does not suppress fallback", () => { const result = evaluateAvalancheRisk({ @@ -24,7 +61,6 @@ test("Anti-Avalanche: small payload without explicit token metric does not suppr provider: "gemini", model: "gemini-3.8-flash", promptTokens: 1200, - fallbackAttemptCount: 0, }); assert.equal(result.isAvalancheRisk, false); @@ -41,7 +77,6 @@ test("Anti-Avalanche: explicit Gemini TPM metric in error text triggers immediat provider: "gemini", model: "gemini-3.8-flash", promptTokens: 25000, - fallbackAttemptCount: 0, }); assert.equal(result.isAvalancheRisk, true); @@ -60,7 +95,6 @@ test("Anti-Avalanche: OpenAI-style tokens per minute 429 triggers immediate supp provider: "openai", model: "gpt-4o", promptTokens: 45000, - fallbackAttemptCount: 0, }); assert.equal(result.isAvalancheRisk, true); @@ -68,53 +102,80 @@ test("Anti-Avalanche: OpenAI-style tokens per minute 429 triggers immediate supp assert.equal(result.shouldSuppressFallback, true); }); -test("Anti-Avalanche: massive payload (>= 100k tokens, incident repro) immediately suppresses failover", () => { - // Incident condition: 115,618 tokens hitting 429 - const result = evaluateAvalancheRisk({ - status: 429, - errorText: "Resource has been exhausted (e.g. check quota).", - provider: "gemini", - model: "gemini-3.8-flash", - promptTokens: 115618, - fallbackAttemptCount: 0, - }); - - assert.equal(result.isAvalancheRisk, true); - assert.equal(result.reason, "oversized_payload_rate_limit"); - assert.equal(result.shouldSuppressFallback, true); - assert.equal(result.suggestedCooldownMs, 60000); - assert.match(result.explanation || "", /Massive payload/); +test("Anti-Avalanche: a quota-exhausted 429 keeps failing over however large the prompt is", () => { + // The incident prompt size (115,618 tokens) and the former 50k-100k band: payload size alone + // says nothing about the sibling accounts, whose quota may be intact. + for (const errorText of [ + "Resource has been exhausted (e.g. check quota).", + "You exceeded your current quota, please check your plan and billing details.", + "Too Many Requests", + ]) { + for (const promptTokens of [115618, 60000]) { + const result = evaluateAvalancheRisk({ + status: 429, + errorText, + provider: "gemini", + model: "gemini-3.8-flash", + promptTokens, + }); + assert.equal(result.isAvalancheRisk, false, errorText); + assert.equal(result.shouldSuppressFallback, false, errorText); + assert.equal(result.suggestedCooldownMs, 0, errorText); + } + } }); -test("Anti-Avalanche: oversized payload (50k-100k tokens) allows 1 retry then suppresses consecutive failure", () => { - // Attempt 0: first key fails on 60k tokens -> allow 1 failover to test if key was just busy - const attempt0 = evaluateAvalancheRisk({ - status: 429, - errorText: "Too Many Requests", - provider: "gemini", - model: "gemini-3.8-flash", - promptTokens: 60000, - fallbackAttemptCount: 0, - }); - - assert.equal(attempt0.isAvalancheRisk, true); - assert.equal(attempt0.reason, "oversized_payload_rate_limit"); - assert.equal(attempt0.shouldSuppressFallback, false); - - // Attempt 1: second key ALSO fails on 60k tokens -> suppress failover to protect keys 3..8 - const attempt1 = evaluateAvalancheRisk({ - status: 429, - errorText: "Too Many Requests", - provider: "gemini", - model: "gemini-3.8-flash", - promptTokens: 60000, - fallbackAttemptCount: 1, - }); +test("Anti-Avalanche: token wording on a daily/plan quota is not a TPM marker", () => { + const cases: Array<[string | null, string]> = [ + [ + "openai", + '{"error":{"code":"rate_limit_exceeded","message":"Rate limit reached for gpt-4o on tokens per day (TPD): Limit 200000, Used 199000, Requested 45000."}}', + ], + ["openai", "Monthly token limit reached"], + ["openai", "Token limit reached for your plan"], + ["openai", "Your input tokens exceeded the monthly allowance"], + ["openai", "Usage limit exceeded: token rate limit for your plan"], + [ + "gemini", + "Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_input_token_count, quotaId: GenerateContentInputTokensPerModelPerDay-FreeTier", + ], + [ + "gemini", + "Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 15", + ], + ]; + + for (const [provider, errorText] of cases) { + const result = evaluateAvalancheRisk({ + status: 429, + errorText, + provider, + model: "some-model", + promptTokens: 115618, + }); + assert.equal(result.isAvalancheRisk, false, errorText); + assert.equal(result.shouldSuppressFallback, false, errorText); + } +}); - assert.equal(attempt1.isAvalancheRisk, true); - assert.equal(attempt1.reason, "oversized_payload_rate_limit"); - assert.equal(attempt1.shouldSuppressFallback, true); - assert.match(attempt1.explanation || "", /Cascading fallback suppressed/); +test("Anti-Avalanche: the local Gemini token counter is not a marker on the 429 itself", () => { + // The counter is per model across every key, so it cannot tell this key's quota 429 apart + // from a pool-wide TPM limit. + resetCounters(); + incrementTokenUsage("gemini-2.5-flash", 250001); + try { + const result = evaluateAvalancheRisk({ + status: 429, + errorText: "Resource has been exhausted (e.g. check quota).", + provider: "gemini", + model: "gemini-2.5-flash", + promptTokens: 115618, + }); + assert.equal(result.isAvalancheRisk, false); + assert.equal(result.shouldSuppressFallback, false); + } finally { + resetCounters(); + } }); test("Anti-Avalanche: HTTP 413 Payload Too Large keeps the #13350 fallback contract (not suppressed)", () => { @@ -126,7 +187,6 @@ test("Anti-Avalanche: HTTP 413 Payload Too Large keeps the #13350 fallback contr provider: "anthropic", model: "claude-3-5-sonnet", promptTokens: 180000, - fallbackAttemptCount: 0, }); assert.equal(result.isAvalancheRisk, false); @@ -145,7 +205,6 @@ test("Anti-Avalanche: generic token_count / token-quota wording on a 429 is not provider: "openai", model: "gpt-4o", promptTokens: 1200, - fallbackAttemptCount: 0, }); assert.equal(result.isAvalancheRisk, false, errorText); assert.equal(result.shouldSuppressFallback, false, errorText); @@ -160,7 +219,6 @@ test("Anti-Avalanche: HTTP 400 context length overflow suppresses failover immed provider: "openai", model: "gpt-4", promptTokens: 35000, - fallbackAttemptCount: 0, }); assert.equal(result.isAvalancheRisk, true); @@ -169,24 +227,17 @@ test("Anti-Avalanche: HTTP 400 context length overflow suppresses failover immed }); test("Anti-Avalanche: disabled via OMNIROUTE_AVALANCHE_PROTECTION_ENABLED bypasses protection", () => { - const previous = process.env.OMNIROUTE_AVALANCHE_PROTECTION_ENABLED; - process.env.OMNIROUTE_AVALANCHE_PROTECTION_ENABLED = "false"; - try { - const result = evaluateAvalancheRisk({ - status: 429, - errorText: "generate_content_free_tier_input_token_count", - provider: "gemini", - model: "gemini-3.8-flash", - promptTokens: 115618, - fallbackAttemptCount: 5, - }); + process.env[ENABLE_ENV] = "false"; + const result = evaluateAvalancheRisk({ + status: 429, + errorText: "generate_content_free_tier_input_token_count", + provider: "gemini", + model: "gemini-3.8-flash", + promptTokens: 115618, + }); - assert.equal(result.isAvalancheRisk, false); - assert.equal(result.shouldSuppressFallback, false); - } finally { - if (previous === undefined) delete process.env.OMNIROUTE_AVALANCHE_PROTECTION_ENABLED; - else process.env.OMNIROUTE_AVALANCHE_PROTECTION_ENABLED = previous; - } + assert.equal(result.isAvalancheRisk, false); + assert.equal(result.shouldSuppressFallback, false); }); test("Anti-Avalanche: formatAntiAvalancheMessage provides actionable guidance", () => { @@ -194,7 +245,7 @@ test("Anti-Avalanche: formatAntiAvalancheMessage provides actionable guidance", "gemini", "gemini-3.8-flash", 115618, - "oversized_payload_rate_limit", + "explicit_tpm_limit", "RESOURCE_EXHAUSTED" ); @@ -204,28 +255,16 @@ test("Anti-Avalanche: formatAntiAvalancheMessage provides actionable guidance", assert.match(msg, /RESOURCE_EXHAUSTED/); }); -test("Anti-Avalanche: resolveAvalancheSettings respects env thresholds", () => { - const keys = [ - "OMNIROUTE_AVALANCHE_TOKEN_THRESHOLD", - "OMNIROUTE_AVALANCHE_MAX_RETRIES", - "OMNIROUTE_AVALANCHE_BURST_COOLDOWN_MS", - ] as const; - const previous = keys.map((key) => process.env[key]); - process.env.OMNIROUTE_AVALANCHE_TOKEN_THRESHOLD = "30000"; - process.env.OMNIROUTE_AVALANCHE_MAX_RETRIES = "2"; +test("Anti-Avalanche: resolveAvalancheSettings respects the cooldown env", () => { + const previous = process.env.OMNIROUTE_AVALANCHE_BURST_COOLDOWN_MS; process.env.OMNIROUTE_AVALANCHE_BURST_COOLDOWN_MS = "45000"; try { const custom = resolveAvalancheSettings(); assert.equal(custom.enabled, true); - assert.equal(custom.tokenThreshold, 30000); - assert.equal(custom.maxRetries, 2); assert.equal(custom.burstCooldownMs, 45000); } finally { - keys.forEach((key, index) => { - const value = previous[index]; - if (value === undefined) delete process.env[key]; - else process.env[key] = value; - }); + if (previous === undefined) delete process.env.OMNIROUTE_AVALANCHE_BURST_COOLDOWN_MS; + else process.env.OMNIROUTE_AVALANCHE_BURST_COOLDOWN_MS = previous; } }); @@ -275,6 +314,109 @@ test("Anti-Avalanche: markAccountUnavailable honors suppressFallback and bounds ); }); +const GEMINI_TPM_ERROR = + "Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_input_token_count, limit: 16000"; +const AVALANCHE_OPTIONS = { + isAvalancheRisk: true, + avalancheCooldownMs: 60000, + suppressFallback: true, +}; +const ONE_HOUR_MS = 3600 * 1000; +// Retry-After is parsed against the wall clock, so the hint can read slightly under the hour. +const CLOCK_TOLERANCE_MS = 5000; + +async function createKey(provider: string, name: string) { + const providersDb = await import("../../src/lib/db/providers.ts"); + const conn = await providersDb.createProviderConnection({ + provider, + authType: "apikey", + name, + apiKey: `test-${name}`, + }); + return (conn as { id: string }).id; +} + +test("Anti-Avalanche: a longer upstream reset beats the burst cooldown on the model lockout", async () => { + const { markAccountUnavailable } = await import("../../src/sse/services/auth.ts"); + const { getModelLockoutInfo } = await import("../../open-sse/services/accountFallback.ts"); + const connId = await createKey("gemini", "avalanche-long-reset"); + + const result = await markAccountUnavailable( + connId, + 429, + GEMINI_TPM_ERROR, + "gemini", + "gemini-3.8-flash", + null, + { ...AVALANCHE_OPTIONS, headers: new Headers({ "retry-after": "3600" }) } + ); + + assert.equal(result.shouldFallback, false); + assert.ok(result.cooldownMs > ONE_HOUR_MS - CLOCK_TOLERANCE_MS, String(result.cooldownMs)); + const lock = getModelLockoutInfo("gemini", connId, "gemini-3.8-flash"); + assert.ok(lock && lock.remainingMs > ONE_HOUR_MS - CLOCK_TOLERANCE_MS, JSON.stringify(lock)); +}); + +test("Anti-Avalanche: a shorter upstream reset keeps the burst cooldown", async () => { + const { markAccountUnavailable } = await import("../../src/sse/services/auth.ts"); + const connId = await createKey("gemini", "avalanche-short-reset"); + + const result = await markAccountUnavailable( + connId, + 429, + GEMINI_TPM_ERROR, + "gemini", + "gemini-3.8-flash", + null, + { ...AVALANCHE_OPTIONS, headers: new Headers({ "retry-after": "10" }) } + ); + + assert.equal(result.shouldFallback, false); + assert.equal(result.cooldownMs, 60000); +}); + +test("Anti-Avalanche: a connection-scoped 429 still records the longer upstream reset", async () => { + const { markAccountUnavailable } = await import("../../src/sse/services/auth.ts"); + const { getModelLockoutInfo } = await import("../../open-sse/services/accountFallback.ts"); + const connId = await createKey("openai", "avalanche-connection-reset"); + + const result = await markAccountUnavailable( + connId, + 429, + OPENAI_TPM_ERROR, + "openai", + "gpt-4o", + null, + { ...AVALANCHE_OPTIONS, headers: new Headers({ "retry-after": "3600" }) } + ); + + assert.equal(result.shouldFallback, false); + assert.ok(result.cooldownMs > ONE_HOUR_MS - CLOCK_TOLERANCE_MS, String(result.cooldownMs)); + const lock = getModelLockoutInfo("openai", connId, "gpt-4o"); + assert.ok(lock && lock.remainingMs > ONE_HOUR_MS - CLOCK_TOLERANCE_MS, JSON.stringify(lock)); +}); + +test("Anti-Avalanche: a connection-scoped 429 without a hint applies the burst cooldown", async () => { + const { markAccountUnavailable } = await import("../../src/sse/services/auth.ts"); + const { getModelLockoutInfo } = await import("../../open-sse/services/accountFallback.ts"); + const connId = await createKey("openai", "avalanche-connection-no-hint"); + + const result = await markAccountUnavailable( + connId, + 429, + OPENAI_TPM_ERROR, + "openai", + "gpt-4o", + null, + AVALANCHE_OPTIONS + ); + + assert.equal(result.shouldFallback, false); + assert.equal(result.cooldownMs, 60000); + const lock = getModelLockoutInfo("openai", connId, "gpt-4o"); + assert.ok(lock && lock.remainingMs > 0 && lock.remainingMs <= 60000, JSON.stringify(lock)); +}); + test("Anti-Avalanche: normal 429 without suppressFallback still allows fallback", async () => { const { markAccountUnavailable } = await import("../../src/sse/services/auth.ts"); const providersDb = await import("../../src/lib/db/providers.ts"); @@ -299,6 +441,8 @@ test("Anti-Avalanche: normal 429 without suppressFallback still allows fallback" }); test.after(async () => { + if (previousEnabled === undefined) delete process.env[ENABLE_ENV]; + else process.env[ENABLE_ENV] = previousEnabled; const core = await import("../../src/lib/db/core.ts"); core.resetDbInstance(); if (previousDataDir === undefined) delete process.env.DATA_DIR;