fix(sse): honor native Claude quota scope

changelog.d/fixes/13888-claude-model-scoped-429.md

@@ -0,0 +1 @@
+- **fix(sse):** Use fresh Anthropic scoped-limit evidence to isolate an exhausted native Claude model while keeping global or unproven quota failures connection-wide ([#13888](https://github.com/diegosouzapw/OmniRoute/pull/13888)) - thanks @riez

open-sse/services/combo/executeTargetAttempt.ts

@@ -15,7 +15,6 @@ import {
   recordModelLockoutFailure,
   recordProviderFailure,
   recordProviderSuccess,
-  retryHintBypassesMaxCooldownMs,
   selectLockoutCooldownMs,
 } from "../accountFallback.ts";
 import {
@@ -860,34 +859,7 @@ export async function executeTargetAttempt(opts: {
       await resolveComboDailyReset(provider)
     );
     const { cooldownMs } = fallbackResult;
-    // #6863: a parsed upstream quota reset (e.g. Antigravity "Resets in 92h27m28s")
-    // arrives in `quotaResetHintMs` — it bypasses the operator-gated
-    // `useUpstreamRetryHints` connection-cooldown setting. Mirror the
-    // single-model path (src/sse/services/auth.ts): when the retry hint was
-    // already honored, `cooldownMs` IS the upstream value; otherwise prefer the
-    // parsed quota reset — even when it is SHORTER than the fallback cooldown
-    // (e.g. subscription-quota 1h default vs a real "resets in 10m").
-    // `selectLockoutCooldownMs` still ignores hints at/below the base cooldown,
-    // so absent/tiny hints keep the #1308 exponential-backoff behavior.
-    const lockoutHintMs =
-      fallbackResult.usedUpstreamRetryHint === true
-        ? cooldownMs
-        : (fallbackResult.quotaResetHintMs ?? 0);
-    // Only a transport header or google.rpc.RetryInfo is authoritative enough
-    // to bypass maxCooldownMs. Prose and generic JSON remain useful exact hints,
-    // but the operator cap still bounds them.
-    const lockoutHintVerified = retryHintBypassesMaxCooldownMs(fallbackResult.retryHintSource);
-    const selectedConnectionId =
-      result.headers?.get("X-OmniRoute-Selected-Connection-Id") ||
-      result.headers?.get("x-omniroute-selected-connection-id") ||
-      undefined;
-    const targetWithConnection = selectedConnectionId
-      ? { ...target, connectionId: selectedConnectionId }
-      : target;
-
-    // #1731 / #1731v2: classify the upstream error and update the exhaustion sets
-    // (shared with handleRoundRobinCombo). Returns whether the provider is fully exhausted.
-    const providerExhausted = applyComboTargetExhaustion(targetWithConnection, {
+    const targetFailure = applyComboTargetExhaustion(target, {
       result,
       fallbackResult,
       errorText,
@@ -905,6 +877,15 @@ export async function executeTargetAttempt(opts: {
       exhaustedLogLevel: "info",
       structuredError,
     });
+    const {
+      target: targetWithConnection,
+      providerExhausted,
+      isModelScopedClaudeQuota,
+      isConnectionScopedClaudeQuota,
+      modelScopedClaudeCooldownMs,
+      lockoutHintMs,
+      lockoutHintVerified,
+    } = targetFailure;
     // #6692: this connection was just classified as provider/connection-level
     // exhausted — if it's the currently sticky-bound one, release the pin now
     // rather than waiting for the next turn's lazy headroom/status recheck.
@@ -1096,7 +1077,14 @@ export async function executeTargetAttempt(opts: {
       // once the model is cooling down, retrying it would waste an upstream
       // call and extend the cooldown via exponential backoff.
       let lockoutRecorded = false;
-      if (!protectedPriorityTarget && provider && rawModel && retry === 0 && !scopedFailure) {
+      if (
+        !protectedPriorityTarget &&
+        provider &&
+        rawModel &&
+        retry === 0 &&
+        !scopedFailure &&
+        !isConnectionScopedClaudeQuota
+      ) {
         const mlSettings = resolveModelLockoutSettings(deps.settings);
         if (mlSettings.enabled && mlSettings.errorCodes.includes(result.status)) {
           recordModelLockoutFailure(
@@ -1111,8 +1099,9 @@ export async function executeTargetAttempt(opts: {
               // #1308/#6863: honor a long upstream reset (e.g. "Resets in 160h") over
               // the short base cooldown / exponential backoff when present. #7940's
               // maxCooldownMs cap only applies to synthetic values — a verified
-              // upstream reset (lockoutHintVerified) bypasses it.
-              exactCooldownMs: selectLockoutCooldownMs(lockoutHintMs, mlSettings),
+              // upstream reset (including a cached Claude reset) bypasses it.
+              exactCooldownMs:
+                modelScopedClaudeCooldownMs ?? selectLockoutCooldownMs(lockoutHintMs, mlSettings),
               maxCooldownMs: mlSettings.maxCooldownMs,
               // Preserve authoritative structured/header resets; clamp body prose.
               exactCooldownIsUpstreamReset: lockoutHintVerified,
@@ -1182,7 +1171,7 @@ export async function executeTargetAttempt(opts: {
     if (i > 0) state.fallbackCount++;
     // Wire combo failures into the resilience dashboard (model-level lockout)
     // alongside the provider-level cooldown below — they govern different scopes.
-    if (provider && rawModel && !scopedFailure) {
+    if (provider && rawModel && !scopedFailure && !isConnectionScopedClaudeQuota) {
       const mlSettings = resolveModelLockoutSettings(deps.settings);
       if (mlSettings.enabled && mlSettings.errorCodes.includes(result.status)) {
         recordModelLockoutFailure(
@@ -1196,8 +1185,9 @@ export async function executeTargetAttempt(opts: {
           {
             // #1308/#6863: honor a long upstream reset over base/exponential cooldown.
             // #7940's maxCooldownMs cap only applies to synthetic values — a verified
-            // upstream reset (lockoutHintVerified) bypasses it.
-            exactCooldownMs: selectLockoutCooldownMs(lockoutHintMs, mlSettings),
+            // upstream reset (including a cached Claude reset) bypasses it.
+            exactCooldownMs:
+              modelScopedClaudeCooldownMs ?? selectLockoutCooldownMs(lockoutHintMs, mlSettings),
             maxCooldownMs: mlSettings.maxCooldownMs,
             // Preserve authoritative structured/header resets; clamp body prose.
             exactCooldownIsUpstreamReset: lockoutHintVerified,
@@ -1219,6 +1209,7 @@ export async function executeTargetAttempt(opts: {
       provider &&
       provider !== "unknown" &&
       !scopedFailure &&
+      !isModelScopedClaudeQuota &&
       !((result.status === 500 || result.status === 429) && hasPerModelQuota(provider, rawModel))
     ) {
       recordProviderCooldown(

open-sse/services/combo/roundRobinCombo.ts

@@ -963,31 +963,18 @@ export async function handleRoundRobinCombo({
             await resolveComboDailyReset(provider)
           );
           const { cooldownMs } = fallbackResult;
-          const selectedConnectionId =
-            result.headers?.get("X-OmniRoute-Selected-Connection-Id") ||
-            result.headers?.get("x-omniroute-selected-connection-id") ||
-            undefined;
-          const targetWithConnection = selectedConnectionId
-            ? { ...target, connectionId: selectedConnectionId }
-            : target;
-
+          const rawModel = parseModel(modelStr).model || modelStr;
           const isAllAccountsRateLimited = isAllAccountsRateLimitedResponse(
             result.status,
             result.headers?.get("content-type") ?? null,
             errorText
           );
 
-          // #1731: If the entire provider quota is exhausted, mark it so subsequent
-          // same-provider targets are skipped immediately. API-key 429s still use
-          // the short resilience cooldown, but explicit quota text should stop the
-          // combo from trying another target for the same provider in this request.
-          // #1731 / #1731v2: classify the upstream error and update the exhaustion sets
-          // (shared with handleComboChat). Returns whether the provider is fully exhausted.
-          const providerExhausted = applyComboTargetExhaustion(targetWithConnection, {
+          const targetFailure = applyComboTargetExhaustion(target, {
             result,
             fallbackResult,
             errorText,
-            rawModel: parseModel(modelStr).model || modelStr,
+            rawModel,
             isTokenLimitBreach,
             allAccountsRateLimited: isAllAccountsRateLimited,
             requestScopedFailure: scopedFailure,
@@ -997,6 +984,12 @@ export async function handleRoundRobinCombo({
             exhaustedLogLevel: "debug",
             structuredError,
           });
+          const {
+            target: targetWithConnection,
+            providerExhausted,
+            isModelScopedClaudeQuota,
+            effectiveTargetCooldownMs: targetCooldownMs,
+          } = targetFailure;
           // #6692: mirrors handleComboChat's exhaustion-point release above.
           releaseStickyPinOnFailure(
             _rrSessionSticky.messageHash,
@@ -1024,10 +1017,13 @@ export async function handleRoundRobinCombo({
             !isTokenLimitBreach &&
             !scopedFailure &&
             TRANSIENT_FOR_SEMAPHORE.includes(result.status) &&
-            cooldownMs > 0
+            targetCooldownMs > 0
           ) {
-            semaphore.markRateLimited(semaphoreKey, cooldownMs);
-            log.warn("COMBO-RR", `${modelStr} error ${result.status}, cooldown ${cooldownMs}ms`);
+            semaphore.markRateLimited(semaphoreKey, targetCooldownMs);
+            log.warn(
+              "COMBO-RR",
+              `${modelStr} error ${result.status}, cooldown ${targetCooldownMs}ms`
+            );
           }
 
           if (isAllAccountsRateLimited) {
@@ -1056,6 +1052,7 @@ export async function handleRoundRobinCombo({
             retry < maxRetries &&
             isTransient &&
             !providerExhausted &&
+            !isModelScopedClaudeQuota &&
             (!config.failoverBeforeRetryExplicit || !hasNextRrTarget)
           ) {
             continue;
@@ -1102,9 +1099,10 @@ export async function handleRoundRobinCombo({
             provider &&
             provider !== "unknown" &&
             !scopedFailure &&
+            !isModelScopedClaudeQuota &&
             !(
               (result.status === 500 || result.status === 429) &&
-              hasPerModelQuota(provider, parseModel(modelStr).model || modelStr)
+              hasPerModelQuota(provider, rawModel)
             )
           ) {
             recordProviderCooldown(