fix(combos): hold a spent usage window for ten minutes, not sixty seconds

docs-site/src/content/docs/guides/combos.md

@@ -325,14 +325,20 @@ If the shared request send budget refuses the first target, the combo returns a
 the combo returns the last real upstream failure without sending to that target.
 
 When `cooldownMs` is unset, a hopped target uses an upstream fallback: 5 seconds for request-rate
-429s with upstream code `1302` or `1305`, and 60 seconds otherwise. When it is set, `cooldownMs`
+429s with upstream code `1302` or `1305`, 10 minutes for a spent account usage window, and 60
+seconds otherwise. A usage window is recognised by upstream code (`usage_limit_exceeded`,
+`usage_limit_reached`, `1308`) or by the prose `usage limit reached`, independent of HTTP status —
+the ChatGPT Codex backend reports a spent window as a 502 rather than the documented 429. When it is set, `cooldownMs`
 applies whenever no usable upstream `Retry-After` or Codex reset signal exists, including those
 request-rate 429s. Numeric `Retry-After` seconds and HTTP-date values are accepted. Explicit
 server delays are capped at 24 hours; reset-derived, configured, and fallback cooldowns are capped
 at 10 minutes. The precedence is, from strongest to weakest, explicit
 `Retry-After` → Codex reset headers (`x-codex-primary-reset-at`, `x-codex-secondary-reset-at`, or
-`x-codex-tertiary-reset-at`) → the combo's `cooldownMs` (when set) → the 5-second request-rate
-fallback for upstream rate-limit codes `1302`/`1305` → the 60-second default. A valid immediate
+`x-codex-tertiary-reset-at`) → the combo's `cooldownMs` (when set) → the
+10-minute hold for a spent usage window → the 5-second request-rate fallback for upstream
+rate-limit codes `1302`/`1305` → the 60-second default. The usage-window hold is tested first, so a
+failure that carries a request-rate code *and* usage-limit prose is held for ten minutes rather than
+five seconds. A valid immediate
 `Retry-After: 0` remains an immediate upstream directive rather than being replaced by a configured
 cooldown.
 

src/combos/failover.ts

@@ -230,11 +230,13 @@ export function coolComboTarget(
   const cooldownMs = serverDelayMs
     ?? parseResetCooldownMs(options?.resetAt, now)
     ?? options?.cooldownMs
-    ?? (isTransientRequestRateLimit({
-      status: options?.status,
-      code: options?.code,
-      message: options?.message,
-    }) ? COMBO_REQUEST_RATE_COOLDOWN_MS : DEFAULT_COOLDOWN_MS);
+    ?? (isAccountWindowExhausted(options?.message ?? "", options?.code)
+      ? MAX_COOLDOWN_MS
+      : isTransientRequestRateLimit({
+        status: options?.status,
+        code: options?.code,
+        message: options?.message,
+      }) ? COMBO_REQUEST_RATE_COOLDOWN_MS : DEFAULT_COOLDOWN_MS);
   targetCooldowns.set(cooldownMapKey(comboId, target), {
     // Local fallbacks are capped at ten minutes; explicit server delays at one day.
     cooldownUntil: now + (serverDelayMs ?? Math.min(Math.max(cooldownMs, 1), MAX_COOLDOWN_MS)),
@@ -306,6 +308,27 @@ function normalizedFailureCode(code?: string | null): string {
   return code?.trim().toLowerCase().replaceAll("-", "_") ?? "";
 }
 
+/**
+ * A spent account window, by structured code or upstream prose. Status is deliberately not
+ * consulted: the ChatGPT Codex backend reports a depleted plan window as HTTP 502
+ * `upstream_server_error` carrying `The usage limit has been reached`, never the documented 429,
+ * so any status gate misses it. Read in exactly ONE place -- the cooldown DURATION fallback. A
+ * status-blind prose match is safe for choosing how long to wait; it is not safe for choosing
+ * what to black out, so `isProviderScopedQuotaCap` and the scope/decision paths stay untouched
+ * and a Codex 502 still resolves `target` scope and `hop` through `status >= 500`.
+ */
+// `1308` is the vendor code for a spent five-hour window and carries no prose of its own when the
+// upstream reports it bare, so it belongs here too. The rest of QUOTA_LIMIT_CODES stays out: those
+// are quota-limit codes whose window length this gateway has no evidence for, and guessing long on
+// them would hold a target that may clear sooner.
+const ACCOUNT_EXHAUSTION_CODES = new Set(["usage_limit_exceeded", "usage_limit_reached", "1308"]);
+const ACCOUNT_EXHAUSTION_TEXT = /usage limit (?:has been )?reached/;
+
+function isAccountWindowExhausted(message: string, code?: string | null): boolean {
+  return ACCOUNT_EXHAUSTION_CODES.has(normalizedFailureCode(code))
+    || ACCOUNT_EXHAUSTION_TEXT.test(message.toLowerCase());
+}
+
 function isProviderScopedQuotaCap(
   status: number | undefined,
   message: string,

tests/codex-integration/combo-codex-exhaustion-cooldown.test.ts

@@ -0,0 +1,72 @@
+import { afterEach, describe, expect, test } from "bun:test";
+import {
+  clearComboTargetCooldowns,
+  comboFailureCooldownScope,
+  comboFailureDecision,
+  coolComboTarget,
+  isComboTargetInCooldown,
+} from "../../src/combos/failover";
+
+/**
+ * The ChatGPT Codex backend reports a depleted plan window as HTTP 502 `upstream_server_error`
+ * carrying the prose `The usage limit has been reached`, never the documented 429. The
+ * quota-cap predicate is gated on 429, so a dead account took the generic 60-second cooldown and
+ * was re-offered every minute: 942 doomed sends in 72h of production ledger, each surfacing as
+ * `adapter_eof`, plus 94 downstream `503 No available targets`. See #5860.
+ *
+ * The fix is duration only. Delete the exhaustion arm of `coolComboTarget`'s `cooldownMs` chain
+ * and the ten-minute assertions below go back to 60 seconds.
+ */
+const combo = "codex-exhaustion-burn-test";
+const target = { provider: "openai", model: "gpt-6-astra" };
+const now = Date.UTC(2026, 8, 24, 12, 0, 0);
+const EXHAUSTED = "The usage limit has been reached";
+
+afterEach(() => clearComboTargetCooldowns(combo));
+
+describe("a depleted Codex plan window", () => {
+  test.each([
+    ["502 prose", 502, "upstream_server_error", EXHAUSTED],
+    ["429 structured code", 429, "usage_limit_exceeded", "quota"],
+    ["structured usage_limit_reached", 502, "usage-limit-reached", "upstream error"],
+    ["vendor 5-hour window", 429, "1308", "Usage limit reached for 5 hour"],
+  ])("cools the target for ten minutes, not 60s (%s)", (_label, status, code, message) => {
+    coolComboTarget(combo, target, { now, status, code, message });
+    // The whole point: still cooling a minute later, when the 60s default would have expired.
+    expect(isComboTargetInCooldown(combo, target, now + 60_000)).toBe(true);
+    expect(isComboTargetInCooldown(combo, target, now + 10 * 60_000 - 1)).toBe(true);
+    expect(isComboTargetInCooldown(combo, target, now + 10 * 60_000)).toBe(false);
+  });
+
+  test("hops without blacking out the whole provider", () => {
+    // Scope and decision are untouched by this change: a Codex 502 still resolves `target` scope
+    // and `hop` through the existing `status >= 500` path, which is why the status-blind prose
+    // match is kept out of `isProviderScopedQuotaCap`.
+    expect(comboFailureCooldownScope(502, EXHAUSTED, { code: "upstream_server_error" })).toBe("target");
+    expect(comboFailureDecision(502, EXHAUSTED, { code: "upstream_server_error" })).toBe("hop");
+  });
+
+  test("an unrelated 502 keeps the 60-second default", () => {
+    coolComboTarget(combo, target, { now, status: 502, code: "upstream_server_error", message: "bad gateway" });
+    expect(isComboTargetInCooldown(combo, target, now + 60_000 - 1)).toBe(true);
+    expect(isComboTargetInCooldown(combo, target, now + 60_000)).toBe(false);
+  });
+
+  test("a bare 1308 code with no prose still takes the exhaustion hold", () => {
+    coolComboTarget(combo, target, { now, status: 429, code: "1308", message: "" });
+    expect(isComboTargetInCooldown(combo, target, now + 10 * 60_000 - 1)).toBe(true);
+    expect(isComboTargetInCooldown(combo, target, now + 10 * 60_000)).toBe(false);
+  });
+
+  test("an explicit server Retry-After still outranks the exhaustion default", () => {
+    coolComboTarget(combo, target, { now, status: 502, code: "upstream_server_error", message: EXHAUSTED, retryAfter: "30" });
+    expect(isComboTargetInCooldown(combo, target, now + 30_000 - 1)).toBe(true);
+    expect(isComboTargetInCooldown(combo, target, now + 30_000)).toBe(false);
+  });
+
+  test("an operator's configured cooldownMs still outranks the exhaustion default", () => {
+    coolComboTarget(combo, target, { now, status: 502, code: "upstream_server_error", message: EXHAUSTED, cooldownMs: 5_000 });
+    expect(isComboTargetInCooldown(combo, target, now + 5_000 - 1)).toBe(true);
+    expect(isComboTargetInCooldown(combo, target, now + 5_000)).toBe(false);
+  });
+});

tests/codex-integration/combos.test.ts

@@ -633,14 +633,21 @@ describe("combo target cooldowns", () => {
     expect(isComboTargetInCooldown("free", configuredPick.target, 1_000 + 7_000)).toBe(false);
   });
 
-  test("keeps the default cooldown for usage-window 1308", () => {
+  // Changed in #5860. This arrived as the negative control proving 1308 does NOT take the
+  // 5-second request-rate cooldown, so the 60-second value was a side effect of that contrast,
+  // not a position on exhaustion. 1308 IS a 5-hour usage window: a 60-second cooldown re-probes
+  // it about 300 times before it can possibly succeed. The contrast it was written to prove is
+  // preserved -- 1308 is still not COMBO_REQUEST_RATE_COOLDOWN_MS -- it now holds the full
+  // ten-minute exhaustion cooldown instead of the generic default.
+  test("holds the exhaustion cooldown for usage-window 1308", () => {
     coolComboTarget("free", target, {
       now: 1_000,
       code: "1308",
       message: "Usage limit reached for 5 hour",
     });
-    expect(isComboTargetInCooldown("free", target, 1_000 + 59_999)).toBe(true);
-    expect(isComboTargetInCooldown("free", target, 1_000 + 60_000)).toBe(false);
+    expect(isComboTargetInCooldown("free", target, 1_000 + 60_000)).toBe(true);
+    expect(isComboTargetInCooldown("free", target, 1_000 + 10 * 60_000 - 1)).toBe(true);
+    expect(isComboTargetInCooldown("free", target, 1_000 + 10 * 60_000)).toBe(false);
   });
 
   test("honors explicit Retry-After over the request-rate default", () => {