fix(resilience): don't clear an active rate-limit cooldown for non-quota_exhausted errors (#11277)

src/lib/usage/providerLimits.ts

@@ -499,6 +499,25 @@ export async function maybeClearRecoveredQuotaState(
       // the previous synthetic-cooldown guard.
       return connection;
     }
+  } else if (
+    connection.rateLimitedUntil &&
+    new Date(connection.rateLimitedUntil).getTime() > Date.now()
+  ) {
+    // Universal fallback guard for every lastErrorType other than
+    // "quota_exhausted" (which gets the more precise per-window check above,
+    // and may legitimately release early once the REAL window has reset even
+    // while a synthetic rateLimitedUntil is still in the future). A future
+    // rateLimitedUntil is a hard statement made by the 429/error handler that
+    // persisted it (src/sse/services/auth.ts, src/app/api/providers/[id]/test/
+    // route.ts) — no quota poll finding *some* usable window elsewhere should
+    // be able to overrule it. Before this fix, ANY lastErrorType other than
+    // "quota_exhausted" skipped straight to hasTransientState/
+    // clearRecoveredProviderState() below with no rateLimitedUntil check at
+    // all, so a multi-day cooldown (observed: 146h, Z.AI weekly quota) got
+    // cleared on the very next quota sync a few minutes later — a
+    // self-restart/burn loop that kept burning real upstream calls against a
+    // known-exhausted connection (#11277).
+    return connection;
   }
 
   const hasTransientState =

tests/unit/provider-limits-recovery.test.ts

@@ -83,7 +83,25 @@ test.after(async () => {
 });
 
 test("successful GLM quota refresh clears transient rate-limit state", async () => {
-  const connection = await createGlmConnectionWithTransientCooldown();
+  // The cooldown must already be EXPIRED for a successful refresh to clear it
+  // (#11277: a rateLimitedUntil still in the future is a hard statement from
+  // the error handler that persisted it — no quota poll may overrule it,
+  // regardless of lastErrorType). Before #11277's fix this test used a
+  // still-future rateLimitedUntil and asserted it got cleared anyway, which
+  // was the same defect class as the reported bug, just a shorter window.
+  const connection = await providersDb.createProviderConnection({
+    provider: "glm",
+    authType: "apikey",
+    name: `GLM Recovery ${Date.now()}`,
+    apiKey: "glm-test-key",
+    testStatus: "unavailable",
+    rateLimitedUntil: new Date(Date.now() - 60_000).toISOString(),
+    lastError: "rate limit exceeded",
+    lastErrorType: "rate_limited",
+    lastErrorSource: "executor",
+    errorCode: 429,
+    backoffLevel: 2,
+  });
   const connectionId = (connection as { id: string }).id;
 
   await withMockedFetch((() => glmQuotaResponse()) as typeof fetch, async () => {
@@ -101,6 +119,39 @@ test("successful GLM quota refresh clears transient rate-limit state", async ()
   assert.equal(updated.backoffLevel, 0, "backoffLevel should be reset to 0");
 });
 
+test("a still-future rateLimitedUntil is not cleared by a successful quota refresh, regardless of lastErrorType (#11277)", async () => {
+  const stillFutureRateLimitedUntil = new Date(Date.now() + 60_000).toISOString();
+  const connection = await providersDb.createProviderConnection({
+    provider: "glm",
+    authType: "apikey",
+    name: `GLM Still Cooling ${Date.now()}`,
+    apiKey: "glm-test-key",
+    testStatus: "unavailable",
+    rateLimitedUntil: stillFutureRateLimitedUntil,
+    lastError: "rate limit exceeded",
+    lastErrorType: "rate_limited",
+    lastErrorSource: "executor",
+    errorCode: 429,
+    backoffLevel: 2,
+  });
+  const connectionId = (connection as { id: string }).id;
+
+  await withMockedFetch((() => glmQuotaResponse()) as typeof fetch, async () => {
+    await providerLimits.fetchAndPersistProviderLimits(connectionId, "manual");
+  });
+
+  const updated = (await providersDb.getProviderConnectionById(connectionId)) as Record<
+    string,
+    unknown
+  >;
+  assert.equal(
+    updated.testStatus,
+    "unavailable",
+    "an active cooldown must stay locked even though the quota fetch succeeded"
+  );
+  assert.equal(updated.rateLimitedUntil, stillFutureRateLimitedUntil);
+});
+
 async function createGlmConnectionWithStatus(status: string) {
   return providersDb.createProviderConnection({
     provider: "glm",
@@ -334,6 +385,52 @@ test("Claude subscription quota still exhausted keeps the connection locked (no
   assert.equal(after.rateLimitedUntil, syntheticRateLimitedUntil);
 });
 
+test("rate_limit_exceeded cooldown is not cleared early by an unrelated quota window looking usable (#11277)", async () => {
+  // Reproduces #11277: a connection-scoped cooldown persisted with
+  // lastErrorType "rate_limit_exceeded" (RateLimitReason.RATE_LIMIT_EXCEEDED)
+  // and a long rateLimitedUntil (derived from an upstream reset hint — the
+  // reported production case was ~146h) must NOT be cleared just because the
+  // next scheduled quota sync reports hasUsableQuota()===true from some
+  // unrelated window. Before the fix, only lastErrorType==="quota_exhausted"
+  // reached the rateLimitedUntil guard, so every other reason (including
+  // rate_limit_exceeded) skipped straight to clearRecoveredProviderState(),
+  // producing a self-restart/burn loop on a multi-day cooldown.
+  const farFutureRateLimitedUntil = new Date(Date.now() + 146 * 60 * 60 * 1000).toISOString();
+  const created = await providersDb.createProviderConnection({
+    provider: "opencode",
+    authType: "apikey",
+    name: `OpenCode RateLimitExceeded ${Date.now()}`,
+    apiKey: "opencode-test-key",
+    testStatus: "unavailable",
+    isActive: true,
+    lastError: "Account quota exhausted (opencode)",
+    lastErrorType: "rate_limit_exceeded",
+    errorCode: 429,
+    rateLimitedUntil: farFutureRateLimitedUntil,
+    backoffLevel: 1,
+  });
+  const connectionId = (created as { id: string }).id;
+  const connection = await providersDb.getProviderConnectionById(connectionId);
+
+  // No `quotas` object at all (degraded/partial fetch shape) — this is the
+  // exact shape that, pre-fix, fell straight through to hasTransientState
+  // and cleared the cooldown for any lastErrorType other than quota_exhausted.
+  const result = await providerLimits.maybeClearRecoveredQuotaState(connection, {
+    quotas: { unrelated: { unlimited: true } },
+  });
+
+  assert.equal(
+    result.testStatus,
+    "unavailable",
+    "an active rate_limit_exceeded cooldown must stay locked"
+  );
+
+  const after = await providersDb.getProviderConnectionById(connectionId);
+  assert.equal(after.testStatus, "unavailable");
+  assert.equal(after.lastErrorType, "rate_limit_exceeded");
+  assert.equal(after.rateLimitedUntil, farFutureRateLimitedUntil);
+});
+
 test("CAS primitive clears when expected state matches", async () => {
   const created = await createGlmConnectionWithTransientCooldown();
   const connectionId = (created as { id: string }).id;