fix(auth): do not park healthy quota accounts as expired

open-sse/services/errorClassifier.ts

@@ -256,7 +256,7 @@ export function classifyProviderError(
   const oauthInvalid = isOAuthInvalidToken(bodyStr);
   const preserveQuota429 = shouldPreserveQuotaSignalsFor429(provider);
 
-  if ((creditsExhausted || subscriptionQuotaExhausted) && [400, 402, 403].includes(statusCode)) {
+  if ((creditsExhausted || subscriptionQuotaExhausted) && [400, 401, 402, 403].includes(statusCode)) {
     return PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED;
   }
 

src/lib/quota/connectionRecovery.ts

@@ -63,6 +63,7 @@ export interface RecoverableConnectionInput {
   testStatus?: string | null;
   rateLimitedUntil?: string | null;
   lastErrorAt?: string | null;
+  lastErrorType?: string | null;
 }
 
 function normalizeStatus(value: string | null | undefined): string {
@@ -158,6 +159,37 @@ export function isRecoverableCooldownConnection(
  *
  * Pure — `nowMs` and `reprobeMs` are injected so callers/tests control the clock.
  */
+
+const EXPIRED_REPROBE_BLOCKLIST = new Set([
+  "account_deactivated",
+  "invalid_grant",
+  "unrecoverable_refresh_error",
+  "provider_deprecated",
+  "no_refresh_token",
+]);
+
+/**
+ * Re-probe `expired` after the same window as credits_exhausted.
+ * API-key 401s and OAuth races were persisted as expired and then never
+ * retried (combo pre-skip + health-check skip). Do not reopen a real
+ * deactivation / invalid_grant.
+ */
+export function isExpiredReprobeCandidate(
+  connection: RecoverableConnectionInput | null | undefined,
+  nowMs: number,
+  reprobeMs: number = DEFAULT_CREDITS_REPROBE_MS
+): boolean {
+  if (!connection || typeof connection.id !== "string" || connection.id.length === 0) {
+    return false;
+  }
+  if (normalizeStatus(connection.testStatus) !== "expired") return false;
+  const err = (connection.lastErrorType || "").trim().toLowerCase();
+  if (EXPIRED_REPROBE_BLOCKLIST.has(err)) return false;
+  const sinceMs = cooldownUntilMs(connection.lastErrorAt || connection.rateLimitedUntil || "");
+  if (!Number.isFinite(sinceMs) || sinceMs <= 0) return true;
+  return nowMs - sinceMs >= reprobeMs;
+}
+
 export function isCreditsExhaustedReprobeCandidate(
   connection: RecoverableConnectionInput | null | undefined,
   nowMs: number,
@@ -188,7 +220,8 @@ export function selectRecoverableConnections<T extends RecoverableConnectionInpu
   return connections.filter(
     (connection) =>
       isRecoverableCooldownConnection(connection, nowMs) ||
-      isCreditsExhaustedReprobeCandidate(connection, nowMs)
+      isCreditsExhaustedReprobeCandidate(connection, nowMs) ||
+      isExpiredReprobeCandidate(connection, nowMs)
   );
 }
 
@@ -245,6 +278,7 @@ export async function runConnectionRecoveryTick(
           testStatus: typeof row.testStatus === "string" ? row.testStatus : null,
           rateLimitedUntil: typeof row.rateLimitedUntil === "string" ? row.rateLimitedUntil : null,
           lastErrorAt: typeof row.lastErrorAt === "string" ? row.lastErrorAt : null,
+          lastErrorType: typeof row.lastErrorType === "string" ? row.lastErrorType : null,
         }));
       });
     connections = await load();

src/lib/tokenHealthCheck.ts

@@ -564,18 +564,11 @@ export async function checkConnection(conn) {
     }
   }
 
-  // #8182: skip terminal connections (credits_exhausted / banned / expired).
-  // These can never self-heal via a token refresh — probing them wastes
-  // CPU and network on every sweep cycle. Mirrors isTerminalConnectionStatus
-  // in src/sse/services/auth.ts and TERMINAL_CONNECTION_STATUSES in
-  // src/lib/quota/connectionRecovery.ts.
-  //
-  // #5326 exception: a GitHub Copilot access-token-only connection parked in
-  // "expired" with errorCode "no_refresh_token" is NOT actually terminal — it's
-  // the exact target of the self-heal below (canClearGitHubNoRefreshTokenState),
-  // which clears that stale status back to "active" once the Copilot sub-token
-  // proves usable. Treating it as terminal here made that self-heal unreachable,
-  // leaving healthy Copilot connections stuck at "expired" forever.
+  // #8182: skip banned/expired (dead credentials). credits_exhausted is a
+  // renewing window — keep sweeping so OAuth refresh can clear a false mark.
+  // #5326: GitHub Copilot access-token-only "expired" + no_refresh_token is
+  // the self-heal target below (canClearGitHubNoRefreshTokenState). Treating
+  // it as terminal made that heal unreachable and stuck healthy Copilot rows.
   const isRecoverableGithubCopilotNoRefresh =
     conn.testStatus === "expired" &&
     conn.errorCode === "no_refresh_token" &&
@@ -596,7 +589,8 @@ export async function checkConnection(conn) {
     conn.testStatus === "expired" &&
     conn.lastErrorType !== "account_deactivated" &&
     getExpiredRetryCount(conn) < EXPIRED_RETRY_MAX;
-  const terminalStatuses = new Set(["credits_exhausted", "banned", "expired"]);
+  // Skip only banned/expired. Combo pre-skip still hides exhausted rows.
+  const terminalStatuses = new Set(["banned", "expired"]);
   if (
     typeof conn.testStatus === "string" &&
     terminalStatuses.has(conn.testStatus.toLowerCase()) &&

src/sse/services/auth.ts

@@ -93,6 +93,7 @@ import {
   classifyProviderError,
   PROVIDER_ERROR_TYPES,
 } from "@omniroute/open-sse/services/errorClassifier.ts";
+import { resolveTerminalConnectionStatus } from "./authTerminalStatus.ts";
 import {
   ALIBABA_FREE_DRAINED_LOCK_MS,
   getAlibabaBillingMode,
@@ -325,71 +326,6 @@ function isTerminalConnectionStatusForModel(
   return true;
 }
 
-// #8200: cookie-auth providers (perplexity-web, grok-web, ...) use a rotating browser
-// session, not a static API key — a 401 means "session needs a refresh", not "dead".
-function isRecoverableCookieAuth401(
-  provider: string | null,
-  providerErrorType: string | null
-): boolean {
-  return (
-    providerErrorType !== PROVIDER_ERROR_TYPES.ACCOUNT_DEACTIVATED &&
-    provider != null &&
-    resolveProviderId(provider) in WEB_COOKIE_PROVIDERS
-  );
-}
-// #12242 (402 variant of #3027): a bare 402 on a passthrough/gateway
-// provider that multiplexes many models behind one credential
-// (kilo-gateway, ollama-cloud, etc.) is a PER-MODEL billing signal, not
-// proof the credential itself is dead — free models on the same connection
-// remain perfectly usable. Only terminalize the whole connection for a 402
-// when the provider is NOT a per-model-quota provider; the caller lets it
-// fall through to the per-model lockout branch instead.
-// `result.creditsExhausted` is a provider's own explicit classification
-// (independent of HTTP status) and stays unconditionally terminal — it is
-// not scoped by this check.
-function isConnectionWideCreditsExhausted(
-  status: number,
-  result: { permanent?: boolean; creditsExhausted?: boolean },
-  isPerModelQuotaProvider: boolean
-): boolean {
-  return result.creditsExhausted || (status === 402 && !isPerModelQuotaProvider);
-}
-function resolveTerminalConnectionStatus(
-  status: number,
-  result: { permanent?: boolean; creditsExhausted?: boolean },
-  providerErrorType: string | null = null,
-  provider: string | null = null,
-  isPerModelQuotaProvider = false
-): string | null {
-  if (isConnectionWideCreditsExhausted(status, result, isPerModelQuotaProvider)) {
-    return "credits_exhausted";
-  }
-  if (
-    providerErrorType === PROVIDER_ERROR_TYPES.PROJECT_ROUTE_ERROR ||
-    providerErrorType === PROVIDER_ERROR_TYPES.GEO_BLOCKED ||
-    providerErrorType === PROVIDER_ERROR_TYPES.OAUTH_INVALID_TOKEN ||
-    // #1010: Cloudflare fingerprint rejection is the CDN refusing the CLIENT's
-    // signature, not the account's credentials — never a terminal account state.
-    // A different client on the same key succeeds (measured 2026-08-08: curl 200,
-    // urllib 403 on byte-identical body), so banning the account here would flip a
-    // healthy free pool to ALL_ACCOUNTS_INACTIVE after two such calls.
-    providerErrorType === PROVIDER_ERROR_TYPES.FINGERPRINT_REJECTION
-  ) {
-    return null;
-  }
-  if (result.permanent || providerErrorType === PROVIDER_ERROR_TYPES.FORBIDDEN) {
-    return "banned";
-  }
-  if (
-    (providerErrorType === PROVIDER_ERROR_TYPES.ACCOUNT_DEACTIVATED ||
-      providerErrorType === PROVIDER_ERROR_TYPES.UNAUTHORIZED ||
-      status === 401) &&
-    !isRecoverableCookieAuth401(provider, providerErrorType)
-  ) {
-    return "expired";
-  }
-  return null;
-}
 export function resolveQuotaLimitPolicy(
   provider: string,
   providerSpecificData: JsonRecord
@@ -3038,13 +2974,24 @@ export async function markAccountUnavailable(
       return { shouldFallback: true, cooldownMs: lockout.cooldownMs };
     }
 
-    const terminalStatus = resolveTerminalConnectionStatus(
+    let terminalStatus = resolveTerminalConnectionStatus(
       status,
       result as { permanent?: boolean; creditsExhausted?: boolean },
       providerErrorType,
       provider,
-      isPerModelQuotaProvider
+      isPerModelQuotaProvider,
+      errorText
     );
+    // A still-valid access token after a successful refresh is not "expired".
+    // A follow-up 401 (timeout, hop, race) must cooldown, not park the account.
+    const tokenExpiryMs = Date.parse(String(conn?.tokenExpiresAt || conn?.expiresAt || ""));
+    if (
+      terminalStatus === "expired" &&
+      Number.isFinite(tokenExpiryMs) &&
+      tokenExpiryMs > Date.now() + 60_000
+    ) {
+      terminalStatus = null;
+    }
     const cachedQuotaResetAt =
       providerErrorType === PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED ||
       reason === RateLimitReason.QUOTA_EXHAUSTED

src/sse/services/authTerminalStatus.ts

@@ -0,0 +1,93 @@
+import { PROVIDER_ERROR_TYPES } from "@omniroute/open-sse/services/errorClassifier.ts";
+import { isCreditsExhausted } from "@omniroute/open-sse/services/accountFallback.ts";
+import { resolveProviderId, WEB_COOKIE_PROVIDERS } from "@/shared/constants/providers";
+
+// #8200: cookie-auth providers (perplexity-web, grok-web, ...) use a rotating browser
+// session, not a static API key — a 401 means "session needs a refresh", not "dead".
+export function isRecoverableCookieAuth401(
+  provider: string | null,
+  providerErrorType: string | null
+): boolean {
+  return (
+    providerErrorType !== PROVIDER_ERROR_TYPES.ACCOUNT_DEACTIVATED &&
+    provider != null &&
+    resolveProviderId(provider) in WEB_COOKIE_PROVIDERS
+  );
+}
+// #12242 (402 variant of #3027): a bare 402 on a passthrough/gateway
+// provider that multiplexes many models behind one credential
+// (kilo-gateway, ollama-cloud, etc.) is a PER-MODEL billing signal, not
+// proof the credential itself is dead — free models on the same connection
+// remain perfectly usable. Only terminalize the whole connection for a 402
+// when the provider is NOT a per-model-quota provider; the caller lets it
+// fall through to the per-model lockout branch instead.
+// `result.creditsExhausted` is a provider's own explicit classification
+// (independent of HTTP status) and stays unconditionally terminal — it is
+// not scoped by this check.
+export function isConnectionWideCreditsExhausted(
+  status: number,
+  result: { permanent?: boolean; creditsExhausted?: boolean },
+  isPerModelQuotaProvider: boolean
+): boolean {
+  return result.creditsExhausted || (status === 402 && !isPerModelQuotaProvider);
+}
+
+/** Credits-depleted bodies park; renewing billing-cycle quota does not. */
+export function shouldParkCreditsExhausted(
+  status: number,
+  result: { permanent?: boolean; creditsExhausted?: boolean },
+  isPerModelQuotaProvider: boolean,
+  errorText: string
+): boolean {
+  return (
+    isConnectionWideCreditsExhausted(status, result, isPerModelQuotaProvider) ||
+    (!isPerModelQuotaProvider && isCreditsExhausted(errorText))
+  );
+}
+
+function isNonTerminalProviderError(providerErrorType: string | null): boolean {
+  return (
+    providerErrorType === PROVIDER_ERROR_TYPES.PROJECT_ROUTE_ERROR ||
+    providerErrorType === PROVIDER_ERROR_TYPES.GEO_BLOCKED ||
+    providerErrorType === PROVIDER_ERROR_TYPES.OAUTH_INVALID_TOKEN ||
+    // #1010: Cloudflare fingerprint rejection is the CDN refusing the CLIENT's
+    // signature, not the account's credentials — never a terminal account state.
+    providerErrorType === PROVIDER_ERROR_TYPES.FINGERPRINT_REJECTION
+  );
+}
+
+function isExpiredAuthFailure(
+  status: number,
+  providerErrorType: string | null,
+  provider: string | null
+): boolean {
+  return (
+    (providerErrorType === PROVIDER_ERROR_TYPES.ACCOUNT_DEACTIVATED ||
+      providerErrorType === PROVIDER_ERROR_TYPES.UNAUTHORIZED ||
+      status === 401) &&
+    !isRecoverableCookieAuth401(provider, providerErrorType)
+  );
+}
+
+export function resolveTerminalConnectionStatus(
+  status: number,
+  result: { permanent?: boolean; creditsExhausted?: boolean },
+  providerErrorType: string | null = null,
+  provider: string | null = null,
+  isPerModelQuotaProvider = false,
+  errorText: string = ""
+): string | null {
+  if (shouldParkCreditsExhausted(status, result, isPerModelQuotaProvider, errorText)) {
+    return "credits_exhausted";
+  }
+  if (isNonTerminalProviderError(providerErrorType)) {
+    return null;
+  }
+  if (result.permanent || providerErrorType === PROVIDER_ERROR_TYPES.FORBIDDEN) {
+    return "banned";
+  }
+  if (isExpiredAuthFailure(status, providerErrorType, provider)) {
+    return "expired";
+  }
+  return null;
+}

stryker.conf.json

@@ -250,6 +250,7 @@
       "tests/unit/executor-contract-violation-terminal.test.ts",
       "tests/unit/executor-devin-cli-agentic-acp.test.ts",
       "tests/unit/executor-web-cookie-sweep.test.ts",
+      "tests/unit/false-terminal-401-quota.test.ts",
       "tests/unit/format-provider-error-cause.test.ts",
       "tests/unit/forwarded-header-budget.test.ts",
       "tests/unit/fusion-vision-panel-3378.test.ts",