From f6fd5e79ad6126b665e30dd2c24abe20dac2c348 Mon Sep 17 00:00:00 2001 From: Ravi Tharuma Date: Wed, 2 Sep 2026 11:17:45 +0200 Subject: [PATCH 1/4] fix(auth): do not park healthy quota accounts as expired Co-authored-by: Ravi Tharuma --- open-sse/services/errorClassifier.ts | 2 +- src/lib/quota/connectionRecovery.ts | 36 ++++++- src/lib/tokenHealthCheck.ts | 13 +-- src/sse/services/auth.ts | 27 ++++- tests/unit/false-terminal-401-quota.test.ts | 100 +++++++++++++++++++ tests/unit/quota-connection-recovery.test.ts | 34 +++++++ 6 files changed, 200 insertions(+), 12 deletions(-) create mode 100644 tests/unit/false-terminal-401-quota.test.ts diff --git a/open-sse/services/errorClassifier.ts b/open-sse/services/errorClassifier.ts index 2bdbbbc8c2f..5e600176629 100644 --- a/open-sse/services/errorClassifier.ts +++ b/open-sse/services/errorClassifier.ts @@ -256,7 +256,7 @@ export function classifyProviderError( const oauthInvalid = isOAuthInvalidToken(bodyStr); const preserveQuota429 = shouldPreserveQuotaSignalsFor429(provider); - if ((creditsExhausted || subscriptionQuotaExhausted) && [400, 402, 403].includes(statusCode)) { + if ((creditsExhausted || subscriptionQuotaExhausted) && [400, 401, 402, 403].includes(statusCode)) { return PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED; } diff --git a/src/lib/quota/connectionRecovery.ts b/src/lib/quota/connectionRecovery.ts index e7c09be1bb4..26952a262df 100644 --- a/src/lib/quota/connectionRecovery.ts +++ b/src/lib/quota/connectionRecovery.ts @@ -63,6 +63,7 @@ export interface RecoverableConnectionInput { testStatus?: string | null; rateLimitedUntil?: string | null; lastErrorAt?: string | null; + lastErrorType?: string | null; } function normalizeStatus(value: string | null | undefined): string { @@ -158,6 +159,37 @@ export function isRecoverableCooldownConnection( * * Pure — `nowMs` and `reprobeMs` are injected so callers/tests control the clock. */ + +const EXPIRED_REPROBE_BLOCKLIST = new Set([ + "account_deactivated", + "invalid_grant", + "unrecoverable_refresh_error", + "provider_deprecated", + "no_refresh_token", +]); + +/** + * Re-probe `expired` after the same window as credits_exhausted. + * API-key 401s and OAuth races were persisted as expired and then never + * retried (combo pre-skip + health-check skip). Do not reopen a real + * deactivation / invalid_grant. + */ +export function isExpiredReprobeCandidate( + connection: RecoverableConnectionInput | null | undefined, + nowMs: number, + reprobeMs: number = DEFAULT_CREDITS_REPROBE_MS +): boolean { + if (!connection || typeof connection.id !== "string" || connection.id.length === 0) { + return false; + } + if (normalizeStatus(connection.testStatus) !== "expired") return false; + const err = (connection.lastErrorType || "").trim().toLowerCase(); + if (EXPIRED_REPROBE_BLOCKLIST.has(err)) return false; + const sinceMs = cooldownUntilMs(connection.lastErrorAt || connection.rateLimitedUntil || ""); + if (!Number.isFinite(sinceMs) || sinceMs <= 0) return true; + return nowMs - sinceMs >= reprobeMs; +} + export function isCreditsExhaustedReprobeCandidate( connection: RecoverableConnectionInput | null | undefined, nowMs: number, @@ -188,7 +220,8 @@ export function selectRecoverableConnections isRecoverableCooldownConnection(connection, nowMs) || - isCreditsExhaustedReprobeCandidate(connection, nowMs) + isCreditsExhaustedReprobeCandidate(connection, nowMs) || + isExpiredReprobeCandidate(connection, nowMs) ); } @@ -245,6 +278,7 @@ export async function runConnectionRecoveryTick( testStatus: typeof row.testStatus === "string" ? row.testStatus : null, rateLimitedUntil: typeof row.rateLimitedUntil === "string" ? row.rateLimitedUntil : null, lastErrorAt: typeof row.lastErrorAt === "string" ? row.lastErrorAt : null, + lastErrorType: typeof row.lastErrorType === "string" ? row.lastErrorType : null, })); }); connections = await load(); diff --git a/src/lib/tokenHealthCheck.ts b/src/lib/tokenHealthCheck.ts index 2820a28e234..20f075c56a0 100644 --- a/src/lib/tokenHealthCheck.ts +++ b/src/lib/tokenHealthCheck.ts @@ -564,11 +564,9 @@ export async function checkConnection(conn) { } } - // #8182: skip terminal connections (credits_exhausted / banned / expired). - // These can never self-heal via a token refresh — probing them wastes - // CPU and network on every sweep cycle. Mirrors isTerminalConnectionStatus - // in src/sse/services/auth.ts and TERMINAL_CONNECTION_STATUSES in - // src/lib/quota/connectionRecovery.ts. + // #8182: skip banned/expired connections (real dead credentials). + // credits_exhausted is a renewing window — keep sweeping so OAuth refresh + // can clear a false no-quota mark. // // #5326 exception: a GitHub Copilot access-token-only connection parked in // "expired" with errorCode "no_refresh_token" is NOT actually terminal — it's @@ -596,7 +594,10 @@ export async function checkConnection(conn) { conn.testStatus === "expired" && conn.lastErrorType !== "account_deactivated" && getExpiredRetryCount(conn) < EXPIRED_RETRY_MAX; - const terminalStatuses = new Set(["credits_exhausted", "banned", "expired"]); + // credits_exhausted is a renewing window, not a dead credential — skip + // only banned/expired here so OAuth refresh can still clear a false + // no-quota mark. Combo pre-skip still hides exhausted rows until recovery. + const terminalStatuses = new Set(["banned", "expired"]); if ( typeof conn.testStatus === "string" && terminalStatuses.has(conn.testStatus.toLowerCase()) && diff --git a/src/sse/services/auth.ts b/src/sse/services/auth.ts index b1218763762..fbcc9e5e6d4 100644 --- a/src/sse/services/auth.ts +++ b/src/sse/services/auth.ts @@ -57,6 +57,7 @@ import { getCreditsMode } from "@omniroute/open-sse/services/antigravityCredits. import { preferAntigravityConnectionsWithStoredProject } from "@omniroute/open-sse/services/antigravityProjectPersistence.ts"; import { isAccountUnavailable, + isCreditsExhausted, getUnavailableUntil, getEarliestRateLimitedUntil, cooldownUntilMs, @@ -359,9 +360,16 @@ function resolveTerminalConnectionStatus( result: { permanent?: boolean; creditsExhausted?: boolean }, providerErrorType: string | null = null, provider: string | null = null, - isPerModelQuotaProvider = false + isPerModelQuotaProvider = false, + errorText: string = "" ): string | null { - if (isConnectionWideCreditsExhausted(status, result, isPerModelQuotaProvider)) { + // Credits-depleted bodies (and explicit 402) park the connection. A renewing + // quota window (billing-cycle / usage-limit QUOTA_EXHAUSTED) must stay on the + // cached-reset cooldown path — not credits_exhausted with cooldownMs=0. + if ( + isConnectionWideCreditsExhausted(status, result, isPerModelQuotaProvider) || + (!isPerModelQuotaProvider && isCreditsExhausted(errorText)) + ) { return "credits_exhausted"; } if ( @@ -3038,13 +3046,24 @@ export async function markAccountUnavailable( return { shouldFallback: true, cooldownMs: lockout.cooldownMs }; } - const terminalStatus = resolveTerminalConnectionStatus( + let terminalStatus = resolveTerminalConnectionStatus( status, result as { permanent?: boolean; creditsExhausted?: boolean }, providerErrorType, provider, - isPerModelQuotaProvider + isPerModelQuotaProvider, + errorText ); + // A still-valid access token after a successful refresh is not "expired". + // A follow-up 401 (timeout, hop, race) must cooldown, not park the account. + const tokenExpiryMs = Date.parse(String(conn?.tokenExpiresAt || conn?.expiresAt || "")); + if ( + terminalStatus === "expired" && + Number.isFinite(tokenExpiryMs) && + tokenExpiryMs > Date.now() + 60_000 + ) { + terminalStatus = null; + } const cachedQuotaResetAt = providerErrorType === PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED || reason === RateLimitReason.QUOTA_EXHAUSTED diff --git a/tests/unit/false-terminal-401-quota.test.ts b/tests/unit/false-terminal-401-quota.test.ts new file mode 100644 index 00000000000..393e2fdd688 --- /dev/null +++ b/tests/unit/false-terminal-401-quota.test.ts @@ -0,0 +1,100 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-false-terminal-")); +process.env.DATA_DIR = TEST_DATA_DIR; + +const core = await import("../../src/lib/db/core.ts"); +const providersDb = await import("../../src/lib/db/providers.ts"); +const auth = await import("../../src/sse/services/auth.ts"); + +async function resetStorage() { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); +} + +test.after(() => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); +}); + +test("401 credits-exhausted body is credits_exhausted, not expired", async () => { + await resetStorage(); + const conn = await providersDb.createProviderConnection({ + provider: "chutes", + authType: "apikey", + apiKey: "sk-chutes-live", + isActive: true, + testStatus: "active", + }); + const connId = String(conn.id); + await auth.markAccountUnavailable( + connId, + 401, + "[chutes] All 3 connection(s) credits exhausted — please reconnect in the dashboard", + "chutes", + "moonshotai/Kimi-K3-TEE" + ); + const after = await providersDb.getProviderConnectionById(connId); + assert.equal(after.testStatus, "credits_exhausted"); + assert.notEqual(after.testStatus, "expired"); +}); + +test("billing-cycle quota 403 stays unavailable until the cached reset", async () => { + await resetStorage(); + const quotaCache = await import("../../src/domain/quotaCache.ts"); + const conn = await providersDb.createProviderConnection({ + provider: "kimi-coding", + authType: "oauth", + accessToken: "kimi-access-token", + refreshToken: "kimi-refresh-token", + isActive: true, + testStatus: "active", + }); + const connId = String(conn.id); + const resetAt = new Date(Date.now() + 30 * 60 * 1000).toISOString(); + quotaCache.setQuotaCache(connId, "kimi-coding", { + Ratelimit: { remainingPercentage: 0, resetAt }, + Weekly: { + remainingPercentage: 62, + resetAt: new Date(Date.now() + 5 * 24 * 60 * 60 * 1000).toISOString(), + }, + }); + const result = await auth.markAccountUnavailable( + connId, + 403, + "You've reached your usage limit for this billing cycle. Your quota will be refreshed in the next cycle.", + "kimi-coding", + "kimi-for-coding" + ); + const after = await providersDb.getProviderConnectionById(connId); + assert.equal(result.shouldFallback, true); + assert.ok(Math.abs(result.cooldownMs - 30 * 60 * 1000) < 2_000); + assert.equal(after.testStatus, "unavailable"); + assert.notEqual(after.testStatus, "credits_exhausted"); + assert.equal(after.lastErrorType, "quota_exhausted"); + quotaCache.__clearForTests(); +}); + +test("401 with a still-valid access token does not expire the connection", async () => { + await resetStorage(); + const conn = await providersDb.createProviderConnection({ + provider: "claude", + authType: "oauth", + accessToken: "sk-ant-fresh", + refreshToken: "rt-fresh", + isActive: true, + testStatus: "active", + tokenExpiresAt: new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), + expiresAt: new Date(Date.now() + 8 * 60 * 60 * 1000).toISOString(), + }); + const connId = String(conn.id); + await auth.markAccountUnavailable(connId, 401, "unauthorized", "claude", "claude-opus-4-8"); + const after = await providersDb.getProviderConnectionById(connId); + assert.notEqual(after.testStatus, "expired"); + assert.equal(after.isActive, true); +}); diff --git a/tests/unit/quota-connection-recovery.test.ts b/tests/unit/quota-connection-recovery.test.ts index fa96ae8f932..ddd535c4e1c 100644 --- a/tests/unit/quota-connection-recovery.test.ts +++ b/tests/unit/quota-connection-recovery.test.ts @@ -3,6 +3,7 @@ import assert from "node:assert/strict"; import { CREDITS_EXHAUSTED_STATUS, isCreditsExhaustedReprobeCandidate, + isExpiredReprobeCandidate, isRecoverableCooldownConnection, selectRecoverableConnections, runConnectionRecoveryTick, @@ -216,3 +217,36 @@ describe("connectionRecovery — mixed timestamp encodings", () => { assert.equal(isRecoverableCooldownConnection(conn, nowMs), false); }); }); + +describe("connectionRecovery — expired reprobe", () => { + const nowMs = 1_700_000_000_000; + const thirtyMinMs = 30 * 60 * 1000; + + it("should reprobe expired after 30m unless lastErrorType is a real deactivation", () => { + const old = { + id: "e-1", + testStatus: "expired", + lastErrorAt: new Date(nowMs - thirtyMinMs - 1000).toISOString(), + lastErrorType: "unauthorized", + }; + assert.equal(isExpiredReprobeCandidate(old, nowMs), true); + assert.equal( + isExpiredReprobeCandidate({ ...old, lastErrorType: "invalid_grant" }, nowMs), + false + ); + }); + + it("selectRecoverableConnections includes stale expired rows", () => { + const selected = selectRecoverableConnections( + [ + { + id: "e-1", + testStatus: "expired", + lastErrorAt: new Date(nowMs - thirtyMinMs - 1000).toISOString(), + }, + ], + nowMs + ); + assert.deepEqual(selected.map((c) => c.id), ["e-1"]); + }); +}); From 3dd852479694a7e5e77a4f5bc50091b9ab7719a4 Mon Sep 17 00:00:00 2001 From: Ravi Tharuma Date: Wed, 2 Sep 2026 12:22:39 +0200 Subject: [PATCH 2/4] fix: extract terminal status helpers so auth.ts stays under the file-size cap --- src/sse/services/auth.ts | 73 +------------------ src/sse/services/authTerminalStatus.ts | 76 ++++++++++++++++++++ tests/unit/token-health-check-cursor.test.ts | 6 +- 3 files changed, 81 insertions(+), 74 deletions(-) create mode 100644 src/sse/services/authTerminalStatus.ts diff --git a/src/sse/services/auth.ts b/src/sse/services/auth.ts index fbcc9e5e6d4..c9e49af6067 100644 --- a/src/sse/services/auth.ts +++ b/src/sse/services/auth.ts @@ -94,6 +94,7 @@ import { classifyProviderError, PROVIDER_ERROR_TYPES, } from "@omniroute/open-sse/services/errorClassifier.ts"; +import { resolveTerminalConnectionStatus } from "./authTerminalStatus.ts"; import { ALIBABA_FREE_DRAINED_LOCK_MS, getAlibabaBillingMode, @@ -326,78 +327,6 @@ function isTerminalConnectionStatusForModel( return true; } -// #8200: cookie-auth providers (perplexity-web, grok-web, ...) use a rotating browser -// session, not a static API key — a 401 means "session needs a refresh", not "dead". -function isRecoverableCookieAuth401( - provider: string | null, - providerErrorType: string | null -): boolean { - return ( - providerErrorType !== PROVIDER_ERROR_TYPES.ACCOUNT_DEACTIVATED && - provider != null && - resolveProviderId(provider) in WEB_COOKIE_PROVIDERS - ); -} -// #12242 (402 variant of #3027): a bare 402 on a passthrough/gateway -// provider that multiplexes many models behind one credential -// (kilo-gateway, ollama-cloud, etc.) is a PER-MODEL billing signal, not -// proof the credential itself is dead — free models on the same connection -// remain perfectly usable. Only terminalize the whole connection for a 402 -// when the provider is NOT a per-model-quota provider; the caller lets it -// fall through to the per-model lockout branch instead. -// `result.creditsExhausted` is a provider's own explicit classification -// (independent of HTTP status) and stays unconditionally terminal — it is -// not scoped by this check. -function isConnectionWideCreditsExhausted( - status: number, - result: { permanent?: boolean; creditsExhausted?: boolean }, - isPerModelQuotaProvider: boolean -): boolean { - return result.creditsExhausted || (status === 402 && !isPerModelQuotaProvider); -} -function resolveTerminalConnectionStatus( - status: number, - result: { permanent?: boolean; creditsExhausted?: boolean }, - providerErrorType: string | null = null, - provider: string | null = null, - isPerModelQuotaProvider = false, - errorText: string = "" -): string | null { - // Credits-depleted bodies (and explicit 402) park the connection. A renewing - // quota window (billing-cycle / usage-limit QUOTA_EXHAUSTED) must stay on the - // cached-reset cooldown path — not credits_exhausted with cooldownMs=0. - if ( - isConnectionWideCreditsExhausted(status, result, isPerModelQuotaProvider) || - (!isPerModelQuotaProvider && isCreditsExhausted(errorText)) - ) { - return "credits_exhausted"; - } - if ( - providerErrorType === PROVIDER_ERROR_TYPES.PROJECT_ROUTE_ERROR || - providerErrorType === PROVIDER_ERROR_TYPES.GEO_BLOCKED || - providerErrorType === PROVIDER_ERROR_TYPES.OAUTH_INVALID_TOKEN || - // #1010: Cloudflare fingerprint rejection is the CDN refusing the CLIENT's - // signature, not the account's credentials — never a terminal account state. - // A different client on the same key succeeds (measured 2026-08-08: curl 200, - // urllib 403 on byte-identical body), so banning the account here would flip a - // healthy free pool to ALL_ACCOUNTS_INACTIVE after two such calls. - providerErrorType === PROVIDER_ERROR_TYPES.FINGERPRINT_REJECTION - ) { - return null; - } - if (result.permanent || providerErrorType === PROVIDER_ERROR_TYPES.FORBIDDEN) { - return "banned"; - } - if ( - (providerErrorType === PROVIDER_ERROR_TYPES.ACCOUNT_DEACTIVATED || - providerErrorType === PROVIDER_ERROR_TYPES.UNAUTHORIZED || - status === 401) && - !isRecoverableCookieAuth401(provider, providerErrorType) - ) { - return "expired"; - } - return null; -} export function resolveQuotaLimitPolicy( provider: string, providerSpecificData: JsonRecord diff --git a/src/sse/services/authTerminalStatus.ts b/src/sse/services/authTerminalStatus.ts new file mode 100644 index 00000000000..4cc079b2217 --- /dev/null +++ b/src/sse/services/authTerminalStatus.ts @@ -0,0 +1,76 @@ +import { PROVIDER_ERROR_TYPES } from "@omniroute/open-sse/services/errorClassifier.ts"; +import { isCreditsExhausted } from "@omniroute/open-sse/services/accountFallback.ts"; +import { resolveProviderId, WEB_COOKIE_PROVIDERS } from "@/shared/constants/providers"; + +// #8200: cookie-auth providers (perplexity-web, grok-web, ...) use a rotating browser +// session, not a static API key — a 401 means "session needs a refresh", not "dead". +export function isRecoverableCookieAuth401( + provider: string | null, + providerErrorType: string | null +): boolean { + return ( + providerErrorType !== PROVIDER_ERROR_TYPES.ACCOUNT_DEACTIVATED && + provider != null && + resolveProviderId(provider) in WEB_COOKIE_PROVIDERS + ); +} +// #12242 (402 variant of #3027): a bare 402 on a passthrough/gateway +// provider that multiplexes many models behind one credential +// (kilo-gateway, ollama-cloud, etc.) is a PER-MODEL billing signal, not +// proof the credential itself is dead — free models on the same connection +// remain perfectly usable. Only terminalize the whole connection for a 402 +// when the provider is NOT a per-model-quota provider; the caller lets it +// fall through to the per-model lockout branch instead. +// `result.creditsExhausted` is a provider's own explicit classification +// (independent of HTTP status) and stays unconditionally terminal — it is +// not scoped by this check. +export function isConnectionWideCreditsExhausted( + status: number, + result: { permanent?: boolean; creditsExhausted?: boolean }, + isPerModelQuotaProvider: boolean +): boolean { + return result.creditsExhausted || (status === 402 && !isPerModelQuotaProvider); +} +export function resolveTerminalConnectionStatus( + status: number, + result: { permanent?: boolean; creditsExhausted?: boolean }, + providerErrorType: string | null = null, + provider: string | null = null, + isPerModelQuotaProvider = false, + errorText: string = "" +): string | null { + // Credits-depleted bodies (and explicit 402) park the connection. A renewing + // quota window (billing-cycle / usage-limit QUOTA_EXHAUSTED) must stay on the + // cached-reset cooldown path — not credits_exhausted with cooldownMs=0. + if ( + isConnectionWideCreditsExhausted(status, result, isPerModelQuotaProvider) || + (!isPerModelQuotaProvider && isCreditsExhausted(errorText)) + ) { + return "credits_exhausted"; + } + if ( + providerErrorType === PROVIDER_ERROR_TYPES.PROJECT_ROUTE_ERROR || + providerErrorType === PROVIDER_ERROR_TYPES.GEO_BLOCKED || + providerErrorType === PROVIDER_ERROR_TYPES.OAUTH_INVALID_TOKEN || + // #1010: Cloudflare fingerprint rejection is the CDN refusing the CLIENT's + // signature, not the account's credentials — never a terminal account state. + // A different client on the same key succeeds (measured 2026-08-08: curl 200, + // urllib 403 on byte-identical body), so banning the account here would flip a + // healthy free pool to ALL_ACCOUNTS_INACTIVE after two such calls. + providerErrorType === PROVIDER_ERROR_TYPES.FINGERPRINT_REJECTION + ) { + return null; + } + if (result.permanent || providerErrorType === PROVIDER_ERROR_TYPES.FORBIDDEN) { + return "banned"; + } + if ( + (providerErrorType === PROVIDER_ERROR_TYPES.ACCOUNT_DEACTIVATED || + providerErrorType === PROVIDER_ERROR_TYPES.UNAUTHORIZED || + status === 401) && + !isRecoverableCookieAuth401(provider, providerErrorType) + ) { + return "expired"; + } + return null; +} diff --git a/tests/unit/token-health-check-cursor.test.ts b/tests/unit/token-health-check-cursor.test.ts index f05933e2bb3..12284bc3aed 100644 --- a/tests/unit/token-health-check-cursor.test.ts +++ b/tests/unit/token-health-check-cursor.test.ts @@ -430,7 +430,7 @@ test("checkConnection: a banned Cursor connection stays skipped regardless of la }); }); -test("checkConnection: a credits_exhausted Cursor connection stays skipped regardless of lastErrorType", async () => { +test("checkConnection: a credits_exhausted Cursor connection is still swept", async () => { await resetStorage(); await withCursorEnv(async () => { const id = await createCursorConnection({ @@ -444,7 +444,9 @@ test("checkConnection: a credits_exhausted Cursor connection stays skipped regar await tokenHealthCheck.checkConnection(before); const after = await freshConn(id); - assert.deepEqual(after, before); + assert.notEqual(after.testStatus, "credits_exhausted"); + assert.ok(after.lastHealthCheckAt); + assert.notEqual(after.updatedAt, before.updatedAt); }); }); From e9278156642815126ceff7b5097b9668b92b6e8b Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 2 Sep 2026 10:37:54 +0000 Subject: [PATCH 3/4] fix(auth): satisfy file-size, complexity, and Stryker tap gates Shrink tokenHealthCheck comments under the frozen LOC cap, split authTerminalStatus helpers so cyclomatic stays at the base count, and register the new 401-quota unit file in tap.testFiles. Co-authored-by: Ravi Tharuma --- src/lib/tokenHealthCheck.ts | 19 +++----- src/sse/services/authTerminalStatus.ts | 65 ++++++++++++++++---------- stryker.conf.json | 1 + 3 files changed, 48 insertions(+), 37 deletions(-) diff --git a/src/lib/tokenHealthCheck.ts b/src/lib/tokenHealthCheck.ts index 20f075c56a0..eff72dfd9cf 100644 --- a/src/lib/tokenHealthCheck.ts +++ b/src/lib/tokenHealthCheck.ts @@ -564,16 +564,11 @@ export async function checkConnection(conn) { } } - // #8182: skip banned/expired connections (real dead credentials). - // credits_exhausted is a renewing window — keep sweeping so OAuth refresh - // can clear a false no-quota mark. - // - // #5326 exception: a GitHub Copilot access-token-only connection parked in - // "expired" with errorCode "no_refresh_token" is NOT actually terminal — it's - // the exact target of the self-heal below (canClearGitHubNoRefreshTokenState), - // which clears that stale status back to "active" once the Copilot sub-token - // proves usable. Treating it as terminal here made that self-heal unreachable, - // leaving healthy Copilot connections stuck at "expired" forever. + // #8182: skip banned/expired (dead credentials). credits_exhausted is a + // renewing window — keep sweeping so OAuth refresh can clear a false mark. + // #5326: GitHub Copilot access-token-only "expired" + no_refresh_token is + // the self-heal target below (canClearGitHubNoRefreshTokenState). Treating + // it as terminal made that heal unreachable and stuck healthy Copilot rows. const isRecoverableGithubCopilotNoRefresh = conn.testStatus === "expired" && conn.errorCode === "no_refresh_token" && @@ -594,9 +589,7 @@ export async function checkConnection(conn) { conn.testStatus === "expired" && conn.lastErrorType !== "account_deactivated" && getExpiredRetryCount(conn) < EXPIRED_RETRY_MAX; - // credits_exhausted is a renewing window, not a dead credential — skip - // only banned/expired here so OAuth refresh can still clear a false - // no-quota mark. Combo pre-skip still hides exhausted rows until recovery. + // Skip only banned/expired. Combo pre-skip still hides exhausted rows. const terminalStatuses = new Set(["banned", "expired"]); if ( typeof conn.testStatus === "string" && diff --git a/src/sse/services/authTerminalStatus.ts b/src/sse/services/authTerminalStatus.ts index 4cc079b2217..b8afed537bd 100644 --- a/src/sse/services/authTerminalStatus.ts +++ b/src/sse/services/authTerminalStatus.ts @@ -31,45 +31,62 @@ export function isConnectionWideCreditsExhausted( ): boolean { return result.creditsExhausted || (status === 402 && !isPerModelQuotaProvider); } -export function resolveTerminalConnectionStatus( + +/** Credits-depleted bodies park; renewing billing-cycle quota does not. */ +export function shouldParkCreditsExhausted( status: number, result: { permanent?: boolean; creditsExhausted?: boolean }, - providerErrorType: string | null = null, - provider: string | null = null, - isPerModelQuotaProvider = false, - errorText: string = "" -): string | null { - // Credits-depleted bodies (and explicit 402) park the connection. A renewing - // quota window (billing-cycle / usage-limit QUOTA_EXHAUSTED) must stay on the - // cached-reset cooldown path — not credits_exhausted with cooldownMs=0. - if ( + isPerModelQuotaProvider: boolean, + errorText: string +): boolean { + return ( isConnectionWideCreditsExhausted(status, result, isPerModelQuotaProvider) || (!isPerModelQuotaProvider && isCreditsExhausted(errorText)) - ) { - return "credits_exhausted"; - } - if ( + ); +} + +function isNonTerminalProviderError(providerErrorType: string | null): boolean { + return ( providerErrorType === PROVIDER_ERROR_TYPES.PROJECT_ROUTE_ERROR || providerErrorType === PROVIDER_ERROR_TYPES.GEO_BLOCKED || providerErrorType === PROVIDER_ERROR_TYPES.OAUTH_INVALID_TOKEN || // #1010: Cloudflare fingerprint rejection is the CDN refusing the CLIENT's // signature, not the account's credentials — never a terminal account state. - // A different client on the same key succeeds (measured 2026-08-08: curl 200, - // urllib 403 on byte-identical body), so banning the account here would flip a - // healthy free pool to ALL_ACCOUNTS_INACTIVE after two such calls. providerErrorType === PROVIDER_ERROR_TYPES.FINGERPRINT_REJECTION - ) { + ); +} + +function isExpiredAuthFailure( + status: number, + providerErrorType: string | null, + provider: string | null +): boolean { + return ( + (providerErrorType === PROVIDER_ERROR_TYPES.ACCOUNT_DEACTIVATED || + providerErrorType === PROVIDER_ERROR_TYPES.UNAUTHORIZED || + status === 401) && + !isRecoverableCookieAuth401(provider, providerErrorType) + ); +} + +export function resolveTerminalConnectionStatus( + status: number, + result: { permanent?: boolean; creditsExhausted?: boolean }, + providerErrorType: string | null = null, + provider: string | null = null, + isPerModelQuotaProvider = false, + errorText: string = "" +): string | null { + if (shouldParkCreditsExhausted(status, result, isPerModelQuotaProvider, errorText)) { + return "credits_exhausted"; + } + if (isNonTerminalProviderError(providerErrorType)) { return null; } if (result.permanent || providerErrorType === PROVIDER_ERROR_TYPES.FORBIDDEN) { return "banned"; } - if ( - (providerErrorType === PROVIDER_ERROR_TYPES.ACCOUNT_DEACTIVATED || - providerErrorType === PROVIDER_ERROR_TYPES.UNAUTHORIZED || - status === 401) && - !isRecoverableCookieAuth401(provider, providerErrorType) - ) { + if (isExpiredAuthFailure(status, providerErrorType, provider)) { return "expired"; } return null; diff --git a/stryker.conf.json b/stryker.conf.json index 96f421f6248..8aabfd5928a 100644 --- a/stryker.conf.json +++ b/stryker.conf.json @@ -250,6 +250,7 @@ "tests/unit/executor-contract-violation-terminal.test.ts", "tests/unit/executor-devin-cli-agentic-acp.test.ts", "tests/unit/executor-web-cookie-sweep.test.ts", + "tests/unit/false-terminal-401-quota.test.ts", "tests/unit/format-provider-error-cause.test.ts", "tests/unit/forwarded-header-budget.test.ts", "tests/unit/fusion-vision-panel-3378.test.ts", From ccf25b9af745b995f0c2220a6d36036455df908b Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 2 Sep 2026 10:42:05 +0000 Subject: [PATCH 4/4] fix(auth): drop unused isCreditsExhausted import from auth.ts The helper moved to authTerminalStatus.ts; the leftover import trips @typescript-eslint/no-unused-vars on parent Quality Gates. Co-authored-by: Ravi Tharuma --- src/sse/services/auth.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/sse/services/auth.ts b/src/sse/services/auth.ts index c9e49af6067..605c40cf26b 100644 --- a/src/sse/services/auth.ts +++ b/src/sse/services/auth.ts @@ -57,7 +57,6 @@ import { getCreditsMode } from "@omniroute/open-sse/services/antigravityCredits. import { preferAntigravityConnectionsWithStoredProject } from "@omniroute/open-sse/services/antigravityProjectPersistence.ts"; import { isAccountUnavailable, - isCreditsExhausted, getUnavailableUntil, getEarliestRateLimitedUntil, cooldownUntilMs,