From 39e388a013a7dcc1adf48b35fb2c5d7e961643a0 Mon Sep 17 00:00:00 2001 From: Minxi Hou Date: Tue, 8 Sep 2026 12:05:41 -0400 Subject: [PATCH] fix(grok-cli): park a 402 on the empty Grok Build login, not grok-4.6 Grok Build bills Chat/Imagine/Voice/Build/API against one weekly pool, but the registry still sets passthroughModels so a 402 was treated as a per-model billing miss. Combo then locked grok-4.6 and skipped every other login in the same set. Classify that 402 as connection-wide credits exhaustion so the next Grok Build login can still run. Signed-off-by: Minxi Hou --- .../fixes/grok-cli-shared-wallet-402.md | 1 + open-sse/services/accountFallback.ts | 4 +- .../accountFallback/sharedWalletCredits.ts | 39 ++++ open-sse/services/combo/targetExhaustion.ts | 28 +++ src/sse/services/auth.ts | 14 +- .../auth-grok-cli-402-shared-wallet.test.ts | 186 ++++++++++++++++++ .../combo/combo-target-exhaustion.test.ts | 56 ++++++ 7 files changed, 325 insertions(+), 3 deletions(-) create mode 100644 changelog.d/fixes/grok-cli-shared-wallet-402.md create mode 100644 open-sse/services/accountFallback/sharedWalletCredits.ts create mode 100644 tests/unit/auth-grok-cli-402-shared-wallet.test.ts diff --git a/changelog.d/fixes/grok-cli-shared-wallet-402.md b/changelog.d/fixes/grok-cli-shared-wallet-402.md new file mode 100644 index 00000000000..5559902c6b8 --- /dev/null +++ b/changelog.d/fixes/grok-cli-shared-wallet-402.md @@ -0,0 +1 @@ +- **fix(grok-cli):** a 402 "Grok Build usage balance exhausted" parks that Grok login as out of credit (Grok Build CLI, grok.com cookie, and xAI OAuth share the weekly pool). Combo routing then tries the next login instead of locking the model for every account in the pool diff --git a/open-sse/services/accountFallback.ts b/open-sse/services/accountFallback.ts index c6465321f94..06964caa6c8 100644 --- a/open-sse/services/accountFallback.ts +++ b/open-sse/services/accountFallback.ts @@ -98,6 +98,7 @@ export { MODEL_LOCKOUT_EVICTION_CAP } from "./accountFallback/lockoutEviction.ts import { capScaledCooldownMs } from "./accountFallback/cooldownCap.ts"; import { resolveApiKeyForbiddenFallback } from "./accountFallback/nonRetryableUpstream.ts"; import * as exactModelLock from "./accountFallback/exactModelLock.ts"; +import { isCreditsExhaustedWithSharedWallet } from "./accountFallback/sharedWalletCredits.ts"; export type ProviderProfile = { baseCooldownMs: number; useUpstreamRetryHints: boolean; @@ -484,8 +485,7 @@ export function isAccountDeactivated(errorText: string): boolean { * T10: Returns true if response body indicates credits/quota are permanently exhausted. */ export function isCreditsExhausted(errorText: string): boolean { - const lower = String(errorText || "").toLowerCase(); - return CREDITS_EXHAUSTED_SIGNALS.some((sig) => lower.includes(sig)); + return isCreditsExhaustedWithSharedWallet(errorText, CREDITS_EXHAUSTED_SIGNALS); } /** diff --git a/open-sse/services/accountFallback/sharedWalletCredits.ts b/open-sse/services/accountFallback/sharedWalletCredits.ts new file mode 100644 index 00000000000..d24341bfc06 --- /dev/null +++ b/open-sse/services/accountFallback/sharedWalletCredits.ts @@ -0,0 +1,39 @@ +/** + * Providers whose 402 is a shared account wallet, not a per-model billing miss. + * + * Grok Build (`grok-cli`), grok.com cookie sessions (`grok-web`), and xAI + * OAuth (`xai-oauth`) bill Chat/Imagine/Voice/Build/API against one weekly + * percent pool. `passthroughModels: true` still stands for catalog/404 + * behaviour; it must not send this 402 through the #12242 model-only lockout, + * or a combo of five grok-4.6 steps parks the empty account and then skips the + * remaining live accounts as "model locked". + * + * `matchesSharedWalletCreditsBody` expects a pre-lowercased string. + */ +const SHARED_WALLET_402_PROVIDERS = new Set(["grok-cli", "grok-web", "xai-oauth"]); + +export const GROK_BUILD_USAGE_BALANCE_SIGNAL = "usage balance exhausted"; + +export function matchesSharedWalletCreditsBody(loweredErrorText: string): boolean { + return loweredErrorText.includes(GROK_BUILD_USAGE_BALANCE_SIGNAL); +} + +export function isSharedWalletCredits402( + provider: string | null | undefined, + status: number, + errorText?: string | null +): boolean { + if (status !== 402 || typeof provider !== "string" || !SHARED_WALLET_402_PROVIDERS.has(provider)) { + return false; + } + if (errorText == null || String(errorText).trim() === "") return true; + return matchesSharedWalletCreditsBody(String(errorText).toLowerCase()); +} + +export function isCreditsExhaustedWithSharedWallet( + errorText: string, + signals: readonly string[] +): boolean { + const lower = String(errorText || "").toLowerCase(); + return signals.some((sig) => lower.includes(sig)) || matchesSharedWalletCreditsBody(lower); +} diff --git a/open-sse/services/combo/targetExhaustion.ts b/open-sse/services/combo/targetExhaustion.ts index 11e8efa6543..ca8c4ca916e 100644 --- a/open-sse/services/combo/targetExhaustion.ts +++ b/open-sse/services/combo/targetExhaustion.ts @@ -31,6 +31,7 @@ import { isCloudflareFingerprintRejection } from "../errorClassifier.ts"; // (markAccountUnavailable) so the same-request combo skip and the persisted // connection cooldown agree on exactly which fallbackResult shapes qualify. import { isAgentrouterConnectionQuotaScope } from "@/sse/services/auth"; +import { isSharedWalletCredits402 } from "../accountFallback/sharedWalletCredits.ts"; import type { ComboLogger, ResolvedComboTarget } from "./types.ts"; // Connection-level failure statuses: the provider connection itself is likely bad (upstream @@ -165,6 +166,11 @@ export function applyComboTargetExhaustion( return true; } + if (isSharedWalletCredits402(provider, result.status, opts.errorText)) { + markSharedWalletCreditsExhaustion(target, { sets, log, tag }); + return true; + } + // #8133/#8137: auth-level failures (401/403) mean that connection's credentials are bad. // Split out to keep applyComboTargetExhaustion under the complexity ceiling. // Cloudflare 1010 (a 403 carrying error_code 1010 / browser_signature_banned) is NOT an @@ -340,6 +346,28 @@ function markAuthLevelExhaustion( } } +function markSharedWalletCreditsExhaustion( + target: ResolvedComboTarget, + opts: Pick +): void { + const { sets, log, tag } = opts; + const provider = target.provider; + const connId = target.connectionId ?? undefined; + if (connId) { + sets.exhaustedConnections.add(`${provider}:${connId}`); + log.info( + tag, + `Provider ${provider} connection ${connId} shared-wallet 402 — marking for skip on remaining targets` + ); + } else { + sets.exhaustedProviders.add(provider as string); + log.info( + tag, + `Provider ${provider} shared-wallet 402 (no connectionId) — marking for skip on remaining targets` + ); + } +} + /** * #10334: agentrouter-exclusive connection-scope account quota exhaustion. Mirrors * markAuthLevelExhaustion's connectionId-present/absent split — when the target carries a diff --git a/src/sse/services/auth.ts b/src/sse/services/auth.ts index 79fc31dfecc..a3d0866379d 100644 --- a/src/sse/services/auth.ts +++ b/src/sse/services/auth.ts @@ -75,6 +75,7 @@ import { retryHintBypassesMaxCooldownMs, isProviderModelUnsupported400, } from "@omniroute/open-sse/services/accountFallback.ts"; +import { isSharedWalletCredits402 } from "@omniroute/open-sse/services/accountFallback/sharedWalletCredits.ts"; import { isLocalProvider } from "@omniroute/open-sse/config/providerRegistry.ts"; import { COOLDOWN_MS, RateLimitReason } from "@omniroute/open-sse/config/constants.ts"; import { sanitizeErrorMessage } from "@omniroute/open-sse/utils/errorSanitization.ts"; @@ -2943,6 +2944,11 @@ export async function markAccountUnavailable( return { shouldFallback: true, cooldownMs: lockout.cooldownMs }; } const result = fallbackResult; + if (isSharedWalletCredits402(provider, status, errorText)) { + result.creditsExhausted = true; + result.reason = result.reason || RateLimitReason.QUOTA_EXHAUSTED; + result.shouldFallback = true; + } const { shouldFallback, cooldownMs: rawCooldownMs, newBackoffLevel, reason } = result; if (!shouldFallback) return { shouldFallback: false, cooldownMs: 0 }; const providerErrorType = classifyProviderError(status, errorText, provider); @@ -3061,6 +3067,7 @@ export async function markAccountUnavailable( provider && model && !terminalStatus && + !isSharedWalletCredits402(provider, status, errorText) && !(provider === "vertex" && isVertexConnectionWidePermissionDenied(errorText)) ) { const lockoutReason = status === 402 ? "credits" : "forbidden"; @@ -3182,7 +3189,12 @@ export async function markAccountUnavailable( // the DB, but record an in-memory model lockout so credential selection // skips this exact provider+connection+model while it cools down — other // models on the same connection stay usable. - if (provider && model && cooldownMs > 0) { + if ( + provider && + model && + cooldownMs > 0 && + !isSharedWalletCredits402(provider, status, errorText) + ) { lockModel(provider, connectionId, model, reason || "unknown", cooldownMs); } await updateProviderConnection(connectionId, { diff --git a/tests/unit/auth-grok-cli-402-shared-wallet.test.ts b/tests/unit/auth-grok-cli-402-shared-wallet.test.ts new file mode 100644 index 00000000000..0f3fcf5ecaa --- /dev/null +++ b/tests/unit/auth-grok-cli-402-shared-wallet.test.ts @@ -0,0 +1,186 @@ +// Grok Build (`grok-cli`) bills Chat/Imagine/Voice/Build/API against one +// weekly credit pool. A 402 "Grok Build usage balance exhausted" is therefore +// a connection-wide wallet signal, not a per-model billing miss. The +// passthroughModels flag still stands for catalog/404 behaviour; it must not +// route this 402 through the #12242 model-only lockout, or a combo of five +// grok-4.6 steps parks the first empty account and then skips the four +// remaining live accounts as "model locked". +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-grok-cli-402-")); +process.env.DATA_DIR = TEST_DATA_DIR; + +const core = await import("../../src/lib/db/core.ts"); +const providersDb = await import("../../src/lib/db/providers.ts"); +const auth = await import("../../src/sse/services/auth.ts"); +const accountFallback = await import("../../open-sse/services/accountFallback.ts"); + +const GROK_BUILD_402 = "Grok Build usage balance exhausted"; + +async function resetStorage() { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); +} + +async function seedGrokCli(name: string) { + return seedSharedWallet("grok-cli", name); +} + +async function seedSharedWallet(provider: string, name: string) { + const oauth = provider === "grok-cli" || provider === "xai-oauth"; + return providersDb.createProviderConnection({ + provider, + authType: oauth ? "oauth" : "apikey", + name, + email: name, + ...(oauth + ? { accessToken: `${provider}-${name}` } + : { apiKey: `${provider}-${name}` }), + isActive: true, + testStatus: "active", + }); +} + +test.after(() => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); +}); + +test("grok-cli 402 parks the connection as credits_exhausted, not a model lock", async () => { + await resetStorage(); + const conn = await seedGrokCli("empty@qq.com"); + const id = (conn as { id: string }).id; + + const result = await auth.markAccountUnavailable(id, 402, GROK_BUILD_402, "grok-cli", "grok-4.6"); + + assert.equal(result.shouldFallback, true); + + const after = await providersDb.getProviderConnectionById(id); + assert.equal(after.testStatus, "credits_exhausted"); + + const lockout = accountFallback.getModelLockoutInfo("grok-cli", id, "grok-4.6"); + assert.equal(lockout, null, "shared-wallet 402 must not lock grok-4.6 on this account"); +}); + +test("a sibling grok-cli account stays eligible after another account's 402", async () => { + await resetStorage(); + const empty = await seedGrokCli("empty@qq.com"); + const live = await seedGrokCli("live@hotmail.com"); + const emptyId = (empty as { id: string }).id; + const liveId = (live as { id: string }).id; + + await auth.markAccountUnavailable(emptyId, 402, GROK_BUILD_402, "grok-cli", "grok-4.6"); + + assert.equal( + accountFallback.isModelLocked("grok-cli", liveId, "grok-4.6"), + false, + "sibling account must not inherit the empty account's model lock" + ); + + const selected = await auth.getProviderCredentials("grok-cli"); + assert.ok(selected); + assert.equal(selected.connectionId, liveId); + assert.notEqual(selected.connectionId, emptyId); +}); + +test("grok-cli 402 still parks the connection when disableCooling is set", async () => { + await resetStorage(); + const conn = await providersDb.createProviderConnection({ + provider: "grok-cli", + authType: "oauth", + accessToken: "gcli-disabled-cooling", + isActive: true, + testStatus: "active", + providerSpecificData: { disableCooling: true }, + }); + const id = (conn as { id: string }).id; + + await auth.markAccountUnavailable(id, 402, GROK_BUILD_402, "grok-cli", "grok-4.6"); + + const after = await providersDb.getProviderConnectionById(id); + assert.equal(after.testStatus, "credits_exhausted"); +}); + +test("passthrough 402 on ollama-cloud still locks only the paid model (#12242)", async () => { + await resetStorage(); + const conn = await providersDb.createProviderConnection({ + provider: "ollama-cloud", + authType: "apikey", + apiKey: "ollama-cloud-test-key", + isActive: true, + testStatus: "active", + }); + const id = (conn as { id: string }).id; + + await auth.markAccountUnavailable( + id, + 402, + "Add credits to continue, or switch to a free model", + "ollama-cloud", + "gpt-chat-latest" + ); + + const after = await providersDb.getProviderConnectionById(id); + assert.equal(after.testStatus, "active"); + assert.equal( + accountFallback.getModelLockoutInfo("ollama-cloud", id, "gpt-chat-latest")?.reason, + "credits" + ); +}); + +test("Grok Build usage balance exhausted matches the credits-exhausted signal", () => { + assert.equal(accountFallback.isCreditsExhausted(GROK_BUILD_402), true); +}); + +test("a grok-cli 402 with an unrelated body does not park the connection", async () => { + await resetStorage(); + const conn = await seedGrokCli("empty-unrelated@qq.com"); + const id = (conn as { id: string }).id; + await auth.markAccountUnavailable( + id, + 402, + "Add credits to continue, or switch to a free model", + "grok-cli", + "grok-4.6" + ); + const after = await providersDb.getProviderConnectionById(id); + assert.equal(after.testStatus, "active"); + assert.equal( + accountFallback.getModelLockoutInfo("grok-cli", id, "grok-4.6")?.reason, + "credits" + ); +}); + +for (const provider of ["grok-web", "xai-oauth"] as const) { + test(`${provider} 402 parks the connection as credits_exhausted, not a model lock`, async () => { + await resetStorage(); + const conn = await seedSharedWallet(provider, `empty@${provider}.example`); + const id = (conn as { id: string }).id; + const model = provider === "grok-web" ? "fast" : "grok-4.5"; + + await auth.markAccountUnavailable(id, 402, GROK_BUILD_402, provider, model); + + const after = await providersDb.getProviderConnectionById(id); + assert.equal(after.testStatus, "credits_exhausted"); + assert.equal( + accountFallback.getModelLockoutInfo(provider, id, model), + null, + `${provider} shares the Grok weekly wallet` + ); + }); +} + +test("a grok-cli 402 with empty body parks the connection as credits_exhausted", async () => { + await resetStorage(); + const conn = await seedGrokCli("empty-nobody@qq.com"); + const id = (conn as { id: string }).id; + await auth.markAccountUnavailable(id, 402, "", "grok-cli", "grok-4.6"); + const after = await providersDb.getProviderConnectionById(id); + assert.equal(after.testStatus, "credits_exhausted"); + assert.equal(accountFallback.getModelLockoutInfo("grok-cli", id, "grok-4.6"), null); +}); diff --git a/tests/unit/combo/combo-target-exhaustion.test.ts b/tests/unit/combo/combo-target-exhaustion.test.ts index 8d4e3104d1c..5f8f98501ae 100644 --- a/tests/unit/combo/combo-target-exhaustion.test.ts +++ b/tests/unit/combo/combo-target-exhaustion.test.ts @@ -36,6 +36,7 @@ const baseOpts = { rawModel: "m1", isTokenLimitBreach: false, allAccountsRateLimited: false, + requestScopedFailure: false, log, tag: "COMBO", exhaustedLogLevel: "info" as const, @@ -683,6 +684,61 @@ test("sibling connection on the same provider is NOT skipped after a different c assert.ok(s.exhaustedConnections.has(`${failingTarget.provider}:${failingTarget.connectionId}`)); }); +test("grok-cli 402 marks only the empty connection, not the whole provider", () => { + const s = sets(); + const empty = target({ + provider: "grok-cli", + connectionId: "qq-empty", + modelStr: "grok-cli/grok-4.6", + }); + const sibling = target({ + provider: "grok-cli", + connectionId: "hotmail-live", + modelStr: "grok-cli/grok-4.6", + }); + + const exhausted = applyComboTargetExhaustion(empty, { + ...baseOpts, + result: { status: 402 }, + fallbackResult: { creditsExhausted: true, reason: "quota_exhausted" }, + errorText: "Grok Build usage balance exhausted", + rawModel: "grok-4.6", + sets: s, + }); + + assert.equal(exhausted, true); + assert.ok(s.exhaustedConnections.has("grok-cli:qq-empty")); + assert.equal( + s.exhaustedProviders.has("grok-cli"), + false, + "sibling grok-cli accounts still have weekly credits" + ); + assert.equal(s.exhaustedConnections.has("grok-cli:hotmail-live"), false); + void sibling; +}); + +for (const provider of ["grok-web", "xai-oauth"] as const) { + test(`${provider} 402 with empty body marks only that connection`, () => { + const s = sets(); + const empty = target({ + provider, + connectionId: "empty", + modelStr: `${provider}/m`, + }); + const exhausted = applyComboTargetExhaustion(empty, { + ...baseOpts, + result: { status: 402 }, + fallbackResult: {}, + errorText: "", + rawModel: "m", + sets: s, + }); + assert.equal(exhausted, true); + assert.ok(s.exhaustedConnections.has(`${provider}:empty`)); + assert.equal(s.exhaustedProviders.has(provider), false); + }); +} + test("401 carrying a real fingerprint signal still marks auth-level (exemption is 403-only)", () => { // Round 4 finding: Cloudflare 1010 is a 403-only CDN signal. A 401 invalid-credential // whose errorText carries a genuinely Cloudflare-keyed 1010 (error_code: 1010) must still