fix(grok-cli): park a 402 on the empty Grok Build login, not grok-4.6

changelog.d/fixes/grok-cli-shared-wallet-402.md

@@ -0,0 +1 @@
+- **fix(grok-cli):** a 402 "Grok Build usage balance exhausted" parks that Grok login as out of credit (Grok Build CLI, grok.com cookie, and xAI OAuth share the weekly pool). Combo routing then tries the next login instead of locking the model for every account in the pool

open-sse/services/accountFallback.ts

@@ -98,6 +98,7 @@ export { MODEL_LOCKOUT_EVICTION_CAP } from "./accountFallback/lockoutEviction.ts
 import { capScaledCooldownMs } from "./accountFallback/cooldownCap.ts";
 import { resolveApiKeyForbiddenFallback } from "./accountFallback/nonRetryableUpstream.ts";
 import * as exactModelLock from "./accountFallback/exactModelLock.ts";
+import { isCreditsExhaustedWithSharedWallet } from "./accountFallback/sharedWalletCredits.ts";
 export type ProviderProfile = {
   baseCooldownMs: number;
   useUpstreamRetryHints: boolean;
@@ -484,8 +485,7 @@ export function isAccountDeactivated(errorText: string): boolean {
  * T10: Returns true if response body indicates credits/quota are permanently exhausted.
  */
 export function isCreditsExhausted(errorText: string): boolean {
-  const lower = String(errorText || "").toLowerCase();
-  return CREDITS_EXHAUSTED_SIGNALS.some((sig) => lower.includes(sig));
+  return isCreditsExhaustedWithSharedWallet(errorText, CREDITS_EXHAUSTED_SIGNALS);
 }
 
 /**

open-sse/services/accountFallback/sharedWalletCredits.ts

@@ -0,0 +1,39 @@
+/**
+ * Providers whose 402 is a shared account wallet, not a per-model billing miss.
+ *
+ * Grok Build (`grok-cli`), grok.com cookie sessions (`grok-web`), and xAI
+ * OAuth (`xai-oauth`) bill Chat/Imagine/Voice/Build/API against one weekly
+ * percent pool. `passthroughModels: true` still stands for catalog/404
+ * behaviour; it must not send this 402 through the #12242 model-only lockout,
+ * or a combo of five grok-4.6 steps parks the empty account and then skips the
+ * remaining live accounts as "model locked".
+ *
+ * `matchesSharedWalletCreditsBody` expects a pre-lowercased string.
+ */
+const SHARED_WALLET_402_PROVIDERS = new Set(["grok-cli", "grok-web", "xai-oauth"]);
+
+export const GROK_BUILD_USAGE_BALANCE_SIGNAL = "usage balance exhausted";
+
+export function matchesSharedWalletCreditsBody(loweredErrorText: string): boolean {
+  return loweredErrorText.includes(GROK_BUILD_USAGE_BALANCE_SIGNAL);
+}
+
+export function isSharedWalletCredits402(
+  provider: string | null | undefined,
+  status: number,
+  errorText?: string | null
+): boolean {
+  if (status !== 402 || typeof provider !== "string" || !SHARED_WALLET_402_PROVIDERS.has(provider)) {
+    return false;
+  }
+  if (errorText == null || String(errorText).trim() === "") return true;
+  return matchesSharedWalletCreditsBody(String(errorText).toLowerCase());
+}
+
+export function isCreditsExhaustedWithSharedWallet(
+  errorText: string,
+  signals: readonly string[]
+): boolean {
+  const lower = String(errorText || "").toLowerCase();
+  return signals.some((sig) => lower.includes(sig)) || matchesSharedWalletCreditsBody(lower);
+}

open-sse/services/combo/targetExhaustion.ts

@@ -31,6 +31,7 @@ import { isCloudflareFingerprintRejection } from "../errorClassifier.ts";
 // (markAccountUnavailable) so the same-request combo skip and the persisted
 // connection cooldown agree on exactly which fallbackResult shapes qualify.
 import { isAgentrouterConnectionQuotaScope } from "@/sse/services/auth";
+import { isSharedWalletCredits402 } from "../accountFallback/sharedWalletCredits.ts";
 import type { ComboLogger, ResolvedComboTarget } from "./types.ts";
 
 // Connection-level failure statuses: the provider connection itself is likely bad (upstream
@@ -165,6 +166,11 @@ export function applyComboTargetExhaustion(
     return true;
   }
 
+  if (isSharedWalletCredits402(provider, result.status, opts.errorText)) {
+    markSharedWalletCreditsExhaustion(target, { sets, log, tag });
+    return true;
+  }
+
   // #8133/#8137: auth-level failures (401/403) mean that connection's credentials are bad.
   // Split out to keep applyComboTargetExhaustion under the complexity ceiling.
   // Cloudflare 1010 (a 403 carrying error_code 1010 / browser_signature_banned) is NOT an
@@ -340,6 +346,28 @@ function markAuthLevelExhaustion(
   }
 }
 
+function markSharedWalletCreditsExhaustion(
+  target: ResolvedComboTarget,
+  opts: Pick<ApplyComboTargetExhaustionOptions, "sets" | "log" | "tag">
+): void {
+  const { sets, log, tag } = opts;
+  const provider = target.provider;
+  const connId = target.connectionId ?? undefined;
+  if (connId) {
+    sets.exhaustedConnections.add(`${provider}:${connId}`);
+    log.info(
+      tag,
+      `Provider ${provider} connection ${connId} shared-wallet 402 — marking for skip on remaining targets`
+    );
+  } else {
+    sets.exhaustedProviders.add(provider as string);
+    log.info(
+      tag,
+      `Provider ${provider} shared-wallet 402 (no connectionId) — marking for skip on remaining targets`
+    );
+  }
+}
+
 /**
  * #10334: agentrouter-exclusive connection-scope account quota exhaustion. Mirrors
  * markAuthLevelExhaustion's connectionId-present/absent split — when the target carries a

src/sse/services/auth.ts

@@ -75,6 +75,7 @@ import {
   retryHintBypassesMaxCooldownMs,
   isProviderModelUnsupported400,
 } from "@omniroute/open-sse/services/accountFallback.ts";
+import { isSharedWalletCredits402 } from "@omniroute/open-sse/services/accountFallback/sharedWalletCredits.ts";
 import { isLocalProvider } from "@omniroute/open-sse/config/providerRegistry.ts";
 import { COOLDOWN_MS, RateLimitReason } from "@omniroute/open-sse/config/constants.ts";
 import { sanitizeErrorMessage } from "@omniroute/open-sse/utils/errorSanitization.ts";
@@ -2943,6 +2944,11 @@ export async function markAccountUnavailable(
       return { shouldFallback: true, cooldownMs: lockout.cooldownMs };
     }
     const result = fallbackResult;
+    if (isSharedWalletCredits402(provider, status, errorText)) {
+      result.creditsExhausted = true;
+      result.reason = result.reason || RateLimitReason.QUOTA_EXHAUSTED;
+      result.shouldFallback = true;
+    }
     const { shouldFallback, cooldownMs: rawCooldownMs, newBackoffLevel, reason } = result;
     if (!shouldFallback) return { shouldFallback: false, cooldownMs: 0 };
     const providerErrorType = classifyProviderError(status, errorText, provider);
@@ -3061,6 +3067,7 @@ export async function markAccountUnavailable(
       provider &&
       model &&
       !terminalStatus &&
+      !isSharedWalletCredits402(provider, status, errorText) &&
       !(provider === "vertex" && isVertexConnectionWidePermissionDenied(errorText))
     ) {
       const lockoutReason = status === 402 ? "credits" : "forbidden";
@@ -3182,7 +3189,12 @@ export async function markAccountUnavailable(
       // the DB, but record an in-memory model lockout so credential selection
       // skips this exact provider+connection+model while it cools down — other
       // models on the same connection stay usable.
-      if (provider && model && cooldownMs > 0) {
+      if (
+        provider &&
+        model &&
+        cooldownMs > 0 &&
+        !isSharedWalletCredits402(provider, status, errorText)
+      ) {
         lockModel(provider, connectionId, model, reason || "unknown", cooldownMs);
       }
       await updateProviderConnection(connectionId, {

tests/unit/auth-grok-cli-402-shared-wallet.test.ts

@@ -0,0 +1,186 @@
+// Grok Build (`grok-cli`) bills Chat/Imagine/Voice/Build/API against one
+// weekly credit pool. A 402 "Grok Build usage balance exhausted" is therefore
+// a connection-wide wallet signal, not a per-model billing miss. The
+// passthroughModels flag still stands for catalog/404 behaviour; it must not
+// route this 402 through the #12242 model-only lockout, or a combo of five
+// grok-4.6 steps parks the first empty account and then skips the four
+// remaining live accounts as "model locked".
+import test from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-grok-cli-402-"));
+process.env.DATA_DIR = TEST_DATA_DIR;
+
+const core = await import("../../src/lib/db/core.ts");
+const providersDb = await import("../../src/lib/db/providers.ts");
+const auth = await import("../../src/sse/services/auth.ts");
+const accountFallback = await import("../../open-sse/services/accountFallback.ts");
+
+const GROK_BUILD_402 = "Grok Build usage balance exhausted";
+
+async function resetStorage() {
+  core.resetDbInstance();
+  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 });
+  fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
+}
+
+async function seedGrokCli(name: string) {
+  return seedSharedWallet("grok-cli", name);
+}
+
+async function seedSharedWallet(provider: string, name: string) {
+  const oauth = provider === "grok-cli" || provider === "xai-oauth";
+  return providersDb.createProviderConnection({
+    provider,
+    authType: oauth ? "oauth" : "apikey",
+    name,
+    email: name,
+    ...(oauth
+      ? { accessToken: `${provider}-${name}` }
+      : { apiKey: `${provider}-${name}` }),
+    isActive: true,
+    testStatus: "active",
+  });
+}
+
+test.after(() => {
+  core.resetDbInstance();
+  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+});
+
+test("grok-cli 402 parks the connection as credits_exhausted, not a model lock", async () => {
+  await resetStorage();
+  const conn = await seedGrokCli("empty@qq.com");
+  const id = (conn as { id: string }).id;
+
+  const result = await auth.markAccountUnavailable(id, 402, GROK_BUILD_402, "grok-cli", "grok-4.6");
+
+  assert.equal(result.shouldFallback, true);
+
+  const after = await providersDb.getProviderConnectionById(id);
+  assert.equal(after.testStatus, "credits_exhausted");
+
+  const lockout = accountFallback.getModelLockoutInfo("grok-cli", id, "grok-4.6");
+  assert.equal(lockout, null, "shared-wallet 402 must not lock grok-4.6 on this account");
+});
+
+test("a sibling grok-cli account stays eligible after another account's 402", async () => {
+  await resetStorage();
+  const empty = await seedGrokCli("empty@qq.com");
+  const live = await seedGrokCli("live@hotmail.com");
+  const emptyId = (empty as { id: string }).id;
+  const liveId = (live as { id: string }).id;
+
+  await auth.markAccountUnavailable(emptyId, 402, GROK_BUILD_402, "grok-cli", "grok-4.6");
+
+  assert.equal(
+    accountFallback.isModelLocked("grok-cli", liveId, "grok-4.6"),
+    false,
+    "sibling account must not inherit the empty account's model lock"
+  );
+
+  const selected = await auth.getProviderCredentials("grok-cli");
+  assert.ok(selected);
+  assert.equal(selected.connectionId, liveId);
+  assert.notEqual(selected.connectionId, emptyId);
+});
+
+test("grok-cli 402 still parks the connection when disableCooling is set", async () => {
+  await resetStorage();
+  const conn = await providersDb.createProviderConnection({
+    provider: "grok-cli",
+    authType: "oauth",
+    accessToken: "gcli-disabled-cooling",
+    isActive: true,
+    testStatus: "active",
+    providerSpecificData: { disableCooling: true },
+  });
+  const id = (conn as { id: string }).id;
+
+  await auth.markAccountUnavailable(id, 402, GROK_BUILD_402, "grok-cli", "grok-4.6");
+
+  const after = await providersDb.getProviderConnectionById(id);
+  assert.equal(after.testStatus, "credits_exhausted");
+});
+
+test("passthrough 402 on ollama-cloud still locks only the paid model (#12242)", async () => {
+  await resetStorage();
+  const conn = await providersDb.createProviderConnection({
+    provider: "ollama-cloud",
+    authType: "apikey",
+    apiKey: "ollama-cloud-test-key",
+    isActive: true,
+    testStatus: "active",
+  });
+  const id = (conn as { id: string }).id;
+
+  await auth.markAccountUnavailable(
+    id,
+    402,
+    "Add credits to continue, or switch to a free model",
+    "ollama-cloud",
+    "gpt-chat-latest"
+  );
+
+  const after = await providersDb.getProviderConnectionById(id);
+  assert.equal(after.testStatus, "active");
+  assert.equal(
+    accountFallback.getModelLockoutInfo("ollama-cloud", id, "gpt-chat-latest")?.reason,
+    "credits"
+  );
+});
+
+test("Grok Build usage balance exhausted matches the credits-exhausted signal", () => {
+  assert.equal(accountFallback.isCreditsExhausted(GROK_BUILD_402), true);
+});
+
+test("a grok-cli 402 with an unrelated body does not park the connection", async () => {
+  await resetStorage();
+  const conn = await seedGrokCli("empty-unrelated@qq.com");
+  const id = (conn as { id: string }).id;
+  await auth.markAccountUnavailable(
+    id,
+    402,
+    "Add credits to continue, or switch to a free model",
+    "grok-cli",
+    "grok-4.6"
+  );
+  const after = await providersDb.getProviderConnectionById(id);
+  assert.equal(after.testStatus, "active");
+  assert.equal(
+    accountFallback.getModelLockoutInfo("grok-cli", id, "grok-4.6")?.reason,
+    "credits"
+  );
+});
+
+for (const provider of ["grok-web", "xai-oauth"] as const) {
+  test(`${provider} 402 parks the connection as credits_exhausted, not a model lock`, async () => {
+    await resetStorage();
+    const conn = await seedSharedWallet(provider, `empty@${provider}.example`);
+    const id = (conn as { id: string }).id;
+    const model = provider === "grok-web" ? "fast" : "grok-4.5";
+
+    await auth.markAccountUnavailable(id, 402, GROK_BUILD_402, provider, model);
+
+    const after = await providersDb.getProviderConnectionById(id);
+    assert.equal(after.testStatus, "credits_exhausted");
+    assert.equal(
+      accountFallback.getModelLockoutInfo(provider, id, model),
+      null,
+      `${provider} shares the Grok weekly wallet`
+    );
+  });
+}
+
+test("a grok-cli 402 with empty body parks the connection as credits_exhausted", async () => {
+  await resetStorage();
+  const conn = await seedGrokCli("empty-nobody@qq.com");
+  const id = (conn as { id: string }).id;
+  await auth.markAccountUnavailable(id, 402, "", "grok-cli", "grok-4.6");
+  const after = await providersDb.getProviderConnectionById(id);
+  assert.equal(after.testStatus, "credits_exhausted");
+  assert.equal(accountFallback.getModelLockoutInfo("grok-cli", id, "grok-4.6"), null);
+});