fix(combo): classify Cloudflare 1010 fingerprint rejection as non-auth

open-sse/services/combo/targetExhaustion.ts

@@ -22,6 +22,7 @@ import {
 } from "../accountFallback.ts";
 import { RateLimitReason } from "../../config/constants.ts";
 import { isProviderCircuitOpenResult, isRequestScopedUpstreamFailure } from "./comboPredicates.ts";
+import { isCloudflareFingerprintRejection } from "../errorClassifier.ts";
 import type { ComboLogger, ResolvedComboTarget } from "./types.ts";
 
 // Connection-level failure statuses: the provider connection itself is likely bad (upstream
@@ -77,12 +78,45 @@ export function applyComboTargetExhaustion(
   target: ResolvedComboTarget,
   opts: ApplyComboTargetExhaustionOptions
 ): boolean {
-  const { result, sets, log, tag } = opts;
+  const { result, sets, log, tag, errorText, structuredError } = opts;
   const provider = target.provider;
 
   // #8133/#8137: auth-level failures (401/403) mean that connection's credentials are bad.
   // Split out to keep applyComboTargetExhaustion under the complexity ceiling.
-  if (AUTH_LEVEL_ERROR_STATUSES.includes(result.status) && provider && provider !== "unknown") {
+  // Cloudflare 1010 (a 403 carrying error_code 1010 / browser_signature_banned) is NOT an
+  // auth failure: the CDN in front of the upstream refused the client's TLS/UA signature,
+  // and a different client on the same key succeeds. Treating it as auth-level would mark
+  // every connection in the pool exhausted on the first 1010 and, with a multi-target combo,
+  // crystallize a misleading ALL_ACCOUNTS_INACTIVE after two such calls — see
+  // errorClassifier.isCloudflareFingerprintRejection. The signal may arrive via the
+  // upstream JSON's structuredError.message (nested "error_code":1010 / browser_signature_banned)
+  // when the raw errorText is generic, so inspect both. A normalized structuredError.code/type
+  // ("1010" / browser_signature_banned / fingerprint_rejection) is matched directly — it arrives
+  // without the error_code key that the text regex keys on. The comparison is case-insensitive
+  // (matching isCloudflareFingerprintRejection's lowercase) and exact: a numeric 10101
+  // (port/count/request id) is a different token, never a 1010.
+  const fingerprintToken = [structuredError?.code, structuredError?.type].some((value) =>
+    ["1010", "browser_signature_banned", "fingerprint_rejection"].includes(
+      value == null ? "" : String(value).toLowerCase()
+    )
+  );
+  // code/type can also carry the signal in a non-normalized form (e.g. a gateway stuffing
+  // "error_code: 1010" into the code field verbatim), so the shared text matcher sees every
+  // candidate string — the exact allowlist above is not the only path in.
+  const fingerprintText = isCloudflareFingerprintRejection(
+    [structuredError?.message, structuredError?.code, structuredError?.type, errorText]
+      .filter(Boolean)
+      .join(" ")
+  );
+  if (
+    AUTH_LEVEL_ERROR_STATUSES.includes(result.status) &&
+    // Cloudflare 1010 is a 403-ONLY fingerprint rejection. A 401 that merely happens to
+    // mention "1010" or "fingerprint_rejection" in a port/count/model token must NOT skip
+    // auth-level exhaustion — only a 403 carrying the Cloudflare fingerprint signal does.
+    !(result.status === 403 && (fingerprintToken || fingerprintText)) &&
+    provider &&
+    provider !== "unknown"
+  ) {
     markAuthLevelExhaustion(target, { result, sets, log, tag });
     return true;
   }

open-sse/services/errorClassifier.ts

@@ -78,6 +78,7 @@ export const PROVIDER_ERROR_TYPES = {
   OAUTH_INVALID_TOKEN: "oauth_invalid_token",
   EMPTY_CONTENT: "empty_content",
   MODEL_NOT_FOUND: "model_not_found",
+  FINGERPRINT_REJECTION: "fingerprint_rejection",
 };
 
 export const CONTEXT_OVERFLOW_SIGNALS = [
@@ -113,6 +114,31 @@ export function containsModelUnavailableMessage(errorMessage: string): boolean {
   return MODEL_NAMED_UNSUPPORTED_REGEX.test(String(errorMessage || "").toLowerCase());
 }
 
+// Cloudflare 1010 "Access denied ... blocked based on your browser's signature" —
+// a fingerprint/browser-like rejection issued by the CDN in front of an upstream
+// (e.g. opencode.ai/zen/v1), carrying error_code 1010 or error_name
+// "browser_signature_banned". Distinct from an auth 403: the account is healthy,
+// the CLIENT's TLS/UA signature was refused.
+//
+// IMPORTANT: the bare number 1010 is NOT matched on its own — a 403 body can
+// legitimately contain "1010" as a port, count, request id, or model token
+// ("model foo-1010 is not supported", "retry after 1010 seconds"). 1010 is only
+// treated as a fingerprint rejection when it appears with an explicit Cloudflare
+// key (`error_code` / `error-code`) or the unique `browser_signature_banned` /
+// `fingerprint_rejection` tokens. `\\?` tolerates the escaped-quote form that
+// appears when the upstream body is nested inside the gateway's error.message JSON.
+const CLOUDFLARE_1010_REGEX =
+  /(?<![A-Za-z0-9_-])error[\s_-]?code[\\"':=\s]{0,12}1010(?!\w)|(?<![A-Za-z0-9_-])error[-_]\s?1010(?!\w)\/?/i;
+
+export function isCloudflareFingerprintRejection(errorText: string): boolean {
+  const text = String(errorText || "").toLowerCase();
+  return (
+    CLOUDFLARE_1010_REGEX.test(text) ||
+    text.includes("browser_signature_banned") ||
+    text.includes("fingerprint_rejection")
+  );
+}
+
 function responseBodyToString(responseBody: unknown): string {
   if (typeof responseBody === "string") return responseBody;
   if (responseBody !== null && typeof responseBody === "object") {
@@ -216,6 +242,16 @@ export function classifyProviderError(
   }
 
   if (statusCode === 402) return PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED;
+  if (statusCode === 403 && isCloudflareFingerprintRejection(bodyStr)) {
+    // Cloudflare 1010 / error_name "browser_signature_banned": the CDN in front of the
+    // upstream (e.g. opencode.ai/zen/v1) rejected the CLIENT's TLS/UA signature, not the
+    // account's credentials. It says nothing about account health — a different client on
+    // the same key succeeds (measured 2026-08-08: curl 200, urllib 403 on byte-identical
+    // body). Marking it FORBIDDEN would flow through markAccountUnavailable to the
+    // terminal "banned" state and, after two such calls, flip the whole free pool to
+    // ALL_ACCOUNTS_INACTIVE. Classify it separately so account state stays untouched.
+    return PROVIDER_ERROR_TYPES.FINGERPRINT_REJECTION;
+  }
   if (statusCode === 403 && accountDeactivated) {
     return PROVIDER_ERROR_TYPES.ACCOUNT_DEACTIVATED;
   }

src/sse/services/auth.ts

@@ -364,7 +364,13 @@ function resolveTerminalConnectionStatus(
   if (result.creditsExhausted || status === 402) return "credits_exhausted";
   if (
     providerErrorType === PROVIDER_ERROR_TYPES.PROJECT_ROUTE_ERROR ||
-    providerErrorType === PROVIDER_ERROR_TYPES.OAUTH_INVALID_TOKEN
+    providerErrorType === PROVIDER_ERROR_TYPES.OAUTH_INVALID_TOKEN ||
+    // #1010: Cloudflare fingerprint rejection is the CDN refusing the CLIENT's
+    // signature, not the account's credentials — never a terminal account state.
+    // A different client on the same key succeeds (measured 2026-08-08: curl 200,
+    // urllib 403 on byte-identical body), so banning the account here would flip a
+    // healthy free pool to ALL_ACCOUNTS_INACTIVE after two such calls.
+    providerErrorType === PROVIDER_ERROR_TYPES.FINGERPRINT_REJECTION
   ) {
     return null;
   }
@@ -2046,7 +2052,10 @@ export async function markAccountUnavailable(
         ? "model"
         : getQuotaScopeLabelForProvider(provider, model);
       const antigravityFamilyInferredBaseCooldownMs =
-        !usesExactAntigravityLock && provider === "antigravity" && quotaScope === "family" && status === 429
+        !usesExactAntigravityLock &&
+        provider === "antigravity" &&
+        quotaScope === "family" &&
+        status === 429
           ? ANTIGRAVITY_FAMILY_INFERRED_BASE_COOLDOWN_MS
           : null;
       const lockout = recordModelLockoutFailure(

tests/unit/combo/combo-target-exhaustion.test.ts

@@ -516,6 +516,144 @@ test("401 on per-model-quota provider marks only the failing connection (auth is
   );
 });
 
+test("Cloudflare 1010 (403 fingerprint rejection) does NOT mark auth-level exhaustion", () => {
+  // #1010 incident: urllib's Python-urllib UA is refused by Cloudflare in front of
+  // opencode.ai/zen/v1 with error_code 1010 while curl on the SAME key/body succeeds.
+  // Treating it as auth-level marks every connection exhausted and, after two such
+  // calls, flips the pool to ALL_ACCOUNTS_INACTIVE. It must fall through to the
+  // transient path (no exhaustion marking) so remaining targets still get tried.
+  const s = sets();
+  const exhausted = applyComboTargetExhaustion(target(), {
+    ...baseOpts,
+    errorText:
+      '[openai/deepseek-v4-flash-free] [403]: {"type":"https://developers.cloudflare.com/support/troubleshooting/http-status-codes/cloudflare-1xxx-errors/error-1010/","title":"Error 1010: Access denied","status":403,"detail":"The site owner has blocked access based on your browser\'s signature.","instance":"a283cb68eb52bda8","error_code":1010,"error_name":"browser_signature_banned"}',
+    result: { status: 403 },
+    fallbackResult: { creditsExhausted: false },
+    sets: s,
+  });
+  assert.equal(exhausted, false, "a fingerprint rejection must not exhaust the connection");
+  assert.equal(
+    s.exhaustedConnections.size,
+    0,
+    "a Cloudflare 1010 must not mark the connection exhausted for remaining targets"
+  );
+  assert.equal(
+    s.exhaustedProviders.size,
+    0,
+    "a Cloudflare 1010 must not exhaust the whole provider"
+  );
+});
+
+test("plain 403 still marks auth-level exhaustion (1010 detection is specific)", () => {
+  const s = sets();
+  const exhausted = applyComboTargetExhaustion(target(), {
+    ...baseOpts,
+    errorText: "you do not have permission to access this model",
+    result: { status: 403 },
+    fallbackResult: { creditsExhausted: false },
+    sets: s,
+  });
+  assert.equal(exhausted, true);
+  assert.ok(s.exhaustedConnections.has("test-dedup-provider:conn-1"));
+});
+
+test("Cloudflare 1010 arriving via structuredError (not raw errorText) still avoids auth exhaustion", () => {
+  // The 1010 signal may surface in structuredError.message (nested JSON) while errorText
+  // stays generic. The auth-level guard must inspect both, or the #1010 failure recurs.
+  const s = sets();
+  const exhausted = applyComboTargetExhaustion(target(), {
+    ...baseOpts,
+    errorText: "[403] forbidden",
+    structuredError: { code: "browser_signature_banned" },
+    result: { status: 403 },
+    fallbackResult: { creditsExhausted: false },
+    sets: s,
+  });
+  assert.equal(exhausted, false, "structuredError 1010 must not mark auth-level exhaustion");
+  assert.equal(s.exhaustedConnections.size, 0);
+});
+
+test("Cloudflare 1010 in structuredError.code survives a generic structuredError.message (no || short-circuit)", () => {
+  // Review finding: structuredError.message being a generic value (e.g. "Forbidden") must
+  // NOT mask a 1010/browser_signature_banned signal carried in .code. The guard must check
+  // every candidate string, not short-circuit on the first truthy one.
+  const s = sets();
+  const exhausted = applyComboTargetExhaustion(target(), {
+    ...baseOpts,
+    errorText: "You do not have permission",
+    structuredError: { message: "Forbidden", code: "browser_signature_banned" },
+    result: { status: 403 },
+    fallbackResult: { creditsExhausted: false },
+    sets: s,
+  });
+  assert.equal(exhausted, false, "a code-carried 1010 must not be masked by a generic message");
+  assert.equal(s.exhaustedConnections.size, 0);
+});
+
+test("Cloudflare 1010 via structuredError.type still avoids auth exhaustion", () => {
+  // Round 6 finding: the guard promised a normalized structuredError.type of "1010" is
+  // matched directly, but only code/type named browser_signature_banned were checked. A 403
+  // whose only fingerprint signal is type === "1010" fell through to auth-level exhaustion.
+  // combo.ts coerces upstream numeric codes via String() before building structuredError, so
+  // the string form is the runtime contract this path must honor.
+  const s = sets();
+  const exhausted = applyComboTargetExhaustion(target(), {
+    ...baseOpts,
+    errorText: "Forbidden",
+    structuredError: { message: "generic", type: "1010" },
+    result: { status: 403 },
+    fallbackResult: { creditsExhausted: false },
+    sets: s,
+  });
+  assert.equal(exhausted, false, 'a 403 with structuredError.type "1010" must not mark auth-level');
+  assert.equal(s.exhaustedConnections.size, 0);
+});
+
+test("structuredError code/type fingerprint match is case-insensitive like the text matcher", () => {
+  // Round 6 finding: fingerprintCode compared code/type case-sensitively while
+  // isCloudflareFingerprintRejection lowercases text — a 403 carrying
+  // "BROWSER_SIGNATURE_BANNED" (all-caps from a normalizing gateway) was treated as
+  // auth-level. Normalize code/type before comparing, mirroring the text matcher.
+  for (const structuredError of [
+    { code: "BROWSER_SIGNATURE_BANNED" },
+    { type: "Browser_Signature_Banned" },
+    { code: "Fingerprint_Rejection" },
+  ] as const) {
+    const s = sets();
+    const exhausted = applyComboTargetExhaustion(target(), {
+      ...baseOpts,
+      errorText: "Forbidden",
+      structuredError,
+      result: { status: 403 },
+      fallbackResult: { creditsExhausted: false },
+      sets: s,
+    });
+    assert.equal(exhausted, false, "a mixed-case fingerprint token must not mark auth-level");
+    assert.equal(s.exhaustedConnections.size, 0);
+  }
+});
+
+test("Cloudflare 1010 inside a non-normalized structuredError.code still avoids auth exhaustion", () => {
+  // Round 7 finding: a gateway can stuff the raw Cloudflare body ("error_code: 1010") into
+  // the code field verbatim, so the exact token allowlist misses it and the shared text
+  // matcher never saw code/type. Feed every candidate string to the text matcher.
+  const s = sets();
+  const exhausted = applyComboTargetExhaustion(target(), {
+    ...baseOpts,
+    errorText: "Forbidden",
+    structuredError: { message: "generic", code: "error_code: 1010" },
+    result: { status: 403 },
+    fallbackResult: { creditsExhausted: false },
+    sets: s,
+  });
+  assert.equal(
+    exhausted,
+    false,
+    "a 403 whose structuredError.code embeds error_code: 1010 must not mark auth-level"
+  );
+  assert.equal(s.exhaustedConnections.size, 0);
+});
+
 // #8137 regression: a SIBLING connection on the SAME provider must NOT be skipped when a
 // DIFFERENT connection on that provider returned 401/403 — proves the fix at the call-site
 // level (getExhaustedTargetSkipReason-style check), not just the raw Set contents above.
@@ -544,3 +682,20 @@ test("sibling connection on the same provider is NOT skipped after a different c
   // The failing connection itself IS marked.
   assert.ok(s.exhaustedConnections.has(`${failingTarget.provider}:${failingTarget.connectionId}`));
 });
+
+test("401 carrying a real fingerprint signal still marks auth-level (exemption is 403-only)", () => {
+  // Round 4 finding: Cloudflare 1010 is a 403-only CDN signal. A 401 invalid-credential
+  // whose errorText carries a genuinely Cloudflare-keyed 1010 (error_code: 1010) must still
+  // mark auth-level exhaustion on the 401 — only a 403 earns the fingerprint exemption.
+  // Otherwise a 401 echoing an upstream 1010 would leave a bad credential retryable.
+  const s = sets();
+  const exhausted = applyComboTargetExhaustion(target(), {
+    ...baseOpts,
+    errorText: "error_code: 1010, token expired",
+    result: { status: 401 },
+    fallbackResult: { creditsExhausted: false },
+    sets: s,
+  });
+  assert.equal(exhausted, true, "a 401 with a fingerprint-looking body must still mark auth-level");
+  assert.ok(s.exhaustedConnections.has("test-dedup-provider:conn-1"));
+});