fix(combo): detect invalid model errors via structured error codes + regex fallback

open-sse/services/accountFallback.ts

@@ -136,6 +136,44 @@ const CONTEXT_OVERFLOW_PATTERNS = [
   /\brequest too large\b/i,
 ];
 
+// Structured error codes that reliably indicate model access denied
+// (more reliable than regex on human-readable messages).
+// OpenAI:  { error: { code: "model_not_found", ... } }
+// Anthropic: { error: { type: "not_found_error", ... } }
+const MODEL_ACCESS_DENIED_CODES = new Set([
+  "model_not_found", // OpenAI, OpenAI-compatible (Kiro, Together, Fireworks, etc.)
+  "deployment_not_found", // Azure OpenAI
+]);
+
+const MODEL_ACCESS_DENIED_TYPES = new Set([
+  "not_found_error", // Anthropic: model doesn't exist — reliably model-scoped
+]);
+
+// Anthropic's permission_error is NOT exclusively model-access related: it also
+// covers API-key scope, organization restrictions and feature gating. Treating it
+// as model-access-denied unconditionally would make a genuinely auth-restricted key
+// silently exhaust every combo target and hide the real error from the caller.
+// So it only counts when the message text confirms it refers to the model.
+const MODEL_ACCESS_AMBIGUOUS_TYPES = new Set([
+  "permission_error", // Anthropic: could be model access OR key/org/feature scope
+]);
+
+// Model access patterns — the account does not have access to the requested model
+// but a different account (e.g. PRO vs free tier) may support it.
+const MODEL_ACCESS_DENIED_PATTERNS = [
+  /\binvalid model\b/i,
+  /\bmodel.*not.*(?:available|found|supported|accessible)\b/i,
+  /\bmodel.*(?:does not exist|doesn't exist)\b/i,
+  /\baccess.*denied.*model\b/i,
+  /\bmodel.*access.*denied\b/i,
+  /\bplease select a different model\b/i,
+  // "...access to the requested model" / "model ... access" — bounded lookahead
+  // (no nested quantifiers) so it stays ReDoS-safe while requiring BOTH an
+  // access/permission word and "model" so a pure auth error never matches.
+  /\b(?:access|permission)[\s\S]{0,60}?\bmodel\b/i,
+  /\bmodel[\s\S]{0,60}?\b(?:access|permission)\b/i,
+];
+
 // Malformed request patterns — the model rejected the message format but a different
 // provider/model in the combo may accept it.
 const MALFORMED_REQUEST_PATTERNS = [
@@ -1007,7 +1045,8 @@ export function checkFallbackError(
   _model: string | null = null,
   provider: string | null = null,
   headers: Headers | Record<string, string> | null = null,
-  profileOverride: ProviderProfile | null = null
+  profileOverride: ProviderProfile | null = null,
+  structuredError?: { code?: string | null; type?: string | null } | null
 ): {
   shouldFallback: boolean;
   cooldownMs: number;
@@ -1222,12 +1261,30 @@ export function checkFallbackError(
     return buildRetryableFallback(RateLimitReason.SERVER_ERROR);
   }
 
-  // 400 — context overflow / malformed request may succeed on another model in the combo
+  // 400 — context overflow / malformed request / model access denied
   if (status === HTTP_STATUS.BAD_REQUEST) {
+    // Check structured error codes first (more reliable, no false positives)
+    // OpenAI:  error.code === "model_not_found"
+    // Anthropic: error.type === "not_found_error" / "permission_error"
+    const structuredCode =
+      typeof structuredError?.code === "string" ? structuredError.code.toLowerCase() : "";
+    const structuredType =
+      typeof structuredError?.type === "string" ? structuredError.type.toLowerCase() : "";
+    const matchesModelAccessPattern = MODEL_ACCESS_DENIED_PATTERNS.some((p) => p.test(errorStr));
+
+    const isModelAccessDeniedStructured =
+      !!structuredError &&
+      (MODEL_ACCESS_DENIED_CODES.has(structuredCode) ||
+        MODEL_ACCESS_DENIED_TYPES.has(structuredType) ||
+        // Ambiguous types (e.g. Anthropic permission_error) only count as a model
+        // access denial when the message text confirms it is about the model.
+        (MODEL_ACCESS_AMBIGUOUS_TYPES.has(structuredType) && matchesModelAccessPattern));
+
     const isOverflow = CONTEXT_OVERFLOW_PATTERNS.some((p) => p.test(errorStr));
     const isMalformed = MALFORMED_REQUEST_PATTERNS.some((p) => p.test(errorStr));
+    const isModelAccessDenied = isModelAccessDeniedStructured || matchesModelAccessPattern;
 
-    if (isOverflow || isMalformed) {
+    if (isOverflow || isMalformed || isModelAccessDenied) {
       return {
         shouldFallback: true,
         cooldownMs: 0,

open-sse/services/combo.ts

@@ -2258,14 +2258,23 @@ export async function handleComboChat({
         // treated as local to that target and the combo continues to the next target.
         // Error classification is retained only for retry/cooldown pacing; it must
         // not decide whether fallback happens, including for generic 400 responses.
+        const rawError = errorBody?.error;
+        const structuredError =
+          rawError && typeof rawError === "object"
+            ? {
+                code: (rawError as Record<string, unknown>).code as string,
+                type: (rawError as Record<string, unknown>).type as string,
+              }
+            : undefined;
         const fallbackResult = checkFallbackError(
           result.status,
           errorText,
           0,
           null,
           provider,
           result.headers,
-          profile
+          profile,
+          structuredError
         );
         const { cooldownMs } = fallbackResult;
 
@@ -2643,14 +2652,23 @@ async function handleRoundRobinCombo({
         // strategies: non-ok target responses fall through to the next target.
         // Classification stays here only to support cooldown/semaphore pacing,
         // not to decide whether fallback is allowed.
+        const rawError = errorBody?.error;
+        const structuredError =
+          rawError && typeof rawError === "object"
+            ? {
+                code: (rawError as Record<string, unknown>).code as string,
+                type: (rawError as Record<string, unknown>).type as string,
+              }
+            : undefined;
         const fallbackResult = checkFallbackError(
           result.status,
           errorText,
           0,
           null,
           provider,
           result.headers,
-          profile
+          profile,
+          structuredError
         );
         const { cooldownMs } = fallbackResult;
 

tests/unit/account-fallback-service.test.ts

@@ -829,3 +829,100 @@ test("classifyErrorText handles hour quota messages", () => {
   assert.equal(classifyErrorText("hour quota has been exceeded"), RateLimitReason.QUOTA_EXHAUSTED);
   assert.equal(classifyErrorText("quota has been exceeded"), RateLimitReason.QUOTA_EXHAUSTED);
 });
+
+// ─── Model Access Denied (structured error codes + regex fallback) ─────
+
+test("checkFallbackError detects model access denied via structured error code (OpenAI)", () => {
+  const result = checkFallbackError(
+    400,
+    "The model `gpt-5` does not exist",
+    0,
+    null,
+    "openai",
+    null,
+    null,
+    { code: "model_not_found", type: null }
+  );
+  assert.equal(result.shouldFallback, true);
+  assert.equal(result.cooldownMs, 0);
+  assert.equal(result.reason, RateLimitReason.MODEL_CAPACITY);
+});
+
+test("checkFallbackError detects model access denied via structured error type (Anthropic not_found_error)", () => {
+  const result = checkFallbackError(
+    400,
+    "model: claude-sonnet-4-7-20260515",
+    0,
+    null,
+    "anthropic",
+    null,
+    null,
+    { code: null, type: "not_found_error" }
+  );
+  assert.equal(result.shouldFallback, true);
+  assert.equal(result.cooldownMs, 0);
+  assert.equal(result.reason, RateLimitReason.MODEL_CAPACITY);
+});
+
+test("checkFallbackError detects model access denied via structured error type (Anthropic permission_error) when the message confirms the model", () => {
+  const result = checkFallbackError(
+    400,
+    "you do not have access to the requested model",
+    0,
+    null,
+    "anthropic",
+    null,
+    null,
+    { code: null, type: "permission_error" }
+  );
+  assert.equal(result.shouldFallback, true);
+  assert.equal(result.cooldownMs, 0);
+  assert.equal(result.reason, RateLimitReason.MODEL_CAPACITY);
+});
+
+test("checkFallbackError does NOT fallback on a permission_error that is a key/feature scope issue (not model access)", () => {
+  // permission_error is ambiguous on Anthropic — also raised for API-key scope,
+  // org restrictions and feature gating. Without a model-related message it must
+  // surface the real error instead of silently exhausting every combo target.
+  const result = checkFallbackError(
+    400,
+    "Your API key does not have permission to use the Message Batches API",
+    0,
+    null,
+    "anthropic",
+    null,
+    null,
+    { code: null, type: "permission_error" }
+  );
+  assert.equal(result.shouldFallback, false);
+});
+
+test("checkFallbackError detects model access denied via regex fallback (invalid model)", () => {
+  const result = checkFallbackError(
+    400,
+    "Invalid model: gpt-5-turbo",
+    0,
+    null,
+    "some-provider",
+    null,
+    null
+  );
+  assert.equal(result.shouldFallback, true);
+  assert.equal(result.cooldownMs, 0);
+  assert.equal(result.reason, RateLimitReason.MODEL_CAPACITY);
+});
+
+test("checkFallbackError does NOT fallback on generic 400 without model access denied", () => {
+  const result = checkFallbackError(400, "bad request payload", 0, null, "openai", null, null);
+  assert.equal(result.shouldFallback, false);
+});
+
+test("checkFallbackError ignores structured error with unrelated code on 400", () => {
+  const result = checkFallbackError(400, "something went wrong", 0, null, "openai", null, null, {
+    code: "invalid_api_key",
+    type: null,
+  });
+  // "invalid_api_key" is not in MODEL_ACCESS_DENIED_CODES,
+  // no MODEL_ACCESS_DENIED_PATTERNS match either → shouldFallback: false
+  assert.equal(result.shouldFallback, false);
+});