merge queue: checking release/v3.8.49 (7123236), #6983 and #7004 together

open-sse/config/providerErrorRules.ts

@@ -130,6 +130,31 @@ function buildMinimaxRules(): ProviderErrorRule[] {
   ];
 }
 
+// ─── Cloudflare Workers AI ─────────────────────────────────────────────────────
+// Free tier = 10,000 Neurons/day, shared across the WHOLE account
+// (docs/reference/FREE_TIERS.md; official: developers.cloudflare.com/
+// workers-ai/platform/errors/). The exhaustion body doesn't match any
+// QUOTA_PATTERNS keyword so it falls through to rate_limit and gets
+// retried every ~60s against a budget that only resets at UTC midnight.
+// Issue #6980.
+function buildCloudflareAiRules(): ProviderErrorRule[] {
+  return [
+    {
+      id: "cloudflare-ai-daily-neuron-allocation",
+      match: ({ status, body }) => {
+        if (status !== 429) return null;
+        const text = JSON.stringify(body ?? "").toLowerCase();
+        // Body: "you have used up your daily free allocation of 10,000 neurons,
+        //        please upgrade to Cloudflare's Workers Paid plan..."
+        if (!text.includes("daily free allocation")) return null;
+        // No cooldownMs: recordModelLockoutFailure already sets
+        // quota_exhausted without one to "next UTC midnight".
+        return { reason: "quota_exhausted", scope: "connection" };
+      },
+    },
+  ];
+}
+
 /**
  * Global registry. Provider name → ordered list of rules (first match wins).
  * Add new providers here; the matcher in classifyError will pick them up
@@ -141,6 +166,7 @@ export const providerRuleRegistry = new Map<string, ProviderErrorRule[]>([
   ["opencode-cli", buildOpencodeRules()],
   ["minimax", buildMinimaxRules()],
   ["minimax-passthrough", buildMinimaxRules()],
+  ["cloudflare-ai", buildCloudflareAiRules()],
 ]);
 
 /**
@@ -194,7 +220,9 @@ export function getProviderErrorRuleMatch(
  */
 export function parseResetCountdownMs(text: string): number | null {
   if (typeof text !== "string" || text.length === 0) return null;
-  const match = text.match(/resets?\s+in\s+(\d+)\s+(day|days|hour|hours|minute|minutes|second|seconds)\b/);
+  const match = text.match(
+    /resets?\s+in\s+(\d+)\s+(day|days|hour|hours|minute|minutes|second|seconds)\b/
+  );
   if (!match) return null;
   const n = Number(match[1]);
   if (!Number.isFinite(n) || n <= 0) return null;

src/app/api/v1/models/catalog.ts

@@ -860,12 +860,17 @@ async function buildUnifiedModelsResponseCore(
           // #6457: some upstream discovery catalogs (e.g. HuggingFace's live
           // `/v1/models`) return image/diffusion models with no modality info,
           // so `endpoints` below would default to ["chat"] and misrepresent
-          // them as chat-capable. Skip any synced model that is already a
-          // registered image model for this provider — getAllImageModels()
-          // below adds the correctly-typed `type: "image"` entry instead.
+          // them as chat-capable. Skip a registered image model only when its
+          // synced metadata does not explicitly advertise a chat endpoint.
+          // Multi-capability models may intentionally share an id between the
+          // chat and image catalogs; getAllImageModels() adds the image entry.
+          const explicitlySupportsChat = sm.supportedEndpoints?.some(
+            (endpoint) => endpoint === "chat" || endpoint === "responses"
+          );
           if (
-            isRegisteredImageModel(canonicalProviderId, sm.id) ||
-            isRegisteredImageModel(providerId, sm.id)
+            !explicitlySupportsChat &&
+            (isRegisteredImageModel(canonicalProviderId, sm.id) ||
+              isRegisteredImageModel(providerId, sm.id))
           ) {
             continue;
           }

src/shared/utils/classify429.ts

@@ -53,6 +53,14 @@ const QUOTA_PATTERNS: ReadonlyArray<RegExp> = [
   /individual quota reached/i,
   /enable overages/i,
   /INSUFFICIENT_G1_CREDITS_BALANCE/i,
+
+  // Cloudflare Workers AI daily neuron exhaustion (Issue #6980).
+  // Body: "you have used up your daily free allocation of 10,000 neurons,
+  //        please upgrade to Cloudflare's Workers Paid plan..."
+  // No existing pattern matches "daily free allocation" — without this,
+  // the 429 is misclassified as transient rate_limit and retried every
+  // ~60s against a budget that only resets at UTC midnight.
+  /daily free allocation/i,
 ];
 
 /**

tests/unit/cloudflare-ai-neuron-exhaustion-6980.test.ts

@@ -0,0 +1,108 @@
+/**
+ * Issue #6980 — Cloudflare Workers AI daily neuron exhaustion 429 must be
+ * classified as quota_exhausted (not transient rate_limit).
+ *
+ * Two layers of defense:
+ *  1. Provider-specific rule in providerErrorRules.ts → getProviderErrorRuleMatch
+ *  2. Global QUOTA_PATTERNS in classify429.ts → looksLikeQuotaExhausted
+ *
+ * Without these, the 429 body "you have used up your daily free allocation of
+ * 10,000 neurons" matches no keyword, falls through to rate_limit (~60s cooldown),
+ * and the combo router keeps cycling through every cloudflare model on retry
+ * against a budget that only resets at UTC midnight.
+ */
+
+import { describe, test } from "node:test";
+import assert from "node:assert/strict";
+
+import {
+  getProviderErrorRuleMatch,
+  providerRuleRegistry,
+} from "../../open-sse/config/providerErrorRules.ts";
+import { classify429, looksLikeQuotaExhausted } from "../../src/shared/utils/classify429.ts";
+
+// ─── Fixtures ────────────────────────────────────────────────────────────────
+
+const CF_NEURON_BODY =
+  "you have used up your daily free allocation of 10,000 neurons, please upgrade to Cloudflare's Workers Paid plan";
+
+const CF_NEURON_BODY_JSON = {
+  errors: [
+    {
+      code: 4006,
+      message:
+        "you have used up your daily free allocation of 10,000 neurons, please upgrade to Cloudflare's Workers Paid plan",
+    },
+  ],
+};
+
+// ─── Tests: provider-specific rule (primary path) ───────────────────────────
+
+describe("#6980 provider rule: cloudflare-ai neuron exhaustion", () => {
+  test("cloudflare-ai is registered in providerRuleRegistry", () => {
+    assert.ok(providerRuleRegistry.has("cloudflare-ai"));
+  });
+
+  test("429 with plain-string neuron body → quota_exhausted, scope connection", () => {
+    const result = getProviderErrorRuleMatch("cloudflare-ai", 429, {}, CF_NEURON_BODY);
+    assert.ok(result, "expected a match");
+    assert.equal(result!.reason, "quota_exhausted");
+    assert.equal(result!.scope, "connection");
+    // No explicit cooldownMs — recordModelLockoutFailure resolves to next UTC midnight.
+    assert.equal(result!.cooldownMs, undefined);
+  });
+
+  test("429 with JSON-structured neuron body → quota_exhausted", () => {
+    const result = getProviderErrorRuleMatch("cloudflare-ai", 429, {}, CF_NEURON_BODY_JSON);
+    assert.ok(result);
+    assert.equal(result!.reason, "quota_exhausted");
+    assert.equal(result!.scope, "connection");
+  });
+
+  test("non-429 status does not match even with neuron body", () => {
+    const result = getProviderErrorRuleMatch("cloudflare-ai", 500, {}, CF_NEURON_BODY);
+    assert.equal(result, null);
+  });
+
+  test("429 with unrelated body does not match", () => {
+    const result = getProviderErrorRuleMatch(
+      "cloudflare-ai",
+      429,
+      {},
+      {
+        error: "rate limited, try again later",
+      }
+    );
+    assert.equal(result, null);
+  });
+
+  test("provider name matching is case-insensitive", () => {
+    const result = getProviderErrorRuleMatch("Cloudflare-AI", 429, {}, CF_NEURON_BODY);
+    assert.ok(result);
+    assert.equal(result!.reason, "quota_exhausted");
+  });
+});
+
+// ─── Tests: classify429 defense-in-depth (fallback path) ────────────────────
+
+describe("#6980 classify429: daily free allocation pattern", () => {
+  test("looksLikeQuotaExhausted matches neuron body string", () => {
+    assert.ok(looksLikeQuotaExhausted(CF_NEURON_BODY));
+  });
+
+  test("looksLikeQuotaExhausted matches neuron body JSON-stringified", () => {
+    assert.ok(looksLikeQuotaExhausted(CF_NEURON_BODY_JSON));
+  });
+
+  test("classify429 returns quota_exhausted for neuron body", () => {
+    assert.equal(classify429({ status: 429, body: CF_NEURON_BODY }), "quota_exhausted");
+  });
+
+  test("classify429 returns quota_exhausted for neuron JSON body", () => {
+    assert.equal(classify429({ status: 429, body: CF_NEURON_BODY_JSON }), "quota_exhausted");
+  });
+
+  test("classify429 returns rate_limit for generic 429 without quota keywords", () => {
+    assert.equal(classify429({ status: 429, body: "Too many requests" }), "rate_limit");
+  });
+});

tests/unit/image-model-not-in-chat-catalog-6457.test.ts

@@ -11,10 +11,10 @@
 // `type: "image"` by the imageRegistry loop — and catalogDedupe.ts keys on
 // (id, type, subtype), so the two distinct-`type` entries both survived.
 //
-// Fix: skip a synced model in the chat-catalog loop when it is already a registered
-// image model for that exact provider (open-sse/config/imageRegistry.ts
-// isRegisteredImageModel()) — the imageRegistry loop still adds the correctly-typed
-// `type: "image"` entry.
+// Fix: skip an exact-provider registered image model from the chat-catalog loop only
+// when synced metadata does not explicitly advertise `chat` or `responses`. The image
+// registry loop still adds the correctly typed image entry, while multi-capability
+// models keep both entries.
 
 import test from "node:test";
 import assert from "node:assert/strict";
@@ -38,6 +38,7 @@ async function resetStorage() {
 }
 
 test.beforeEach(async () => {
+  v1ModelsCatalog.__resetCatalogBuilderRunsForTest();
   await resetStorage();
 });
 
@@ -46,19 +47,19 @@ test.after(async () => {
   fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
 });
 
-async function seedHuggingFaceConnection() {
+async function seedProviderConnection(provider: string) {
   return providersDb.createProviderConnection({
-    provider: "huggingface",
+    provider,
     authType: "apikey",
-    name: `huggingface-${Math.random().toString(16).slice(2, 8)}`,
-    apiKey: "hf-key",
+    name: `${provider}-${Math.random().toString(16).slice(2, 8)}`,
+    apiKey: `${provider}-key`,
     isActive: true,
     testStatus: "active",
   });
 }
 
 test("#6457 image/diffusion model discovered via live sync is NOT listed as a chat model", async () => {
-  const connection = await seedHuggingFaceConnection();
+  const connection = await seedProviderConnection("huggingface");
 
   // Simulate what HuggingFace's live `/v1/models` discovery persists for an
   // image/diffusion model: no supportedEndpoints/modality info at all — the exact
@@ -100,3 +101,36 @@ test("#6457 image/diffusion model discovered via live sync is NOT listed as a ch
     assert.equal(entry.type, undefined, "the real chat model must not carry a non-chat type");
   }
 });
+
+test("registered image model with explicit chat endpoints keeps both catalog entries", async () => {
+  const connection = await seedProviderConnection("codex");
+
+  await modelsDb.replaceSyncedAvailableModelsForConnection("codex", connection.id, [
+    {
+      id: "gpt-5.6-sol",
+      name: "GPT 5.6 Sol",
+      supportedEndpoints: ["responses"],
+    },
+  ]);
+
+  const response = await v1ModelsCatalog.getUnifiedModelsResponse(
+    new Request("http://localhost/api/v1/models?prefix=alias")
+  );
+  assert.equal(response.status, 200);
+
+  const body = (await response.json()) as {
+    data: Array<{ id: string; type?: string; supported_endpoints?: string[] }>;
+  };
+  const entries = body.data.filter((model) => model.id.endsWith("/gpt-5.6-sol"));
+
+  assert.ok(
+    entries.some(
+      (model) => model.type !== "image" && model.supported_endpoints?.includes("responses")
+    ),
+    "explicit responses support must keep the synced chat entry"
+  );
+  assert.ok(
+    entries.some((model) => model.type === "image"),
+    "the registered image entry must remain available under the same model id"
+  );
+});