Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions open-sse/config/freeModelCatalog.data.ts
Original file line number Diff line number Diff line change
Expand Up @@ -114,11 +114,11 @@ export const FREE_MODEL_BUDGETS: FreeModelBudget[] = [
{ provider: "bytez", modelId: "Qwen/Qwen2.5-72B-Instruct", displayName: "Qwen/Qwen2.5-72B-Instruct", monthlyTokens: 0, creditTokens: 1000000, freeType: "recurring-credit", poolKey: "bytez", tos: "ambiguous" },
{ provider: "cerebras", modelId: "zai-glm-4.7", displayName: "GLM 4.7", monthlyTokens: 30000000, creditTokens: 0, freeType: "recurring-daily", poolKey: "cerebras", tos: "caution" },
{ provider: "cerebras", modelId: "gpt-oss-120b", displayName: "GPT OSS 120B", monthlyTokens: 30000000, creditTokens: 0, freeType: "recurring-daily", poolKey: "cerebras", tos: "caution" },
{ provider: "cloudflare-ai", modelId: "@cf/meta/llama-3.3-70b-instruct", displayName: "Llama 3.3 70B (🆓 ~150 resp/day)", monthlyTokens: 30000000, creditTokens: 0, freeType: "recurring-daily", poolKey: "cloudflare-ai", tos: "caution" },
{ provider: "cloudflare-ai", modelId: "@cf/google/gemma-3-12b-it", displayName: "Gemma 3 12B (🆓)", monthlyTokens: 0, creditTokens: 0, freeType: "recurring-daily", poolKey: "cloudflare-ai", tos: "caution" },
// #8717: drop dead Workers AI ids (400/403/410). Keep Neurons/day budget on fp8-fast.
{ provider: "cloudflare-ai", modelId: "@cf/mistral/mistral-7b-instruct-v0.2-lora", displayName: "Mistral 7B (🆓)", monthlyTokens: 0, creditTokens: 0, freeType: "recurring-daily", poolKey: "cloudflare-ai", tos: "caution" },
{ provider: "cloudflare-ai", modelId: "@cf/qwen/qwen2.5-coder-32b-instruct", displayName: "Qwen 2.5 Coder 32B (🆓)", monthlyTokens: 0, creditTokens: 0, freeType: "recurring-daily", poolKey: "cloudflare-ai", tos: "caution" },
{ provider: "cloudflare-ai", modelId: "@cf/deepseek-ai/deepseek-r1-distill-qwen-32b", displayName: "DeepSeek R1 Distill 32B (🆓)", monthlyTokens: 0, creditTokens: 0, freeType: "recurring-daily", poolKey: "cloudflare-ai", tos: "caution" },
{ provider: "cloudflare-ai", modelId: "@cf/meta/llama-3.3-70b-instruct-fp8-fast", displayName: "Llama 3.3 70B (FP8 Fast 🆓)", monthlyTokens: 0, creditTokens: 0, freeType: "recurring-daily", poolKey: "cloudflare-ai", tos: "caution" },
{ provider: "cloudflare-ai", modelId: "@cf/meta/llama-3.3-70b-instruct-fp8-fast", displayName: "Llama 3.3 70B (FP8 Fast 🆓 ~150 resp/day)", monthlyTokens: 30000000, creditTokens: 0, freeType: "recurring-daily", poolKey: "cloudflare-ai", tos: "caution" },
{ provider: "cloudflare-ai", modelId: "@cf/meta/llama-3.2-3b-instruct", displayName: "Llama 3.2 3B (🆓)", monthlyTokens: 0, creditTokens: 0, freeType: "recurring-daily", poolKey: "cloudflare-ai", tos: "caution" },
{ provider: "cloudflare-ai", modelId: "@cf/qwen/qwq-32b", displayName: "QwQ 32B (🆓)", monthlyTokens: 0, creditTokens: 0, freeType: "recurring-daily", poolKey: "cloudflare-ai", tos: "caution" },
{ provider: "cloudflare-ai", modelId: "@cf/zai-org/glm-4.7-flash", displayName: "GLM 4.7 Flash (🆓)", monthlyTokens: 0, creditTokens: 0, freeType: "recurring-daily", poolKey: "cloudflare-ai", tos: "caution" },
Expand Down
10 changes: 5 additions & 5 deletions open-sse/config/providers/registry/cloudflare-ai/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,14 @@ export const cloudflare_aiProvider: RegistryEntry = {
baseUrl: "https://api.cloudflare.com/client/v4/accounts",
authType: "apikey",
authHeader: "bearer",
// 10K Neurons/day free: ~150 LLM responses or 500s Whisper audio — global edge
// 10K Neurons/day free: ~150 LLM responses or 500s Whisper audio — global edge.
// #8717: omit dead ids (llama-3.3-70b-instruct, llama-3.1-8b-instruct,
// gemma-3-12b-it, qwen2.5-coder-15b-instruct) — Workers AI returns 400/403/410.
models: [
{ id: "@cf/meta/llama-3.3-70b-instruct", name: "Llama 3.3 70B (🆓 ~150 resp/day)" },
{ id: "@cf/google/gemma-3-12b-it", name: "Gemma 3 12B (🆓)" },
{ id: "@cf/mistral/mistral-7b-instruct-v0.2-lora", name: "Mistral 7B (🆓)" },
{ id: "@cf/qwen/qwen2.5-coder-32b-instruct", name: "Qwen 2.5 Coder 32B (🆓)" },
{ id: "@cf/deepseek-ai/deepseek-r1-distill-qwen-32b", name: "DeepSeek R1 Distill 32B (🆓)" },
// Sweep 2026-06-19: + current Workers AI catalog ids (developers.cloudflare.com/workers-ai/models).
{ id: "@cf/meta/llama-3.3-70b-instruct-fp8-fast", name: "Llama 3.3 70B (FP8 Fast 🆓)" },
{ id: "@cf/meta/llama-3.3-70b-instruct-fp8-fast", name: "Llama 3.3 70B (FP8 Fast 🆓 ~150 resp/day)" },
{ id: "@cf/meta/llama-3.2-3b-instruct", name: "Llama 3.2 3B (🆓)" },
{ id: "@cf/qwen/qwq-32b", name: "QwQ 32B (🆓)" },
{
Expand Down
101 changes: 101 additions & 0 deletions tests/unit/cloudflare-workers-ai-catalog-8717.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
/**
* #8717 — Cloudflare Workers AI free catalog must not advertise dead model IDs.
*
* Four of the original six free-catalog entries return 400/403/410 from
* Workers AI (`No such model`, deprecated, or forbidden). #8763 added live
* replacements but left the dead IDs in place — free-model routing / combo
* builders still pick them and burn quota on guaranteed failures.
*
* Regression: dead IDs absent from FREE_MODEL_BUDGETS + cloudflare-ai registry;
* live replacements present; 30M monthlyTokens budget stays on the live
* Llama 3.3 70B FP8 Fast entry (poolKey cloudflare-ai dedupes by max).
*/
import test from "node:test";
import assert from "node:assert/strict";
import { FREE_MODEL_BUDGETS } from "../../open-sse/config/freeModelCatalog.ts";
import { cloudflare_aiProvider } from "../../open-sse/config/providers/registry/cloudflare-ai/index.ts";

const DEAD_CLOUDFLARE_MODEL_IDS = [
"@cf/meta/llama-3.3-70b-instruct",
"@cf/meta/llama-3.1-8b-instruct",
"@cf/google/gemma-3-12b-it",
"@cf/qwen/qwen2.5-coder-15b-instruct",
] as const;

const LIVE_CLOUDFLARE_MODEL_IDS = [
"@cf/mistral/mistral-7b-instruct-v0.2-lora",
"@cf/deepseek-ai/deepseek-r1-distill-qwen-32b",
"@cf/qwen/qwen2.5-coder-32b-instruct",
"@cf/meta/llama-3.3-70b-instruct-fp8-fast",
"@cf/meta/llama-3.2-3b-instruct",
"@cf/qwen/qwq-32b",
"@cf/zai-org/glm-4.7-flash",
"@cf/moonshotai/kimi-k2.6",
"@cf/google/gemma-4-26b-a4b-it",
] as const;

function cloudflareCatalogIds(): string[] {
return FREE_MODEL_BUDGETS.filter((m) => m.provider === "cloudflare-ai").map((m) => m.modelId);
}

function cloudflareRegistryIds(): string[] {
assert.equal(cloudflare_aiProvider.id, "cloudflare-ai");
return (cloudflare_aiProvider.models ?? []).map((m) => m.id);
}

test("#8717: FREE_MODEL_BUDGETS must not list dead Cloudflare Workers AI model IDs", () => {
const ids = cloudflareCatalogIds();
for (const dead of DEAD_CLOUDFLARE_MODEL_IDS) {
assert.equal(
ids.includes(dead),
false,
`dead model ${dead} must be removed from freeModelCatalog (still present)`
);
}
});

test("#8717: FREE_MODEL_BUDGETS must list live Cloudflare Workers AI replacements", () => {
const ids = cloudflareCatalogIds();
for (const live of LIVE_CLOUDFLARE_MODEL_IDS) {
assert.ok(ids.includes(live), `live model ${live} missing from freeModelCatalog`);
}
});

test("#8717: cloudflare-ai registry must not list dead model IDs", () => {
const ids = cloudflareRegistryIds();
for (const dead of DEAD_CLOUDFLARE_MODEL_IDS) {
assert.equal(
ids.includes(dead),
false,
`dead model ${dead} must be removed from cloudflare-ai registry (still present)`
);
}
});

test("#8717: cloudflare-ai registry must list live replacements", () => {
const ids = cloudflareRegistryIds();
for (const live of LIVE_CLOUDFLARE_MODEL_IDS) {
assert.ok(ids.includes(live), `live model ${live} missing from cloudflare-ai registry`);
}
});

test("#8717: 30M monthlyTokens budget moves to live Llama 3.3 70B FP8 Fast", () => {
const fp8 = FREE_MODEL_BUDGETS.find(
(m) =>
m.provider === "cloudflare-ai" && m.modelId === "@cf/meta/llama-3.3-70b-instruct-fp8-fast"
);
assert.ok(fp8, "fp8-fast entry must exist");
assert.equal(
fp8.monthlyTokens,
30_000_000,
"Neurons/day free budget (~30M tok/mo) must stay on the live Llama 3.3 entry"
);

// No other cloudflare-ai catalog row should still claim the 30M (poolKey dedupes by max,
// but advertising it on a dead id was the original bug surface).
const withBudget = FREE_MODEL_BUDGETS.filter(
(m) => m.provider === "cloudflare-ai" && m.monthlyTokens === 30_000_000
);
assert.equal(withBudget.length, 1);
assert.equal(withBudget[0].modelId, "@cf/meta/llama-3.3-70b-instruct-fp8-fast");
});
20 changes: 10 additions & 10 deletions tests/unit/executor-cloudflare-ai.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import { CloudflareAIExecutor } from "../../open-sse/executors/cloudflare-ai.ts"

test("CloudflareAIExecutor.buildUrl prefers providerSpecificData.accountId", () => {
const executor = new CloudflareAIExecutor();
const url = executor.buildUrl("@cf/meta/llama-3.3-70b-instruct", true, 0, {
const url = executor.buildUrl("@cf/meta/llama-3.3-70b-instruct-fp8-fast", true, 0, {
accountId: "top-level-id",
providerSpecificData: { accountId: "provider-id" },
});
Expand All @@ -22,10 +22,10 @@ test("CloudflareAIExecutor.buildUrl falls back to top-level credentials and envi
process.env.CLOUDFLARE_ACCOUNT_ID = "env-account-id";

try {
const fromTopLevel = executor.buildUrl("@cf/meta/llama-3.3-70b-instruct", false, 0, {
const fromTopLevel = executor.buildUrl("@cf/meta/llama-3.3-70b-instruct-fp8-fast", false, 0, {
accountId: "top-level-id",
});
const fromEnv = executor.buildUrl("@cf/meta/llama-3.3-70b-instruct", false, 0, {});
const fromEnv = executor.buildUrl("@cf/meta/llama-3.3-70b-instruct-fp8-fast", false, 0, {});

assert.equal(
fromTopLevel,
Expand All @@ -48,7 +48,7 @@ test("CloudflareAIExecutor.buildUrl throws when account ID is missing", () => {

try {
assert.throws(
() => executor.buildUrl("@cf/meta/llama-3.3-70b-instruct", true, 0, {}),
() => executor.buildUrl("@cf/meta/llama-3.3-70b-instruct-fp8-fast", true, 0, {}),
/Account ID/
);
} finally {
Expand Down Expand Up @@ -76,10 +76,10 @@ test("CloudflareAIExecutor.buildHeaders uses API key or access token and stream
test("CloudflareAIExecutor.transformRequest preserves plain-string content", () => {
const executor = new CloudflareAIExecutor();
const body = {
model: "@cf/meta/llama-3.3-70b-instruct",
model: "@cf/meta/llama-3.3-70b-instruct-fp8-fast",
messages: [{ role: "user", content: "hi" }],
};
const out = executor.transformRequest("@cf/meta/llama-3.3-70b-instruct", body, true, {} as any);
const out = executor.transformRequest("@cf/meta/llama-3.3-70b-instruct-fp8-fast", body, true, {} as any);
assert.deepEqual((out as any).messages, [{ role: "user", content: "hi" }]);
});

Expand All @@ -88,7 +88,7 @@ test("CloudflareAIExecutor.transformRequest preserves plain-string content", ()
test("CloudflareAIExecutor.transformRequest flattens content-part arrays to strings (#2539)", () => {
const executor = new CloudflareAIExecutor();
const body = {
model: "@cf/meta/llama-3.3-70b-instruct",
model: "@cf/meta/llama-3.3-70b-instruct-fp8-fast",
messages: [
{
role: "user",
Expand All @@ -101,7 +101,7 @@ test("CloudflareAIExecutor.transformRequest flattens content-part arrays to stri
],
};
const out = executor.transformRequest(
"@cf/meta/llama-3.3-70b-instruct",
"@cf/meta/llama-3.3-70b-instruct-fp8-fast",
body,
false,
{} as any
Expand Down Expand Up @@ -132,11 +132,11 @@ test("CloudflareAIExecutor.execute uses inherited BaseExecutor flow successfully

try {
const body = {
model: "@cf/meta/llama-3.3-70b-instruct",
model: "@cf/meta/llama-3.3-70b-instruct-fp8-fast",
messages: [{ role: "user", content: "hello" }],
};
const result = await executor.execute({
model: "@cf/meta/llama-3.3-70b-instruct",
model: "@cf/meta/llama-3.3-70b-instruct-fp8-fast",
body,
stream: false,
credentials: {
Expand Down
Loading