diff --git a/packages/ai/test/empty.test.ts b/packages/ai/test/empty.test.ts index 4539b3f39a..f0ffb5ee0a 100644 --- a/packages/ai/test/empty.test.ts +++ b/packages/ai/test/empty.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from "vitest"; import { getModel } from "../src/models.js"; import { complete } from "../src/stream.js"; import type { Api, AssistantMessage, Context, Model, StreamOptions, UserMessage } from "../src/types.js"; -import { getKimiCodingTestModel } from "./kimi-test-model.js"; +import { getCloudflareGatewayWorkersAiTestModel, getKimiCodingTestModel } from "./kimi-test-model.js"; import { getZaiTestModel } from "./zai-test-model.js"; type StreamOptionsWithExtras = StreamOptions & Record; @@ -329,25 +329,31 @@ describe("AI Providers Empty Message Tests", () => { }); }); - describe.skipIf(!hasCloudflareAiGatewayCredentials())("Cloudflare AI Gateway Provider Empty Messages", () => { - const llm = getModel("cloudflare-ai-gateway", "workers-ai/@cf/moonshotai/kimi-k2.6"); + // models.dev drops Workers AI ids between catalog revisions, so resolve instead of pin. + const cloudflareGatewayWorkersAi = getCloudflareGatewayWorkersAiTestModel(); - it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => { - await testEmptyMessage(llm); - }); + describe.skipIf(!hasCloudflareAiGatewayCredentials() || !cloudflareGatewayWorkersAi)( + "Cloudflare AI Gateway Provider Empty Messages", + () => { + const llm = cloudflareGatewayWorkersAi; - it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => { - await testEmptyStringMessage(llm); - }); + it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => { + await testEmptyMessage(llm); + }); - it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => { - await testWhitespaceOnlyMessage(llm); - }); + it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => { + await testEmptyStringMessage(llm); + }); - it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => { - await testEmptyAssistantMessage(llm); - }); - }); + it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => { + await testWhitespaceOnlyMessage(llm); + }); + + it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => { + await testEmptyAssistantMessage(llm); + }); + }, + ); describe.skipIf(!process.env.HF_TOKEN)("Hugging Face Provider Empty Messages", () => { const llm = getModel("huggingface", "moonshotai/Kimi-K2.5"); diff --git a/packages/ai/test/fireworks-models.test.ts b/packages/ai/test/fireworks-models.test.ts index 897968df4b..c6346e16d4 100644 --- a/packages/ai/test/fireworks-models.test.ts +++ b/packages/ai/test/fireworks-models.test.ts @@ -1,6 +1,6 @@ import { afterEach, describe, expect, it } from "vitest"; import { findEnvKeys, getEnvApiKey } from "../src/env-api-keys.js"; -import { getModel } from "../src/models.js"; +import { getModel, getModels } from "../src/models.js"; const originalFireworksApiKey = process.env.FIREWORKS_API_KEY; @@ -32,13 +32,19 @@ describe("Fireworks models", () => { }); }); - it("registers the Fire Pass turbo router model", () => { - const model = getModel("fireworks", "accounts/fireworks/routers/kimi-k2p6-turbo"); - - expect(model).toBeDefined(); - expect(model.api).toBe("anthropic-messages"); - expect(model.baseUrl).toBe("https://api.fireworks.ai/inference"); - expect(model.input).toEqual(["text", "image"]); + // models.dev swaps router ids between catalog revisions (kimi-k2p6-turbo → + // kimi-k3-fast), so resolve the current router model instead of pinning one. + // api and baseUrl are generator invariants for every fireworks entry; input + // mirrors live modality data, so only its text floor is asserted. + const routerModel = getModels("fireworks") + .filter((model) => model.id.startsWith("accounts/fireworks/routers/")) + .sort((a, b) => b.id.localeCompare(a.id))[0]; + + it.skipIf(!routerModel)("registers Fire Pass router models", () => { + expect(routerModel).toBeDefined(); + expect(routerModel.api).toBe("anthropic-messages"); + expect(routerModel.baseUrl).toBe("https://api.fireworks.ai/inference"); + expect(routerModel.input).toContain("text"); }); it("resolves FIREWORKS_API_KEY from the environment", () => { diff --git a/packages/ai/test/kimi-test-model.ts b/packages/ai/test/kimi-test-model.ts index 43e9b35ad7..21d506c0c6 100644 --- a/packages/ai/test/kimi-test-model.ts +++ b/packages/ai/test/kimi-test-model.ts @@ -1,8 +1,28 @@ import { getModels } from "../src/models.js"; -import type { Model } from "../src/types.js"; +import type { Api, Model } from "../src/types.js"; const KIMI_TEST_MODEL_PREFERENCE = ["kimi-k2-thinking", "kimi-for-coding", "k2p7", "k3", "kimi-for-coding-highspeed"]; +// models.dev drops and renames Workers AI ids between catalog revisions +// (workers-ai/@cf/moonshotai/kimi-k2.6 vanished from the cloudflare-ai-gateway +// listing), so resolve the current /compat model at runtime rather than pinning +// an id the next revision invalidates. Kimi ids are preferred to keep the +// exercised model comparable across revisions; any workers-ai /compat model +// keeps the transport tests alive when no Kimi is listed. Callers must guard +// with skipIf: the result is undefined when the catalog lists no workers-ai +// model at all. Everything here is typed against the Api base rather than the +// provider's current api union — that union is itself regenerated from the +// catalog, so narrowing against it breaks exactly when the catalog moves. The +// final cast is safe because the generator emits every workers-ai/ gateway +// entry as an openai-completions /compat route. +export function getCloudflareGatewayWorkersAiTestModel(): Model<"openai-completions"> { + const gatewayModels: Model[] = getModels("cloudflare-ai-gateway"); + const models = gatewayModels.filter((model) => model.id.startsWith("workers-ai/")); + const kimis = models.filter((model) => model.id.includes("/moonshotai/kimi-")); + const pool = kimis.length > 0 ? kimis : models; + return pool.sort((a, b) => b.id.localeCompare(a.id))[0] as Model<"openai-completions">; +} + export function getKimiCodingTestModel(options: { image?: boolean } = {}): Model<"anthropic-messages"> { const models = getModels("kimi-coding") as Model<"anthropic-messages">[]; const eligible = options.image ? models.filter((model) => model.input.includes("image")) : models; diff --git a/packages/ai/test/openai-completions-empty-tools.test.ts b/packages/ai/test/openai-completions-empty-tools.test.ts index 0fbdc7ab34..484067d665 100644 --- a/packages/ai/test/openai-completions-empty-tools.test.ts +++ b/packages/ai/test/openai-completions-empty-tools.test.ts @@ -2,6 +2,10 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import { getModel } from "../src/models.js"; import { streamSimple } from "../src/stream.js"; import type { Model } from "../src/types.js"; +import { getCloudflareGatewayWorkersAiTestModel } from "./kimi-test-model.js"; + +// models.dev drops Workers AI ids between catalog revisions, so resolve instead of pin. +const cloudflareGatewayWorkersAi = getCloudflareGatewayWorkersAiTestModel(); // Empty tools arrays must NOT be serialized as `tools: []` — some OpenAI-compatible // backends (e.g. DashScope / Aliyun Qwen via compatible-mode) reject the request with @@ -94,41 +98,44 @@ describe("openai-completions empty tools handling", () => { expect("tools" in (params as object)).toBe(false); }); - it("uses conservative OpenAI-compatible fields for Cloudflare AI Gateway /compat models", async () => { - process.env.CLOUDFLARE_ACCOUNT_ID = "account-id"; - process.env.CLOUDFLARE_GATEWAY_ID = "gateway-id"; - const model = getModel("cloudflare-ai-gateway", "workers-ai/@cf/moonshotai/kimi-k2.6")!; + it.skipIf(!cloudflareGatewayWorkersAi)( + "uses conservative OpenAI-compatible fields for Cloudflare AI Gateway /compat models", + async () => { + process.env.CLOUDFLARE_ACCOUNT_ID = "account-id"; + process.env.CLOUDFLARE_GATEWAY_ID = "gateway-id"; + const model = cloudflareGatewayWorkersAi; - await streamSimple( - model, - { - systemPrompt: "You are helpful.", - messages: [{ role: "user", content: "hi", timestamp: Date.now() }], - }, - { apiKey: "test", reasoning: "high" }, - ).result(); + await streamSimple( + model, + { + systemPrompt: "You are helpful.", + messages: [{ role: "user", content: "hi", timestamp: Date.now() }], + }, + { apiKey: "test", reasoning: "high" }, + ).result(); - const params = mockState.lastParams as { - messages: Array<{ role: string }>; - max_tokens?: number; - max_completion_tokens?: number; - reasoning_effort?: string; - store?: boolean; - }; - expect(params.messages[0].role).toBe("system"); - expect(params.max_tokens).toBeDefined(); - expect(params.max_completion_tokens).toBeUndefined(); - expect(params.reasoning_effort).toBeUndefined(); - expect(params.store).toBeUndefined(); - - const clientOptions = mockState.lastClientOptions as { - baseURL?: string; - defaultHeaders?: Record; - }; - expect(clientOptions.baseURL).toBe("https://gateway.ai.cloudflare.com/v1/account-id/gateway-id/compat"); - expect(clientOptions.defaultHeaders?.Authorization).toBeNull(); - expect(clientOptions.defaultHeaders?.["cf-aig-authorization"]).toBe("Bearer test"); - }); + const params = mockState.lastParams as { + messages: Array<{ role: string }>; + max_tokens?: number; + max_completion_tokens?: number; + reasoning_effort?: string; + store?: boolean; + }; + expect(params.messages[0].role).toBe("system"); + expect(params.max_tokens).toBeDefined(); + expect(params.max_completion_tokens).toBeUndefined(); + expect(params.reasoning_effort).toBeUndefined(); + expect(params.store).toBeUndefined(); + + const clientOptions = mockState.lastClientOptions as { + baseURL?: string; + defaultHeaders?: Record; + }; + expect(clientOptions.baseURL).toBe("https://gateway.ai.cloudflare.com/v1/account-id/gateway-id/compat"); + expect(clientOptions.defaultHeaders?.Authorization).toBeNull(); + expect(clientOptions.defaultHeaders?.["cf-aig-authorization"]).toBe("Bearer test"); + }, + ); it("uses OpenAI reasoning fields for an explicitly configured private Prime Inference route", async () => { const model: Model<"openai-completions"> = { @@ -179,24 +186,27 @@ describe("openai-completions empty tools handling", () => { expect(clientOptions.defaultHeaders?.["cf-aig-authorization"]).toBe("Bearer cf-token"); }); - it("sends session affinity headers for Workers AI through Cloudflare AI Gateway", async () => { - process.env.CLOUDFLARE_ACCOUNT_ID = "account-id"; - process.env.CLOUDFLARE_GATEWAY_ID = "gateway-id"; - const workersModel = getModel("cloudflare-ai-gateway", "workers-ai/@cf/moonshotai/kimi-k2.6")!; + it.skipIf(!cloudflareGatewayWorkersAi)( + "sends session affinity headers for Workers AI through Cloudflare AI Gateway", + async () => { + process.env.CLOUDFLARE_ACCOUNT_ID = "account-id"; + process.env.CLOUDFLARE_GATEWAY_ID = "gateway-id"; + const workersModel = cloudflareGatewayWorkersAi; - await streamSimple( - workersModel, - { - messages: [{ role: "user", content: "hi", timestamp: Date.now() }], - }, - { apiKey: "test", sessionId: "session-1" }, - ).result(); + await streamSimple( + workersModel, + { + messages: [{ role: "user", content: "hi", timestamp: Date.now() }], + }, + { apiKey: "test", sessionId: "session-1" }, + ).result(); - const clientOptions = mockState.lastClientOptions as { defaultHeaders?: Record }; - expect(clientOptions.defaultHeaders?.session_id).toBe("session-1"); - expect(clientOptions.defaultHeaders?.["x-client-request-id"]).toBe("session-1"); - expect(clientOptions.defaultHeaders?.["x-session-affinity"]).toBe("session-1"); - }); + const clientOptions = mockState.lastClientOptions as { defaultHeaders?: Record }; + expect(clientOptions.defaultHeaders?.session_id).toBe("session-1"); + expect(clientOptions.defaultHeaders?.["x-client-request-id"]).toBe("session-1"); + expect(clientOptions.defaultHeaders?.["x-session-affinity"]).toBe("session-1"); + }, + ); it("still emits tools: [] for Anthropic/LiteLLM proxy when conversation has tool history", async () => { const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini")!; diff --git a/packages/ai/test/prime-inference-models.test.ts b/packages/ai/test/prime-inference-models.test.ts index f28ac159aa..f17032f6bd 100644 --- a/packages/ai/test/prime-inference-models.test.ts +++ b/packages/ai/test/prime-inference-models.test.ts @@ -82,8 +82,10 @@ describe("Prime Inference models", () => { expect(model.input).toEqual(["text", "image"]); expect(model.contextWindow).toBe(1048576); expect(model.maxTokens).toBe(1048576); - expect(model.cost.input).toBe(3); - expect(model.cost.output).toBe(15); + // Live catalogs reprice between revisions (3 → 3.45 broke an exact pin), + // so assert the cost shape rather than a price models.dev controls. + expect(model.cost.input).toBeGreaterThan(0); + expect(model.cost.output).toBeGreaterThan(0); } }); diff --git a/packages/ai/test/stream.test.ts b/packages/ai/test/stream.test.ts index 6309455992..8fae3f3028 100644 --- a/packages/ai/test/stream.test.ts +++ b/packages/ai/test/stream.test.ts @@ -8,7 +8,7 @@ import { getEnvApiKey } from "../src/env-api-keys.js"; import { getModel, getModels } from "../src/models.js"; import { complete, stream } from "../src/stream.js"; import type { Api, Context, ImageContent, Model, StreamOptions, Tool, ToolResultMessage } from "../src/types.js"; -import { getKimiCodingTestModel } from "./kimi-test-model.js"; +import { getCloudflareGatewayWorkersAiTestModel, getKimiCodingTestModel } from "./kimi-test-model.js"; import { getZaiTestModel } from "./zai-test-model.js"; type StreamOptionsWithExtras = StreamOptions & Record; @@ -666,10 +666,14 @@ describe("Generate E2E Tests", () => { }, ); - describe.skipIf(!hasCloudflareAiGatewayCredentials())( - "Cloudflare AI Gateway → Workers AI (Kimi K2.6 via /compat)", + // models.dev also drops Workers AI ids between catalog revisions + // (workers-ai/@cf/moonshotai/kimi-k2.6 vanished), so resolve instead of pin. + const cloudflareGatewayWorkersAi = getCloudflareGatewayWorkersAiTestModel(); + + describe.skipIf(!hasCloudflareAiGatewayCredentials() || !cloudflareGatewayWorkersAi)( + "Cloudflare AI Gateway → Workers AI (via /compat)", () => { - const llm = getModel("cloudflare-ai-gateway", "workers-ai/@cf/moonshotai/kimi-k2.6"); + const llm = cloudflareGatewayWorkersAi; it("should complete basic text generation", { retry: 3 }, async () => { await basicTextGeneration(llm); diff --git a/packages/ai/test/tokens.test.ts b/packages/ai/test/tokens.test.ts index 5fceca1e06..1c8c5b6c02 100644 --- a/packages/ai/test/tokens.test.ts +++ b/packages/ai/test/tokens.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from "vitest"; import { getModel } from "../src/models.js"; import { stream } from "../src/stream.js"; import type { Api, Context, Model, StreamOptions } from "../src/types.js"; -import { getKimiCodingTestModel } from "./kimi-test-model.js"; +import { getCloudflareGatewayWorkersAiTestModel, getKimiCodingTestModel } from "./kimi-test-model.js"; import { getZaiTestModel } from "./zai-test-model.js"; type StreamOptionsWithExtras = StreamOptions & Record; @@ -166,13 +166,19 @@ describe("Token Statistics on Abort", () => { }); }); - describe.skipIf(!hasCloudflareAiGatewayCredentials())("Cloudflare AI Gateway Provider", () => { - const llm = getModel("cloudflare-ai-gateway", "workers-ai/@cf/moonshotai/kimi-k2.6"); + // models.dev drops Workers AI ids between catalog revisions, so resolve instead of pin. + const cloudflareGatewayWorkersAi = getCloudflareGatewayWorkersAiTestModel(); - it("should include token stats when aborted mid-stream", { retry: 3, timeout: 30000 }, async () => { - await testTokensOnAbort(llm); - }); - }); + describe.skipIf(!hasCloudflareAiGatewayCredentials() || !cloudflareGatewayWorkersAi)( + "Cloudflare AI Gateway Provider", + () => { + const llm = cloudflareGatewayWorkersAi; + + it("should include token stats when aborted mid-stream", { retry: 3, timeout: 30000 }, async () => { + await testTokensOnAbort(llm); + }); + }, + ); describe.skipIf(!process.env.HF_TOKEN)("Hugging Face Provider", () => { const llm = getModel("huggingface", "moonshotai/Kimi-K2.5"); diff --git a/packages/ai/test/tool-call-without-result.test.ts b/packages/ai/test/tool-call-without-result.test.ts index 39c314a9a2..51881a2a6a 100644 --- a/packages/ai/test/tool-call-without-result.test.ts +++ b/packages/ai/test/tool-call-without-result.test.ts @@ -3,7 +3,7 @@ import { describe, expect, it } from "vitest"; import { getModel } from "../src/models.js"; import { complete } from "../src/stream.js"; import type { Api, Context, Model, StreamOptions, Tool } from "../src/types.js"; -import { getKimiCodingTestModel } from "./kimi-test-model.js"; +import { getCloudflareGatewayWorkersAiTestModel, getKimiCodingTestModel } from "./kimi-test-model.js"; import { getZaiTestModel } from "./zai-test-model.js"; type StreamOptionsWithExtras = StreamOptions & Record; @@ -177,13 +177,23 @@ describe("Tool Call Without Result Tests", () => { }); }); - describe.skipIf(!hasCloudflareAiGatewayCredentials())("Cloudflare AI Gateway Provider", () => { - const model = getModel("cloudflare-ai-gateway", "workers-ai/@cf/moonshotai/kimi-k2.6"); - - it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => { - await testToolCallWithoutResult(model); - }); - }); + // models.dev drops Workers AI ids between catalog revisions, so resolve instead of pin. + const cloudflareGatewayWorkersAi = getCloudflareGatewayWorkersAiTestModel(); + + describe.skipIf(!hasCloudflareAiGatewayCredentials() || !cloudflareGatewayWorkersAi)( + "Cloudflare AI Gateway Provider", + () => { + const model = cloudflareGatewayWorkersAi; + + it( + "should filter out tool calls without corresponding tool results", + { retry: 3, timeout: 30000 }, + async () => { + await testToolCallWithoutResult(model); + }, + ); + }, + ); describe.skipIf(!process.env.HF_TOKEN)("Hugging Face Provider", () => { const model = getModel("huggingface", "moonshotai/Kimi-K2.5"); diff --git a/packages/ai/test/total-tokens.test.ts b/packages/ai/test/total-tokens.test.ts index 4d79c64352..76ffa72fc0 100644 --- a/packages/ai/test/total-tokens.test.ts +++ b/packages/ai/test/total-tokens.test.ts @@ -16,7 +16,7 @@ import { describe, expect, it } from "vitest"; import { getModel } from "../src/models.js"; import { complete } from "../src/stream.js"; import type { Api, Context, Model, StreamOptions, Usage } from "../src/types.js"; -import { getKimiCodingTestModel } from "./kimi-test-model.js"; +import { getCloudflareGatewayWorkersAiTestModel, getKimiCodingTestModel } from "./kimi-test-model.js"; import { getZaiTestModel } from "./zai-test-model.js"; type StreamOptionsWithExtras = StreamOptions & Record; @@ -332,12 +332,15 @@ describe("totalTokens field", () => { // Cloudflare AI Gateway // ========================================================================= - describe.skipIf(!hasCloudflareAiGatewayCredentials())("Cloudflare AI Gateway", () => { + // models.dev drops Workers AI ids between catalog revisions, so resolve instead of pin. + const cloudflareGatewayWorkersAi = getCloudflareGatewayWorkersAiTestModel(); + + describe.skipIf(!hasCloudflareAiGatewayCredentials() || !cloudflareGatewayWorkersAi)("Cloudflare AI Gateway", () => { it( - "workers-ai/@cf/moonshotai/kimi-k2.6 - should return totalTokens equal to sum of components", + "workers-ai model - should return totalTokens equal to sum of components", { retry: 3, timeout: 60000 }, async () => { - const llm = getModel("cloudflare-ai-gateway", "workers-ai/@cf/moonshotai/kimi-k2.6"); + const llm = cloudflareGatewayWorkersAi; console.log(`\nCloudflare AI Gateway / ${llm.id}:`); const { first, second } = await testTotalTokensWithCache(llm, { diff --git a/packages/ai/test/unicode-surrogate.test.ts b/packages/ai/test/unicode-surrogate.test.ts index f923ee77d7..2056c9bea1 100644 --- a/packages/ai/test/unicode-surrogate.test.ts +++ b/packages/ai/test/unicode-surrogate.test.ts @@ -3,7 +3,7 @@ import { describe, expect, it } from "vitest"; import { getModel } from "../src/models.js"; import { complete } from "../src/stream.js"; import type { Api, Context, Model, StreamOptions, ToolResultMessage } from "../src/types.js"; -import { getKimiCodingTestModel } from "./kimi-test-model.js"; +import { getCloudflareGatewayWorkersAiTestModel, getKimiCodingTestModel } from "./kimi-test-model.js"; import { getZaiTestModel } from "./zai-test-model.js"; type StreamOptionsWithExtras = StreamOptions & Record; @@ -516,21 +516,31 @@ describe("AI Providers Unicode Surrogate Pair Tests", () => { }); }); - describe.skipIf(!hasCloudflareAiGatewayCredentials())("Cloudflare AI Gateway Provider Unicode Handling", () => { - const llm = getModel("cloudflare-ai-gateway", "workers-ai/@cf/moonshotai/kimi-k2.6"); + // models.dev drops Workers AI ids between catalog revisions, so resolve instead of pin. + const cloudflareGatewayWorkersAi = getCloudflareGatewayWorkersAiTestModel(); - it("should handle emoji in tool results", { retry: 3, timeout: 30000 }, async () => { - await testEmojiInToolResults(llm); - }); + describe.skipIf(!hasCloudflareAiGatewayCredentials() || !cloudflareGatewayWorkersAi)( + "Cloudflare AI Gateway Provider Unicode Handling", + () => { + const llm = cloudflareGatewayWorkersAi; - it("should handle real-world LinkedIn comment data with emoji", { retry: 3, timeout: 30000 }, async () => { - await testRealWorldLinkedInData(llm); - }); + it("should handle emoji in tool results", { retry: 3, timeout: 30000 }, async () => { + await testEmojiInToolResults(llm); + }); - it("should handle unpaired high surrogate (0xD83D) in tool results", { retry: 3, timeout: 30000 }, async () => { - await testUnpairedHighSurrogate(llm); - }); - }); + it("should handle real-world LinkedIn comment data with emoji", { retry: 3, timeout: 30000 }, async () => { + await testRealWorldLinkedInData(llm); + }); + + it( + "should handle unpaired high surrogate (0xD83D) in tool results", + { retry: 3, timeout: 30000 }, + async () => { + await testUnpairedHighSurrogate(llm); + }, + ); + }, + ); describe.skipIf(!process.env.HF_TOKEN)("Hugging Face Provider Unicode Handling", () => { const llm = getModel("huggingface", "moonshotai/Kimi-K2.5");