From f1a7bdbda26d771828db3a36a77b5af60e5f3884 Mon Sep 17 00:00:00 2001 From: Li Tai Chen Date: Mon, 10 Aug 2026 12:53:14 -0400 Subject: [PATCH 1/2] feat(models): add InclusionAI Ling-3.0-flash via DeepInfra and NovitaAI Adds the Ling-3.0-flash mapping (DeepInfra + NovitaAI) with chatTemplateThinkingKey reasoning handling plus unit tests covering thinking translation, tool calling, and JSON output on both providers. Co-Authored-By: Claude --- apps/gateway/src/api.spec.ts | 454 ++++++++++++++++++ .../tools/validate-model-capabilities.spec.ts | 32 ++ .../chat/tools/validate-model-capabilities.ts | 8 +- .../actions/src/prepare-request-body.spec.ts | 207 ++++++++ packages/actions/src/prepare-request-body.ts | 43 +- packages/models/src/models.ts | 14 + packages/models/src/models/inclusionai.ts | 78 +++ 7 files changed, 818 insertions(+), 18 deletions(-) create mode 100644 packages/models/src/models/inclusionai.ts diff --git a/apps/gateway/src/api.spec.ts b/apps/gateway/src/api.spec.ts index 6aad957c1c..91fe13341f 100644 --- a/apps/gateway/src/api.spec.ts +++ b/apps/gateway/src/api.spec.ts @@ -8592,4 +8592,458 @@ describe("api", () => { expect(JSON.stringify(json)).toContain("input_schema"); }); }); + + describe("native /v1/messages reasoning controls for Ling-3.0-flash", () => { + // Ling-3.0-flash is a hybrid reasoning model that thinks by default; its + // only reasoning control is the vLLM chat-template flag `enable_thinking` + // (declared via `chatTemplateThinkingKey` on the DeepInfra mapping). + // These tests exercise the full Anthropic -> unified reasoning + // -> chat_template_kwargs chain: `thinking` controls on the native + // /v1/messages lane must reach the upstream body as + // `chat_template_kwargs.enable_thinking` and never leak `reasoning_effort`. + // Novita is an exception: its backend ignores the chat-template flag + // (verified live 2026-08-10), so its tests assert no kwargs and thinking + // stays on. + const lingProviders = ["deepinfra", "novita"] as const; + const upstreamModels = { + deepinfra: "inclusionAI/Ling-3.0-flash", + novita: "inclusionai/ling-3.0-flash", + } as const; + + // Anthropic custom tool + the OpenAI function tool it translates to + // (mirrors the mapping in anthropic.ts so the forwarded tools can be + // asserted verbatim). + const weatherTool = { + name: "get_weather", + description: "Get the weather for a city", + input_schema: { + type: "object", + properties: { city: { type: "string" } }, + required: ["city"], + }, + }; + const expectedOpenaiTool = [ + { + type: "function", + function: { + name: "get_weather", + description: "Get the weather for a city", + parameters: { + type: "object", + properties: { city: { type: "string" } }, + required: ["city"], + }, + }, + }, + ]; + + // The upstream OpenAI-compatible body the gateway sends for a Ling + // request, plus the chat-template flag that controls thinking. + interface CapturedLingBody { + model?: string; + reasoning_effort?: string; + chat_template_kwargs?: Record; + tools?: unknown[]; + thinking?: unknown; + anthropic_version?: unknown; + } + + // Insert the API key + provider key(s) pinned to the mock server, then + // capture the body the gateway sends upstream. deepinfra/novita POST to + // `${baseUrl}/chat/completions` (no `/v1/` prefix), which the mock server + // doesn't serve, so the spy returns the canned completion itself. + // `model` defaults to the prefixed id; pass a bare id to exercise standard + // routing, which needs both providers' keys present to resolve a provider. + async function exerciseLingMessages( + provider: (typeof lingProviders)[number], + body: Record, + options: { + model?: string; + insertBothProviderKeys?: boolean; + insertNoProviderKeys?: boolean; + } = {}, + ) { + await db.insert(tables.apiKey).values({ + id: "token-id", + token: "real-token", + projectId: "project-id", + description: "Test API Key", + createdBy: "user-id", + }); + + const providerIds = options.insertNoProviderKeys + ? [] + : options.insertBothProviderKeys + ? lingProviders + : [provider]; + + for (const [index, providerId] of providerIds.entries()) { + await db.insert(tables.providerKey).values({ + id: `provider-key-id-${index}`, + token: "sk-test-key", + provider: providerId, + organizationId: "org-id", + baseUrl: mockServerUrl, + }); + } + + let capturedBody: CapturedLingBody | undefined; + const originalFetch = globalThis.fetch; + const fetchSpy = vi + .spyOn(globalThis, "fetch") + .mockImplementation(async (input, init) => { + const url = + typeof input === "string" + ? input + : input instanceof URL + ? input.toString() + : input.url; + + if (url === `${mockServerUrl}/chat/completions`) { + capturedBody = JSON.parse( + String(init?.body ?? ""), + ) as CapturedLingBody; + return new Response( + JSON.stringify({ + id: "chatcmpl-ling-3.0-flash", + object: "chat.completion", + created: 1774549411, + model: options.model + ? "inclusionAI/Ling-3.0-flash" + : upstreamModels[provider], + choices: [ + { + index: 0, + message: { + role: "assistant", + content: "It's sunny in Paris.", + }, + finish_reason: "stop", + }, + ], + usage: { + prompt_tokens: 5, + completion_tokens: 3, + total_tokens: 8, + }, + }), + { + status: 200, + headers: { "Content-Type": "application/json" }, + }, + ); + } + + return await originalFetch(input as RequestInfo | URL, init); + }); + + try { + const res = await app.request("/v1/messages", { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: "Bearer real-token", + "x-no-fallback": "true", + }, + body: JSON.stringify({ + model: options.model ?? `${provider}/ling-3.0-flash`, + max_tokens: 1024, + messages: [ + { role: "user", content: "What is the weather in Paris?" }, + ], + ...body, + }), + }); + + return { res, capturedBody }; + } finally { + fetchSpy.mockRestore(); + } + } + + test.each(lingProviders)( + "maps thinking disabled to enable_thinking false upstream on %s", + async (provider) => { + const { res, capturedBody } = await exerciseLingMessages(provider, { + thinking: { type: "disabled" }, + }); + + expect(res.status).toBe(200); + // DeepInfra honors the chat-template flag; Novita ignores it + // (verified live 2026-08-10), so thinking stays on and no flag is + // sent upstream. + if (provider === "deepinfra") { + expect(capturedBody?.chat_template_kwargs).toEqual({ + enable_thinking: false, + }); + } else { + expect(capturedBody?.chat_template_kwargs).toBeUndefined(); + } + expect(capturedBody?.reasoning_effort).toBeUndefined(); + // The Anthropic-only thinking block must not leak into the upstream + // OpenAI-compatible body. + expect(capturedBody?.thinking).toBeUndefined(); + expect(capturedBody?.anthropic_version).toBeUndefined(); + }, + ); + + test.each(lingProviders)( + "maps thinking adaptive to enable_thinking true upstream on %s", + async (provider) => { + const { res, capturedBody } = await exerciseLingMessages(provider, { + thinking: { type: "adaptive" }, + }); + + expect(res.status).toBe(200); + if (provider === "deepinfra") { + expect(capturedBody?.chat_template_kwargs).toEqual({ + enable_thinking: true, + }); + } else { + expect(capturedBody?.chat_template_kwargs).toBeUndefined(); + } + expect(capturedBody?.reasoning_effort).toBeUndefined(); + expect(capturedBody?.thinking).toBeUndefined(); + expect(capturedBody?.anthropic_version).toBeUndefined(); + }, + ); + + // The most common Anthropic shape (what Claude Code sends): extended + // thinking with a token budget. Ling controls thinking through a binary + // chat-template flag, so the budget is dropped and thinking is enabled. + // DeepInfra honors the chat-template flag and accepts a budget (dropped to + // a binary toggle). Novita has no thinking control at all, so the budget + // request is rejected with Anthropic's "thinking not supported" 400 — the + // honest outcome for a backend that always thinks. + test.each(["deepinfra"] as const)( + "maps thinking enabled with budget to enable_thinking true upstream on %s", + async (provider) => { + const { res, capturedBody } = await exerciseLingMessages(provider, { + thinking: { type: "enabled", budget_tokens: 2048 }, + }); + + expect(res.status).toBe(200); + expect(capturedBody?.chat_template_kwargs).toEqual({ + enable_thinking: true, + }); + expect(capturedBody?.reasoning_effort).toBeUndefined(); + expect(capturedBody?.thinking).toBeUndefined(); + expect(capturedBody?.anthropic_version).toBeUndefined(); + }, + ); + + test("rejects thinking enabled with budget on novita (thinking is always on, no control)", async () => { + const { res } = await exerciseLingMessages("novita", { + thinking: { type: "enabled", budget_tokens: 2048 }, + }); + + expect(res.status).toBe(400); + const json = (await res.json()) as { + error?: { message?: string }; + }; + expect(json.error?.message).toContain( + 'Remove the "thinking" parameter or use a model that supports extended thinking', + ); + }); + + test.each(lingProviders)( + "forwards tools intact and applies enable_thinking false with thinking disabled on %s", + async (provider) => { + const { res, capturedBody } = await exerciseLingMessages(provider, { + thinking: { type: "disabled" }, + tools: [weatherTool], + }); + + expect(res.status).toBe(200); + expect(capturedBody?.tools).toEqual(expectedOpenaiTool); + if (provider === "deepinfra") { + expect(capturedBody?.chat_template_kwargs).toEqual({ + enable_thinking: false, + }); + } else { + expect(capturedBody?.chat_template_kwargs).toBeUndefined(); + } + expect(capturedBody?.reasoning_effort).toBeUndefined(); + expect(capturedBody?.thinking).toBeUndefined(); + expect(capturedBody?.anthropic_version).toBeUndefined(); + }, + ); + + test.each(lingProviders)( + "keeps the provider default (thinking on) with no thinking control on %s", + async (provider) => { + const { res, capturedBody } = await exerciseLingMessages(provider, {}); + + expect(res.status).toBe(200); + expect(capturedBody?.chat_template_kwargs).toBeUndefined(); + expect(capturedBody?.reasoning_effort).toBeUndefined(); + expect(capturedBody?.thinking).toBeUndefined(); + expect(capturedBody?.anthropic_version).toBeUndefined(); + }, + ); + + // A client that can't send a provider prefix (e.g. devpass) relies on the + // gateway's standard routing to resolve a bare catalog id to a provider. + // Both DeepInfra and Novita mappings are active and keyed, so routing + // picks the cheapest (DeepInfra, $0.045 vs $0.06 input) and the request + // still goes through the same chat-template thinking translation. + test("resolves a bare ling-3.0-flash id to the cheapest Ling provider and applies enable_thinking false", async () => { + const { res, capturedBody } = await exerciseLingMessages( + "deepinfra", + { thinking: { type: "disabled" } }, + { model: "ling-3.0-flash", insertBothProviderKeys: true }, + ); + + expect(res.status).toBe(200); + // Standard routing lands on DeepInfra (cheapest of the two keyed + // Ling mappings), so the upstream model is its external id. + expect(capturedBody?.model).toBe("inclusionAI/Ling-3.0-flash"); + expect(capturedBody?.chat_template_kwargs).toEqual({ + enable_thinking: false, + }); + expect(capturedBody?.reasoning_effort).toBeUndefined(); + }); + + // With only one of the two Ling providers keyed, bare-id routing resolves + // to that single provider instead of failing. + test.each(lingProviders)( + "resolves a bare ling-3.0-flash id to the only keyed provider (%s) and applies enable_thinking false", + async (provider) => { + const { res, capturedBody } = await exerciseLingMessages( + provider, + { thinking: { type: "disabled" } }, + { model: "ling-3.0-flash" }, + ); + + expect(res.status).toBe(200); + expect(capturedBody?.model).toBe(upstreamModels[provider]); + if (provider === "deepinfra") { + expect(capturedBody?.chat_template_kwargs).toEqual({ + enable_thinking: false, + }); + } else { + expect(capturedBody?.chat_template_kwargs).toBeUndefined(); + } + expect(capturedBody?.reasoning_effort).toBeUndefined(); + }, + ); + + // With neither provider keyed, bare-id routing has no candidate provider + // and the request fails with the standard no-provider-key error. + test("rejects a bare ling-3.0-flash id when no Ling provider has a key", async () => { + const { res, capturedBody } = await exerciseLingMessages( + "deepinfra", + { thinking: { type: "disabled" } }, + { model: "ling-3.0-flash", insertNoProviderKeys: true }, + ); + + expect(res.status).toBe(400); + const json = await res.json(); + expect(JSON.stringify(json)).toContain( + "No provider key set for any of the providers that support model ling-3.0-flash", + ); + expect(capturedBody).toBeUndefined(); + }); + + // Native /v1/chat/completions twin of the bare-id /v1/messages test: a + // bare catalog id resolves through standard routing to the cheapest Ling + // provider (DeepInfra), and `reasoning_effort: "none"` (the OpenAI-path + // thinking control) reaches the upstream body as `enable_thinking: false`. + test("native OpenAI path resolves a bare ling-3.0-flash id to DeepInfra and maps reasoning none to enable_thinking false", async () => { + await db.insert(tables.apiKey).values({ + id: "token-id", + token: "real-token", + projectId: "project-id", + description: "Test API Key", + createdBy: "user-id", + }); + + for (const [index, providerId] of lingProviders.entries()) { + await db.insert(tables.providerKey).values({ + id: `provider-key-id-${index}`, + token: "sk-test-key", + provider: providerId, + organizationId: "org-id", + baseUrl: mockServerUrl, + }); + } + + let capturedBody: CapturedLingBody | undefined; + const originalFetch = globalThis.fetch; + const fetchSpy = vi + .spyOn(globalThis, "fetch") + .mockImplementation(async (input, init) => { + const url = + typeof input === "string" + ? input + : input instanceof URL + ? input.toString() + : input.url; + + if (url === `${mockServerUrl}/chat/completions`) { + capturedBody = JSON.parse( + String(init?.body ?? ""), + ) as CapturedLingBody; + return new Response( + JSON.stringify({ + id: "chatcmpl-ling-3.0-flash", + object: "chat.completion", + created: 1774549411, + model: "inclusionAI/Ling-3.0-flash", + choices: [ + { + index: 0, + message: { + role: "assistant", + content: "It's sunny in Paris.", + }, + finish_reason: "stop", + }, + ], + usage: { + prompt_tokens: 5, + completion_tokens: 3, + total_tokens: 8, + }, + }), + { + status: 200, + headers: { "Content-Type": "application/json" }, + }, + ); + } + + return await originalFetch(input as RequestInfo | URL, init); + }); + + try { + const res = await app.request("/v1/chat/completions", { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: "Bearer real-token", + "x-no-fallback": "true", + }, + body: JSON.stringify({ + model: "ling-3.0-flash", + reasoning_effort: "none", + messages: [ + { role: "user", content: "What is the weather in Paris?" }, + ], + }), + }); + + expect(res.status).toBe(200); + const json = await res.json(); + expect(json.metadata?.used_provider).toBe("deepinfra"); + expect(capturedBody?.model).toBe("inclusionAI/Ling-3.0-flash"); + expect(capturedBody?.chat_template_kwargs).toEqual({ + enable_thinking: false, + }); + expect(capturedBody?.reasoning_effort).toBeUndefined(); + } finally { + fetchSpy.mockRestore(); + } + }); + }); }); diff --git a/apps/gateway/src/chat/tools/validate-model-capabilities.spec.ts b/apps/gateway/src/chat/tools/validate-model-capabilities.spec.ts index 816c4ec128..2f14fbd2b3 100644 --- a/apps/gateway/src/chat/tools/validate-model-capabilities.spec.ts +++ b/apps/gateway/src/chat/tools/validate-model-capabilities.spec.ts @@ -324,3 +324,35 @@ describe("validateModelCapabilities - output capability", () => { ).not.toThrow(); }); }); + +describe("validateModelCapabilities - reasoning.max_tokens", () => { + const lingModel = getModel("ling-3.0-flash"); + const nonBudgetModel = getModel("gpt-4o-mini"); + + it("rejects reasoning.max_tokens for models with no budget support", () => { + expect(() => + validateModelCapabilities(nonBudgetModel, nonBudgetModel.id, undefined, { + reasoning_max_tokens: 2048, + }), + ).toThrow(HTTPException); + }); + + it("allows reasoning.max_tokens on chatTemplateThinkingKey mappings (budget is dropped to a binary toggle)", () => { + expect(() => + validateModelCapabilities(lingModel, lingModel.id, undefined, { + reasoning_max_tokens: 2048, + }), + ).not.toThrow(); + }); + + it("rejects reasoning.max_tokens on a provider with no thinking control (thinking is always on)", () => { + // Novita's Ling mapping has no chatTemplateThinkingKey and no + // reasoningMaxTokens — its backend ignores the chat-template flag, so a + // budget request is rejected rather than silently accepted. + expect(() => + validateModelCapabilities(lingModel, lingModel.id, "novita", { + reasoning_max_tokens: 2048, + }), + ).toThrow(HTTPException); + }); +}); diff --git a/apps/gateway/src/chat/tools/validate-model-capabilities.ts b/apps/gateway/src/chat/tools/validate-model-capabilities.ts index 587618f83c..58a75157ea 100644 --- a/apps/gateway/src/chat/tools/validate-model-capabilities.ts +++ b/apps/gateway/src/chat/tools/validate-model-capabilities.ts @@ -260,9 +260,15 @@ export function validateModelCapabilities( ) : modelInfo.providers; + // A mapping that thinks through a binary chat-template flag + // (`chatTemplateThinkingKey`) also accepts a budget: the budget is dropped + // and only the on/off state is conveyed, so it is not "unsupported" the way + // it is on a provider with no thinking control at all. const reasoningMaxTokens = providersToCheck.some( (provider) => - (provider as ProviderModelMapping).reasoningMaxTokens === true, + (provider as ProviderModelMapping).reasoningMaxTokens === true || + (provider as ProviderModelMapping).chatTemplateThinkingKey !== + undefined, ); if (!reasoningMaxTokens) { diff --git a/packages/actions/src/prepare-request-body.spec.ts b/packages/actions/src/prepare-request-body.spec.ts index 9ce67089c2..9e92b6e7f9 100644 --- a/packages/actions/src/prepare-request-body.spec.ts +++ b/packages/actions/src/prepare-request-body.spec.ts @@ -11,9 +11,20 @@ import { import type { AnthropicRequestBody, + OpenAIRequestBody, ProviderModelMapping, } from "@llmgateway/models"; +/** + * The OpenAI-compatible body Ling-3.0-flash requests get, plus the vLLM + * chat-template flag the gateway injects to control thinking. That flag is not + * part of `OpenAIRequestBody` (it is added dynamically in prepare-request-body), + * so tests reach it through this local intersection instead of `as any`. + */ +type LingRequestBody = OpenAIRequestBody & { + chat_template_kwargs?: Record; +}; + function getCacheControl(block: unknown): unknown { if (block && typeof block === "object" && "cache_control" in block) { return (block as { cache_control: unknown }).cache_control; @@ -1246,6 +1257,202 @@ describe("prepareRequestBody - Moonshot thinking", () => { ); }); +describe("prepareRequestBody - InclusionAI Ling-3.0-flash thinking", () => { + // prepareRequestBody always receives the canonical (bare) catalog id as + // usedInternalModel — the provider prefix is stripped by parseModelInput + // before chat.ts calls it. These tests therefore exercise the same body + // path a bare "ling-3.0-flash" request goes through; the gateway-level + // resolution of that bare id to a provider is covered in api.spec.ts. + async function prepare(options: { + provider: Parameters[0]; + reasoningEffort?: "none" | "minimal" | "low" | "medium" | "high" | "max"; + tools?: Parameters[12]; + toolChoice?: Parameters[13]; + responseFormat?: Parameters[11]; + }) { + // Derive supportsReasoning from the catalog the same way chat.ts does + // (`selectedProviderMapping?.reasoning === true`) so a regression in the + // derivation can't silently drop the thinking-disable translation. + const mapping = models + .find((m) => m.id === "ling-3.0-flash") + ?.providers.find((p) => p.providerId === options.provider); + const supportsReasoning = mapping?.reasoning === true; + + return (await prepareRequestBody( + options.provider, + "ling-3.0-flash", + null, + options.provider === "deepinfra" + ? "inclusionAI/Ling-3.0-flash" + : "inclusionai/ling-3.0-flash", + [{ role: "user", content: "Hello!" }], + false, // stream + undefined, // temperature + undefined, // max_tokens + undefined, // top_p + undefined, // frequency_penalty + undefined, // presence_penalty + options.responseFormat, // response_format + options.tools, // tools + options.toolChoice, // tool_choice + options.reasoningEffort, // reasoning_effort + supportsReasoning, + )) as LingRequestBody; + } + + test.each(["deepinfra", "novita"] as const)( + "maps none to enable_thinking disabled on %s and never forwards reasoning_effort", + async (provider) => { + const requestBody = await prepare({ + provider, + reasoningEffort: "none", + }); + // DeepInfra honors the chat-template flag; Novita ignores it (verified + // live 2026-08-10), so it gets no chat_template_kwargs and thinking + // stays on. + if (provider === "deepinfra") { + expect(requestBody.chat_template_kwargs).toEqual({ + enable_thinking: false, + }); + } else { + expect(requestBody.chat_template_kwargs).toBeUndefined(); + } + expect(requestBody.reasoning_effort).toBeUndefined(); + }, + ); + + test.each(["deepinfra", "novita"] as const)( + "maps high to enable_thinking enabled on %s and never forwards reasoning_effort", + async (provider) => { + const requestBody = await prepare({ + provider, + reasoningEffort: "high", + }); + if (provider === "deepinfra") { + expect(requestBody.chat_template_kwargs).toEqual({ + enable_thinking: true, + }); + } else { + expect(requestBody.chat_template_kwargs).toBeUndefined(); + } + expect(requestBody.reasoning_effort).toBeUndefined(); + }, + ); + + test.each(["deepinfra", "novita"] as const)( + "keeps the provider default (thinking on) when no reasoning_effort is requested on %s", + async (provider) => { + const requestBody = await prepare({ provider }); + expect(requestBody.chat_template_kwargs).toBeUndefined(); + expect(requestBody.reasoning_effort).toBeUndefined(); + }, + ); + + const tools = [ + { + type: "function" as const, + function: { + name: "get_weather", + description: "Get the weather for a city", + parameters: { + type: "object", + properties: { + city: { type: "string" }, + }, + required: ["city"], + }, + }, + }, + ]; + + test.each(["deepinfra", "novita"] as const)( + "forwards function tools with parameters and tool_choice intact on %s", + async (provider) => { + const requestBody = await prepare({ + provider, + tools, + toolChoice: { + type: "function", + function: { name: "get_weather" }, + }, + }); + expect(requestBody.tools).toEqual(tools); + expect(requestBody.tool_choice).toEqual({ + type: "function", + function: { name: "get_weather" }, + }); + }, + ); + + test.each(["deepinfra", "novita"] as const)( + "leaves tool_choice absent when none is requested on %s", + async (provider) => { + const requestBody = await prepare({ provider, tools }); + expect(requestBody.tools).toEqual(tools); + expect(requestBody.tool_choice).toBeUndefined(); + }, + ); + + test.each(["deepinfra", "novita"] as const)( + "passes through json_object response_format on %s", + async (provider) => { + const requestBody = await prepare({ + provider, + responseFormat: { type: "json_object" }, + }); + expect(requestBody.response_format).toEqual({ type: "json_object" }); + }, + ); + + test.each(["deepinfra", "novita"] as const)( + "passes through json_schema response_format on %s", + async (provider) => { + const schema = { + name: "weather", + strict: true, + schema: { + type: "object", + properties: { + city: { type: "string" }, + }, + required: ["city"], + additionalProperties: false, + }, + }; + const requestBody = await prepare({ + provider, + responseFormat: { type: "json_schema", json_schema: schema }, + }); + expect(requestBody.response_format).toEqual({ + type: "json_schema", + json_schema: schema, + }); + }, + ); + + test.each(["deepinfra", "novita"] as const)( + "maps none to enable_thinking disabled while tools and response_format are present on %s", + async (provider) => { + const requestBody = await prepare({ + provider, + reasoningEffort: "none", + tools, + responseFormat: { type: "json_object" }, + }); + if (provider === "deepinfra") { + expect(requestBody.chat_template_kwargs).toEqual({ + enable_thinking: false, + }); + } else { + expect(requestBody.chat_template_kwargs).toBeUndefined(); + } + expect(requestBody.reasoning_effort).toBeUndefined(); + expect(requestBody.tools).toEqual(tools); + expect(requestBody.response_format).toEqual({ type: "json_object" }); + }, + ); +}); + describe("prepareRequestBody - embercloud reasoning shape", () => { // embercloud's documented request body uses a nested `reasoning` object // (reasoning.enabled + reasoning.effort) — the flat OpenAI-style diff --git a/packages/actions/src/prepare-request-body.ts b/packages/actions/src/prepare-request-body.ts index ee1b0b9f8f..74d8b50ea4 100644 --- a/packages/actions/src/prepare-request-body.ts +++ b/packages/actions/src/prepare-request-body.ts @@ -4181,26 +4181,35 @@ export async function prepareRequestBody( } } } - // Hybrid models that keep thinking off by default (e.g. DeepSeek V3.2 on - // Novita) ignore `reasoning_effort` and require the vLLM chat-template - // flag to turn reasoning on. Only set it when the caller asked for - // reasoning so plain requests stay non-thinking. - if (supportsReasoning && (reasoning_effort || reasoning_max_tokens)) { - const thinkingMapping = modelDef?.providers.find( - (p) => - p.providerId === usedProvider && - ((p as ProviderModelMapping).region ?? null) === usedRegion, - ) as ProviderModelMapping | undefined; - if (thinkingMapping?.requiresEnableThinking) { - requestBody.chat_template_kwargs = { - ...(requestBody.chat_template_kwargs ?? {}), - thinking: true, - }; - } - } break; } } + // vLLM chat-template thinking flags are handled after the provider switch so + // every provider branch (not just the OpenAI-compatible default) applies them + // uniformly. Hybrid models that keep thinking off by default (e.g. DeepSeek + // V3.2 on Novita) ignore `reasoning_effort` and require the chat-template flag + // to turn reasoning on; mappings that think by default (e.g. InclusionAI + // Ling-3.0-flash) declare `chatTemplateThinkingKey` for the flag that controls + // thinking. `reasoning_effort: "none"` flips it to false to turn thinking off, + // and any other effort flips it to true. A `reasoning_max_tokens` budget (from + // an Anthropic `thinking.budget_tokens`) is intentionally dropped for these + // mappings: the chat-template flag is a binary on/off toggle with no budget + // dimension, so the gateway only conveys whether thinking is on. + if (supportsReasoning && (reasoning_effort || reasoning_max_tokens)) { + if (providerMappingForOptions?.chatTemplateThinkingKey) { + requestBody.chat_template_kwargs = { + ...(requestBody.chat_template_kwargs ?? {}), + [providerMappingForOptions.chatTemplateThinkingKey]: + reasoning_effort !== "none", + }; + } else if (providerMappingForOptions?.requiresEnableThinking) { + requestBody.chat_template_kwargs = { + ...(requestBody.chat_template_kwargs ?? {}), + thinking: true, + }; + } + } + return requestBody; } diff --git a/packages/models/src/models.ts b/packages/models/src/models.ts index 7ebe846572..6501d47f24 100644 --- a/packages/models/src/models.ts +++ b/packages/models/src/models.ts @@ -6,6 +6,7 @@ import { bytedanceModels } from "./models/bytedance.js"; import { deepseekModels } from "./models/deepseek.js"; import { elevenlabsModels } from "./models/elevenlabs.js"; import { googleModels } from "./models/google.js"; +import { inclusionaiModels } from "./models/inclusionai.js"; import { llmgatewayModels } from "./models/llmgateway.js"; import { metaModels } from "./models/meta.js"; import { microsoftModels } from "./models/microsoft.js"; @@ -391,6 +392,18 @@ export interface ProviderModelMapping { * declare `none` in `reasoningEfforts`. */ requiresDisableThinkingParam?: boolean; + /** + * Name of the chat-template kwargs key used to control thinking on + * mappings that think by default and expose only a chat-template flag + * (e.g. vLLM-hosted hybrid models). When set, `reasoning_effort: "none"` + * sends `chat_template_kwargs: { [key]: false }` to turn thinking off, and + * any other effort sends `{ [key]: true }` — a boolean value under the + * named key. Differs from `requiresEnableThinking`, which always sends + * `chat_template_kwargs: { thinking: true }`, and + * `requiresDisableThinkingParam`, which sends a top-level + * `thinking: { type: "disabled" }` object. + */ + chatTemplateThinkingKey?: string; /** * Whether this model supports the OpenAI responses API (defaults to true if reasoning is true) */ @@ -741,6 +754,7 @@ export const models = [ ...openaiModels, ...anthropicModels, ...googleModels, + ...inclusionaiModels, ...perplexityModels, ...xaiModels, ...xiaomiModels, diff --git a/packages/models/src/models/inclusionai.ts b/packages/models/src/models/inclusionai.ts new file mode 100644 index 0000000000..b339cf57ce --- /dev/null +++ b/packages/models/src/models/inclusionai.ts @@ -0,0 +1,78 @@ +import type { ModelDefinition } from "@/models.js"; + +export const inclusionaiModels = [ + { + id: "ling-3.0-flash", + name: "InclusionAI Ling 3.0 Flash", + description: + "InclusionAI's native hybrid-reasoning MoE model (124B total, 5.1B active) with strong agentic and coding performance.", + family: "inclusionai", + releasedAt: new Date("2026-08-02"), + providers: [ + { + providerId: "deepinfra", + externalId: "inclusionAI/Ling-3.0-flash", + inputPrice: "0.045e-6", + cachedInputPrice: "0.008e-6", + outputPrice: "0.1e-6", + requestPrice: "0", + contextSize: 262144, + maxOutput: 32768, + streaming: true, + // Thinking is enabled by default via the vLLM chat template; only + // disabling is exposed through the chat-template flag, so only + // `none` is advertised as a supported reasoning effort. + reasoning: true, + reasoningEfforts: ["none"], + chatTemplateThinkingKey: "enable_thinking", + // Thinking is controlled solely via the chat-template flag, so + // `reasoning_effort` must NOT be forwarded to the provider. + supportedParameters: [ + "temperature", + "max_tokens", + "top_p", + "frequency_penalty", + "presence_penalty", + "stop", + "stream", + "response_format", + "tools", + "tool_choice", + ], + vision: false, + tools: true, + jsonOutput: true, + }, + { + providerId: "novita", + externalId: "inclusionai/ling-3.0-flash", + inputPrice: "0.06e-6", + cachedInputPrice: "0.012e-6", + outputPrice: "0.18e-6", + requestPrice: "0", + contextSize: 262144, + maxOutput: 32768, + streaming: true, + reasoning: true, + // Novita backend ignores chat_template_kwargs.enable_thinking (verified + // live 2026-08-10) — thinking is always on, no client control. + reasoningEfforts: [], + supportedParameters: [ + "temperature", + "max_tokens", + "top_p", + "frequency_penalty", + "presence_penalty", + "stop", + "stream", + "response_format", + "tools", + "tool_choice", + ], + vision: false, + tools: true, + jsonOutput: true, + }, + ], + }, +] as const satisfies ModelDefinition[]; From ab676e59f0ea11a59e55e27d4e78da393c295ba7 Mon Sep 17 00:00:00 2001 From: Luca Steeb Date: Wed, 12 Aug 2026 14:26:50 +0200 Subject: [PATCH 2/2] fix(models): correct Ling-3.0-flash pricing DeepInfra serves this model at $0.06/$0.18 per 1M tokens (cached $0.012), not $0.045/$0.10/$0.008. Confirmed against DeepInfra's model listing and the usage.estimated_cost returned on live completions, which matches the corrected rates exactly. The old figures would have under-billed output by 44%. Novita also rejects response_format for this model outright ("does not support feature: structured-outputs") for both json_object and json_schema, so mark the mapping jsonOutput: false and drop response_format from its supportedParameters. This was failing two scoped e2e cases; with the flag corrected the gateway rejects such requests up front and routes JSON traffic for the bare model id to DeepInfra instead. Co-Authored-By: Claude Opus 5 (1M context) --- .../actions/src/prepare-request-body.spec.ts | 16 ++++++++++++---- packages/models/src/models/inclusionai.ts | 12 +++++++----- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/packages/actions/src/prepare-request-body.spec.ts b/packages/actions/src/prepare-request-body.spec.ts index 9e92b6e7f9..21174da477 100644 --- a/packages/actions/src/prepare-request-body.spec.ts +++ b/packages/actions/src/prepare-request-body.spec.ts @@ -1393,7 +1393,10 @@ describe("prepareRequestBody - InclusionAI Ling-3.0-flash thinking", () => { }, ); - test.each(["deepinfra", "novita"] as const)( + // Only DeepInfra serves this model with structured outputs — Novita rejects + // any response_format ("does not support feature: structured-outputs"), so + // its mapping is jsonOutput: false and such requests never reach this layer. + test.each(["deepinfra"] as const)( "passes through json_object response_format on %s", async (provider) => { const requestBody = await prepare({ @@ -1404,7 +1407,7 @@ describe("prepareRequestBody - InclusionAI Ling-3.0-flash thinking", () => { }, ); - test.each(["deepinfra", "novita"] as const)( + test.each(["deepinfra"] as const)( "passes through json_schema response_format on %s", async (provider) => { const schema = { @@ -1433,11 +1436,16 @@ describe("prepareRequestBody - InclusionAI Ling-3.0-flash thinking", () => { test.each(["deepinfra", "novita"] as const)( "maps none to enable_thinking disabled while tools and response_format are present on %s", async (provider) => { + // response_format is only valid on DeepInfra (see above). + const responseFormat = + provider === "deepinfra" + ? ({ type: "json_object" } as const) + : undefined; const requestBody = await prepare({ provider, reasoningEffort: "none", tools, - responseFormat: { type: "json_object" }, + responseFormat, }); if (provider === "deepinfra") { expect(requestBody.chat_template_kwargs).toEqual({ @@ -1448,7 +1456,7 @@ describe("prepareRequestBody - InclusionAI Ling-3.0-flash thinking", () => { } expect(requestBody.reasoning_effort).toBeUndefined(); expect(requestBody.tools).toEqual(tools); - expect(requestBody.response_format).toEqual({ type: "json_object" }); + expect(requestBody.response_format).toEqual(responseFormat); }, ); }); diff --git a/packages/models/src/models/inclusionai.ts b/packages/models/src/models/inclusionai.ts index b339cf57ce..1c1babf696 100644 --- a/packages/models/src/models/inclusionai.ts +++ b/packages/models/src/models/inclusionai.ts @@ -12,9 +12,9 @@ export const inclusionaiModels = [ { providerId: "deepinfra", externalId: "inclusionAI/Ling-3.0-flash", - inputPrice: "0.045e-6", - cachedInputPrice: "0.008e-6", - outputPrice: "0.1e-6", + inputPrice: "0.06e-6", + cachedInputPrice: "0.012e-6", + outputPrice: "0.18e-6", requestPrice: "0", contextSize: 262144, maxOutput: 32768, @@ -57,6 +57,9 @@ export const inclusionaiModels = [ // Novita backend ignores chat_template_kwargs.enable_thinking (verified // live 2026-08-10) — thinking is always on, no client control. reasoningEfforts: [], + // Novita rejects response_format outright for this model + // ("does not support feature: structured-outputs"), for both + // json_object and json_schema. supportedParameters: [ "temperature", "max_tokens", @@ -65,13 +68,12 @@ export const inclusionaiModels = [ "presence_penalty", "stop", "stream", - "response_format", "tools", "tool_choice", ], vision: false, tools: true, - jsonOutput: true, + jsonOutput: false, }, ], },