From 8c06229e1ca04ed7ed7cf63be6b5e4e4250cde94 Mon Sep 17 00:00:00 2001 From: Luca Steeb Date: Mon, 27 Jul 2026 21:42:29 +0100 Subject: [PATCH 01/12] feat(models): add RanoAI provider Add the RanoAI OpenAI-compatible provider (https://api.ranoai.com/v1) serving Gemma 4 31B on Furiosa RNGD NPU hardware. Co-Authored-By: Claude --- .env.example | 3 +++ .env.unified.example | 3 +++ .github/workflows/e2e.yml | 1 + .../tools/transform-streaming-to-openai.ts | 1 + .../components/provider-keys/provider-logo.ts | 1 + infra/helm/llmgateway/values.yaml | 1 + packages/actions/src/get-provider-endpoint.ts | 2 ++ packages/models/src/models/google.ts | 20 ++++++++++++++ packages/models/src/providers.ts | 26 +++++++++++++++++++ .../shared/src/components/provider-icons.tsx | 17 ++++++++++++ 10 files changed, 75 insertions(+) diff --git a/.env.example b/.env.example index 2997d82af8..8cf4bfb892 100644 --- a/.env.example +++ b/.env.example @@ -223,6 +223,9 @@ LLM_GONKA_24_API_KEY=your_gonka24_key_here LLM_FIREWORKS_API_KEY=your_fireworks_key_here LLM_FIREWORKS_BASE_URL=https://api.fireworks.ai/inference +# RanoAI +LLM_RANOAI_API_KEY=your_ranoai_key_here + # Google Vertex AI (Gemini models) LLM_GOOGLE_VERTEX_API_KEY=your_google_vertex_api_key_here LLM_GOOGLE_CLOUD_PROJECT=your_google_cloud_project_id diff --git a/.env.unified.example b/.env.unified.example index 457981fdd9..fe711c59b6 100644 --- a/.env.unified.example +++ b/.env.unified.example @@ -81,6 +81,9 @@ LLM_TOGETHER_AI_API_KEY=your_together_ai_key_here # SCX.ai LLM_SCX_AI_API_KEY=your_scx_ai_key_here +# RanoAI +LLM_RANOAI_API_KEY=your_ranoai_key_here + # Mistral LLM_MISTRAL_API_KEY=your_mistral_key_here diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 8480148609..e415dfc99c 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -144,6 +144,7 @@ jobs: LLM_META_API_KEY: ${{ secrets.LLM_META_API_KEY }} LLM_GONKA_24_API_KEY: ${{ secrets.LLM_GONKA_24_API_KEY }} LLM_FIREWORKS_API_KEY: ${{ secrets.LLM_FIREWORKS_API_KEY }} + LLM_RANOAI_API_KEY: ${{ secrets.LLM_RANOAI_API_KEY }} - name: Upload shard results if: always() diff --git a/apps/gateway/src/chat/tools/transform-streaming-to-openai.ts b/apps/gateway/src/chat/tools/transform-streaming-to-openai.ts index 8081de0491..f8b3ededc1 100644 --- a/apps/gateway/src/chat/tools/transform-streaming-to-openai.ts +++ b/apps/gateway/src/chat/tools/transform-streaming-to-openai.ts @@ -1438,6 +1438,7 @@ export function transformStreamingToOpenai( case "embercloud": case "runware": case "gonka24": + case "ranoai": case "granite": case "tundra": case "xiaomi": diff --git a/apps/ui/src/components/provider-keys/provider-logo.ts b/apps/ui/src/components/provider-keys/provider-logo.ts index e388f6ed3d..79d30a624c 100644 --- a/apps/ui/src/components/provider-keys/provider-logo.ts +++ b/apps/ui/src/components/provider-keys/provider-logo.ts @@ -47,6 +47,7 @@ export const providerLogoUrls: Partial< gonka24: ProviderIcons.gonka24, fireworks: ProviderIcons.fireworks, runware: ProviderIcons.runware, + ranoai: ProviderIcons.ranoai, }; export const getProviderLogoDarkModeClasses = () => { diff --git a/infra/helm/llmgateway/values.yaml b/infra/helm/llmgateway/values.yaml index 9af56c707d..9392d36c28 100644 --- a/infra/helm/llmgateway/values.yaml +++ b/infra/helm/llmgateway/values.yaml @@ -460,6 +460,7 @@ llmProviders: # LLM_PERPLEXITY_API_KEY: "" # LLM_TOGETHER_AI_API_KEY: "" # LLM_SCX_AI_API_KEY: "" + # LLM_RANOAI_API_KEY: "" # LLM_ALIBABA_API_KEY: "" # LLM_NEBIUS_API_KEY: "" # LLM_CEREBRAS_API_KEY: "" diff --git a/packages/actions/src/get-provider-endpoint.ts b/packages/actions/src/get-provider-endpoint.ts index f40ab17df9..bdc88042d6 100644 --- a/packages/actions/src/get-provider-endpoint.ts +++ b/packages/actions/src/get-provider-endpoint.ts @@ -135,6 +135,7 @@ const PROVIDER_DEFAULT_BASE_URLS: Partial> = { deepinfra: "https://api.deepinfra.com/v1/openai", gonka24: "https://api.gonka24.com", fireworks: "https://api.fireworks.ai/inference", + ranoai: "https://api.ranoai.com", }; export function getProviderDefaultBaseUrl( @@ -865,6 +866,7 @@ export function getProviderEndpoint( case "tundra": case "scx-ai": case "scx-ai-gp": + case "ranoai": case "custom": default: return `${url}/v1/chat/completions`; diff --git a/packages/models/src/models/google.ts b/packages/models/src/models/google.ts index 15ce341d43..00e449416d 100644 --- a/packages/models/src/models/google.ts +++ b/packages/models/src/models/google.ts @@ -2450,6 +2450,26 @@ export const googleModels = [ tools: true, jsonOutput: true, }, + { + // RanoAI serves Gemma 4 on Furiosa RNGD NPUs. Their docs advertise a + // 131K window and multimodal input, but the deployment measurably + // caps prompts at 4096 tokens and prompt + max_tokens at 8192, drops + // image parts (the model receives `[object Object]`), emits tool + // calls as raw `<|tool_call>…` text in `content` instead of OpenAI + // `tool_calls`, and returns no reasoning field (verified 2026-07-27). + providerId: "ranoai", + externalId: "gemma-4-31b", + inputPrice: "0.1e-6", + outputPrice: "0.3e-6", + requestPrice: "0", + contextSize: 8192, + maxOutput: 4096, + streaming: true, + reasoning: false, + vision: false, + tools: false, + jsonOutput: true, + }, ], }, { diff --git a/packages/models/src/providers.ts b/packages/models/src/providers.ts index d30c854e00..a95b6a97cc 100644 --- a/packages/models/src/providers.ts +++ b/packages/models/src/providers.ts @@ -1781,6 +1781,32 @@ export const providers: ProviderDefinition[] = [ gdpr: true, }, }, + { + id: "ranoai", + name: "RanoAI", + description: + "RanoAI serves open-weight large language models on Furiosa RNGD NPU hardware via an OpenAI-compatible inference API.", + env: { + required: { + apiKey: "LLM_RANOAI_API_KEY", + }, + }, + streaming: true, + cancellation: true, + color: "#000000", + website: "https://ranoai.com", + statusPageUrl: null, + announcement: null, + termsUrl: "https://ranoai.com/terms", + privacyPolicyUrl: "https://ranoai.com/privacy", + headquarters: "US", + dataPolicy: { + apiTraining: false, + consumerTraining: false, + promptLogging: false, + retentionPeriod: "0 days", + }, + }, ] as const satisfies ProviderDefinition[]; export type ProviderId = (typeof providers)[number]["id"]; diff --git a/packages/shared/src/components/provider-icons.tsx b/packages/shared/src/components/provider-icons.tsx index c1a4646bf9..8a5b6ddb7f 100644 --- a/packages/shared/src/components/provider-icons.tsx +++ b/packages/shared/src/components/provider-icons.tsx @@ -1330,6 +1330,21 @@ export const FireworksIconStatic: React.FC> = ( ); +export const RanoAIIcon: React.FC> = (props) => ( + + + + + + + + +); + export const DeepInfraIcon: React.FC> = ( props, ) => ( @@ -1586,6 +1601,7 @@ export const ProviderIcons = { gonka24: Gonka24Icon, fireworks: FireworksIcon, runware: RunwareIcon, + ranoai: RanoAIIcon, } as const; // Type for provider icon keys @@ -1637,6 +1653,7 @@ export const providerLogoUrls: Partial< gonka24: ProviderIcons.gonka24, fireworks: ProviderIcons.fireworks, runware: ProviderIcons.runware, + ranoai: ProviderIcons.ranoai, }; export const getProviderLogoDarkModeClasses = () => { From 6e90fea777acd3ce8845a3944b993cdbc88e77cb Mon Sep 17 00:00:00 2001 From: Luca Steeb Date: Tue, 28 Jul 2026 11:31:23 +0100 Subject: [PATCH 02/12] feat(models): enable vision, raise RanoAI context to 20K RanoAI fixed image input and non-streaming tool calls, and expanded the NPU deployment window from 8K to 20480 tokens. Verified against the live deployment: - vision works (data URIs and remote URLs) -> vision: true - window is 20480 (prompt + max_tokens), enforced with a proper OpenAI-style error -> contextSize/maxOutput 20480 - Gemma 4 has no reasoning feature -> reasoning stays false Tools stay disabled: streaming tool calls emit finish_reason "tool_calls" with no tool_calls delta, and tool-result follow-ups leak raw <|channel>thought markers into content or drop it to null. Co-Authored-By: Claude --- packages/models/src/models/google.ts | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/packages/models/src/models/google.ts b/packages/models/src/models/google.ts index 00e449416d..4d7c13b811 100644 --- a/packages/models/src/models/google.ts +++ b/packages/models/src/models/google.ts @@ -2451,22 +2451,27 @@ export const googleModels = [ jsonOutput: true, }, { - // RanoAI serves Gemma 4 on Furiosa RNGD NPUs. Their docs advertise a - // 131K window and multimodal input, but the deployment measurably - // caps prompts at 4096 tokens and prompt + max_tokens at 8192, drops - // image parts (the model receives `[object Object]`), emits tool - // calls as raw `<|tool_call>…` text in `content` instead of OpenAI - // `tool_calls`, and returns no reasoning field (verified 2026-07-27). + // RanoAI serves Gemma 4 on Furiosa RNGD NPUs. The NPU deployment + // enforces a 20480-token window (prompt + max_tokens), not the + // 131K its /v1/models still advertises. Gemma 4 has no reasoning + // feature, so no reasoning field is ever returned. + // + // Tools stay disabled: non-streaming tool calls are correct, but + // streaming ones emit `finish_reason: "tool_calls"` with no + // `tool_calls` delta at all, and tool-result follow-ups leak raw + // `<|channel>thought` template markers into `content` + // (or drop it to null when `tools` is not re-declared). Re-enable + // once those are fixed upstream (verified 2026-07-28). providerId: "ranoai", externalId: "gemma-4-31b", inputPrice: "0.1e-6", outputPrice: "0.3e-6", requestPrice: "0", - contextSize: 8192, - maxOutput: 4096, + contextSize: 20480, + maxOutput: 20480, streaming: true, reasoning: false, - vision: false, + vision: true, tools: false, jsonOutput: true, }, From 94d8b566556e59a5589ec02e2025d4c228d338ef Mon Sep 17 00:00:00 2001 From: Luca Steeb Date: Tue, 28 Jul 2026 12:55:15 +0100 Subject: [PATCH 03/12] feat(models): enable tools for RanoAI Gemma 4 RanoAI fixed streaming tool calls (a proper tool_calls delta is now emitted) and tool-result follow-ups (clean content, no <|channel> template leak, with or without tools re-declared). Verified against the live deployment: auto tool_choice, parallel multi-tool calls, streaming deltas and tool-result round-trips all behave correctly. supportedToolChoices is pinned to ["auto"] because the other modes are still wrong upstream: "none" emits raw call:name{args} text into content whenever the model wants a tool, "required" returns finish_reason "tool_calls" with an empty tool_calls array, and a named function choice is ignored. Unsupported modes downgrade to auto. Co-Authored-By: Claude --- packages/models/src/models/google.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/models/src/models/google.ts b/packages/models/src/models/google.ts index 4d7c13b811..7b18bcb719 100644 --- a/packages/models/src/models/google.ts +++ b/packages/models/src/models/google.ts @@ -2456,12 +2456,11 @@ export const googleModels = [ // 131K its /v1/models still advertises. Gemma 4 has no reasoning // feature, so no reasoning field is ever returned. // - // Tools stay disabled: non-streaming tool calls are correct, but - // streaming ones emit `finish_reason: "tool_calls"` with no - // `tool_calls` delta at all, and tool-result follow-ups leak raw - // `<|channel>thought` template markers into `content` - // (or drop it to null when `tools` is not re-declared). Re-enable - // once those are fixed upstream (verified 2026-07-28). + // Only `auto` tool_choice is sound: "none" still emits the raw + // `call:name{args}` template text into `content` whenever the + // model wants a tool, "required" returns `finish_reason: + // "tool_calls"` with an empty `tool_calls` array, and a named + // function choice is ignored (verified 2026-07-28). providerId: "ranoai", externalId: "gemma-4-31b", inputPrice: "0.1e-6", @@ -2472,8 +2471,9 @@ export const googleModels = [ streaming: true, reasoning: false, vision: true, - tools: false, + tools: true, jsonOutput: true, + supportedToolChoices: ["auto"], }, ], }, From fcbf71f5cf83e33e7af8817b5444347c2cfc11cf Mon Sep 17 00:00:00 2001 From: Luca Steeb Date: Tue, 28 Jul 2026 15:37:43 +0100 Subject: [PATCH 04/12] feat(models): honour tool_choice none for RanoAI RanoAI fixed tool_choice "none": it now returns finish_reason "stop" with no tool_calls and a clean refusal, instead of leaking the raw call:name{args} template text into content. Pass "none" through instead of downgrading it to "auto", which was turning an explicit "do not call tools" into a tool call. "required" (empty tool_calls array with finish_reason "tool_calls") and named function choices (get_time request yields get_weather) are still wrong upstream, so they continue to downgrade to "auto". Co-Authored-By: Claude --- packages/models/src/models/google.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/models/src/models/google.ts b/packages/models/src/models/google.ts index 7b18bcb719..59a1643edb 100644 --- a/packages/models/src/models/google.ts +++ b/packages/models/src/models/google.ts @@ -2456,11 +2456,11 @@ export const googleModels = [ // 131K its /v1/models still advertises. Gemma 4 has no reasoning // feature, so no reasoning field is ever returned. // - // Only `auto` tool_choice is sound: "none" still emits the raw - // `call:name{args}` template text into `content` whenever the - // model wants a tool, "required" returns `finish_reason: - // "tool_calls"` with an empty `tool_calls` array, and a named - // function choice is ignored (verified 2026-07-28). + // "required" returns `finish_reason: "tool_calls"` with an empty + // `tool_calls` array, and a named function choice is ignored + // (asking for get_time yields get_weather), so both downgrade to + // "auto". "auto" and "none" are honoured correctly + // (verified 2026-07-28). providerId: "ranoai", externalId: "gemma-4-31b", inputPrice: "0.1e-6", @@ -2473,7 +2473,7 @@ export const googleModels = [ vision: true, tools: true, jsonOutput: true, - supportedToolChoices: ["auto"], + supportedToolChoices: ["auto", "none"], }, ], }, From c6a499aefa013cd2d75a3fe6509fc2367b94b52d Mon Sep 17 00:00:00 2001 From: Luca Steeb Date: Tue, 28 Jul 2026 16:24:39 +0100 Subject: [PATCH 05/12] docs(models): clarify RanoAI tool_choice behaviour MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "required" is accepted rather than rejected: it behaves like "auto" and only misreports finish_reason as "tool_calls" (with an empty tool_calls array) when the model declines to call a tool. With a tool-inviting prompt it returns a correct tool call. The previous comment implied the empty array was unconditional, which overstated the defect. No behaviour change — downgrading "required" and named choices to "auto" remains correct, since it leaves tool selection identical while restoring an accurate finish_reason. Co-Authored-By: Claude --- packages/models/src/models/google.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/models/src/models/google.ts b/packages/models/src/models/google.ts index 59a1643edb..98cdc83811 100644 --- a/packages/models/src/models/google.ts +++ b/packages/models/src/models/google.ts @@ -2456,10 +2456,13 @@ export const googleModels = [ // 131K its /v1/models still advertises. Gemma 4 has no reasoning // feature, so no reasoning field is ever returned. // - // "required" returns `finish_reason: "tool_calls"` with an empty - // `tool_calls` array, and a named function choice is ignored - // (asking for get_time yields get_weather), so both downgrade to - // "auto". "auto" and "none" are honoured correctly + // "auto" and "none" are honoured correctly. "required" is accepted + // but not enforced — it behaves like "auto", and when the model + // declines to call a tool it still reports `finish_reason: + // "tool_calls"` alongside an empty `tool_calls` array. A named + // function choice is likewise ignored (asking for get_time + // yields get_weather). Downgrading both to "auto" leaves tool + // selection unchanged while restoring a correct finish_reason // (verified 2026-07-28). providerId: "ranoai", externalId: "gemma-4-31b", From 241c25dcccc6f70b7bb89d7cc7eedaab6512e853 Mon Sep 17 00:00:00 2001 From: Luca Steeb Date: Tue, 28 Jul 2026 16:55:41 +0100 Subject: [PATCH 06/12] docs(models): pin down RanoAI required tool_choice bug Narrow the description of the tool_choice "required" defect: it is specific to the non-streaming response path, and the message carries no tool_calls key at all rather than an empty array. Streaming returns the correct "stop" for an identical request. Co-Authored-By: Claude --- packages/models/src/models/google.ts | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/packages/models/src/models/google.ts b/packages/models/src/models/google.ts index 98cdc83811..15745bc606 100644 --- a/packages/models/src/models/google.ts +++ b/packages/models/src/models/google.ts @@ -2457,13 +2457,14 @@ export const googleModels = [ // feature, so no reasoning field is ever returned. // // "auto" and "none" are honoured correctly. "required" is accepted - // but not enforced — it behaves like "auto", and when the model - // declines to call a tool it still reports `finish_reason: - // "tool_calls"` alongside an empty `tool_calls` array. A named - // function choice is likewise ignored (asking for get_time - // yields get_weather). Downgrading both to "auto" leaves tool - // selection unchanged while restoring a correct finish_reason - // (verified 2026-07-28). + // but not enforced — it behaves like "auto", and on the + // non-streaming path only, when the model declines to call a + // tool, it reports `finish_reason: "tool_calls"` on a message + // carrying no `tool_calls` key at all. The streaming path + // returns "stop" for the same request. A named function choice + // is likewise ignored (asking for get_time yields get_weather). + // Downgrading both to "auto" leaves tool selection unchanged + // while restoring a correct finish_reason (verified 2026-07-28). providerId: "ranoai", externalId: "gemma-4-31b", inputPrice: "0.1e-6", From 3b11181106bdc7d494e0d2ca85929e8f9f707d86 Mon Sep 17 00:00:00 2001 From: Luca Steeb Date: Tue, 28 Jul 2026 18:26:28 +0100 Subject: [PATCH 07/12] docs(models): note ranoai finish_reason fix upstream MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RanoAI fixed the non-streaming finish_reason defect: "required" with a prompt that needs no tool now returns "stop" instead of "tool_calls" on a message with no tool_calls key. /v1/models also now reports context_length 20480 and modality text+image->text. "required" and named function choices are still not enforced, so they continue to downgrade to "auto" — behaviourally identical, since the upstream disregards both constraints either way. Co-Authored-By: Claude --- packages/models/src/models/google.ts | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/packages/models/src/models/google.ts b/packages/models/src/models/google.ts index 15745bc606..0a0ca13dcb 100644 --- a/packages/models/src/models/google.ts +++ b/packages/models/src/models/google.ts @@ -2456,15 +2456,14 @@ export const googleModels = [ // 131K its /v1/models still advertises. Gemma 4 has no reasoning // feature, so no reasoning field is ever returned. // - // "auto" and "none" are honoured correctly. "required" is accepted - // but not enforced — it behaves like "auto", and on the - // non-streaming path only, when the model declines to call a - // tool, it reports `finish_reason: "tool_calls"` on a message - // carrying no `tool_calls` key at all. The streaming path - // returns "stop" for the same request. A named function choice - // is likewise ignored (asking for get_time yields get_weather). - // Downgrading both to "auto" leaves tool selection unchanged - // while restoring a correct finish_reason (verified 2026-07-28). + // "auto" and "none" are honoured correctly. "required" and named + // function choices are accepted and now return well-formed + // responses, but neither is enforced: "required" lets the model + // answer without calling a tool, and a named choice is ignored + // (asking for get_time yields get_weather). Since the upstream + // simply disregards both constraints, downgrading them to "auto" + // is behaviourally identical and keeps the mapping from implying + // a guarantee it cannot make (verified 2026-07-28). providerId: "ranoai", externalId: "gemma-4-31b", inputPrice: "0.1e-6", From bd707caad83bee56747cb8962eaf2b7f46142a16 Mon Sep 17 00:00:00 2001 From: Luca Steeb Date: Tue, 28 Jul 2026 22:46:45 +0100 Subject: [PATCH 08/12] feat(models): enable json_schema and n for ranoai Verified against the live deployment: strict json_schema conforms to a nested schema with enums and additionalProperties:false (3/3, identical output), and n=3 returns three distinct correctly-indexed choices with the prompt billed once and output summed across choices. Streaming n also emits index 0/1/2. Also correct the reasoning note: the deepinfra, together-ai, cerebras and runware deployments of this same model do emit reasoning, so the absence here is a deployment gap rather than a Gemma 4 limitation. RanoAI accepts reasoning_effort but returns no reasoning field and no reasoning_tokens. Co-Authored-By: Claude --- packages/models/src/models/google.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/models/src/models/google.ts b/packages/models/src/models/google.ts index 0a0ca13dcb..ad70e6640e 100644 --- a/packages/models/src/models/google.ts +++ b/packages/models/src/models/google.ts @@ -2452,9 +2452,10 @@ export const googleModels = [ }, { // RanoAI serves Gemma 4 on Furiosa RNGD NPUs. The NPU deployment - // enforces a 20480-token window (prompt + max_tokens), not the - // 131K its /v1/models still advertises. Gemma 4 has no reasoning - // feature, so no reasoning field is ever returned. + // enforces a 20480-token window (prompt + max_tokens). Unlike the + // deepinfra/together-ai/cerebras/runware deployments of this same + // model, it emits no reasoning: `reasoning_effort` is accepted but + // no reasoning field or reasoning_tokens ever comes back. // // "auto" and "none" are honoured correctly. "required" and named // function choices are accepted and now return well-formed @@ -2476,6 +2477,8 @@ export const googleModels = [ vision: true, tools: true, jsonOutput: true, + jsonOutputSchema: true, + supportsN: true, supportedToolChoices: ["auto", "none"], }, ], From 1e49ff0da00ea8109114926a479548811f203b7f Mon Sep 17 00:00:00 2001 From: Luca Steeb Date: Wed, 29 Jul 2026 13:54:15 +0100 Subject: [PATCH 09/12] feat(models): enable reasoning for ranoai gemma 4 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RanoAI now emits reasoning as `reasoning_content`, streamed as deltas, across all seven effort tiers (none/minimal/low/medium/high/xhigh/max); an invalid tier is rejected with a validation error. Reasoning only appears when reasoning_effort is passed explicitly — a request without it returns none — and "none" suppresses it, so the mapping declares the full reasoningEfforts list. Usage still does not break out reasoning_tokens. This also fixes a real defect: with reasoning:false the streaming transform folds reasoning_content into content, so the reasoning text was leaking into the assistant message. Co-Authored-By: Claude --- packages/models/src/models/google.ts | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/packages/models/src/models/google.ts b/packages/models/src/models/google.ts index ad70e6640e..771ae8aab8 100644 --- a/packages/models/src/models/google.ts +++ b/packages/models/src/models/google.ts @@ -2452,10 +2452,11 @@ export const googleModels = [ }, { // RanoAI serves Gemma 4 on Furiosa RNGD NPUs. The NPU deployment - // enforces a 20480-token window (prompt + max_tokens). Unlike the - // deepinfra/together-ai/cerebras/runware deployments of this same - // model, it emits no reasoning: `reasoning_effort` is accepted but - // no reasoning field or reasoning_tokens ever comes back. + // enforces a 20480-token window (prompt + max_tokens). Reasoning + // arrives as `reasoning_content` (streamed as deltas) only when + // `reasoning_effort` is passed explicitly — a request without it + // returns no reasoning at all, and "none" suppresses it. Usage + // never breaks out `reasoning_tokens`. // // "auto" and "none" are honoured correctly. "required" and named // function choices are accepted and now return well-formed @@ -2473,7 +2474,16 @@ export const googleModels = [ contextSize: 20480, maxOutput: 20480, streaming: true, - reasoning: false, + reasoning: true, + reasoningEfforts: [ + "none", + "minimal", + "low", + "medium", + "high", + "xhigh", + "max", + ], vision: true, tools: true, jsonOutput: true, From 418ceeebed359f80c79334541e079f951290475d Mon Sep 17 00:00:00 2001 From: Luca Steeb Date: Wed, 5 Aug 2026 13:07:07 +0100 Subject: [PATCH 10/12] fix(gateway): stop double-billing ranoai reasoning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RanoAI now reports reasoning_tokens inside completion_tokens_details. The streaming transform hoists that to a top-level reasoning_tokens, and costs.ts adds reasoning on top of completion tokens for any provider not in the completionIncludesReasoning list — but RanoAI already counts reasoning inside completion_tokens. Measured on the live deployment, reasoning is 81-86% of completion tokens, so a streaming reasoning request was billed roughly 1.8x its real output (330 output tokens billed as 597). Adds a regression test, verified to fail without the fix: expected 0.0001791 to be close to 0.000099. Co-Authored-By: Claude --- apps/gateway/src/lib/costs.spec.ts | 23 +++++++++++++++++++++++ apps/gateway/src/lib/costs.ts | 9 +++++++-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/apps/gateway/src/lib/costs.spec.ts b/apps/gateway/src/lib/costs.spec.ts index 700b53706b..81ed7f47c6 100644 --- a/apps/gateway/src/lib/costs.spec.ts +++ b/apps/gateway/src/lib/costs.spec.ts @@ -692,6 +692,29 @@ describe("calculateCosts", () => { expect(result.estimatedCost).toBe(false); }); + it("should not double-bill RanoAI reasoning tokens", async () => { + // RanoAI reports reasoning in completion_tokens_details (hoisted to a + // top-level reasoning_tokens by the streaming transform) while already + // counting it inside completion_tokens, so it must not be added again. + const result = await calculateCosts( + "gemma-4-31b-it", + "ranoai", + null, + 1000, + 330, // completionTokens already includes the 267 reasoning tokens + null, + undefined, + 267, + ); + + // inputPrice 0.1e-6, outputPrice 0.3e-6. Precision matters here: billing + // 330 vs 597 output tokens differs by only 8e-5, which the default + // two-decimal toBeCloseTo would not catch. + expect(result.inputCost).toBeCloseTo(0.0001, 10); + expect(result.outputCost).toBeCloseTo(0.000099, 10); // 330 * 0.3e-6, not 597 + expect(result.completionTokens).toBe(330); + }); + it("should handle null reasoning tokens gracefully", async () => { const result = await calculateCosts( "gemini-2.5-pro", diff --git a/apps/gateway/src/lib/costs.ts b/apps/gateway/src/lib/costs.ts index 41189b649f..c631bdfc24 100644 --- a/apps/gateway/src/lib/costs.ts +++ b/apps/gateway/src/lib/costs.ts @@ -643,8 +643,12 @@ export async function calculateCosts( // For Google models, completionTokens already includes reasoning tokens // (merged during extraction). The same holds for OpenAI-style Responses API // providers (OpenAI, Azure, Sakana, Meta), whose `output_tokens` counts - // reasoning — their `reasoning_tokens` detail is informational only. For - // remaining providers, add reasoning separately. + // reasoning — their `reasoning_tokens` detail is informational only. RanoAI + // reports reasoning in `completion_tokens_details` (which the streaming + // transform hoists to a top-level `reasoning_tokens`) while already counting + // it inside `completion_tokens`, so adding it again would roughly double the + // billed output on reasoning requests. For remaining providers, add + // reasoning separately. const completionIncludesReasoning = provider === "google-ai-studio" || provider === "glacier" || @@ -655,6 +659,7 @@ export async function calculateCosts( provider === "azure" || provider === "sakana" || provider === "meta" || + provider === "ranoai" || provider === "aws-mantle"; const totalOutputTokens = completionIncludesReasoning ? calculatedCompletionTokens From 31812316b9efc239d42f3c9fdcabaaf42a16599c Mon Sep 17 00:00:00 2001 From: Luca Steeb Date: Wed, 5 Aug 2026 18:18:28 +0100 Subject: [PATCH 11/12] feat(models): honour all tool_choice modes on ranoai RanoAI now enforces every tool_choice mode, so the ["auto","none"] restriction is dropped and requests pass through unchanged. Verified on the live deployment: "required" forces a tool call on five varied prompts that need no tool (5/5, previously 0/5), a named function choice returns exactly the requested function on five weather-biased prompts (5/5, previously always get_weather), and "none"/"auto" are unchanged with no raw call: template text leaking into content. logprobs are now returned as well. Prompt caching is still not offered. Co-Authored-By: Claude --- packages/models/src/models/google.ts | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/packages/models/src/models/google.ts b/packages/models/src/models/google.ts index 771ae8aab8..d7d74c5ba1 100644 --- a/packages/models/src/models/google.ts +++ b/packages/models/src/models/google.ts @@ -2455,17 +2455,14 @@ export const googleModels = [ // enforces a 20480-token window (prompt + max_tokens). Reasoning // arrives as `reasoning_content` (streamed as deltas) only when // `reasoning_effort` is passed explicitly — a request without it - // returns no reasoning at all, and "none" suppresses it. Usage - // never breaks out `reasoning_tokens`. + // returns no reasoning at all, and "none" suppresses it. + // `reasoning_tokens` is reported inside completion_tokens, so + // costs.ts lists this provider in completionIncludesReasoning. // - // "auto" and "none" are honoured correctly. "required" and named - // function choices are accepted and now return well-formed - // responses, but neither is enforced: "required" lets the model - // answer without calling a tool, and a named choice is ignored - // (asking for get_time yields get_weather). Since the upstream - // simply disregards both constraints, downgrading them to "auto" - // is behaviourally identical and keeps the mapping from implying - // a guarantee it cannot make (verified 2026-07-28). + // All four tool_choice modes are honoured, so none are declared + // here (verified 2026-08-05). Prompt caching is still not offered: + // repeating an identical 2.5k-token prompt reports no + // prompt_tokens_details, hence no cachedInputPrice. providerId: "ranoai", externalId: "gemma-4-31b", inputPrice: "0.1e-6", @@ -2489,7 +2486,6 @@ export const googleModels = [ jsonOutput: true, jsonOutputSchema: true, supportsN: true, - supportedToolChoices: ["auto", "none"], }, ], }, From 42ad794a3b173b4c83d35f0e76f6e1181438950f Mon Sep 17 00:00:00 2001 From: Luca Steeb Date: Wed, 5 Aug 2026 19:07:37 +0100 Subject: [PATCH 12/12] feat(models): price ranoai prompt caching RanoAI now does automatic prefix caching and advertises the rate as input_cache_read in /v1/models: 5e-8, exactly half the input price. Verified live: the first request misses, subsequent identical requests report cached_tokens 2496/2521, and an 8k prompt reuses the shorter prefix before caching the full 8000. Without cachedInputPrice the cost engine falls back to inputPrice, so cached tokens were billed at 2x what RanoAI charges. Adds a regression test, verified to fail without the price (0.0002496 vs 0.0001248). The e2e prompt-caching suite now covers this mapping under TEST_CACHE_MODE. Co-Authored-By: Claude --- apps/gateway/src/lib/costs.spec.ts | 18 ++++++++++++++++++ packages/models/src/models/google.ts | 8 +++++--- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/apps/gateway/src/lib/costs.spec.ts b/apps/gateway/src/lib/costs.spec.ts index 81ed7f47c6..8771ae06ca 100644 --- a/apps/gateway/src/lib/costs.spec.ts +++ b/apps/gateway/src/lib/costs.spec.ts @@ -692,6 +692,24 @@ describe("calculateCosts", () => { expect(result.estimatedCost).toBe(false); }); + it("should bill RanoAI cached tokens at the cache-read rate", async () => { + // RanoAI does automatic prefix caching and charges input_cache_read + // (0.05/M), half the input rate. Without cachedInputPrice the engine + // falls back to inputPrice and would bill cached tokens at 2x. + const result = await calculateCosts( + "gemma-4-31b-it", + "ranoai", + null, + 2521, + 10, + 2496, // cachedTokens + ); + + // 25 uncached * 0.1e-6 + 2496 cached * 0.05e-6 + expect(result.inputCost).toBeCloseTo(0.0000025, 10); + expect(result.cachedInputCost).toBeCloseTo(0.0001248, 10); + }); + it("should not double-bill RanoAI reasoning tokens", async () => { // RanoAI reports reasoning in completion_tokens_details (hoisted to a // top-level reasoning_tokens by the streaming transform) while already diff --git a/packages/models/src/models/google.ts b/packages/models/src/models/google.ts index d7d74c5ba1..cbc64c3a1c 100644 --- a/packages/models/src/models/google.ts +++ b/packages/models/src/models/google.ts @@ -2460,13 +2460,15 @@ export const googleModels = [ // costs.ts lists this provider in completionIncludesReasoning. // // All four tool_choice modes are honoured, so none are declared - // here (verified 2026-08-05). Prompt caching is still not offered: - // repeating an identical 2.5k-token prompt reports no - // prompt_tokens_details, hence no cachedInputPrice. + // here. Prompt caching is automatic prefix caching — the first + // request misses and later ones report `cached_tokens` — priced + // at the `input_cache_read` rate /v1/models advertises, half the + // input rate (verified 2026-08-05). providerId: "ranoai", externalId: "gemma-4-31b", inputPrice: "0.1e-6", outputPrice: "0.3e-6", + cachedInputPrice: "0.05e-6", requestPrice: "0", contextSize: 20480, maxOutput: 20480,