diff --git a/.env.example b/.env.example index 2997d82af8..8cf4bfb892 100644 --- a/.env.example +++ b/.env.example @@ -223,6 +223,9 @@ LLM_GONKA_24_API_KEY=your_gonka24_key_here LLM_FIREWORKS_API_KEY=your_fireworks_key_here LLM_FIREWORKS_BASE_URL=https://api.fireworks.ai/inference +# RanoAI +LLM_RANOAI_API_KEY=your_ranoai_key_here + # Google Vertex AI (Gemini models) LLM_GOOGLE_VERTEX_API_KEY=your_google_vertex_api_key_here LLM_GOOGLE_CLOUD_PROJECT=your_google_cloud_project_id diff --git a/.env.unified.example b/.env.unified.example index 457981fdd9..fe711c59b6 100644 --- a/.env.unified.example +++ b/.env.unified.example @@ -81,6 +81,9 @@ LLM_TOGETHER_AI_API_KEY=your_together_ai_key_here # SCX.ai LLM_SCX_AI_API_KEY=your_scx_ai_key_here +# RanoAI +LLM_RANOAI_API_KEY=your_ranoai_key_here + # Mistral LLM_MISTRAL_API_KEY=your_mistral_key_here diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 8480148609..e415dfc99c 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -144,6 +144,7 @@ jobs: LLM_META_API_KEY: ${{ secrets.LLM_META_API_KEY }} LLM_GONKA_24_API_KEY: ${{ secrets.LLM_GONKA_24_API_KEY }} LLM_FIREWORKS_API_KEY: ${{ secrets.LLM_FIREWORKS_API_KEY }} + LLM_RANOAI_API_KEY: ${{ secrets.LLM_RANOAI_API_KEY }} - name: Upload shard results if: always() diff --git a/apps/gateway/src/chat/tools/transform-streaming-to-openai.ts b/apps/gateway/src/chat/tools/transform-streaming-to-openai.ts index 8081de0491..f8b3ededc1 100644 --- a/apps/gateway/src/chat/tools/transform-streaming-to-openai.ts +++ b/apps/gateway/src/chat/tools/transform-streaming-to-openai.ts @@ -1438,6 +1438,7 @@ export function transformStreamingToOpenai( case "embercloud": case "runware": case "gonka24": + case "ranoai": case "granite": case "tundra": case "xiaomi": diff --git a/apps/gateway/src/lib/costs.spec.ts b/apps/gateway/src/lib/costs.spec.ts index 700b53706b..8771ae06ca 100644 --- a/apps/gateway/src/lib/costs.spec.ts +++ b/apps/gateway/src/lib/costs.spec.ts @@ -692,6 +692,47 @@ describe("calculateCosts", () => { expect(result.estimatedCost).toBe(false); }); + it("should bill RanoAI cached tokens at the cache-read rate", async () => { + // RanoAI does automatic prefix caching and charges input_cache_read + // (0.05/M), half the input rate. Without cachedInputPrice the engine + // falls back to inputPrice and would bill cached tokens at 2x. + const result = await calculateCosts( + "gemma-4-31b-it", + "ranoai", + null, + 2521, + 10, + 2496, // cachedTokens + ); + + // 25 uncached * 0.1e-6 + 2496 cached * 0.05e-6 + expect(result.inputCost).toBeCloseTo(0.0000025, 10); + expect(result.cachedInputCost).toBeCloseTo(0.0001248, 10); + }); + + it("should not double-bill RanoAI reasoning tokens", async () => { + // RanoAI reports reasoning in completion_tokens_details (hoisted to a + // top-level reasoning_tokens by the streaming transform) while already + // counting it inside completion_tokens, so it must not be added again. + const result = await calculateCosts( + "gemma-4-31b-it", + "ranoai", + null, + 1000, + 330, // completionTokens already includes the 267 reasoning tokens + null, + undefined, + 267, + ); + + // inputPrice 0.1e-6, outputPrice 0.3e-6. Precision matters here: billing + // 330 vs 597 output tokens differs by only 8e-5, which the default + // two-decimal toBeCloseTo would not catch. + expect(result.inputCost).toBeCloseTo(0.0001, 10); + expect(result.outputCost).toBeCloseTo(0.000099, 10); // 330 * 0.3e-6, not 597 + expect(result.completionTokens).toBe(330); + }); + it("should handle null reasoning tokens gracefully", async () => { const result = await calculateCosts( "gemini-2.5-pro", diff --git a/apps/gateway/src/lib/costs.ts b/apps/gateway/src/lib/costs.ts index 41189b649f..c631bdfc24 100644 --- a/apps/gateway/src/lib/costs.ts +++ b/apps/gateway/src/lib/costs.ts @@ -643,8 +643,12 @@ export async function calculateCosts( // For Google models, completionTokens already includes reasoning tokens // (merged during extraction). The same holds for OpenAI-style Responses API // providers (OpenAI, Azure, Sakana, Meta), whose `output_tokens` counts - // reasoning — their `reasoning_tokens` detail is informational only. For - // remaining providers, add reasoning separately. + // reasoning — their `reasoning_tokens` detail is informational only. RanoAI + // reports reasoning in `completion_tokens_details` (which the streaming + // transform hoists to a top-level `reasoning_tokens`) while already counting + // it inside `completion_tokens`, so adding it again would roughly double the + // billed output on reasoning requests. For remaining providers, add + // reasoning separately. const completionIncludesReasoning = provider === "google-ai-studio" || provider === "glacier" || @@ -655,6 +659,7 @@ export async function calculateCosts( provider === "azure" || provider === "sakana" || provider === "meta" || + provider === "ranoai" || provider === "aws-mantle"; const totalOutputTokens = completionIncludesReasoning ? calculatedCompletionTokens diff --git a/apps/ui/src/components/provider-keys/provider-logo.ts b/apps/ui/src/components/provider-keys/provider-logo.ts index e388f6ed3d..79d30a624c 100644 --- a/apps/ui/src/components/provider-keys/provider-logo.ts +++ b/apps/ui/src/components/provider-keys/provider-logo.ts @@ -47,6 +47,7 @@ export const providerLogoUrls: Partial< gonka24: ProviderIcons.gonka24, fireworks: ProviderIcons.fireworks, runware: ProviderIcons.runware, + ranoai: ProviderIcons.ranoai, }; export const getProviderLogoDarkModeClasses = () => { diff --git a/infra/helm/llmgateway/values.yaml b/infra/helm/llmgateway/values.yaml index 9af56c707d..9392d36c28 100644 --- a/infra/helm/llmgateway/values.yaml +++ b/infra/helm/llmgateway/values.yaml @@ -460,6 +460,7 @@ llmProviders: # LLM_PERPLEXITY_API_KEY: "" # LLM_TOGETHER_AI_API_KEY: "" # LLM_SCX_AI_API_KEY: "" + # LLM_RANOAI_API_KEY: "" # LLM_ALIBABA_API_KEY: "" # LLM_NEBIUS_API_KEY: "" # LLM_CEREBRAS_API_KEY: "" diff --git a/packages/actions/src/get-provider-endpoint.ts b/packages/actions/src/get-provider-endpoint.ts index f40ab17df9..bdc88042d6 100644 --- a/packages/actions/src/get-provider-endpoint.ts +++ b/packages/actions/src/get-provider-endpoint.ts @@ -135,6 +135,7 @@ const PROVIDER_DEFAULT_BASE_URLS: Partial> = { deepinfra: "https://api.deepinfra.com/v1/openai", gonka24: "https://api.gonka24.com", fireworks: "https://api.fireworks.ai/inference", + ranoai: "https://api.ranoai.com", }; export function getProviderDefaultBaseUrl( @@ -865,6 +866,7 @@ export function getProviderEndpoint( case "tundra": case "scx-ai": case "scx-ai-gp": + case "ranoai": case "custom": default: return `${url}/v1/chat/completions`; diff --git a/packages/models/src/models/google.ts b/packages/models/src/models/google.ts index 15ce341d43..cbc64c3a1c 100644 --- a/packages/models/src/models/google.ts +++ b/packages/models/src/models/google.ts @@ -2450,6 +2450,45 @@ export const googleModels = [ tools: true, jsonOutput: true, }, + { + // RanoAI serves Gemma 4 on Furiosa RNGD NPUs. The NPU deployment + // enforces a 20480-token window (prompt + max_tokens). Reasoning + // arrives as `reasoning_content` (streamed as deltas) only when + // `reasoning_effort` is passed explicitly — a request without it + // returns no reasoning at all, and "none" suppresses it. + // `reasoning_tokens` is reported inside completion_tokens, so + // costs.ts lists this provider in completionIncludesReasoning. + // + // All four tool_choice modes are honoured, so none are declared + // here. Prompt caching is automatic prefix caching — the first + // request misses and later ones report `cached_tokens` — priced + // at the `input_cache_read` rate /v1/models advertises, half the + // input rate (verified 2026-08-05). + providerId: "ranoai", + externalId: "gemma-4-31b", + inputPrice: "0.1e-6", + outputPrice: "0.3e-6", + cachedInputPrice: "0.05e-6", + requestPrice: "0", + contextSize: 20480, + maxOutput: 20480, + streaming: true, + reasoning: true, + reasoningEfforts: [ + "none", + "minimal", + "low", + "medium", + "high", + "xhigh", + "max", + ], + vision: true, + tools: true, + jsonOutput: true, + jsonOutputSchema: true, + supportsN: true, + }, ], }, { diff --git a/packages/models/src/providers.ts b/packages/models/src/providers.ts index d30c854e00..a95b6a97cc 100644 --- a/packages/models/src/providers.ts +++ b/packages/models/src/providers.ts @@ -1781,6 +1781,32 @@ export const providers: ProviderDefinition[] = [ gdpr: true, }, }, + { + id: "ranoai", + name: "RanoAI", + description: + "RanoAI serves open-weight large language models on Furiosa RNGD NPU hardware via an OpenAI-compatible inference API.", + env: { + required: { + apiKey: "LLM_RANOAI_API_KEY", + }, + }, + streaming: true, + cancellation: true, + color: "#000000", + website: "https://ranoai.com", + statusPageUrl: null, + announcement: null, + termsUrl: "https://ranoai.com/terms", + privacyPolicyUrl: "https://ranoai.com/privacy", + headquarters: "US", + dataPolicy: { + apiTraining: false, + consumerTraining: false, + promptLogging: false, + retentionPeriod: "0 days", + }, + }, ] as const satisfies ProviderDefinition[]; export type ProviderId = (typeof providers)[number]["id"]; diff --git a/packages/shared/src/components/provider-icons.tsx b/packages/shared/src/components/provider-icons.tsx index c1a4646bf9..8a5b6ddb7f 100644 --- a/packages/shared/src/components/provider-icons.tsx +++ b/packages/shared/src/components/provider-icons.tsx @@ -1330,6 +1330,21 @@ export const FireworksIconStatic: React.FC> = ( ); +export const RanoAIIcon: React.FC> = (props) => ( + + + + + + + + +); + export const DeepInfraIcon: React.FC> = ( props, ) => ( @@ -1586,6 +1601,7 @@ export const ProviderIcons = { gonka24: Gonka24Icon, fireworks: FireworksIcon, runware: RunwareIcon, + ranoai: RanoAIIcon, } as const; // Type for provider icon keys @@ -1637,6 +1653,7 @@ export const providerLogoUrls: Partial< gonka24: ProviderIcons.gonka24, fireworks: ProviderIcons.fireworks, runware: ProviderIcons.runware, + ranoai: ProviderIcons.ranoai, }; export const getProviderLogoDarkModeClasses = () => {