Open Source AI Gateway for 100+ LLMs. Self-hosted. Enterprise-ready. Call any LLM in OpenAI format.
-
+
-
+
-
-
+
+
@@ -467,7 +467,7 @@ Provider API keys live in AWS Secrets Manager; reference ARNs via `gateway_extra
#### GCP — Cloud Run + Cloud SQL + Memorystore + HTTPS LB
-[](https://ssh.cloud.google.com/cloudshell/editor?cloudshell_git_repo=https%3A%2F%2Fgithub.com%2FBerriAI%2Flitellm&cloudshell_workspace=terraform%2Flitellm%2Fgcp%2Fexamples%2Fdefault&cloudshell_tutorial=TUTORIAL.md&cloudshell_image=gcr.io/ds-artifacts-cloudshell/deploystack_custom_image&shellonly=true)
+[](https://ssh.cloud.google.com/cloudshell/editor?cloudshell_git_repo=https%3A%2F%2Fgithub.com%2FBerriAI%2Flitellm&cloudshell_workspace=terraform%2Flitellm%2Fgcp%2Fexamples%2Fdefault&cloudshell_tutorial=TUTORIAL.md&cloudshell_image=gcr.io/ds-artifacts-cloudshell/deploystack_custom_image&shellonly=true)
Real 1-click. Opens Cloud Shell, clones this repo, and walks you through `terraform apply` via a built-in [DeployStack tutorial](./terraform/litellm/gcp/examples/default/TUTORIAL.md) — pick the project, the tutorial sets up the Artifact Registry remote repo, writes `terraform.tfvars` from your answers, and runs apply.
From 7cf1b263356fa6960da275f360db0031182fb1e2 Mon Sep 17 00:00:00 2001
From: Claude
Date: Sun, 7 Jun 2026 02:10:22 +0000
Subject: [PATCH 12/54] docs(readme): collapse Railway deploy anchor to a
single line
The Railway button wrapped its img across indented lines, so the anchor contained leading and trailing whitespace. GitHub underlines link content, rendering that whitespace as a small blue underline beside the button. Put the anchor on one line like the other three buttons so there is no inner whitespace to underline.
---
README.md | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/README.md b/README.md
index 9e6b4030966f..719f74e59248 100644
--- a/README.md
+++ b/README.md
@@ -7,9 +7,7 @@
Open Source AI Gateway for 100+ LLMs. Self-hosted. Enterprise-ready. Call any LLM in OpenAI format.
-
-
-
+
From 7f57a7a068d125dbe390e665391fe63f8a26dd8e Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Wed, 10 Jun 2026 03:58:08 +0000
Subject: [PATCH 13/54] Add Claude Fable 5 cost map entries as a data-only
hotfix
Backports only the model map changes from #30064 so deployments on
released litellm versions pick up Fable 5 pricing, context window, and
the adaptive thinking flag through the hosted cost map fetch without
upgrading. Includes the supports_sampling_params flag on the 28
Fable 5 / Opus 4.7 / Opus 4.8 entries (ignored by released code, read
by the gating that ships with the next release) and the matching
one-line schema declaration so the map validation test passes.
https://claude.ai/code/session_01MZarYYT3aS7DxaNjoax6Gm
---
...odel_prices_and_context_window_backup.json | 276 ++++++++++++++++++
model_prices_and_context_window.json | 276 ++++++++++++++++++
tests/test_litellm/test_utils.py | 1 +
3 files changed, 553 insertions(+)
diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index 397f96fdb1e8..757aacf1cafa 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -1156,6 +1156,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@@ -1202,6 +1203,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@@ -1233,6 +1235,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@@ -1264,6 +1267,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@@ -1295,6 +1299,139 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_sampling_params": false,
+ "supports_tool_choice": true,
+ "supports_vision": true,
+ "supports_xhigh_reasoning_effort": true,
+ "supports_native_structured_output": true,
+ "supports_max_reasoning_effort": true,
+ "supports_output_config": true,
+ "bedrock_output_config_effort_ceiling": "xhigh"
+ },
+ "anthropic.claude-fable-5": {
+ "cache_creation_input_token_cost": 1.25e-05,
+ "cache_creation_input_token_cost_above_1hr": 2e-05,
+ "cache_read_input_token_cost": 1e-06,
+ "input_cost_per_token": 1e-05,
+ "litellm_provider": "bedrock_converse",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 128000,
+ "max_tokens": 128000,
+ "mode": "chat",
+ "output_cost_per_token": 5e-05,
+ "search_context_cost_per_query": {
+ "search_context_size_high": 0.01,
+ "search_context_size_low": 0.01,
+ "search_context_size_medium": 0.01
+ },
+ "supports_adaptive_thinking": true,
+ "supports_assistant_prefill": false,
+ "supports_computer_use": true,
+ "supports_function_calling": true,
+ "supports_pdf_input": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_sampling_params": false,
+ "supports_tool_choice": true,
+ "supports_vision": true,
+ "supports_xhigh_reasoning_effort": true,
+ "supports_native_structured_output": true,
+ "supports_max_reasoning_effort": true,
+ "supports_output_config": true,
+ "bedrock_output_config_effort_ceiling": "xhigh"
+ },
+ "global.anthropic.claude-fable-5": {
+ "cache_creation_input_token_cost": 1.25e-05,
+ "cache_creation_input_token_cost_above_1hr": 2e-05,
+ "cache_read_input_token_cost": 1e-06,
+ "input_cost_per_token": 1e-05,
+ "litellm_provider": "bedrock_converse",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 128000,
+ "max_tokens": 128000,
+ "mode": "chat",
+ "output_cost_per_token": 5e-05,
+ "search_context_cost_per_query": {
+ "search_context_size_high": 0.01,
+ "search_context_size_low": 0.01,
+ "search_context_size_medium": 0.01
+ },
+ "supports_adaptive_thinking": true,
+ "supports_assistant_prefill": false,
+ "supports_computer_use": true,
+ "supports_function_calling": true,
+ "supports_pdf_input": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_sampling_params": false,
+ "supports_tool_choice": true,
+ "supports_vision": true,
+ "supports_xhigh_reasoning_effort": true,
+ "supports_native_structured_output": true,
+ "supports_max_reasoning_effort": true,
+ "supports_output_config": true,
+ "bedrock_output_config_effort_ceiling": "xhigh"
+ },
+ "us.anthropic.claude-fable-5": {
+ "cache_creation_input_token_cost": 1.375e-05,
+ "cache_creation_input_token_cost_above_1hr": 2.2e-05,
+ "cache_read_input_token_cost": 1.1e-06,
+ "input_cost_per_token": 1.1e-05,
+ "litellm_provider": "bedrock_converse",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 128000,
+ "max_tokens": 128000,
+ "mode": "chat",
+ "output_cost_per_token": 5.5e-05,
+ "search_context_cost_per_query": {
+ "search_context_size_high": 0.01,
+ "search_context_size_low": 0.01,
+ "search_context_size_medium": 0.01
+ },
+ "supports_adaptive_thinking": true,
+ "supports_assistant_prefill": false,
+ "supports_computer_use": true,
+ "supports_function_calling": true,
+ "supports_pdf_input": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_sampling_params": false,
+ "supports_tool_choice": true,
+ "supports_vision": true,
+ "supports_xhigh_reasoning_effort": true,
+ "supports_native_structured_output": true,
+ "supports_max_reasoning_effort": true,
+ "supports_output_config": true,
+ "bedrock_output_config_effort_ceiling": "xhigh"
+ },
+ "eu.anthropic.claude-fable-5": {
+ "cache_creation_input_token_cost": 1.375e-05,
+ "cache_creation_input_token_cost_above_1hr": 2.2e-05,
+ "cache_read_input_token_cost": 1.1e-06,
+ "input_cost_per_token": 1.1e-05,
+ "litellm_provider": "bedrock_converse",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 128000,
+ "max_tokens": 128000,
+ "mode": "chat",
+ "output_cost_per_token": 5.5e-05,
+ "search_context_cost_per_query": {
+ "search_context_size_high": 0.01,
+ "search_context_size_low": 0.01,
+ "search_context_size_medium": 0.01
+ },
+ "supports_adaptive_thinking": true,
+ "supports_assistant_prefill": false,
+ "supports_computer_use": true,
+ "supports_function_calling": true,
+ "supports_pdf_input": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@@ -1327,6 +1464,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@@ -1359,6 +1497,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@@ -1391,6 +1530,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@@ -1423,6 +1563,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@@ -1455,6 +1596,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@@ -1485,6 +1627,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@@ -2208,6 +2351,37 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_sampling_params": false,
+ "supports_tool_choice": true,
+ "supports_vision": true,
+ "supports_xhigh_reasoning_effort": true,
+ "supports_max_reasoning_effort": true
+ },
+ "azure_ai/claude-fable-5": {
+ "input_cost_per_token": 1e-05,
+ "output_cost_per_token": 5e-05,
+ "litellm_provider": "azure_ai",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 128000,
+ "max_tokens": 128000,
+ "mode": "chat",
+ "search_context_cost_per_query": {
+ "search_context_size_high": 0.01,
+ "search_context_size_low": 0.01,
+ "search_context_size_medium": 0.01
+ },
+ "cache_creation_input_token_cost": 1.25e-05,
+ "cache_creation_input_token_cost_above_1hr": 2e-05,
+ "cache_read_input_token_cost": 1e-06,
+ "supports_adaptive_thinking": true,
+ "supports_assistant_prefill": false,
+ "supports_computer_use": true,
+ "supports_function_calling": true,
+ "supports_pdf_input": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@@ -2237,6 +2411,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@@ -10133,6 +10308,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@@ -10167,6 +10343,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@@ -10177,6 +10354,40 @@
},
"supports_output_config": true
},
+ "claude-fable-5": {
+ "cache_creation_input_token_cost": 1.25e-05,
+ "cache_creation_input_token_cost_above_1hr": 2e-05,
+ "cache_read_input_token_cost": 1e-06,
+ "input_cost_per_token": 1e-05,
+ "litellm_provider": "anthropic",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 128000,
+ "max_tokens": 128000,
+ "mode": "chat",
+ "output_cost_per_token": 5e-05,
+ "search_context_cost_per_query": {
+ "search_context_size_high": 0.01,
+ "search_context_size_low": 0.01,
+ "search_context_size_medium": 0.01
+ },
+ "supports_adaptive_thinking": true,
+ "supports_assistant_prefill": false,
+ "supports_computer_use": true,
+ "supports_function_calling": true,
+ "supports_pdf_input": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_sampling_params": false,
+ "supports_tool_choice": true,
+ "supports_vision": true,
+ "supports_xhigh_reasoning_effort": true,
+ "supports_max_reasoning_effort": true,
+ "provider_specific_entry": {
+ "us": 1.1
+ },
+ "supports_output_config": true
+ },
"claude-opus-4-8": {
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
@@ -10201,6 +10412,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@@ -33967,6 +34179,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@@ -33995,6 +34208,67 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_sampling_params": false,
+ "supports_tool_choice": true,
+ "supports_vision": true,
+ "supports_xhigh_reasoning_effort": true,
+ "supports_max_reasoning_effort": true
+ },
+ "vertex_ai/claude-fable-5": {
+ "cache_creation_input_token_cost": 1.25e-05,
+ "cache_creation_input_token_cost_above_1hr": 2e-05,
+ "cache_read_input_token_cost": 1e-06,
+ "input_cost_per_token": 1e-05,
+ "litellm_provider": "vertex_ai-anthropic_models",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 128000,
+ "max_tokens": 128000,
+ "mode": "chat",
+ "output_cost_per_token": 5e-05,
+ "search_context_cost_per_query": {
+ "search_context_size_high": 0.01,
+ "search_context_size_low": 0.01,
+ "search_context_size_medium": 0.01
+ },
+ "supports_adaptive_thinking": true,
+ "supports_assistant_prefill": false,
+ "supports_computer_use": true,
+ "supports_function_calling": true,
+ "supports_pdf_input": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_sampling_params": false,
+ "supports_tool_choice": true,
+ "supports_vision": true,
+ "supports_xhigh_reasoning_effort": true,
+ "supports_max_reasoning_effort": true
+ },
+ "vertex_ai/claude-fable-5@default": {
+ "cache_creation_input_token_cost": 1.25e-05,
+ "cache_creation_input_token_cost_above_1hr": 2e-05,
+ "cache_read_input_token_cost": 1e-06,
+ "input_cost_per_token": 1e-05,
+ "litellm_provider": "vertex_ai-anthropic_models",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 128000,
+ "max_tokens": 128000,
+ "mode": "chat",
+ "output_cost_per_token": 5e-05,
+ "search_context_cost_per_query": {
+ "search_context_size_high": 0.01,
+ "search_context_size_low": 0.01,
+ "search_context_size_medium": 0.01
+ },
+ "supports_adaptive_thinking": true,
+ "supports_assistant_prefill": false,
+ "supports_computer_use": true,
+ "supports_function_calling": true,
+ "supports_pdf_input": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@@ -34024,6 +34298,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@@ -34053,6 +34328,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json
index b2836a096b71..ddd7d51d76b3 100644
--- a/model_prices_and_context_window.json
+++ b/model_prices_and_context_window.json
@@ -1156,6 +1156,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@@ -1202,6 +1203,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@@ -1233,6 +1235,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@@ -1264,6 +1267,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@@ -1295,6 +1299,139 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_sampling_params": false,
+ "supports_tool_choice": true,
+ "supports_vision": true,
+ "supports_xhigh_reasoning_effort": true,
+ "supports_native_structured_output": true,
+ "supports_max_reasoning_effort": true,
+ "supports_output_config": true,
+ "bedrock_output_config_effort_ceiling": "xhigh"
+ },
+ "anthropic.claude-fable-5": {
+ "cache_creation_input_token_cost": 1.25e-05,
+ "cache_creation_input_token_cost_above_1hr": 2e-05,
+ "cache_read_input_token_cost": 1e-06,
+ "input_cost_per_token": 1e-05,
+ "litellm_provider": "bedrock_converse",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 128000,
+ "max_tokens": 128000,
+ "mode": "chat",
+ "output_cost_per_token": 5e-05,
+ "search_context_cost_per_query": {
+ "search_context_size_high": 0.01,
+ "search_context_size_low": 0.01,
+ "search_context_size_medium": 0.01
+ },
+ "supports_adaptive_thinking": true,
+ "supports_assistant_prefill": false,
+ "supports_computer_use": true,
+ "supports_function_calling": true,
+ "supports_pdf_input": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_sampling_params": false,
+ "supports_tool_choice": true,
+ "supports_vision": true,
+ "supports_xhigh_reasoning_effort": true,
+ "supports_native_structured_output": true,
+ "supports_max_reasoning_effort": true,
+ "supports_output_config": true,
+ "bedrock_output_config_effort_ceiling": "xhigh"
+ },
+ "global.anthropic.claude-fable-5": {
+ "cache_creation_input_token_cost": 1.25e-05,
+ "cache_creation_input_token_cost_above_1hr": 2e-05,
+ "cache_read_input_token_cost": 1e-06,
+ "input_cost_per_token": 1e-05,
+ "litellm_provider": "bedrock_converse",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 128000,
+ "max_tokens": 128000,
+ "mode": "chat",
+ "output_cost_per_token": 5e-05,
+ "search_context_cost_per_query": {
+ "search_context_size_high": 0.01,
+ "search_context_size_low": 0.01,
+ "search_context_size_medium": 0.01
+ },
+ "supports_adaptive_thinking": true,
+ "supports_assistant_prefill": false,
+ "supports_computer_use": true,
+ "supports_function_calling": true,
+ "supports_pdf_input": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_sampling_params": false,
+ "supports_tool_choice": true,
+ "supports_vision": true,
+ "supports_xhigh_reasoning_effort": true,
+ "supports_native_structured_output": true,
+ "supports_max_reasoning_effort": true,
+ "supports_output_config": true,
+ "bedrock_output_config_effort_ceiling": "xhigh"
+ },
+ "us.anthropic.claude-fable-5": {
+ "cache_creation_input_token_cost": 1.375e-05,
+ "cache_creation_input_token_cost_above_1hr": 2.2e-05,
+ "cache_read_input_token_cost": 1.1e-06,
+ "input_cost_per_token": 1.1e-05,
+ "litellm_provider": "bedrock_converse",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 128000,
+ "max_tokens": 128000,
+ "mode": "chat",
+ "output_cost_per_token": 5.5e-05,
+ "search_context_cost_per_query": {
+ "search_context_size_high": 0.01,
+ "search_context_size_low": 0.01,
+ "search_context_size_medium": 0.01
+ },
+ "supports_adaptive_thinking": true,
+ "supports_assistant_prefill": false,
+ "supports_computer_use": true,
+ "supports_function_calling": true,
+ "supports_pdf_input": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_sampling_params": false,
+ "supports_tool_choice": true,
+ "supports_vision": true,
+ "supports_xhigh_reasoning_effort": true,
+ "supports_native_structured_output": true,
+ "supports_max_reasoning_effort": true,
+ "supports_output_config": true,
+ "bedrock_output_config_effort_ceiling": "xhigh"
+ },
+ "eu.anthropic.claude-fable-5": {
+ "cache_creation_input_token_cost": 1.375e-05,
+ "cache_creation_input_token_cost_above_1hr": 2.2e-05,
+ "cache_read_input_token_cost": 1.1e-06,
+ "input_cost_per_token": 1.1e-05,
+ "litellm_provider": "bedrock_converse",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 128000,
+ "max_tokens": 128000,
+ "mode": "chat",
+ "output_cost_per_token": 5.5e-05,
+ "search_context_cost_per_query": {
+ "search_context_size_high": 0.01,
+ "search_context_size_low": 0.01,
+ "search_context_size_medium": 0.01
+ },
+ "supports_adaptive_thinking": true,
+ "supports_assistant_prefill": false,
+ "supports_computer_use": true,
+ "supports_function_calling": true,
+ "supports_pdf_input": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@@ -1327,6 +1464,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@@ -1359,6 +1497,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@@ -1391,6 +1530,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@@ -1423,6 +1563,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@@ -1455,6 +1596,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@@ -1485,6 +1627,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@@ -2208,6 +2351,37 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_sampling_params": false,
+ "supports_tool_choice": true,
+ "supports_vision": true,
+ "supports_xhigh_reasoning_effort": true,
+ "supports_max_reasoning_effort": true
+ },
+ "azure_ai/claude-fable-5": {
+ "input_cost_per_token": 1e-05,
+ "output_cost_per_token": 5e-05,
+ "litellm_provider": "azure_ai",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 128000,
+ "max_tokens": 128000,
+ "mode": "chat",
+ "search_context_cost_per_query": {
+ "search_context_size_high": 0.01,
+ "search_context_size_low": 0.01,
+ "search_context_size_medium": 0.01
+ },
+ "cache_creation_input_token_cost": 1.25e-05,
+ "cache_creation_input_token_cost_above_1hr": 2e-05,
+ "cache_read_input_token_cost": 1e-06,
+ "supports_adaptive_thinking": true,
+ "supports_assistant_prefill": false,
+ "supports_computer_use": true,
+ "supports_function_calling": true,
+ "supports_pdf_input": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@@ -2237,6 +2411,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@@ -10133,6 +10308,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@@ -10167,6 +10343,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@@ -10177,6 +10354,40 @@
},
"supports_output_config": true
},
+ "claude-fable-5": {
+ "cache_creation_input_token_cost": 1.25e-05,
+ "cache_creation_input_token_cost_above_1hr": 2e-05,
+ "cache_read_input_token_cost": 1e-06,
+ "input_cost_per_token": 1e-05,
+ "litellm_provider": "anthropic",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 128000,
+ "max_tokens": 128000,
+ "mode": "chat",
+ "output_cost_per_token": 5e-05,
+ "search_context_cost_per_query": {
+ "search_context_size_high": 0.01,
+ "search_context_size_low": 0.01,
+ "search_context_size_medium": 0.01
+ },
+ "supports_adaptive_thinking": true,
+ "supports_assistant_prefill": false,
+ "supports_computer_use": true,
+ "supports_function_calling": true,
+ "supports_pdf_input": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_sampling_params": false,
+ "supports_tool_choice": true,
+ "supports_vision": true,
+ "supports_xhigh_reasoning_effort": true,
+ "supports_max_reasoning_effort": true,
+ "provider_specific_entry": {
+ "us": 1.1
+ },
+ "supports_output_config": true
+ },
"claude-opus-4-8": {
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
@@ -10201,6 +10412,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@@ -34007,6 +34219,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@@ -34035,6 +34248,67 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_sampling_params": false,
+ "supports_tool_choice": true,
+ "supports_vision": true,
+ "supports_xhigh_reasoning_effort": true,
+ "supports_max_reasoning_effort": true
+ },
+ "vertex_ai/claude-fable-5": {
+ "cache_creation_input_token_cost": 1.25e-05,
+ "cache_creation_input_token_cost_above_1hr": 2e-05,
+ "cache_read_input_token_cost": 1e-06,
+ "input_cost_per_token": 1e-05,
+ "litellm_provider": "vertex_ai-anthropic_models",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 128000,
+ "max_tokens": 128000,
+ "mode": "chat",
+ "output_cost_per_token": 5e-05,
+ "search_context_cost_per_query": {
+ "search_context_size_high": 0.01,
+ "search_context_size_low": 0.01,
+ "search_context_size_medium": 0.01
+ },
+ "supports_adaptive_thinking": true,
+ "supports_assistant_prefill": false,
+ "supports_computer_use": true,
+ "supports_function_calling": true,
+ "supports_pdf_input": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_sampling_params": false,
+ "supports_tool_choice": true,
+ "supports_vision": true,
+ "supports_xhigh_reasoning_effort": true,
+ "supports_max_reasoning_effort": true
+ },
+ "vertex_ai/claude-fable-5@default": {
+ "cache_creation_input_token_cost": 1.25e-05,
+ "cache_creation_input_token_cost_above_1hr": 2e-05,
+ "cache_read_input_token_cost": 1e-06,
+ "input_cost_per_token": 1e-05,
+ "litellm_provider": "vertex_ai-anthropic_models",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 128000,
+ "max_tokens": 128000,
+ "mode": "chat",
+ "output_cost_per_token": 5e-05,
+ "search_context_cost_per_query": {
+ "search_context_size_high": 0.01,
+ "search_context_size_low": 0.01,
+ "search_context_size_medium": 0.01
+ },
+ "supports_adaptive_thinking": true,
+ "supports_assistant_prefill": false,
+ "supports_computer_use": true,
+ "supports_function_calling": true,
+ "supports_pdf_input": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@@ -34064,6 +34338,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@@ -34093,6 +34368,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py
index f179e9c8f93b..4c4d9e1133bc 100644
--- a/tests/test_litellm/test_utils.py
+++ b/tests/test_litellm/test_utils.py
@@ -858,6 +858,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
"supports_xhigh_reasoning_effort": {"type": "boolean"},
"supports_max_reasoning_effort": {"type": "boolean"},
"supports_adaptive_thinking": {"type": "boolean"},
+ "supports_sampling_params": {"type": "boolean"},
"supports_service_tier": {"type": "boolean"},
"supports_preset": {"type": "boolean"},
"supports_output_config": {"type": "boolean"},
From 70afb75ff1078a8f9cb1d2eb75e93e6fe4fda736 Mon Sep 17 00:00:00 2001
From: Ishaan Jaffer
Date: Mon, 22 Jun 2026 14:42:13 -0700
Subject: [PATCH 14/54] Add litellm rust workspace with mistral ocr bridge
---
.github/workflows/test-rust.yml | 54 +++++
litellm-rust/.gitignore | 2 +
litellm-rust/CLAUDE.md | 88 ++++++++
litellm-rust/Cargo.toml | 20 ++
litellm-rust/README.md | 34 +++
litellm-rust/crates/core/CLAUDE.md | 38 ++++
litellm-rust/crates/core/Cargo.toml | 11 +
litellm-rust/crates/core/src/error.rs | 27 +++
litellm-rust/crates/core/src/lib.rs | 4 +
litellm-rust/crates/core/src/ocr/mod.rs | 2 +
.../crates/core/src/ocr/transformation.rs | 32 +++
litellm-rust/crates/core/src/ocr/types.rs | 29 +++
litellm-rust/crates/providers/CLAUDE.md | 53 +++++
litellm-rust/crates/providers/Cargo.toml | 13 ++
litellm-rust/crates/providers/src/lib.rs | 2 +
.../crates/providers/src/mistral/mod.rs | 1 +
.../crates/providers/src/mistral/ocr/mod.rs | 1 +
.../src/mistral/ocr/transformation.rs | 212 ++++++++++++++++++
litellm-rust/crates/providers/src/ocr.rs | 156 +++++++++++++
litellm-rust/crates/python-bridge/CLAUDE.md | 36 +++
litellm-rust/crates/python-bridge/Cargo.toml | 15 ++
litellm-rust/crates/python-bridge/src/lib.rs | 31 +++
litellm/ocr/main.py | 6 +
litellm/rust_bridge/CLAUDE.md | 39 ++++
litellm/rust_bridge/__init__.py | 19 ++
litellm/rust_bridge/loader.py | 68 ++++++
litellm/rust_bridge/ocr/__init__.py | 8 +
litellm/rust_bridge/ocr/config.py | 160 +++++++++++++
litellm/rust_bridge/ocr/providers.py | 35 +++
.../interactions/test_openapi_compliance.py | 1 -
.../test_mistral_ocr_rust_bridge.py | 183 +++++++++++++++
31 files changed, 1379 insertions(+), 1 deletion(-)
create mode 100644 .github/workflows/test-rust.yml
create mode 100644 litellm-rust/.gitignore
create mode 100644 litellm-rust/CLAUDE.md
create mode 100644 litellm-rust/Cargo.toml
create mode 100644 litellm-rust/README.md
create mode 100644 litellm-rust/crates/core/CLAUDE.md
create mode 100644 litellm-rust/crates/core/Cargo.toml
create mode 100644 litellm-rust/crates/core/src/error.rs
create mode 100644 litellm-rust/crates/core/src/lib.rs
create mode 100644 litellm-rust/crates/core/src/ocr/mod.rs
create mode 100644 litellm-rust/crates/core/src/ocr/transformation.rs
create mode 100644 litellm-rust/crates/core/src/ocr/types.rs
create mode 100644 litellm-rust/crates/providers/CLAUDE.md
create mode 100644 litellm-rust/crates/providers/Cargo.toml
create mode 100644 litellm-rust/crates/providers/src/lib.rs
create mode 100644 litellm-rust/crates/providers/src/mistral/mod.rs
create mode 100644 litellm-rust/crates/providers/src/mistral/ocr/mod.rs
create mode 100644 litellm-rust/crates/providers/src/mistral/ocr/transformation.rs
create mode 100644 litellm-rust/crates/providers/src/ocr.rs
create mode 100644 litellm-rust/crates/python-bridge/CLAUDE.md
create mode 100644 litellm-rust/crates/python-bridge/Cargo.toml
create mode 100644 litellm-rust/crates/python-bridge/src/lib.rs
create mode 100644 litellm/rust_bridge/CLAUDE.md
create mode 100644 litellm/rust_bridge/__init__.py
create mode 100644 litellm/rust_bridge/loader.py
create mode 100644 litellm/rust_bridge/ocr/__init__.py
create mode 100644 litellm/rust_bridge/ocr/config.py
create mode 100644 litellm/rust_bridge/ocr/providers.py
create mode 100644 tests/test_litellm/test_mistral_ocr_rust_bridge.py
diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml
new file mode 100644
index 000000000000..c085e0a29b3a
--- /dev/null
+++ b/.github/workflows/test-rust.yml
@@ -0,0 +1,54 @@
+name: LiteLLM Rust
+
+on:
+ push:
+ paths:
+ - "litellm-rust/**"
+ - ".github/workflows/test-rust.yml"
+ pull_request:
+ branches:
+ - main
+ - litellm_internal_staging
+ - litellm_oss_branch
+ - "litellm_**"
+ paths:
+ - "litellm-rust/**"
+ - ".github/workflows/test-rust.yml"
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ rust-checks:
+ name: rustfmt, clippy, test
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ defaults:
+ run:
+ working-directory: litellm-rust
+ env:
+ CARGO_TERM_COLOR: always
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+ with:
+ persist-credentials: false
+
+ - name: Set up Rust
+ run: |
+ rustup toolchain install stable --profile minimal --component clippy,rustfmt
+ rustup default stable
+
+ - name: Check Rust formatting
+ run: cargo fmt --check
+
+ - name: Run Clippy
+ run: cargo clippy --workspace --all-targets -- -D warnings
+
+ - name: Run Rust tests
+ run: cargo test --workspace
diff --git a/litellm-rust/.gitignore b/litellm-rust/.gitignore
new file mode 100644
index 000000000000..ca98cd96efdc
--- /dev/null
+++ b/litellm-rust/.gitignore
@@ -0,0 +1,2 @@
+/target/
+Cargo.lock
diff --git a/litellm-rust/CLAUDE.md b/litellm-rust/CLAUDE.md
new file mode 100644
index 000000000000..1d2987e0a1a5
--- /dev/null
+++ b/litellm-rust/CLAUDE.md
@@ -0,0 +1,88 @@
+# CLAUDE.md
+
+This file defines the rules for Rust work in LiteLLM.
+
+## Core Boundary
+
+The `core` and `providers` crates describe work; hosts execute work.
+
+Route-level Rust structure mirrors LiteLLM's Python responsibilities:
+- `core/src//` owns the route contract, shared types, and provider
+ template traits. For OCR, this means `core/src/ocr`.
+- `providers/src///transformation.rs` owns the
+ provider-specific transform. For Mistral OCR, this means
+ `providers/src/mistral/ocr/transformation.rs`.
+- Future network execution belongs in a host/transport layer such as
+ `llm_http_handler`, not inside `core` or `providers`.
+
+Allowed in `core` and `providers`:
+- Pure request transforms
+- Pure response transforms
+- Pure stream chunk normalization
+- Shared data types and validation errors
+- Deterministic token/cost helper logic
+
+Not allowed in `core` or `providers`:
+- Network calls
+- Environment variable or secret reads
+- Filesystem access
+- Database or cache access
+- Provider SDK signing or auth flows
+- Logging callbacks, spend writes, or custom callbacks
+- Global mutable runtime state
+
+Python owns rollout state and fallback while Rust is being introduced. Rust
+paths must be off by default until parity tests prove equivalence with Python.
+
+## Production Bar
+
+Rust code in this workspace is held to a strict parity and robustness bar from
+the first PR:
+
+- Correctness parity is proven with tests. Do not rely on README claims or
+ manual inspection for a port that mirrors Python behavior.
+- Every provider transform must have unit tests for supported-parameter
+ filtering, request body shape, response normalization, missing/null fields,
+ and bad-input errors.
+- When Rust is exposed through Python, add Python tests that prove disabled,
+ enabled, and unavailable-bridge fallback behavior.
+- Avoid panics on user/provider input. Return typed errors and let the host map
+ them to Python exceptions or HTTP responses.
+- OCR handles documents that often contain personal data. Do not log document
+ contents, base64 payloads, provider response bodies, or secrets.
+- Error messages must be useful but data-minimized. Truncate or sanitize any
+ upstream body before it crosses a host boundary.
+- Treat empty or whitespace-only credentials, URLs, and config values as absent
+ at the host/config resolution layer.
+- Preserve Python output shape intentionally. If a field is always serialized as
+ `null` for Python parity, leave a short comment explaining that parity choice.
+
+## Host I/O Rules
+
+These rules apply when adding future crates or modules that execute network I/O,
+such as `ai-gateway`, router hosts, or standalone servers:
+
+- Set connect and full-request timeouts. No unbounded waits.
+- Reuse HTTP clients; do not construct clients per request.
+- Prefer rustls TLS for portable Python wheels and Linux images unless there is
+ a documented reason not to.
+- Add request IDs and structured tracing at the host layer, without logging OCR
+ document contents or secrets.
+- Do not echo raw upstream response bodies to callers. Sanitize and bound them.
+- Avoid `expect`/`unwrap` in server startup and request paths unless the panic is
+ impossible by construction and documented.
+
+## Checks
+
+Run these before pushing Rust changes. The same checks run in GitHub Actions
+for changes under `litellm-rust/`.
+
+```bash
+cd litellm-rust
+cargo fmt --check
+cargo clippy --workspace --all-targets -- -D warnings
+cargo test --workspace
+```
+
+When a Rust path is exposed through Python, add Python parity tests that compare
+the existing Python output with the Rust-backed output.
diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml
new file mode 100644
index 000000000000..4117bd5f6ac6
--- /dev/null
+++ b/litellm-rust/Cargo.toml
@@ -0,0 +1,20 @@
+[workspace]
+members = [
+ "crates/core",
+ "crates/providers",
+ "crates/python-bridge",
+]
+resolver = "2"
+
+[workspace.package]
+edition = "2021"
+license = "MIT"
+repository = "https://github.com/BerriAI/litellm"
+
+[workspace.dependencies]
+litellm-core = { path = "crates/core" }
+litellm-providers = { path = "crates/providers" }
+pyo3 = "0.23.5"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+thiserror = "2.0"
diff --git a/litellm-rust/README.md b/litellm-rust/README.md
new file mode 100644
index 000000000000..15ad18554202
--- /dev/null
+++ b/litellm-rust/README.md
@@ -0,0 +1,34 @@
+# LiteLLM Rust
+
+This workspace contains the staged Rust implementation for LiteLLM.
+
+Rust starts as a pure transform core used by the existing Python host. Python
+continues to own auth, configuration, network I/O, retries, routing, logging,
+callbacks, spend tracking, and customer plugins until each Rust path has parity
+coverage and production evidence.
+
+## Layout
+
+```text
+crates/
+ core/ Route contracts, shared pure types, errors, and templates.
+ src/ocr/
+ providers/ Provider-specific pure transforms.
+ src/mistral/ocr/transformation.rs
+ python-bridge/ PyO3 bridge for Python LiteLLM.
+```
+
+The folder shape should follow the Python provider tree:
+`providers/src///transformation.rs`. The bridge should expose
+one function per top-level route, starting with `ocr(payload)`.
+
+## Checks
+
+Run these before pushing Rust changes. GitHub Actions runs the same checks for
+changes under `litellm-rust/`.
+
+```bash
+cargo fmt --check
+cargo clippy --workspace --all-targets -- -D warnings
+cargo test --workspace
+```
diff --git a/litellm-rust/crates/core/CLAUDE.md b/litellm-rust/crates/core/CLAUDE.md
new file mode 100644
index 000000000000..51e0d215e6aa
--- /dev/null
+++ b/litellm-rust/crates/core/CLAUDE.md
@@ -0,0 +1,38 @@
+# CLAUDE.md
+
+Rules for `litellm-rust/crates/core`.
+
+## Responsibility
+
+`core` owns shared data types, typed errors, and deterministic helper contracts.
+It must stay pure and host-independent.
+
+Allowed:
+- Shared request/response structs.
+- Typed errors with stable, non-sensitive messages.
+- Deterministic validation helpers.
+- Serialization helpers that intentionally mirror Python output shape.
+- Route templates that match Python base config responsibilities, such as
+ `ocr::transformation::OcrProviderConfig`.
+
+Not allowed:
+- Network, filesystem, database, cache, or environment access.
+- Secret reads or auth/header construction.
+- Logging callbacks, tracing spans, spend writes, or customer callbacks.
+- Provider-specific branching that belongs in `providers`.
+- Panics for user/provider-controlled input.
+
+## Structure
+
+Use route names directly under `src/`: `ocr`, future `messages`,
+`chat_completions`, `embeddings`, and similar top-level LiteLLM calls. Do not
+invent broad names like `engine` for route contracts.
+
+## Parity Rules
+
+- Every shared type used by a provider transform needs unit tests for
+ serialization shape.
+- If Python parity requires always emitting a `null` field instead of omitting
+ it, document that in code and pin it with a test.
+- Error enums should preserve enough detail for Python/HTTP hosts to map errors
+ consistently without exposing document contents or upstream bodies.
diff --git a/litellm-rust/crates/core/Cargo.toml b/litellm-rust/crates/core/Cargo.toml
new file mode 100644
index 000000000000..e54002fe5e8d
--- /dev/null
+++ b/litellm-rust/crates/core/Cargo.toml
@@ -0,0 +1,11 @@
+[package]
+name = "litellm-core"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+serde.workspace = true
+serde_json.workspace = true
+thiserror.workspace = true
diff --git a/litellm-rust/crates/core/src/error.rs b/litellm-rust/crates/core/src/error.rs
new file mode 100644
index 000000000000..08a7a2b5ed9e
--- /dev/null
+++ b/litellm-rust/crates/core/src/error.rs
@@ -0,0 +1,27 @@
+use thiserror::Error;
+
+pub type CoreResult = Result;
+
+#[derive(Debug, Error, PartialEq, Eq)]
+pub enum CoreError {
+ #[error("expected {expected}, got {actual}")]
+ InvalidType {
+ expected: &'static str,
+ actual: &'static str,
+ },
+ #[error("missing required field: {0}")]
+ MissingField(&'static str),
+ #[error("invalid response: {0}")]
+ InvalidResponse(String),
+}
+
+pub fn json_type_name(value: &serde_json::Value) -> &'static str {
+ match value {
+ serde_json::Value::Null => "null",
+ serde_json::Value::Bool(_) => "bool",
+ serde_json::Value::Number(_) => "number",
+ serde_json::Value::String(_) => "string",
+ serde_json::Value::Array(_) => "array",
+ serde_json::Value::Object(_) => "object",
+ }
+}
diff --git a/litellm-rust/crates/core/src/lib.rs b/litellm-rust/crates/core/src/lib.rs
new file mode 100644
index 000000000000..a88204867c45
--- /dev/null
+++ b/litellm-rust/crates/core/src/lib.rs
@@ -0,0 +1,4 @@
+pub mod error;
+pub mod ocr;
+
+pub use error::{CoreError, CoreResult};
diff --git a/litellm-rust/crates/core/src/ocr/mod.rs b/litellm-rust/crates/core/src/ocr/mod.rs
new file mode 100644
index 000000000000..ec2fbb969a62
--- /dev/null
+++ b/litellm-rust/crates/core/src/ocr/mod.rs
@@ -0,0 +1,2 @@
+pub mod transformation;
+pub mod types;
diff --git a/litellm-rust/crates/core/src/ocr/transformation.rs b/litellm-rust/crates/core/src/ocr/transformation.rs
new file mode 100644
index 000000000000..7353d9d22c42
--- /dev/null
+++ b/litellm-rust/crates/core/src/ocr/transformation.rs
@@ -0,0 +1,32 @@
+use serde_json::{Map, Value};
+
+use crate::CoreResult;
+
+use super::types::{OcrRequestData, OcrResponseData};
+
+pub trait OcrProviderConfig {
+ fn supported_ocr_params(&self) -> &'static [&'static str];
+
+ fn map_ocr_params(&self, non_default_params: &Map) -> Map {
+ let mut mapped_params = Map::new();
+ for (param, value) in non_default_params {
+ if self.supported_ocr_params().contains(¶m.as_str()) {
+ mapped_params.insert(param.clone(), value.clone());
+ }
+ }
+ mapped_params
+ }
+
+ fn transform_ocr_request(
+ &self,
+ model: &str,
+ document: Value,
+ optional_params: Map,
+ ) -> CoreResult;
+
+ fn transform_ocr_response(
+ &self,
+ model: &str,
+ response_json: Value,
+ ) -> CoreResult;
+}
diff --git a/litellm-rust/crates/core/src/ocr/types.rs b/litellm-rust/crates/core/src/ocr/types.rs
new file mode 100644
index 000000000000..1a72b8f1d666
--- /dev/null
+++ b/litellm-rust/crates/core/src/ocr/types.rs
@@ -0,0 +1,29 @@
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct OcrRequestData {
+ pub data: Value,
+ pub files: Option,
+}
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct OcrResponseData {
+ pub pages: Vec,
+ pub model: String,
+ pub document_annotation: Option,
+ pub usage_info: Option,
+ pub object: String,
+}
+
+impl OcrResponseData {
+ pub fn into_json(self) -> Value {
+ serde_json::json!({
+ "pages": self.pages,
+ "model": self.model,
+ "document_annotation": self.document_annotation,
+ "usage_info": self.usage_info,
+ "object": self.object,
+ })
+ }
+}
diff --git a/litellm-rust/crates/providers/CLAUDE.md b/litellm-rust/crates/providers/CLAUDE.md
new file mode 100644
index 000000000000..0f7fdcda2aa1
--- /dev/null
+++ b/litellm-rust/crates/providers/CLAUDE.md
@@ -0,0 +1,53 @@
+# CLAUDE.md
+
+Rules for `litellm-rust/crates/providers`.
+
+## Responsibility
+
+`providers` owns provider-specific pure transforms. It mirrors the existing
+Python provider modules closely enough that parity review is mechanical.
+
+Provider files should map to the Python provider tree:
+
+```text
+providers/src///transformation.rs
+```
+
+For example, Mistral OCR lives at
+`providers/src/mistral/ocr/transformation.rs`, matching
+`litellm/llms/mistral/ocr/transformation.py`.
+
+Allowed:
+- Provider request transforms.
+- Provider response normalization.
+- Supported-parameter filtering.
+- Provider-specific validation that does not require I/O or secrets.
+
+Not allowed:
+- HTTP clients or provider SDK calls.
+- Environment variable reads.
+- API key resolution or auth header construction.
+- Logging, callbacks, spend tracking, retries, routing, cooldowns, or fallbacks.
+- Panics on bad user/provider input.
+
+## Required Tests
+
+Every provider transform must include focused unit tests for:
+- Supported params matching the Python provider config.
+- Unknown params being dropped or transformed the same way as Python.
+- Request body shape matching Python output.
+- Response normalization with complete, missing, null, and extra fields.
+- Bad input returning typed errors.
+
+For OCR specifically, assume documents can contain personal data. Tests should
+prove transforms do not copy document contents into error messages.
+
+## Implementation Rules
+
+- Prefer static supported-parameter lists over allocating strings on every call.
+- Keep transforms deterministic and allocation-conscious, but choose clarity over
+ premature micro-optimization for tiny parameter lists.
+- Use typed errors from `core`; avoid stringly-typed error plumbing.
+- Add comments only when they explain Python-parity decisions or provider quirks.
+- Put route-level provider dispatch in a route file such as `providers/src/ocr.rs`.
+ Do not move provider-specific transform logic into the Python bridge.
diff --git a/litellm-rust/crates/providers/Cargo.toml b/litellm-rust/crates/providers/Cargo.toml
new file mode 100644
index 000000000000..68483af62667
--- /dev/null
+++ b/litellm-rust/crates/providers/Cargo.toml
@@ -0,0 +1,13 @@
+[package]
+name = "litellm-providers"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+litellm-core.workspace = true
+serde_json.workspace = true
+
+[dev-dependencies]
+serde_json.workspace = true
diff --git a/litellm-rust/crates/providers/src/lib.rs b/litellm-rust/crates/providers/src/lib.rs
new file mode 100644
index 000000000000..1a0ca5b7e42a
--- /dev/null
+++ b/litellm-rust/crates/providers/src/lib.rs
@@ -0,0 +1,2 @@
+pub mod mistral;
+pub mod ocr;
diff --git a/litellm-rust/crates/providers/src/mistral/mod.rs b/litellm-rust/crates/providers/src/mistral/mod.rs
new file mode 100644
index 000000000000..3621ff6a2fdb
--- /dev/null
+++ b/litellm-rust/crates/providers/src/mistral/mod.rs
@@ -0,0 +1 @@
+pub mod ocr;
diff --git a/litellm-rust/crates/providers/src/mistral/ocr/mod.rs b/litellm-rust/crates/providers/src/mistral/ocr/mod.rs
new file mode 100644
index 000000000000..f239b6921fac
--- /dev/null
+++ b/litellm-rust/crates/providers/src/mistral/ocr/mod.rs
@@ -0,0 +1 @@
+pub mod transformation;
diff --git a/litellm-rust/crates/providers/src/mistral/ocr/transformation.rs b/litellm-rust/crates/providers/src/mistral/ocr/transformation.rs
new file mode 100644
index 000000000000..ca9073a14a09
--- /dev/null
+++ b/litellm-rust/crates/providers/src/mistral/ocr/transformation.rs
@@ -0,0 +1,212 @@
+use litellm_core::error::{json_type_name, CoreError, CoreResult};
+use litellm_core::ocr::transformation::OcrProviderConfig;
+use litellm_core::ocr::types::{OcrRequestData, OcrResponseData};
+use serde_json::{Map, Value};
+
+const SUPPORTED_OCR_PARAMS: &[&str] = &[
+ "pages",
+ "include_image_base64",
+ "image_limit",
+ "image_min_size",
+ "bbox_annotation_format",
+ "document_annotation_format",
+ "document_annotation_prompt",
+ "extract_header",
+ "extract_footer",
+ "table_format",
+ "confidence_scores_granularity",
+ "id",
+];
+
+pub struct MistralOcrConfig;
+
+pub const MISTRAL_OCR_CONFIG: MistralOcrConfig = MistralOcrConfig;
+
+impl OcrProviderConfig for MistralOcrConfig {
+ fn supported_ocr_params(&self) -> &'static [&'static str] {
+ SUPPORTED_OCR_PARAMS
+ }
+
+ fn transform_ocr_request(
+ &self,
+ model: &str,
+ document: Value,
+ optional_params: Map,
+ ) -> CoreResult {
+ if !document.is_object() {
+ return Err(CoreError::InvalidType {
+ expected: "object",
+ actual: json_type_name(&document),
+ });
+ }
+
+ let mut data = Map::new();
+ data.insert("model".to_string(), Value::String(model.to_string()));
+ data.insert("document".to_string(), document);
+ for (param, value) in optional_params {
+ data.insert(param, value);
+ }
+
+ Ok(OcrRequestData {
+ data: Value::Object(data),
+ files: None,
+ })
+ }
+
+ fn transform_ocr_response(
+ &self,
+ model: &str,
+ response_json: Value,
+ ) -> CoreResult {
+ let response_object = response_json
+ .as_object()
+ .ok_or_else(|| CoreError::InvalidType {
+ expected: "object",
+ actual: json_type_name(&response_json),
+ })?;
+
+ let pages = response_object
+ .get("pages")
+ .and_then(Value::as_array)
+ .cloned()
+ .unwrap_or_default();
+ let model = response_object
+ .get("model")
+ .and_then(Value::as_str)
+ .unwrap_or(model)
+ .to_string();
+ let document_annotation = response_object.get("document_annotation").cloned();
+ let usage_info = response_object.get("usage_info").cloned();
+
+ Ok(OcrResponseData {
+ pages,
+ model,
+ document_annotation,
+ usage_info,
+ object: "ocr".to_string(),
+ })
+ }
+}
+
+pub fn supported_ocr_params() -> &'static [&'static str] {
+ MISTRAL_OCR_CONFIG.supported_ocr_params()
+}
+
+pub fn map_ocr_params(non_default_params: &Map) -> Map {
+ MISTRAL_OCR_CONFIG.map_ocr_params(non_default_params)
+}
+
+pub fn transform_ocr_request(
+ model: &str,
+ document: Value,
+ optional_params: Map,
+) -> CoreResult {
+ MISTRAL_OCR_CONFIG.transform_ocr_request(model, document, optional_params)
+}
+
+pub fn transform_ocr_response(model: &str, response_json: Value) -> CoreResult {
+ MISTRAL_OCR_CONFIG.transform_ocr_response(model, response_json)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde_json::json;
+
+ #[test]
+ fn supported_params_match_python_mistral_ocr_config() {
+ assert_eq!(
+ supported_ocr_params(),
+ &[
+ "pages",
+ "include_image_base64",
+ "image_limit",
+ "image_min_size",
+ "bbox_annotation_format",
+ "document_annotation_format",
+ "document_annotation_prompt",
+ "extract_header",
+ "extract_footer",
+ "table_format",
+ "confidence_scores_granularity",
+ "id",
+ ]
+ );
+ }
+
+ #[test]
+ fn map_ocr_params_drops_unknown_params() {
+ let params = json!({
+ "extract_header": true,
+ "unsupported_param": "value",
+ "pages": [0, 1]
+ });
+ let mapped = map_ocr_params(params.as_object().unwrap());
+
+ assert_eq!(mapped.get("extract_header"), Some(&json!(true)));
+ assert_eq!(mapped.get("pages"), Some(&json!([0, 1])));
+ assert!(!mapped.contains_key("unsupported_param"));
+ }
+
+ #[test]
+ fn transform_ocr_request_builds_mistral_body() {
+ let document = json!({
+ "type": "document_url",
+ "document_url": "https://example.com/doc.pdf"
+ });
+ let optional_params = json!({
+ "include_image_base64": true,
+ "table_format": "html"
+ })
+ .as_object()
+ .unwrap()
+ .clone();
+
+ let result = transform_ocr_request("mistral-ocr-latest", document.clone(), optional_params)
+ .expect("request should transform");
+
+ assert_eq!(
+ result.data,
+ json!({
+ "model": "mistral-ocr-latest",
+ "document": document,
+ "include_image_base64": true,
+ "table_format": "html"
+ })
+ );
+ assert_eq!(result.files, None);
+ }
+
+ #[test]
+ fn transform_ocr_request_rejects_non_object_document() {
+ let err = transform_ocr_request("mistral-ocr-latest", json!("bad"), Map::new())
+ .expect_err("string document should be rejected");
+
+ assert_eq!(
+ err,
+ CoreError::InvalidType {
+ expected: "object",
+ actual: "string",
+ }
+ );
+ }
+
+ #[test]
+ fn transform_ocr_response_normalizes_mistral_json() {
+ let response = json!({
+ "pages": [{"index": 0, "markdown": "hello"}],
+ "model": "mistral-ocr-2505-completion",
+ "document_annotation": null,
+ "usage_info": {"pages_processed": 1}
+ });
+
+ let result = transform_ocr_response("mistral-ocr-latest", response)
+ .expect("response should transform");
+
+ assert_eq!(result.pages, vec![json!({"index": 0, "markdown": "hello"})]);
+ assert_eq!(result.model, "mistral-ocr-2505-completion");
+ assert_eq!(result.document_annotation, Some(Value::Null));
+ assert_eq!(result.usage_info, Some(json!({"pages_processed": 1})));
+ assert_eq!(result.object, "ocr");
+ }
+}
diff --git a/litellm-rust/crates/providers/src/ocr.rs b/litellm-rust/crates/providers/src/ocr.rs
new file mode 100644
index 000000000000..48e3bbe8894f
--- /dev/null
+++ b/litellm-rust/crates/providers/src/ocr.rs
@@ -0,0 +1,156 @@
+use litellm_core::error::{json_type_name, CoreError};
+use litellm_core::ocr::transformation::OcrProviderConfig;
+use litellm_core::CoreResult;
+use serde_json::{Map, Value};
+
+use crate::mistral::ocr::transformation::MISTRAL_OCR_CONFIG;
+
+pub fn transform(payload: Value) -> CoreResult {
+ let payload = payload_object(&payload)?;
+ let provider = required_string(payload, "provider")?;
+ let operation = required_string(payload, "operation")?;
+
+ match provider {
+ "mistral" => transform_with_provider(&MISTRAL_OCR_CONFIG, operation, payload),
+ _ => Err(CoreError::InvalidResponse(format!(
+ "unsupported OCR provider: {provider}"
+ ))),
+ }
+}
+
+fn transform_with_provider(
+ provider_config: &impl OcrProviderConfig,
+ operation: &str,
+ payload: &Map,
+) -> CoreResult {
+ match operation {
+ "map_params" => {
+ let params = required_object(payload, "non_default_params")?;
+ Ok(Value::Object(provider_config.map_ocr_params(¶ms)))
+ }
+ "transform_request" => {
+ let model = required_string(payload, "model")?;
+ let document = required_value(payload, "document")?;
+ let optional_params = required_object(payload, "optional_params")?;
+ let transformed =
+ provider_config.transform_ocr_request(model, document, optional_params)?;
+ Ok(serde_json::json!({
+ "data": transformed.data,
+ "files": transformed.files,
+ }))
+ }
+ "transform_response" => {
+ let model = required_string(payload, "model")?;
+ let response_json = required_value(payload, "response_json")?;
+ let transformed = provider_config.transform_ocr_response(model, response_json)?;
+ Ok(transformed.into_json())
+ }
+ _ => Err(CoreError::InvalidResponse(format!(
+ "unsupported OCR operation: {operation}"
+ ))),
+ }
+}
+
+fn payload_object(payload: &Value) -> CoreResult<&Map> {
+ payload.as_object().ok_or_else(|| CoreError::InvalidType {
+ expected: "object",
+ actual: json_type_name(payload),
+ })
+}
+
+fn required_string<'a>(payload: &'a Map, key: &'static str) -> CoreResult<&'a str> {
+ let value = payload.get(key).ok_or(CoreError::MissingField(key))?;
+ value.as_str().ok_or_else(|| CoreError::InvalidType {
+ expected: "string",
+ actual: json_type_name(value),
+ })
+}
+
+fn required_object(
+ payload: &Map,
+ key: &'static str,
+) -> CoreResult