From ccd8c4112c2092d680fcfc60a2b20da1b722d5e9 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Sat, 6 Jun 2026 17:38:48 -0700 Subject: [PATCH 01/54] docs(readme): add Deploy on AWS/GCP with Terraform section Adds a quickstart for the two published Terraform modules on the public registry (BerriAI/litellm/aws and BerriAI/litellm/google). Copy-paste main.tf for each cloud, the one-time GCP Artifact Registry remote-repo command, and pointers to the registry pages for the full input surface. Sits inside the Get Started section, between the gateway/SDK table and Run in Developer Mode -- where someone scanning the README for "how do I deploy this" will land. Co-Authored-By: Claude Opus 4.7 --- README.md | 118 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) diff --git a/README.md b/README.md index d600f3952c6d..faf79c3729dc 100644 --- a/README.md +++ b/README.md @@ -404,6 +404,124 @@ You can use LiteLLM through either the Proxy Server or Python SDK. Both give you Support for more providers. Missing a provider or LLM Platform, raise a [feature request](https://github.com/BerriAI/litellm/issues/new?assignees=&labels=enhancement&projects=&template=feature_request.yml&title=%5BFeature%5D%3A+). +### Deploy on AWS or GCP with Terraform + +Run the LiteLLM proxy as a production-ready componentized stack (gateway, backend, UI on separate services; managed Postgres + Redis + object store) using the published Terraform modules. Both modules are on the [public Terraform Registry](https://registry.terraform.io/namespaces/BerriAI) — no auth needed. + +#### AWS — ECS Fargate + Aurora + ElastiCache + ALB + +[Module page →](https://registry.terraform.io/modules/BerriAI/litellm/aws/latest) + +```hcl +# main.tf +terraform { + required_version = ">= 1.6.0" + required_providers { + aws = { source = "hashicorp/aws", version = "~> 5.60" } + } +} + +provider "aws" { + region = "us-west-2" +} + +module "litellm" { + source = "BerriAI/litellm/aws" + version = "~> 1.89" + + region = "us-west-2" + azs = ["us-west-2a", "us-west-2b"] + tenant = "acme" + env = "prod" + + # Production: provide an ACM cert. Without one, set allow_plaintext_alb = true + # (dev/trial only). + # acm_certificate_arn = "arn:aws:acm:us-west-2:111122223333:certificate/..." + allow_plaintext_alb = true +} + +output "litellm_url" { + value = module.litellm.alb_dns_name +} +``` + +```bash +terraform init +terraform apply +``` + +Provider API keys live in AWS Secrets Manager; reference ARNs via `gateway_extra_secrets`. Full input list and architecture diagram on the [registry page](https://registry.terraform.io/modules/BerriAI/litellm/aws/latest?tab=inputs). + +#### GCP — Cloud Run + Cloud SQL + Memorystore + HTTPS LB + +[Module page →](https://registry.terraform.io/modules/BerriAI/litellm/google/latest) + +Cloud Run can't pull from `ghcr.io` directly, so first set up a one-time Artifact Registry remote repo backed by GHCR: + +```bash +gcloud artifacts repositories create litellm \ + --location=us-central1 \ + --repository-format=docker \ + --mode=remote-repository \ + --remote-docker-repo=https://ghcr.io \ + --project=my-gcp-project +``` + +Then: + +```hcl +# main.tf +terraform { + required_version = ">= 1.6.0" + required_providers { + google = { source = "hashicorp/google", version = "~> 6.10" } + google-beta = { source = "hashicorp/google-beta", version = "~> 6.10" } + } +} + +provider "google" { project = "my-gcp-project"; region = "us-central1" } +provider "google-beta" { project = "my-gcp-project"; region = "us-central1" } + +module "litellm" { + source = "BerriAI/litellm/google" + version = "~> 1.89" + + project_id = "my-gcp-project" + region = "us-central1" + tenant = "acme" + env = "prod" + + image_registry = "us-central1-docker.pkg.dev/my-gcp-project/litellm/berriai" + + # Production: provide DNS already pointing at the LB IP for Google-managed certs. + # Without one, set allow_plaintext_lb = true (dev/trial only). + # lb_domains = ["proxy.example.com"] + allow_plaintext_lb = true +} + +output "litellm_url" { + value = module.litellm.load_balancer_url +} +``` + +```bash +terraform init +terraform apply +``` + +Provider API keys live in Secret Manager; reference resource IDs (e.g. `projects/my-gcp-project/secrets/openai-api-key`) via `gateway_extra_secrets`. Full input list and architecture diagram on the [registry page](https://registry.terraform.io/modules/BerriAI/litellm/google/latest?tab=inputs). + +#### Both stacks include + +- The full componentized split (gateway / backend / UI as independent services) +- Managed Postgres (writer + reader) and Redis +- Versioned object store for proxy state + file uploads +- An auto-generated `LITELLM_MASTER_KEY` in your cloud's secret manager +- A one-off migration job that runs `prisma migrate deploy` before the proxy starts +- The same `proxy_config` surface as the [Helm chart](./helm/litellm/) — pass YAML as a typed map + +The Terraform modules live at [`terraform/litellm/aws/`](./terraform/litellm/aws/) and [`terraform/litellm/gcp/`](./terraform/litellm/gcp/) in this repo; the registry entries are read-only mirrors updated on each release. + ### Run in Developer Mode #### Services 1. Setup .env file in root From 260f5f1371e3308810cdaab61daa4da231cc1e76 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Sat, 6 Jun 2026 17:44:49 -0700 Subject: [PATCH 02/54] docs(readme): add 1-click deploy buttons for AWS + GCP GCP gets the real 1-click: Open in Cloud Shell badge that clones the repo and walks through `terraform apply` via the existing DeployStack tutorial (already shipped at terraform/litellm/gcp/examples/default/ TUTORIAL.md). User just picks a project. AWS gets a soft 1-click: a Launch in AWS CloudShell badge that opens an in-browser, already-authenticated shell. User runs four commands (clone + cd + cp tfvars + terraform apply) once inside. There's no native AWS deeplink that pre-clones a repo + runs a tutorial -- CFN "Launch Stack" + CodeBuild would be needed for that, and that's a separate piece of work. Co-Authored-By: Claude Opus 4.7 --- README.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index faf79c3729dc..d304d15f169f 100644 --- a/README.md +++ b/README.md @@ -410,8 +410,19 @@ Run the LiteLLM proxy as a production-ready componentized stack (gateway, backen #### AWS — ECS Fargate + Aurora + ElastiCache + ALB +[![Launch in AWS CloudShell](https://img.shields.io/badge/Launch-AWS_CloudShell-FF9900?logo=amazon-aws&logoColor=white)](https://console.aws.amazon.com/cloudshell/home) — opens an in-browser shell, already authenticated to your AWS account. Once inside, run: + +```bash +git clone https://github.com/BerriAI/litellm.git +cd litellm/terraform/litellm/aws/examples/default +cp terraform.tfvars.example terraform.tfvars # edit region/tenant/env +terraform init && terraform apply +``` + [Module page →](https://registry.terraform.io/modules/BerriAI/litellm/aws/latest) +Or call the module from your own root config: + ```hcl # main.tf terraform { @@ -454,9 +465,13 @@ Provider API keys live in AWS Secrets Manager; reference ARNs via `gateway_extra #### GCP — Cloud Run + Cloud SQL + Memorystore + HTTPS LB +[![Open in Cloud Shell](https://gstatic.com/cloudssh/images/open-btn.svg)](https://ssh.cloud.google.com/cloudshell/editor?cloudshell_git_repo=https%3A%2F%2Fgithub.com%2FBerriAI%2Flitellm&cloudshell_workspace=terraform%2Flitellm%2Fgcp%2Fexamples%2Fdefault&cloudshell_tutorial=TUTORIAL.md&cloudshell_image=gcr.io/ds-artifacts-cloudshell/deploystack_custom_image&shellonly=true) + +Real 1-click. Opens Cloud Shell, clones this repo, and walks you through `terraform apply` via a built-in [DeployStack tutorial](./terraform/litellm/gcp/examples/default/TUTORIAL.md) — pick the project, the tutorial sets up the Artifact Registry remote repo, writes `terraform.tfvars` from your answers, and runs apply. + [Module page →](https://registry.terraform.io/modules/BerriAI/litellm/google/latest) -Cloud Run can't pull from `ghcr.io` directly, so first set up a one-time Artifact Registry remote repo backed by GHCR: +To call the module from your own config instead, Cloud Run can't pull from `ghcr.io` directly, so first set up a one-time Artifact Registry remote repo backed by GHCR: ```bash gcloud artifacts repositories create litellm \ From a505734a2cddcece2d27ae7dea2e192f0f8b7a13 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Sat, 6 Jun 2026 17:50:53 -0700 Subject: [PATCH 03/54] docs(readme): move AWS + GCP deploy buttons next to Render button --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index d304d15f169f..7802e33baee7 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,8 @@ Deploy on Railway + Launch in AWS CloudShell + Open in Cloud Shell

LiteLLM Proxy Server (AI Gateway) | Hosted Proxy | Enterprise Tier | Website

From d0ff1753973ee16b2c9ef88d34d2647e3d7cbec4 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Sat, 6 Jun 2026 17:54:38 -0700 Subject: [PATCH 04/54] docs(readme): unify deploy button sizes and badge styles --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 7802e33baee7..2509b7615634 100644 --- a/README.md +++ b/README.md @@ -6,12 +6,12 @@

Open Source AI Gateway for 100+ LLMs. Self-hosted. Enterprise-ready. Call any LLM in OpenAI format.

- Deploy to Render + Deploy to Render - Deploy on Railway + Deploy on Railway - Launch in AWS CloudShell - Open in Cloud Shell + Deploy on AWS + Deploy on GCP

LiteLLM Proxy Server (AI Gateway) | Hosted Proxy | Enterprise Tier | Website

From daba177a5205f1be70a658a5dd063b23fd67e21f Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Sat, 6 Jun 2026 18:15:36 -0700 Subject: [PATCH 05/54] docs(readme): bump deploy button height to 48 to match Render/Railway --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 2509b7615634..693aee14683f 100644 --- a/README.md +++ b/README.md @@ -6,12 +6,12 @@

Open Source AI Gateway for 100+ LLMs. Self-hosted. Enterprise-ready. Call any LLM in OpenAI format.

- Deploy to Render + Deploy to Render - Deploy on Railway + Deploy on Railway - Deploy on AWS - Deploy on GCP + Deploy on AWS + Deploy on GCP

LiteLLM Proxy Server (AI Gateway) | Hosted Proxy | Enterprise Tier | Website

From 1a18184f0eae4b331946ae564fd51dc7dbf1a0a2 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Sat, 6 Jun 2026 18:25:49 -0700 Subject: [PATCH 06/54] docs(readme): bump AWS/GCP badge height to compensate for SVG padding --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 693aee14683f..d97f07a1b8bc 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,8 @@ Deploy on Railway - Deploy on AWS - Deploy on GCP + Deploy on AWS + Deploy on GCP

LiteLLM Proxy Server (AI Gateway) | Hosted Proxy | Enterprise Tier | Website

From 8ba452c0ecfaf6b81268eb596ba2fb0d463bee06 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Sat, 6 Jun 2026 18:26:21 -0700 Subject: [PATCH 07/54] docs(readme): bump AWS/GCP badge height to 72 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d97f07a1b8bc..91246629ad60 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,8 @@ Deploy on Railway - Deploy on AWS - Deploy on GCP + Deploy on AWS + Deploy on GCP

LiteLLM Proxy Server (AI Gateway) | Hosted Proxy | Enterprise Tier | Website

From 2c56b4d3f64ae941322b78b35df4bf0dfea51f34 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Sat, 6 Jun 2026 18:27:10 -0700 Subject: [PATCH 08/54] docs(readme): bump AWS/GCP badge height to 84 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 91246629ad60..07a74296fd1b 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,8 @@ Deploy on Railway - Deploy on AWS - Deploy on GCP + Deploy on AWS + Deploy on GCP

LiteLLM Proxy Server (AI Gateway) | Hosted Proxy | Enterprise Tier | Website

From 65890b732e1e67af0d3c09a318197534d8966ced Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 7 Jun 2026 01:33:43 +0000 Subject: [PATCH 09/54] fix(readme): make deploy buttons same height (48px) https://claude.ai/code/session_01MxQRMHSDXbqJh74rF86UBc --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 07a74296fd1b..693aee14683f 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,8 @@ Deploy on Railway - Deploy on AWS - Deploy on GCP + Deploy on AWS + Deploy on GCP

LiteLLM Proxy Server (AI Gateway) | Hosted Proxy | Enterprise Tier | Website

From 97f067bc70d7006f7aeb8dc117412c4d0f67e514 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Sat, 6 Jun 2026 18:37:28 -0700 Subject: [PATCH 10/54] docs(readme): flag GCP project ID substitution in image_registry --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 693aee14683f..529e1de33548 100644 --- a/README.md +++ b/README.md @@ -508,6 +508,7 @@ module "litellm" { tenant = "acme" env = "prod" + # Replace my-gcp-project with your GCP project ID (same value as project_id above). image_registry = "us-central1-docker.pkg.dev/my-gcp-project/litellm/berriai" # Production: provide DNS already pointing at the LB IP for Google-managed certs. From 99e1a7612860fe469af44a5ef08434867359fd41 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 7 Jun 2026 02:10:22 +0000 Subject: [PATCH 11/54] docs(readme): equalize deploy button heights and fix Cloud Shell button font GitHub rewrites an image's height attribute to "height: auto; max-height: Npx", which only caps and never stretches, so each image renders at its intrinsic height. The AWS/GCP shields badges are intrinsically 28px while the Render/Railway buttons are 40px, leaving the row uneven regardless of the height="48" we set. Replace the two shields badges with committed 40px PNGs so all four header buttons render at the same 40px. Also swap the Cloud Shell button from open-btn.svg to open-btn.png. The SVG renders its label as live text with font-family "Roboto, Sans" and no generic fallback; since neither font exists in GitHub's render environment, the text fell back to a serif (Times New Roman). The PNG bakes in the correct typeface. --- .github/deploy-on-aws.png | Bin 0 -> 4113 bytes .github/deploy-on-gcp.png | Bin 0 -> 4922 bytes README.md | 10 +++++----- 3 files changed, 5 insertions(+), 5 deletions(-) create mode 100644 .github/deploy-on-aws.png create mode 100644 .github/deploy-on-gcp.png diff --git a/.github/deploy-on-aws.png b/.github/deploy-on-aws.png new file mode 100644 index 0000000000000000000000000000000000000000..06d41f2a5e06cd60aca118d53ea50f0beb234ce8 GIT binary patch literal 4113 zcmc)N)msyQy9aR4fiwf8hor>lPDgi1cS?-zZV)D&0wN;yqoh+A-5^Y20|!!~B9msL zk>h#JA8_u@xp|-O#rNX9df(@h`rJU1oP>!44-b!ATT9It4-Y^7Uw%&X;NN_|FByb~ zM>(ObrfeEsy8kY$fEL9vIS3bM#rAf(dhqKUl{=gEzT{xz0c&JDbyg^RA|A z>#0{c@dF@A;t4P<;5Jr^d|kPC(;i&d-MJopMA7#qYIAn2XMZj1J#2HgPtolV?pOTo z(^>yuD6HVNhm9!FG1-MEkyM2d7^DW&lgEk0P)T22RXMD6j5YVQ!!0YJT^zXB;tdTZ8* zD4<49vO2_+ndh7%j55ppVMME1G(lY;cVu6S^v@lvQ7K}$9#pFE$^_=`oE9UcrOv$O^Mlrb^^4B4RF*)+t?cj`jDyI!nJnnat_Z7h1kWFl&f}(kNUp~O8dRgU+ zO6w@AjOdeH3&L z5)BbCU{O2ad!451r>dcd#AKY}I$Hvz*rFg(*^`B-Ml20S?wFErtCL6GZyZB)k1`R$ zz3{#mabK0$y>p5NsU>RzmI?Y8k}ETK?L4#<4v!Hlgt(u3MDklhwhmA2qeGj}wR*JJ zw>4dQhp_w`Dz&=-=!Hx}kA`Q-GNLWm8^*K-GVuCa^Y^6;wF5rbjH0Vo4V7ekyk*H;9%y#v?mkN4Cx8rVD|c(VAcX&=xbF~r_}VEbT?{f{9-Z9SBJKb zv3?fslP_|o5ocv9O?b!6SzoXgc(7KCT&i`i&7HceUv9tL`&wss_h&waYzB3DU$_iZ z)Q5d|(S|EM!am+y#E ziJY)RwJy}c<;#*aCS>K>f0e?R6>RW6dtKCfsLixK_u2(J-Ywi3C;MHiav~lqt-P`_ ze-Tt5lu%&;nn7vQ?ut4R=IL0%(b2+4#NOp!2WwLO4!1!GI0~9A zl?@e)DssF!GKyxh&xX)EKuwbBrzUh5H9buuNsiZWzV!$#%D&F2Q560GlXr*0QP$9u zTGdN{x%BJE_nw z<*G&HB>dR*ZwSoPnxQp4Hr$Y)i3|a<@b%d6=y*5b zy{TOTqa|l4GZJzs{q%$G{+#fR^o`_R$dmQML++Dz76-TYJpxq(JzH&h1C=e?3g_G4pCIt8O}h2Ia#1?O>Q9RYVymRCHWOLHF+ z2!rt+HqniBVHp}Rc)CrF0FCa$7vdV^?P{0DP=1c9)?oW{>1 z=Jab1i1dm2^2(HA)h}8Dkz3u*DHh{8!o6Dn)5wJhvL* zXgCqA__uXA3#T2Ku%hPYGF>h6p&s!;bl7$;y6Elca5R>xP$6A)Goqv}K4w1r6-n&` z%i04q+=&LOY)EGxr4Osgr#xV=DARhp;7*l-OXv11blSvTRI#+D<+5XKc8Fwh4k1hx zbSc1iCaXEyo_i>?^PJyd;?PfIi(fy9f%25)VutFb=U0RzQzEyb$%|YeLpVZDdE6&Qq(~)dDT*TK$Ydjc2>eR=ioswRaII`1{XT__;Vfx22 z>1v967<=)3njZ-nG1J3U8FSarzWG&&Kv-(Lv@bP3CPdss%Ij}4 zh|Xc)F8u}|hol6uQ8cz3p{##Tl9&0Fobdl;sN&LUXT*Iw<)diA9l@Ohvb>kAM;fp>Jf-ET;a zb!Y_-ffGN9Sb%sc$|`8xqwIo{w<3lnm4p>NhNw^!3x?&uPxHr390O30qG;Wzd*3FZ zcr-kJ_T8OP$fERVjEhfSiqYAhpD^)RelKa50Ie&4E&L*Cu6Y)${HdSX$|OVQm@ zkVv1AwArVHGGOfq*;)o<=k*|KSyDP@Dp={!ptudlP{!9k%}e6rmN>xYiRvI9#Dgy@ zyLePDXZWO1>n?~A3eVM#zoOi^6uNII&QE)NrVg%tD`-#owA8jg{%Kbvl2hD}Mt|ZH zSW-FFxQ!hXnIT`3BlQ@$d{|tw&X-chSeF&=)bY}Jw(FT2g{rzl?1d1d$L3S2PEM}1 zMFV&YXv&+im}rS($~rpts1=@Qi}}gVP?r4ff~G2Q1`zNn5L$IJeJAEggV2ybb^^vf zLy|o1;4{ZrvKznQqtmLn*Ub3i$M2&m(26|eF5GSi3;c{JhSIE$KfYtBVifsAOmhod z&TXZwX_Mtegx2^pG;D!L89yhU4q(OU;9O+cn3f_VgA<#zgn_G-R_brtBVOL%a~(5{ zgjzR1{O|gok7^gqn6N@|y9R8+auKxV+h&YtYme$Cg~heQ$#(brP2* zXA5UCIdRerZU~U7OLvTKO7phMfV~nKaS^j$^RrU_(F0R8w4^1r06w?>*2oNKL$>L; zbpOrA56nVHtY!yF8W{uHBl%MQ2YbF;5;p3A>k8u?E!jPi z>)x07MgnDyh3i^IO_7olx;CRM&1txQ`+9o*UMW?G|6S3r_$WYSMZ!>SFdQM6COfNY zcK$KkJ@ZFP3T6sZp7Px6;`!i@vB(Yv8C@@JzLo>AR%$LKuBZWSB|Yy5-6$0! zPl~~q&j|H*vZ5<77(x+%Bb0jZoMj5B@tFHT!kWCC}*2>>Kr z2cZ$xyxk6-EUpY=Tm0ofo%79u>KEZ*NwV5ja8}7iKWPbXRcTbFH5Y7_d9Ii$sYU(i z4fE?m;@7*KkW^GpZ4!@}c48z2am)h=4RM zEU{HAKZY=*!h_$QX*P*6bSv9z+SIdu*ZMvlU$R+6$IsXp^N^kxq-TTeLUvyDyZW&+ z`1uA_EsqH3WqcS)AC8RiVR{cTd5l=v_l;596rg{VH~!_LnbFg>+h=Dr9IS;>j+cv@ zK)9QStF4}Q@~IaB+ysO_Gbj{gAn7vysyH&@QF5jk98=xKs!6RH4>fkfnxr=o&W)n{ zO6oxyWD{S6o)l(E#8j5;6bMCW|4C#|pPSv9DI%p*nIzYk$(T2LtBIV$HH%f;PXYY< zeXV=uXdbMXX{1O65+}!kNCXe~FfHL7=X-A^bCx{EUWXx+V$0Bz39+^=zg^b*C=T8) z&%9C*X>bi^zS+TeehnSI*5j>ySkT~XxrP5)nYo>+Iufh+q^FCH)d8F8MlB7n5dmWTv1S`V~^p^yps`DjgqhQJ2^8ej?xMn9{kjmn9UKJAcqM7? zTeOSSiiG4unxV!g7I%>!z2_age)xMl|1zjZ6D=B%pyBQ0U*Ky*AH&Yv&rqU}g4!8^ z{wakAG%`%jaK1ghTf;QriNdUMUA~M4f;Ib7MWOcIcRA(@h|(81cmb=&&(@AVirF>! z^~WYzrh94E_wf94 z7riyUDOdRF8LM)%HXeQ@PveH+fPu&D1SRr^q7TtTwbMBZCKF9BPWpaqlEpM|Qo3E$ zXB7@`*r>K|V~Y5cVkea~`lzU$*WOp@DLrd<^56tau(h_;UqhmF$~>LYnY4;$;scX( zMmjQXok8%#uu52bi(Iphdz)~erSY=Y8f|3T#@JK$CW9uz_xMkxEG%Y&c?rgG$n|rO zfPGuroOm5vR3~XZ?TU&NssXqE8au*Cw^ds6|9opY`+PeHH%RGi0nyTKgi=L2Rc0=X z>gXeNiWX^Kqo&pHs}NN7t_~yr1JtW*VuJrKmj3TL-nswh`TKh^iR+Sof$+aY{}2yI XH7&3dhx>acd4x^h`m+psv0$-R&i6K zu}5sGXdd^U@Vt1QH|ILnS=Twg>-XY&eG-fewCL{KyF)@kLZ_px4k95TZMgA`s3~u3 zX#4?ygoKeqM_t7fp0}HSJDp{^v!D1=VN&6Fq>M796DI}B3qHPk>_-0-GFiVZBG1H> zX`vtfv6W2Ncw~rA639~5M;hNnzEmH7NDv~Ae#-%oZomSpAXH9qb8VP*-|l;YCw@y6 zHH592zKigUsg*qY(#k(SsxPl+tRD3Wa%tZI#4!GP?!5FvF)|`rl^PK9o8u1WP0UUW zz<-nR5Q4-os)o|VA{kZVYd@lxRH+YNut?k#b|F+XsR3VSj0XQ}y7zzCRH_cdt;A8T z{w@G)&B+Q)Z*R`Y@@e_q)(z8x1nPqn=Rc$~NcTC7=V-pF7r)tE^6=MKHW_< zx>wFd^Uk={~e5?&eTCKVSyGe7%Ks0f${$W_-y6+C-U$!$NH7Z!lO_?dCmJ zX^<|!2I$GtG~VPyZGZ}v2{3fqbM7v7h? zr<%setjtq^vZlv2pq{|F3Z64jcYt-f0)TjBVeO+5n9^Mv|4n$GO^i2q+ZezZ;F)l; z!z9u1$1i|OduT`$%aAPLtAYO75E|Q~g;@Q!HuiaVm-hAbJKt*PYS5D>Fv91Nkh7t7 z@zc(vTr#l}nPT7k#PmFGb>}k3Vufaqb62>vTzPu$Sbuu+?^o9P!^CWBK?73b?t@|w zzKbUetWEzIdO^JHuh2d<>uGWv+K0e)G>VB0ORvhvd|)>1O&Od*bv>G}$ggzN7$y$L zk*HYJi;F=cxH6q~8b1tpE||C(nRv|eP&=tg5%em?yhL(j*zeB7?!M%{?Xj=okpa`-cpOD?=xedvYS!#OV_8@J_L71sJ5~K#vtAb(X}waa zjbYP)Tj}_~3%G}BSZY>>({aHjuHG-q{sF$u50Kv31-TBw+F{P=IH}gBp4z(Av8)Cv zB@9oyGi}bG*EY@+fSw{gyk0+Q8HHZoi*QzdG9l@GOeh}tTL|~Xg7GN&Cn*2pdJD5z zMonrbIi_Y%2~q!x3T8Do`Xyvx%YHbB{&1_P;04z++*4lAlgo6R?U|9j!PpK8ttRoe zD&DVFQi)x?*>I*1hBW`x22wDT{`<{xYP4S=KWdsPVx#feKFCc+&o)?4z?fm?dusuI z`{8c9pV08%kVYlX^f+hDwf1di8^wBav9LOGTz9ReS2jqn_TRdp?g+21aE=aDMKXp; z&jrqsqh+(T{vy0fiY!nl5Siktz$Vz^a44T)IQj{Wi8dOu8B&vJO>Lf7dR}eojuwqT zP%vodI0qh3Z8#@#uG#>1Qm}10#6KeT?)r{t?blx~=GvlCTgDzqa!u55dNosKc>XB)(5-sl7MdV&3e39Ai{z||i`Ra{ zWspV(0(|r6+7{*KXSJ|6tH)R}jJ$mTYdxk2G2gtG5qPmO>3{Tnnib)?h20dkw_41# z`n;2ePMI(=nGE#LjO3aTEk5l+=Jm7;t;wg(IMZ2e3MjB)^;J~U(b6kOulrjsqR6`Cn$4xS|$Si>53GsfhqfI zclnyjIM(-DUoJs4SLai6;`|HjxGsSkQ%6w1FKD%Ax}l=SJr!`bH0@Yk2N=tf&7I+} zG9@z$C{|>a9&#p+a z8mJ3_sF3G^>EbaRR>zI>c+!^*B{3BAy-Z>weWALT~`xn++pU-e(kQMg7on1@%y>@NS@?QhH+We=I6~1APR`moJfK4^Y+K$}TJR2AB_(;LjkNiX#mf zxpWE%U>&+mZa+;$61STvd%dD^tsGlvaxD3b9HrO2l_2+qgN{dH{pf0X9iPC6@0~H{ zpfP&R_}gkM^-8b>HW?q^WSH9~b6!jQfwD+T<<8@wl2<+nvFL;R#1PClmlct6#!p4M zv&Z=+Gq-2ZqV2}tS)>c9b)f1QGO=%h44pNvCyyNv@Zt#abux9=zWy?xf>H(Y#? ztMKVQuxItlc`H7s(?#x2aUGUBe%#=}yQB$Dron47c`gGvt=I$A92(;A+pq%_J5kCU1$nt{qdb*9u}g9#rd`3FCWq>BlbJ8rrmg&gxgv4TwQY9o;WnA? z<@IGc0*V|=GuuC0*rLnhOQK;pK;?3dpS9@M0KCAMF|m;= zTWtXb!rJJ!lHT2r%$PjZhN$Um$4`zC5*}`sf4FsCl^x2Yv>ZmFbxkbF;~9VGySNx= zr^ARabC{Z^7c!b)fWG(KY{bBLJZs9x^ZQDMZf1)HaiRFZLMkBa)KLmn8nR;6xFNYZ z{$`!KyUJ=T38G(MNxDzl$z@&zs;M-Dj{#Fv#R?AB$Xbys8gj;{VW#Sra;iT9N?!IHQfIf~H^*WBnpVIeM zR4?ic%jEFOThZYT!mE{>TIpQqe|Wki`ONW~sj`_LC`>NY2jBV>0Y^0pB!z99;ewwB zY*w%}e*9pSbMOhJq~aJLt^eu}k&jE{s9F9VkUbM6;3vOZIZ5O>(w_^k26v;&wfU?l zW|0r#SfXQC$bg69w z+OMdP>~`qxY^1Fm_n^Lhj#((#a5L=L1vLFVx=i>qnSC%r)Dpiy;3k==rTI9WMSZ37 zgjUw4@WNWltt$iNFEqx5a)AG87dEFsAVOQ5iubW@_@1~X+VTEP;0A!)zQ!`tl3a;6`$6_u?hZ|dqOXr!NLe!CW5!B z0QSzSrvI`O=`u55`%BN3ce{g|RptYyxIk4fKBMI`nN4`8!PKlkIj2YyKNo@c6}zcf zI^_H~Om6P4?)g=~7HX+Ak>X!E0o;l6mV&n@0wO%2yrjL&270$4wfEPXn6aj*M5eym zs&O_-`Y;vt`j9v-bX=;5{VtR>yHPcs$sRVPRxMm2O>-#u4$AvVTv|Yb=UDcU!)Ctk zJtS4m*6`#l@&L6aGwzeKr2?)mD%9ddhIgkE%lZ89rZJysZiGn3xZ8fe=@}_)O_f=D?j*jXKWQ%u%O-m=X7#`!O{BmwZJ`2j{iY z+QpK5PepKJm{{Sfr1H}VV*eeJm&{>$Li6Wx{QA1N^|xzT`w0A#`P@*#Vk>cZ3c6R? z6YD%#6En{IPIswStsm?p?6+C_I774i6v5rP#&9=>S=Ppx5DU}ZC~racTSM9IvZNxp zHa%C#gXT{<*!yqO;owowC+7yxXY`Q>;s+$ITAM<1)FQY=()HQj&HY@n&z&#)`*;ot zq)S)JTHU-03X7+oSj)U|+krmxqW=S{;i!nf3$L$HRh@6v#{dy-$ZUDydXt*xF_La z1?7Yx!|C?9Vh~sEFO=jAY%*UzL2Ru1T^9{VYnxE2rNe0-Eq||LLK#zr-tO3t8f5s^ zEINl5q_8d~YGI-a-hV)fO$M7o%lZ)eaYNiEI$OXLU&>ccvj$JM3B5b}A(Au&6w>`Y zxB%Sv`4=vV+GbaCV+{6n}1QUMgx87pb4UVN*LW{CbAmM&`b}$U=lh z5F#^$2Wy2(80^cbj{MGb)#AhxJ`hV`%S{y@Z&qSguM+-yx+HsJpz&hRpS9mhB-Xlh z|Ru&Acapd_PSpRn=xr`RHQh(AHtn6=8 zU@U8PvRD1{J&KTjt3QdwzrAN!aY3nfNvz&tV5RSoc<=Errl*aOPcj9zg`VRJy2`e9 z)Ov4d8t#!VLAhO|^O%7OIT~@i@+PAe!4%oTTMVjDB!uwu_y4eyH}&$$%o&@NtF&_Z z4(|pZm(tIl{b})OWt3S@+iRG0!&lh%?JdX6D>T(!kk2;^P|c07pbGmIlU@SbF6!+* z=c%i8)JRj(P@k!M&COZi-eH4|xUxLyY#FyZ7UZr?AyX~z(&SoZHOR7rni94m0})_y z^Gyj0{BKEsYuVDFKeC|s6Xq>ZZoRd-bIwD|^mQRuE95L~zLH^e7j#9EC>nEV-K*EuA= zUIQdnXoIg|-|=<7eJEke9XIn{x~+{y@qxySlDS8d?kX8F+G%I+S~L;TrBs58k6g_m zl)Z7C=gD^bsNolD_u4p!JPDsVV@YEm{yF-WW^?vjQ#9C~iuB}s*8*S&juvvU#r)Gl zVk@O3YAy9^3_{zA_MXA+~u-6(lve$4kwGi^MFX1IT5p`ZDNJLbrH{#*dkc$BBK zcrOq_#qsp{<)_lGHE2~te!XBS6caXa$(*aUO0-SX{E)eq{$MVz

Open Source AI Gateway for 100+ LLMs. Self-hosted. Enterprise-ready. Call any LLM in OpenAI format.

- Deploy to Render + Deploy to Render - Deploy on Railway + Deploy on Railway - Deploy on AWS - Deploy on GCP + Deploy on AWS + Deploy on GCP

LiteLLM Proxy Server (AI Gateway) | Hosted Proxy | Enterprise Tier | Website

@@ -467,7 +467,7 @@ Provider API keys live in AWS Secrets Manager; reference ARNs via `gateway_extra #### GCP — Cloud Run + Cloud SQL + Memorystore + HTTPS LB -[![Open in Cloud Shell](https://gstatic.com/cloudssh/images/open-btn.svg)](https://ssh.cloud.google.com/cloudshell/editor?cloudshell_git_repo=https%3A%2F%2Fgithub.com%2FBerriAI%2Flitellm&cloudshell_workspace=terraform%2Flitellm%2Fgcp%2Fexamples%2Fdefault&cloudshell_tutorial=TUTORIAL.md&cloudshell_image=gcr.io/ds-artifacts-cloudshell/deploystack_custom_image&shellonly=true) +[![Open in Cloud Shell](https://gstatic.com/cloudssh/images/open-btn.png)](https://ssh.cloud.google.com/cloudshell/editor?cloudshell_git_repo=https%3A%2F%2Fgithub.com%2FBerriAI%2Flitellm&cloudshell_workspace=terraform%2Flitellm%2Fgcp%2Fexamples%2Fdefault&cloudshell_tutorial=TUTORIAL.md&cloudshell_image=gcr.io/ds-artifacts-cloudshell/deploystack_custom_image&shellonly=true) Real 1-click. Opens Cloud Shell, clones this repo, and walks you through `terraform apply` via a built-in [DeployStack tutorial](./terraform/litellm/gcp/examples/default/TUTORIAL.md) — pick the project, the tutorial sets up the Artifact Registry remote repo, writes `terraform.tfvars` from your answers, and runs apply. From 7cf1b263356fa6960da275f360db0031182fb1e2 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 7 Jun 2026 02:10:22 +0000 Subject: [PATCH 12/54] docs(readme): collapse Railway deploy anchor to a single line The Railway button wrapped its img across indented lines, so the anchor contained leading and trailing whitespace. GitHub underlines link content, rendering that whitespace as a small blue underline beside the button. Put the anchor on one line like the other three buttons so there is no inner whitespace to underline. --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 9e6b4030966f..719f74e59248 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,7 @@

Open Source AI Gateway for 100+ LLMs. Self-hosted. Enterprise-ready. Call any LLM in OpenAI format.

Deploy to Render - - Deploy on Railway - + Deploy on Railway Deploy on AWS Deploy on GCP

From 7f57a7a068d125dbe390e665391fe63f8a26dd8e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 10 Jun 2026 03:58:08 +0000 Subject: [PATCH 13/54] Add Claude Fable 5 cost map entries as a data-only hotfix Backports only the model map changes from #30064 so deployments on released litellm versions pick up Fable 5 pricing, context window, and the adaptive thinking flag through the hosted cost map fetch without upgrading. Includes the supports_sampling_params flag on the 28 Fable 5 / Opus 4.7 / Opus 4.8 entries (ignored by released code, read by the gating that ships with the next release) and the matching one-line schema declaration so the map validation test passes. https://claude.ai/code/session_01MZarYYT3aS7DxaNjoax6Gm --- ...odel_prices_and_context_window_backup.json | 276 ++++++++++++++++++ model_prices_and_context_window.json | 276 ++++++++++++++++++ tests/test_litellm/test_utils.py | 1 + 3 files changed, 553 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 397f96fdb1e8..757aacf1cafa 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1156,6 +1156,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -1202,6 +1203,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -1233,6 +1235,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -1264,6 +1267,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -1295,6 +1299,139 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" + }, + "anthropic.claude-fable-5": { + "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 2e-05, + "cache_read_input_token_cost": 1e-06, + "input_cost_per_token": 1e-05, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" + }, + "global.anthropic.claude-fable-5": { + "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 2e-05, + "cache_read_input_token_cost": 1e-06, + "input_cost_per_token": 1e-05, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" + }, + "us.anthropic.claude-fable-5": { + "cache_creation_input_token_cost": 1.375e-05, + "cache_creation_input_token_cost_above_1hr": 2.2e-05, + "cache_read_input_token_cost": 1.1e-06, + "input_cost_per_token": 1.1e-05, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" + }, + "eu.anthropic.claude-fable-5": { + "cache_creation_input_token_cost": 1.375e-05, + "cache_creation_input_token_cost_above_1hr": 2.2e-05, + "cache_read_input_token_cost": 1.1e-06, + "input_cost_per_token": 1.1e-05, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -1327,6 +1464,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -1359,6 +1497,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -1391,6 +1530,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -1423,6 +1563,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -1455,6 +1596,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -1485,6 +1627,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -2208,6 +2351,37 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true + }, + "azure_ai/claude-fable-5": { + "input_cost_per_token": 1e-05, + "output_cost_per_token": 5e-05, + "litellm_provider": "azure_ai", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 2e-05, + "cache_read_input_token_cost": 1e-06, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -2237,6 +2411,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -10133,6 +10308,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -10167,6 +10343,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -10177,6 +10354,40 @@ }, "supports_output_config": true }, + "claude-fable-5": { + "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 2e-05, + "cache_read_input_token_cost": 1e-06, + "input_cost_per_token": 1e-05, + "litellm_provider": "anthropic", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "provider_specific_entry": { + "us": 1.1 + }, + "supports_output_config": true + }, "claude-opus-4-8": { "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -10201,6 +10412,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -33967,6 +34179,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -33995,6 +34208,67 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true + }, + "vertex_ai/claude-fable-5": { + "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 2e-05, + "cache_read_input_token_cost": 1e-06, + "input_cost_per_token": 1e-05, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true + }, + "vertex_ai/claude-fable-5@default": { + "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 2e-05, + "cache_read_input_token_cost": 1e-06, + "input_cost_per_token": 1e-05, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -34024,6 +34298,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -34053,6 +34328,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b2836a096b71..ddd7d51d76b3 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1156,6 +1156,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -1202,6 +1203,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -1233,6 +1235,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -1264,6 +1267,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -1295,6 +1299,139 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" + }, + "anthropic.claude-fable-5": { + "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 2e-05, + "cache_read_input_token_cost": 1e-06, + "input_cost_per_token": 1e-05, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" + }, + "global.anthropic.claude-fable-5": { + "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 2e-05, + "cache_read_input_token_cost": 1e-06, + "input_cost_per_token": 1e-05, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" + }, + "us.anthropic.claude-fable-5": { + "cache_creation_input_token_cost": 1.375e-05, + "cache_creation_input_token_cost_above_1hr": 2.2e-05, + "cache_read_input_token_cost": 1.1e-06, + "input_cost_per_token": 1.1e-05, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" + }, + "eu.anthropic.claude-fable-5": { + "cache_creation_input_token_cost": 1.375e-05, + "cache_creation_input_token_cost_above_1hr": 2.2e-05, + "cache_read_input_token_cost": 1.1e-06, + "input_cost_per_token": 1.1e-05, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -1327,6 +1464,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -1359,6 +1497,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -1391,6 +1530,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -1423,6 +1563,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -1455,6 +1596,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -1485,6 +1627,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -2208,6 +2351,37 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true + }, + "azure_ai/claude-fable-5": { + "input_cost_per_token": 1e-05, + "output_cost_per_token": 5e-05, + "litellm_provider": "azure_ai", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 2e-05, + "cache_read_input_token_cost": 1e-06, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -2237,6 +2411,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -10133,6 +10308,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -10167,6 +10343,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -10177,6 +10354,40 @@ }, "supports_output_config": true }, + "claude-fable-5": { + "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 2e-05, + "cache_read_input_token_cost": 1e-06, + "input_cost_per_token": 1e-05, + "litellm_provider": "anthropic", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "provider_specific_entry": { + "us": 1.1 + }, + "supports_output_config": true + }, "claude-opus-4-8": { "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -10201,6 +10412,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -34007,6 +34219,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -34035,6 +34248,67 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true + }, + "vertex_ai/claude-fable-5": { + "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 2e-05, + "cache_read_input_token_cost": 1e-06, + "input_cost_per_token": 1e-05, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true + }, + "vertex_ai/claude-fable-5@default": { + "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 2e-05, + "cache_read_input_token_cost": 1e-06, + "input_cost_per_token": 1e-05, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -34064,6 +34338,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -34093,6 +34368,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index f179e9c8f93b..4c4d9e1133bc 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -858,6 +858,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "supports_xhigh_reasoning_effort": {"type": "boolean"}, "supports_max_reasoning_effort": {"type": "boolean"}, "supports_adaptive_thinking": {"type": "boolean"}, + "supports_sampling_params": {"type": "boolean"}, "supports_service_tier": {"type": "boolean"}, "supports_preset": {"type": "boolean"}, "supports_output_config": {"type": "boolean"}, From 70afb75ff1078a8f9cb1d2eb75e93e6fe4fda736 Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Mon, 22 Jun 2026 14:42:13 -0700 Subject: [PATCH 14/54] Add litellm rust workspace with mistral ocr bridge --- .github/workflows/test-rust.yml | 54 +++++ litellm-rust/.gitignore | 2 + litellm-rust/CLAUDE.md | 88 ++++++++ litellm-rust/Cargo.toml | 20 ++ litellm-rust/README.md | 34 +++ litellm-rust/crates/core/CLAUDE.md | 38 ++++ litellm-rust/crates/core/Cargo.toml | 11 + litellm-rust/crates/core/src/error.rs | 27 +++ litellm-rust/crates/core/src/lib.rs | 4 + litellm-rust/crates/core/src/ocr/mod.rs | 2 + .../crates/core/src/ocr/transformation.rs | 32 +++ litellm-rust/crates/core/src/ocr/types.rs | 29 +++ litellm-rust/crates/providers/CLAUDE.md | 53 +++++ litellm-rust/crates/providers/Cargo.toml | 13 ++ litellm-rust/crates/providers/src/lib.rs | 2 + .../crates/providers/src/mistral/mod.rs | 1 + .../crates/providers/src/mistral/ocr/mod.rs | 1 + .../src/mistral/ocr/transformation.rs | 212 ++++++++++++++++++ litellm-rust/crates/providers/src/ocr.rs | 156 +++++++++++++ litellm-rust/crates/python-bridge/CLAUDE.md | 36 +++ litellm-rust/crates/python-bridge/Cargo.toml | 15 ++ litellm-rust/crates/python-bridge/src/lib.rs | 31 +++ litellm/ocr/main.py | 6 + litellm/rust_bridge/CLAUDE.md | 39 ++++ litellm/rust_bridge/__init__.py | 19 ++ litellm/rust_bridge/loader.py | 68 ++++++ litellm/rust_bridge/ocr/__init__.py | 8 + litellm/rust_bridge/ocr/config.py | 160 +++++++++++++ litellm/rust_bridge/ocr/providers.py | 35 +++ .../interactions/test_openapi_compliance.py | 1 - .../test_mistral_ocr_rust_bridge.py | 183 +++++++++++++++ 31 files changed, 1379 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/test-rust.yml create mode 100644 litellm-rust/.gitignore create mode 100644 litellm-rust/CLAUDE.md create mode 100644 litellm-rust/Cargo.toml create mode 100644 litellm-rust/README.md create mode 100644 litellm-rust/crates/core/CLAUDE.md create mode 100644 litellm-rust/crates/core/Cargo.toml create mode 100644 litellm-rust/crates/core/src/error.rs create mode 100644 litellm-rust/crates/core/src/lib.rs create mode 100644 litellm-rust/crates/core/src/ocr/mod.rs create mode 100644 litellm-rust/crates/core/src/ocr/transformation.rs create mode 100644 litellm-rust/crates/core/src/ocr/types.rs create mode 100644 litellm-rust/crates/providers/CLAUDE.md create mode 100644 litellm-rust/crates/providers/Cargo.toml create mode 100644 litellm-rust/crates/providers/src/lib.rs create mode 100644 litellm-rust/crates/providers/src/mistral/mod.rs create mode 100644 litellm-rust/crates/providers/src/mistral/ocr/mod.rs create mode 100644 litellm-rust/crates/providers/src/mistral/ocr/transformation.rs create mode 100644 litellm-rust/crates/providers/src/ocr.rs create mode 100644 litellm-rust/crates/python-bridge/CLAUDE.md create mode 100644 litellm-rust/crates/python-bridge/Cargo.toml create mode 100644 litellm-rust/crates/python-bridge/src/lib.rs create mode 100644 litellm/rust_bridge/CLAUDE.md create mode 100644 litellm/rust_bridge/__init__.py create mode 100644 litellm/rust_bridge/loader.py create mode 100644 litellm/rust_bridge/ocr/__init__.py create mode 100644 litellm/rust_bridge/ocr/config.py create mode 100644 litellm/rust_bridge/ocr/providers.py create mode 100644 tests/test_litellm/test_mistral_ocr_rust_bridge.py diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml new file mode 100644 index 000000000000..c085e0a29b3a --- /dev/null +++ b/.github/workflows/test-rust.yml @@ -0,0 +1,54 @@ +name: LiteLLM Rust + +on: + push: + paths: + - "litellm-rust/**" + - ".github/workflows/test-rust.yml" + pull_request: + branches: + - main + - litellm_internal_staging + - litellm_oss_branch + - "litellm_**" + paths: + - "litellm-rust/**" + - ".github/workflows/test-rust.yml" + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + rust-checks: + name: rustfmt, clippy, test + runs-on: ubuntu-latest + timeout-minutes: 10 + defaults: + run: + working-directory: litellm-rust + env: + CARGO_TERM_COLOR: always + + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Rust + run: | + rustup toolchain install stable --profile minimal --component clippy,rustfmt + rustup default stable + + - name: Check Rust formatting + run: cargo fmt --check + + - name: Run Clippy + run: cargo clippy --workspace --all-targets -- -D warnings + + - name: Run Rust tests + run: cargo test --workspace diff --git a/litellm-rust/.gitignore b/litellm-rust/.gitignore new file mode 100644 index 000000000000..ca98cd96efdc --- /dev/null +++ b/litellm-rust/.gitignore @@ -0,0 +1,2 @@ +/target/ +Cargo.lock diff --git a/litellm-rust/CLAUDE.md b/litellm-rust/CLAUDE.md new file mode 100644 index 000000000000..1d2987e0a1a5 --- /dev/null +++ b/litellm-rust/CLAUDE.md @@ -0,0 +1,88 @@ +# CLAUDE.md + +This file defines the rules for Rust work in LiteLLM. + +## Core Boundary + +The `core` and `providers` crates describe work; hosts execute work. + +Route-level Rust structure mirrors LiteLLM's Python responsibilities: +- `core/src//` owns the route contract, shared types, and provider + template traits. For OCR, this means `core/src/ocr`. +- `providers/src///transformation.rs` owns the + provider-specific transform. For Mistral OCR, this means + `providers/src/mistral/ocr/transformation.rs`. +- Future network execution belongs in a host/transport layer such as + `llm_http_handler`, not inside `core` or `providers`. + +Allowed in `core` and `providers`: +- Pure request transforms +- Pure response transforms +- Pure stream chunk normalization +- Shared data types and validation errors +- Deterministic token/cost helper logic + +Not allowed in `core` or `providers`: +- Network calls +- Environment variable or secret reads +- Filesystem access +- Database or cache access +- Provider SDK signing or auth flows +- Logging callbacks, spend writes, or custom callbacks +- Global mutable runtime state + +Python owns rollout state and fallback while Rust is being introduced. Rust +paths must be off by default until parity tests prove equivalence with Python. + +## Production Bar + +Rust code in this workspace is held to a strict parity and robustness bar from +the first PR: + +- Correctness parity is proven with tests. Do not rely on README claims or + manual inspection for a port that mirrors Python behavior. +- Every provider transform must have unit tests for supported-parameter + filtering, request body shape, response normalization, missing/null fields, + and bad-input errors. +- When Rust is exposed through Python, add Python tests that prove disabled, + enabled, and unavailable-bridge fallback behavior. +- Avoid panics on user/provider input. Return typed errors and let the host map + them to Python exceptions or HTTP responses. +- OCR handles documents that often contain personal data. Do not log document + contents, base64 payloads, provider response bodies, or secrets. +- Error messages must be useful but data-minimized. Truncate or sanitize any + upstream body before it crosses a host boundary. +- Treat empty or whitespace-only credentials, URLs, and config values as absent + at the host/config resolution layer. +- Preserve Python output shape intentionally. If a field is always serialized as + `null` for Python parity, leave a short comment explaining that parity choice. + +## Host I/O Rules + +These rules apply when adding future crates or modules that execute network I/O, +such as `ai-gateway`, router hosts, or standalone servers: + +- Set connect and full-request timeouts. No unbounded waits. +- Reuse HTTP clients; do not construct clients per request. +- Prefer rustls TLS for portable Python wheels and Linux images unless there is + a documented reason not to. +- Add request IDs and structured tracing at the host layer, without logging OCR + document contents or secrets. +- Do not echo raw upstream response bodies to callers. Sanitize and bound them. +- Avoid `expect`/`unwrap` in server startup and request paths unless the panic is + impossible by construction and documented. + +## Checks + +Run these before pushing Rust changes. The same checks run in GitHub Actions +for changes under `litellm-rust/`. + +```bash +cd litellm-rust +cargo fmt --check +cargo clippy --workspace --all-targets -- -D warnings +cargo test --workspace +``` + +When a Rust path is exposed through Python, add Python parity tests that compare +the existing Python output with the Rust-backed output. diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml new file mode 100644 index 000000000000..4117bd5f6ac6 --- /dev/null +++ b/litellm-rust/Cargo.toml @@ -0,0 +1,20 @@ +[workspace] +members = [ + "crates/core", + "crates/providers", + "crates/python-bridge", +] +resolver = "2" + +[workspace.package] +edition = "2021" +license = "MIT" +repository = "https://github.com/BerriAI/litellm" + +[workspace.dependencies] +litellm-core = { path = "crates/core" } +litellm-providers = { path = "crates/providers" } +pyo3 = "0.23.5" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +thiserror = "2.0" diff --git a/litellm-rust/README.md b/litellm-rust/README.md new file mode 100644 index 000000000000..15ad18554202 --- /dev/null +++ b/litellm-rust/README.md @@ -0,0 +1,34 @@ +# LiteLLM Rust + +This workspace contains the staged Rust implementation for LiteLLM. + +Rust starts as a pure transform core used by the existing Python host. Python +continues to own auth, configuration, network I/O, retries, routing, logging, +callbacks, spend tracking, and customer plugins until each Rust path has parity +coverage and production evidence. + +## Layout + +```text +crates/ + core/ Route contracts, shared pure types, errors, and templates. + src/ocr/ + providers/ Provider-specific pure transforms. + src/mistral/ocr/transformation.rs + python-bridge/ PyO3 bridge for Python LiteLLM. +``` + +The folder shape should follow the Python provider tree: +`providers/src///transformation.rs`. The bridge should expose +one function per top-level route, starting with `ocr(payload)`. + +## Checks + +Run these before pushing Rust changes. GitHub Actions runs the same checks for +changes under `litellm-rust/`. + +```bash +cargo fmt --check +cargo clippy --workspace --all-targets -- -D warnings +cargo test --workspace +``` diff --git a/litellm-rust/crates/core/CLAUDE.md b/litellm-rust/crates/core/CLAUDE.md new file mode 100644 index 000000000000..51e0d215e6aa --- /dev/null +++ b/litellm-rust/crates/core/CLAUDE.md @@ -0,0 +1,38 @@ +# CLAUDE.md + +Rules for `litellm-rust/crates/core`. + +## Responsibility + +`core` owns shared data types, typed errors, and deterministic helper contracts. +It must stay pure and host-independent. + +Allowed: +- Shared request/response structs. +- Typed errors with stable, non-sensitive messages. +- Deterministic validation helpers. +- Serialization helpers that intentionally mirror Python output shape. +- Route templates that match Python base config responsibilities, such as + `ocr::transformation::OcrProviderConfig`. + +Not allowed: +- Network, filesystem, database, cache, or environment access. +- Secret reads or auth/header construction. +- Logging callbacks, tracing spans, spend writes, or customer callbacks. +- Provider-specific branching that belongs in `providers`. +- Panics for user/provider-controlled input. + +## Structure + +Use route names directly under `src/`: `ocr`, future `messages`, +`chat_completions`, `embeddings`, and similar top-level LiteLLM calls. Do not +invent broad names like `engine` for route contracts. + +## Parity Rules + +- Every shared type used by a provider transform needs unit tests for + serialization shape. +- If Python parity requires always emitting a `null` field instead of omitting + it, document that in code and pin it with a test. +- Error enums should preserve enough detail for Python/HTTP hosts to map errors + consistently without exposing document contents or upstream bodies. diff --git a/litellm-rust/crates/core/Cargo.toml b/litellm-rust/crates/core/Cargo.toml new file mode 100644 index 000000000000..e54002fe5e8d --- /dev/null +++ b/litellm-rust/crates/core/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "litellm-core" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true diff --git a/litellm-rust/crates/core/src/error.rs b/litellm-rust/crates/core/src/error.rs new file mode 100644 index 000000000000..08a7a2b5ed9e --- /dev/null +++ b/litellm-rust/crates/core/src/error.rs @@ -0,0 +1,27 @@ +use thiserror::Error; + +pub type CoreResult = Result; + +#[derive(Debug, Error, PartialEq, Eq)] +pub enum CoreError { + #[error("expected {expected}, got {actual}")] + InvalidType { + expected: &'static str, + actual: &'static str, + }, + #[error("missing required field: {0}")] + MissingField(&'static str), + #[error("invalid response: {0}")] + InvalidResponse(String), +} + +pub fn json_type_name(value: &serde_json::Value) -> &'static str { + match value { + serde_json::Value::Null => "null", + serde_json::Value::Bool(_) => "bool", + serde_json::Value::Number(_) => "number", + serde_json::Value::String(_) => "string", + serde_json::Value::Array(_) => "array", + serde_json::Value::Object(_) => "object", + } +} diff --git a/litellm-rust/crates/core/src/lib.rs b/litellm-rust/crates/core/src/lib.rs new file mode 100644 index 000000000000..a88204867c45 --- /dev/null +++ b/litellm-rust/crates/core/src/lib.rs @@ -0,0 +1,4 @@ +pub mod error; +pub mod ocr; + +pub use error::{CoreError, CoreResult}; diff --git a/litellm-rust/crates/core/src/ocr/mod.rs b/litellm-rust/crates/core/src/ocr/mod.rs new file mode 100644 index 000000000000..ec2fbb969a62 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/mod.rs @@ -0,0 +1,2 @@ +pub mod transformation; +pub mod types; diff --git a/litellm-rust/crates/core/src/ocr/transformation.rs b/litellm-rust/crates/core/src/ocr/transformation.rs new file mode 100644 index 000000000000..7353d9d22c42 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/transformation.rs @@ -0,0 +1,32 @@ +use serde_json::{Map, Value}; + +use crate::CoreResult; + +use super::types::{OcrRequestData, OcrResponseData}; + +pub trait OcrProviderConfig { + fn supported_ocr_params(&self) -> &'static [&'static str]; + + fn map_ocr_params(&self, non_default_params: &Map) -> Map { + let mut mapped_params = Map::new(); + for (param, value) in non_default_params { + if self.supported_ocr_params().contains(¶m.as_str()) { + mapped_params.insert(param.clone(), value.clone()); + } + } + mapped_params + } + + fn transform_ocr_request( + &self, + model: &str, + document: Value, + optional_params: Map, + ) -> CoreResult; + + fn transform_ocr_response( + &self, + model: &str, + response_json: Value, + ) -> CoreResult; +} diff --git a/litellm-rust/crates/core/src/ocr/types.rs b/litellm-rust/crates/core/src/ocr/types.rs new file mode 100644 index 000000000000..1a72b8f1d666 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/types.rs @@ -0,0 +1,29 @@ +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct OcrRequestData { + pub data: Value, + pub files: Option, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct OcrResponseData { + pub pages: Vec, + pub model: String, + pub document_annotation: Option, + pub usage_info: Option, + pub object: String, +} + +impl OcrResponseData { + pub fn into_json(self) -> Value { + serde_json::json!({ + "pages": self.pages, + "model": self.model, + "document_annotation": self.document_annotation, + "usage_info": self.usage_info, + "object": self.object, + }) + } +} diff --git a/litellm-rust/crates/providers/CLAUDE.md b/litellm-rust/crates/providers/CLAUDE.md new file mode 100644 index 000000000000..0f7fdcda2aa1 --- /dev/null +++ b/litellm-rust/crates/providers/CLAUDE.md @@ -0,0 +1,53 @@ +# CLAUDE.md + +Rules for `litellm-rust/crates/providers`. + +## Responsibility + +`providers` owns provider-specific pure transforms. It mirrors the existing +Python provider modules closely enough that parity review is mechanical. + +Provider files should map to the Python provider tree: + +```text +providers/src///transformation.rs +``` + +For example, Mistral OCR lives at +`providers/src/mistral/ocr/transformation.rs`, matching +`litellm/llms/mistral/ocr/transformation.py`. + +Allowed: +- Provider request transforms. +- Provider response normalization. +- Supported-parameter filtering. +- Provider-specific validation that does not require I/O or secrets. + +Not allowed: +- HTTP clients or provider SDK calls. +- Environment variable reads. +- API key resolution or auth header construction. +- Logging, callbacks, spend tracking, retries, routing, cooldowns, or fallbacks. +- Panics on bad user/provider input. + +## Required Tests + +Every provider transform must include focused unit tests for: +- Supported params matching the Python provider config. +- Unknown params being dropped or transformed the same way as Python. +- Request body shape matching Python output. +- Response normalization with complete, missing, null, and extra fields. +- Bad input returning typed errors. + +For OCR specifically, assume documents can contain personal data. Tests should +prove transforms do not copy document contents into error messages. + +## Implementation Rules + +- Prefer static supported-parameter lists over allocating strings on every call. +- Keep transforms deterministic and allocation-conscious, but choose clarity over + premature micro-optimization for tiny parameter lists. +- Use typed errors from `core`; avoid stringly-typed error plumbing. +- Add comments only when they explain Python-parity decisions or provider quirks. +- Put route-level provider dispatch in a route file such as `providers/src/ocr.rs`. + Do not move provider-specific transform logic into the Python bridge. diff --git a/litellm-rust/crates/providers/Cargo.toml b/litellm-rust/crates/providers/Cargo.toml new file mode 100644 index 000000000000..68483af62667 --- /dev/null +++ b/litellm-rust/crates/providers/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "litellm-providers" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +litellm-core.workspace = true +serde_json.workspace = true + +[dev-dependencies] +serde_json.workspace = true diff --git a/litellm-rust/crates/providers/src/lib.rs b/litellm-rust/crates/providers/src/lib.rs new file mode 100644 index 000000000000..1a0ca5b7e42a --- /dev/null +++ b/litellm-rust/crates/providers/src/lib.rs @@ -0,0 +1,2 @@ +pub mod mistral; +pub mod ocr; diff --git a/litellm-rust/crates/providers/src/mistral/mod.rs b/litellm-rust/crates/providers/src/mistral/mod.rs new file mode 100644 index 000000000000..3621ff6a2fdb --- /dev/null +++ b/litellm-rust/crates/providers/src/mistral/mod.rs @@ -0,0 +1 @@ +pub mod ocr; diff --git a/litellm-rust/crates/providers/src/mistral/ocr/mod.rs b/litellm-rust/crates/providers/src/mistral/ocr/mod.rs new file mode 100644 index 000000000000..f239b6921fac --- /dev/null +++ b/litellm-rust/crates/providers/src/mistral/ocr/mod.rs @@ -0,0 +1 @@ +pub mod transformation; diff --git a/litellm-rust/crates/providers/src/mistral/ocr/transformation.rs b/litellm-rust/crates/providers/src/mistral/ocr/transformation.rs new file mode 100644 index 000000000000..ca9073a14a09 --- /dev/null +++ b/litellm-rust/crates/providers/src/mistral/ocr/transformation.rs @@ -0,0 +1,212 @@ +use litellm_core::error::{json_type_name, CoreError, CoreResult}; +use litellm_core::ocr::transformation::OcrProviderConfig; +use litellm_core::ocr::types::{OcrRequestData, OcrResponseData}; +use serde_json::{Map, Value}; + +const SUPPORTED_OCR_PARAMS: &[&str] = &[ + "pages", + "include_image_base64", + "image_limit", + "image_min_size", + "bbox_annotation_format", + "document_annotation_format", + "document_annotation_prompt", + "extract_header", + "extract_footer", + "table_format", + "confidence_scores_granularity", + "id", +]; + +pub struct MistralOcrConfig; + +pub const MISTRAL_OCR_CONFIG: MistralOcrConfig = MistralOcrConfig; + +impl OcrProviderConfig for MistralOcrConfig { + fn supported_ocr_params(&self) -> &'static [&'static str] { + SUPPORTED_OCR_PARAMS + } + + fn transform_ocr_request( + &self, + model: &str, + document: Value, + optional_params: Map, + ) -> CoreResult { + if !document.is_object() { + return Err(CoreError::InvalidType { + expected: "object", + actual: json_type_name(&document), + }); + } + + let mut data = Map::new(); + data.insert("model".to_string(), Value::String(model.to_string())); + data.insert("document".to_string(), document); + for (param, value) in optional_params { + data.insert(param, value); + } + + Ok(OcrRequestData { + data: Value::Object(data), + files: None, + }) + } + + fn transform_ocr_response( + &self, + model: &str, + response_json: Value, + ) -> CoreResult { + let response_object = response_json + .as_object() + .ok_or_else(|| CoreError::InvalidType { + expected: "object", + actual: json_type_name(&response_json), + })?; + + let pages = response_object + .get("pages") + .and_then(Value::as_array) + .cloned() + .unwrap_or_default(); + let model = response_object + .get("model") + .and_then(Value::as_str) + .unwrap_or(model) + .to_string(); + let document_annotation = response_object.get("document_annotation").cloned(); + let usage_info = response_object.get("usage_info").cloned(); + + Ok(OcrResponseData { + pages, + model, + document_annotation, + usage_info, + object: "ocr".to_string(), + }) + } +} + +pub fn supported_ocr_params() -> &'static [&'static str] { + MISTRAL_OCR_CONFIG.supported_ocr_params() +} + +pub fn map_ocr_params(non_default_params: &Map) -> Map { + MISTRAL_OCR_CONFIG.map_ocr_params(non_default_params) +} + +pub fn transform_ocr_request( + model: &str, + document: Value, + optional_params: Map, +) -> CoreResult { + MISTRAL_OCR_CONFIG.transform_ocr_request(model, document, optional_params) +} + +pub fn transform_ocr_response(model: &str, response_json: Value) -> CoreResult { + MISTRAL_OCR_CONFIG.transform_ocr_response(model, response_json) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn supported_params_match_python_mistral_ocr_config() { + assert_eq!( + supported_ocr_params(), + &[ + "pages", + "include_image_base64", + "image_limit", + "image_min_size", + "bbox_annotation_format", + "document_annotation_format", + "document_annotation_prompt", + "extract_header", + "extract_footer", + "table_format", + "confidence_scores_granularity", + "id", + ] + ); + } + + #[test] + fn map_ocr_params_drops_unknown_params() { + let params = json!({ + "extract_header": true, + "unsupported_param": "value", + "pages": [0, 1] + }); + let mapped = map_ocr_params(params.as_object().unwrap()); + + assert_eq!(mapped.get("extract_header"), Some(&json!(true))); + assert_eq!(mapped.get("pages"), Some(&json!([0, 1]))); + assert!(!mapped.contains_key("unsupported_param")); + } + + #[test] + fn transform_ocr_request_builds_mistral_body() { + let document = json!({ + "type": "document_url", + "document_url": "https://example.com/doc.pdf" + }); + let optional_params = json!({ + "include_image_base64": true, + "table_format": "html" + }) + .as_object() + .unwrap() + .clone(); + + let result = transform_ocr_request("mistral-ocr-latest", document.clone(), optional_params) + .expect("request should transform"); + + assert_eq!( + result.data, + json!({ + "model": "mistral-ocr-latest", + "document": document, + "include_image_base64": true, + "table_format": "html" + }) + ); + assert_eq!(result.files, None); + } + + #[test] + fn transform_ocr_request_rejects_non_object_document() { + let err = transform_ocr_request("mistral-ocr-latest", json!("bad"), Map::new()) + .expect_err("string document should be rejected"); + + assert_eq!( + err, + CoreError::InvalidType { + expected: "object", + actual: "string", + } + ); + } + + #[test] + fn transform_ocr_response_normalizes_mistral_json() { + let response = json!({ + "pages": [{"index": 0, "markdown": "hello"}], + "model": "mistral-ocr-2505-completion", + "document_annotation": null, + "usage_info": {"pages_processed": 1} + }); + + let result = transform_ocr_response("mistral-ocr-latest", response) + .expect("response should transform"); + + assert_eq!(result.pages, vec![json!({"index": 0, "markdown": "hello"})]); + assert_eq!(result.model, "mistral-ocr-2505-completion"); + assert_eq!(result.document_annotation, Some(Value::Null)); + assert_eq!(result.usage_info, Some(json!({"pages_processed": 1}))); + assert_eq!(result.object, "ocr"); + } +} diff --git a/litellm-rust/crates/providers/src/ocr.rs b/litellm-rust/crates/providers/src/ocr.rs new file mode 100644 index 000000000000..48e3bbe8894f --- /dev/null +++ b/litellm-rust/crates/providers/src/ocr.rs @@ -0,0 +1,156 @@ +use litellm_core::error::{json_type_name, CoreError}; +use litellm_core::ocr::transformation::OcrProviderConfig; +use litellm_core::CoreResult; +use serde_json::{Map, Value}; + +use crate::mistral::ocr::transformation::MISTRAL_OCR_CONFIG; + +pub fn transform(payload: Value) -> CoreResult { + let payload = payload_object(&payload)?; + let provider = required_string(payload, "provider")?; + let operation = required_string(payload, "operation")?; + + match provider { + "mistral" => transform_with_provider(&MISTRAL_OCR_CONFIG, operation, payload), + _ => Err(CoreError::InvalidResponse(format!( + "unsupported OCR provider: {provider}" + ))), + } +} + +fn transform_with_provider( + provider_config: &impl OcrProviderConfig, + operation: &str, + payload: &Map, +) -> CoreResult { + match operation { + "map_params" => { + let params = required_object(payload, "non_default_params")?; + Ok(Value::Object(provider_config.map_ocr_params(¶ms))) + } + "transform_request" => { + let model = required_string(payload, "model")?; + let document = required_value(payload, "document")?; + let optional_params = required_object(payload, "optional_params")?; + let transformed = + provider_config.transform_ocr_request(model, document, optional_params)?; + Ok(serde_json::json!({ + "data": transformed.data, + "files": transformed.files, + })) + } + "transform_response" => { + let model = required_string(payload, "model")?; + let response_json = required_value(payload, "response_json")?; + let transformed = provider_config.transform_ocr_response(model, response_json)?; + Ok(transformed.into_json()) + } + _ => Err(CoreError::InvalidResponse(format!( + "unsupported OCR operation: {operation}" + ))), + } +} + +fn payload_object(payload: &Value) -> CoreResult<&Map> { + payload.as_object().ok_or_else(|| CoreError::InvalidType { + expected: "object", + actual: json_type_name(payload), + }) +} + +fn required_string<'a>(payload: &'a Map, key: &'static str) -> CoreResult<&'a str> { + let value = payload.get(key).ok_or(CoreError::MissingField(key))?; + value.as_str().ok_or_else(|| CoreError::InvalidType { + expected: "string", + actual: json_type_name(value), + }) +} + +fn required_object( + payload: &Map, + key: &'static str, +) -> CoreResult> { + let value = payload.get(key).ok_or(CoreError::MissingField(key))?; + value + .as_object() + .cloned() + .ok_or_else(|| CoreError::InvalidType { + expected: "object", + actual: json_type_name(value), + }) +} + +fn required_value(payload: &Map, key: &'static str) -> CoreResult { + payload + .get(key) + .cloned() + .ok_or(CoreError::MissingField(key)) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn transform_dispatches_mistral_map_params() { + let result = transform(json!({ + "provider": "mistral", + "operation": "map_params", + "non_default_params": { + "extract_header": true, + "unsupported_param": "value" + } + })) + .expect("payload should transform"); + + assert_eq!(result, json!({"extract_header": true})); + } + + #[test] + fn transform_dispatches_mistral_request() { + let document = json!({ + "type": "document_url", + "document_url": "https://example.com/doc.pdf" + }); + + let result = transform(json!({ + "provider": "mistral", + "operation": "transform_request", + "model": "mistral-ocr-latest", + "document": document, + "optional_params": {"include_image_base64": true} + })) + .expect("payload should transform"); + + assert_eq!( + result, + json!({ + "data": { + "model": "mistral-ocr-latest", + "document": { + "type": "document_url", + "document_url": "https://example.com/doc.pdf" + }, + "include_image_base64": true + }, + "files": null + }) + ); + } + + #[test] + fn transform_rejects_unknown_provider() { + let err = transform(json!({ + "provider": "azure_ai", + "operation": "map_params", + "non_default_params": {} + })) + .expect_err("unsupported provider should fail"); + + assert_eq!( + err, + CoreError::InvalidResponse("unsupported OCR provider: azure_ai".to_string()) + ); + } +} diff --git a/litellm-rust/crates/python-bridge/CLAUDE.md b/litellm-rust/crates/python-bridge/CLAUDE.md new file mode 100644 index 000000000000..efa1a554c9c9 --- /dev/null +++ b/litellm-rust/crates/python-bridge/CLAUDE.md @@ -0,0 +1,36 @@ +# CLAUDE.md + +Rules for `litellm-rust/crates/python-bridge`. + +## Responsibility + +`python-bridge` is the PyO3 boundary between Python LiteLLM and Rust transforms. +Keep this crate thin. It adapts Python objects to Rust payloads and returns +Python-compatible dictionaries. + +## Bridge Shape + +- Prefer one stable method per top-level LiteLLM route, for example + `ocr(payload)`. +- Do not add one exported PyO3 function per provider helper unless there is a + measured reason. +- Provider dispatch belongs in Rust route modules such as + `litellm_providers::ocr`, not in this PyO3 crate. +- Python owns rollout state and fallback. Rust should return errors; Python + decides whether to raise or fall back. + +## Data Handling + +- OCR payloads can contain personal data and large base64 images. Do not log + payloads or provider responses. +- Avoid copying large payloads more than needed. The current JSON round-trip is + acceptable for the first scaffold, but future performance work should evaluate + direct PyO3 conversion before expanding Rust coverage to image-heavy paths. +- Do not expose raw Rust errors that include document contents or upstream + bodies. + +## Tests + +- `cargo test --workspace` must compile this crate. +- Python tests must cover bridge disabled, bridge enabled, and module-missing + fallback behavior for every exposed route. diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml new file mode 100644 index 000000000000..40275879af5d --- /dev/null +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "litellm-python-bridge" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[lib] +name = "litellm_python_bridge" +crate-type = ["cdylib"] + +[dependencies] +litellm-providers.workspace = true +pyo3 = { workspace = true, features = ["extension-module"] } +serde_json.workspace = true diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs new file mode 100644 index 000000000000..a723daeb65ab --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -0,0 +1,31 @@ +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; +use pyo3::types::PyAny; +use serde_json::Value; + +fn py_to_json(py: Python<'_>, value: &Bound<'_, PyAny>) -> PyResult { + let json = py.import("json")?; + let encoded: String = json.call_method1("dumps", (value,))?.extract()?; + serde_json::from_str(&encoded).map_err(|err| PyValueError::new_err(err.to_string())) +} + +fn json_to_py(py: Python<'_>, value: Value) -> PyResult> { + let json = py.import("json")?; + let encoded = + serde_json::to_string(&value).map_err(|err| PyValueError::new_err(err.to_string()))?; + Ok(json.call_method1("loads", (encoded,))?.unbind()) +} + +#[pyfunction] +fn ocr(py: Python<'_>, payload: Py) -> PyResult> { + let payload = py_to_json(py, payload.bind(py))?; + let transformed = litellm_providers::ocr::transform(payload) + .map_err(|err| PyValueError::new_err(err.to_string()))?; + json_to_py(py, transformed) +} + +#[pymodule] +fn litellm_python_bridge(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(ocr, module)?)?; + Ok(()) +} diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index b27082c361a0..fda2cf1115fd 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -20,6 +20,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.ocr.transformation import BaseOCRConfig, OCRResponse from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.rust_bridge.ocr import get_rust_ocr_provider_config from litellm.types.router import GenericLiteLLMParams from litellm.utils import ProviderConfigManager, client @@ -278,6 +279,11 @@ def ocr( f"OCR call - model: {model}, provider: {custom_llm_provider}" ) + ocr_provider_config = get_rust_ocr_provider_config( + custom_llm_provider=custom_llm_provider, + fallback_config=ocr_provider_config, + ) + # Get litellm params using GenericLiteLLMParams (same as responses API) litellm_params = GenericLiteLLMParams(**kwargs) diff --git a/litellm/rust_bridge/CLAUDE.md b/litellm/rust_bridge/CLAUDE.md new file mode 100644 index 000000000000..a538defc557b --- /dev/null +++ b/litellm/rust_bridge/CLAUDE.md @@ -0,0 +1,39 @@ +# CLAUDE.md + +Rules for `litellm/rust_bridge`. + +## Responsibility + +This package is the Python-side bridge to optional Rust transforms. It should +route to Rust when explicitly enabled and safely return the existing Python path +when Rust is disabled, unavailable, or unsupported for a provider. + +## Naming And Shape + +- Keep this package named `rust_bridge`; do not reintroduce a vague `_rust` + package. +- Organize by LiteLLM route (`ocr/`, future `rerank/`, etc.). +- Keep route entrypoints such as `litellm/ocr/main.py` small. They should only + ask this package for a Rust-backed config or callable. +- Keep provider rollout explicit with enums or small provider registries. +- Keep rollout controlled by Python bridge APIs such as + `set_rust_core_enabled(...)`; do not add new environment variables here + unless the matching docs-repo update lands in the same rollout. +- For each route, expose a single Python-to-Rust call that passes one payload to + the PyO3 module, such as `ocr(payload)`. Do not split provider transform + operations into multiple PyO3 bridge functions. + +## Fallback Rules + +- Rust paths are off by default. +- Missing PyO3 modules must fall back unless strict mode is enabled. +- Unknown providers must return the original Python config unchanged. +- Tests must cover disabled, enabled, module-missing, and unknown-provider paths. + +## Data Handling + +- OCR inputs frequently contain personal data. Do not log documents, base64 + payloads, provider response bodies, or secrets. +- Bridge errors should be bounded and sanitized. Do not surface raw upstream + OCR bodies through Python exceptions. +- Treat blank configuration values as absent at host/config resolution time. diff --git a/litellm/rust_bridge/__init__.py b/litellm/rust_bridge/__init__.py new file mode 100644 index 000000000000..e1ed9e40e0a2 --- /dev/null +++ b/litellm/rust_bridge/__init__.py @@ -0,0 +1,19 @@ +from litellm.rust_bridge.loader import ( + rust_core_available, + set_rust_core_enabled, + set_rust_core_strict, +) +from litellm.rust_bridge.ocr import ( + RUST_OCR_PROVIDERS, + RustOcrProvider, + get_rust_ocr_provider_config, +) + +__all__ = [ + "RUST_OCR_PROVIDERS", + "RustOcrProvider", + "get_rust_ocr_provider_config", + "rust_core_available", + "set_rust_core_enabled", + "set_rust_core_strict", +] diff --git a/litellm/rust_bridge/loader.py b/litellm/rust_bridge/loader.py new file mode 100644 index 000000000000..51342685e8e7 --- /dev/null +++ b/litellm/rust_bridge/loader.py @@ -0,0 +1,68 @@ +import importlib +from types import ModuleType +from typing import Any, Iterable, Optional, Union + +_rust_module: Optional[ModuleType] = None +_rust_module_load_attempted = False +_enabled_rust_core_scopes: set[str] = set() +_rust_core_strict = False + + +def _load_rust_module() -> Optional[ModuleType]: + global _rust_module, _rust_module_load_attempted + + if _rust_module_load_attempted: + return _rust_module + + _rust_module_load_attempted = True + try: + _rust_module = importlib.import_module("litellm_python_bridge") + except Exception: + _rust_module = None + return _rust_module + + +def rust_core_available() -> bool: + return _load_rust_module() is not None + + +def set_rust_core_enabled(scopes: Union[bool, str, Iterable[str]]) -> None: + global _enabled_rust_core_scopes + + if scopes is True: + _enabled_rust_core_scopes = {"all"} + return + if scopes is False: + _enabled_rust_core_scopes = set() + return + if isinstance(scopes, str): + _enabled_rust_core_scopes = { + scope.strip() + for scope in scopes.replace(";", ",").split(",") + if scope.strip() + } + return + + _enabled_rust_core_scopes = {scope for scope in scopes if scope} + + +def rust_core_enabled(scope: str) -> bool: + return "all" in _enabled_rust_core_scopes or scope in _enabled_rust_core_scopes + + +def set_rust_core_strict(enabled: bool) -> None: + global _rust_core_strict + _rust_core_strict = enabled + + +def call_rust_function(function_name: str, *args: Any) -> Optional[Any]: + module = _load_rust_module() + if module is None: + return None + + try: + return getattr(module, function_name)(*args) + except Exception: + if _rust_core_strict: + raise + return None diff --git a/litellm/rust_bridge/ocr/__init__.py b/litellm/rust_bridge/ocr/__init__.py new file mode 100644 index 000000000000..273179030fbe --- /dev/null +++ b/litellm/rust_bridge/ocr/__init__.py @@ -0,0 +1,8 @@ +from litellm.rust_bridge.ocr.config import get_rust_ocr_provider_config +from litellm.rust_bridge.ocr.providers import RUST_OCR_PROVIDERS, RustOcrProvider + +__all__ = [ + "RUST_OCR_PROVIDERS", + "RustOcrProvider", + "get_rust_ocr_provider_config", +] diff --git a/litellm/rust_bridge/ocr/config.py b/litellm/rust_bridge/ocr/config.py new file mode 100644 index 000000000000..b302a62bdd4d --- /dev/null +++ b/litellm/rust_bridge/ocr/config.py @@ -0,0 +1,160 @@ +from typing import Any, Optional, cast + +import httpx + +from litellm.llms.base_llm.ocr.transformation import ( + BaseOCRConfig, + DocumentType, + OCRRequestData, + OCRResponse, +) +from litellm.rust_bridge.ocr.providers import RustOcrProvider, call_ocr + + +def get_rust_ocr_provider_config( + custom_llm_provider: Optional[str], + fallback_config: BaseOCRConfig, +) -> BaseOCRConfig: + if custom_llm_provider is None: + return fallback_config + + provider_value = getattr(custom_llm_provider, "value", custom_llm_provider) + try: + rust_ocr_provider = RustOcrProvider(str(provider_value)) + except ValueError: + return fallback_config + + return cast( + BaseOCRConfig, + _RustOCRProviderConfig( + rust_ocr_provider=rust_ocr_provider, + fallback_config=fallback_config, + ), + ) + + +class _RustOCRProviderConfig: + def __init__( + self, + rust_ocr_provider: RustOcrProvider, + fallback_config: BaseOCRConfig, + ) -> None: + self.rust_ocr_provider = rust_ocr_provider + self.fallback_config = fallback_config + + def __getattr__(self, name: str) -> Any: + return getattr(self.fallback_config, name) + + def map_ocr_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + ) -> dict: + mapped_params = call_ocr( + { + "provider": self.rust_ocr_provider.value, + "operation": "map_params", + "non_default_params": non_default_params, + } + ) + if mapped_params is not None: + return mapped_params + return self.fallback_config.map_ocr_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=model, + ) + + def transform_ocr_request( + self, + model: str, + document: DocumentType, + optional_params: dict, + headers: dict, + **kwargs, + ) -> OCRRequestData: + if isinstance(document, dict): + transformed_request = call_ocr( + { + "provider": self.rust_ocr_provider.value, + "operation": "transform_request", + "model": model, + "document": document, + "optional_params": optional_params, + }, + ) + if transformed_request is not None: + request_data = transformed_request.get("data") + if not isinstance(request_data, dict): + raise ValueError( + f"Rust OCR provider {self.rust_ocr_provider.value} " + "returned invalid request data" + ) + return OCRRequestData( + data=request_data, + files=transformed_request.get("files"), + ) + + return self.fallback_config.transform_ocr_request( + model=model, + document=document, + optional_params=optional_params, + headers=headers, + **kwargs, + ) + + async def async_transform_ocr_request( + self, + model: str, + document: DocumentType, + optional_params: dict, + headers: dict, + **kwargs, + ) -> OCRRequestData: + return self.transform_ocr_request( + model=model, + document=document, + optional_params=optional_params, + headers=headers, + **kwargs, + ) + + def transform_ocr_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: Any, + **kwargs, + ) -> OCRResponse: + transformed_response = call_ocr( + { + "provider": self.rust_ocr_provider.value, + "operation": "transform_response", + "model": model, + "response_json": raw_response.json(), + }, + ) + if transformed_response is not None: + return OCRResponse(**transformed_response) + + return self.fallback_config.transform_ocr_response( + model=model, + raw_response=raw_response, + logging_obj=logging_obj, + **kwargs, + ) + + async def async_transform_ocr_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: Any, + **kwargs, + ) -> OCRResponse: + return self.transform_ocr_response( + model=model, + raw_response=raw_response, + logging_obj=logging_obj, + **kwargs, + ) diff --git a/litellm/rust_bridge/ocr/providers.py b/litellm/rust_bridge/ocr/providers.py new file mode 100644 index 000000000000..c5f806e31171 --- /dev/null +++ b/litellm/rust_bridge/ocr/providers.py @@ -0,0 +1,35 @@ +from enum import Enum +from typing import Any, Optional + +from litellm.rust_bridge.loader import call_rust_function, rust_core_enabled + + +class RustOcrProvider(str, Enum): + MISTRAL = "mistral" + + +RUST_OCR_PROVIDERS = frozenset({RustOcrProvider.MISTRAL.value}) + + +def call_ocr(payload: dict[str, Any]) -> Optional[dict[str, Any]]: + provider = payload.get("provider") + if not isinstance(provider, str) or provider not in RUST_OCR_PROVIDERS: + return None + + if not _rust_ocr_provider_enabled(provider): + return None + + result = call_rust_function("ocr", payload) + if result is None: + return None + if not isinstance(result, dict): + raise ValueError("Rust OCR bridge returned invalid response") + return result + + +def _rust_ocr_provider_enabled(provider: str) -> bool: + return ( + rust_core_enabled("ocr") + or rust_core_enabled(f"ocr:{provider}") + or rust_core_enabled(f"{provider}_ocr") + ) diff --git a/tests/test_litellm/interactions/test_openapi_compliance.py b/tests/test_litellm/interactions/test_openapi_compliance.py index aededaaca77d..509ded8c704c 100644 --- a/tests/test_litellm/interactions/test_openapi_compliance.py +++ b/tests/test_litellm/interactions/test_openapi_compliance.py @@ -165,7 +165,6 @@ def test_interaction_response_fields(self, spec_dict): "status", "created", "updated", - "role", "steps", "usage", ] diff --git a/tests/test_litellm/test_mistral_ocr_rust_bridge.py b/tests/test_litellm/test_mistral_ocr_rust_bridge.py new file mode 100644 index 000000000000..42ea77586375 --- /dev/null +++ b/tests/test_litellm/test_mistral_ocr_rust_bridge.py @@ -0,0 +1,183 @@ +import httpx +import pytest + +from litellm.llms.mistral.ocr.transformation import MistralOCRConfig +from litellm.rust_bridge import loader +from litellm.rust_bridge import ocr as rust_ocr +from litellm.rust_bridge.ocr import providers +from litellm.rust_bridge.ocr import RustOcrProvider + +MODEL = "mistral-ocr-latest" +DOCUMENT = { + "type": "document_url", + "document_url": "https://example.com/doc.pdf", +} + + +@pytest.fixture(autouse=True) +def reset_rust_bridge_state(): + loader.set_rust_core_enabled(False) + loader.set_rust_core_strict(False) + yield + loader.set_rust_core_enabled(False) + loader.set_rust_core_strict(False) + + +class _FakeRustModule: + @staticmethod + def ocr(payload): + provider = payload["provider"] + operation = payload["operation"] + + assert provider == "mistral" + if operation == "map_params": + return { + key: value + for key, value in payload["non_default_params"].items() + if key != "unsupported_param" + } + if operation == "transform_request": + return { + "data": { + "model": payload["model"], + "document": payload["document"], + **payload["optional_params"], + }, + "files": None, + } + if operation == "transform_response": + response_json = payload["response_json"] + return { + "pages": response_json.get("pages", []), + "model": response_json.get("model", payload["model"]), + "document_annotation": response_json.get("document_annotation"), + "usage_info": response_json.get("usage_info"), + "object": "ocr", + } + raise AssertionError(f"Unexpected operation: {operation}") + + +class _FakeLoggingObj: + pass + + +def test_rust_ocr_provider_enum_is_explicit(): + assert providers.RUST_OCR_PROVIDERS == {RustOcrProvider.MISTRAL.value} + + +def test_unknown_ocr_provider_uses_python_fallback(): + fallback_config = MistralOCRConfig() + + config = rust_ocr.get_rust_ocr_provider_config("azure_ai", fallback_config) + + assert config is fallback_config + + +def test_rust_ocr_provider_returns_none_when_scope_disabled(monkeypatch): + monkeypatch.setattr(loader, "_load_rust_module", lambda: _FakeRustModule) + + assert ( + providers.call_ocr( + { + "provider": RustOcrProvider.MISTRAL.value, + "operation": "map_params", + "non_default_params": {"extract_header": True}, + } + ) + is None + ) + + +def test_mistral_ocr_map_params_uses_provider_gated_rust(monkeypatch): + loader.set_rust_core_enabled("ocr:mistral") + monkeypatch.setattr(loader, "_load_rust_module", lambda: _FakeRustModule) + + result = providers.call_ocr( + { + "provider": RustOcrProvider.MISTRAL.value, + "operation": "map_params", + "non_default_params": { + "extract_header": True, + "unsupported_param": "value", + }, + }, + ) + + assert result == {"extract_header": True} + + +def test_mistral_ocr_provider_wrapper_uses_rust_when_enabled(monkeypatch): + loader.set_rust_core_enabled("ocr:mistral") + monkeypatch.setattr(loader, "_load_rust_module", lambda: _FakeRustModule) + + config = rust_ocr.get_rust_ocr_provider_config("mistral", MistralOCRConfig()) + + request = config.transform_ocr_request( + model=MODEL, + document=DOCUMENT, + optional_params={"include_image_base64": True}, + headers={}, + ) + assert request.data == { + "model": MODEL, + "document": DOCUMENT, + "include_image_base64": True, + } + assert request.files is None + + response = config.transform_ocr_response( + model=MODEL, + raw_response=httpx.Response( + 200, + json={ + "pages": [{"index": 0, "markdown": "hello"}], + "model": "mistral-ocr-2505-completion", + "document_annotation": None, + "usage_info": {"pages_processed": 1}, + }, + ), + logging_obj=_FakeLoggingObj(), + ) + + assert response.pages[0].index == 0 + assert response.model == "mistral-ocr-2505-completion" + assert response.usage_info.pages_processed == 1 + + +def test_mistral_ocr_provider_wrapper_falls_back_when_rust_module_missing( + monkeypatch, +): + loader.set_rust_core_enabled("ocr:mistral") + monkeypatch.setattr(loader, "_load_rust_module", lambda: None) + + config = rust_ocr.get_rust_ocr_provider_config("mistral", MistralOCRConfig()) + + result = config.transform_ocr_request( + model=MODEL, + document=DOCUMENT, + optional_params={"include_image_base64": True}, + headers={}, + ) + + assert result.data == { + "model": MODEL, + "document": DOCUMENT, + "include_image_base64": True, + } + + +def test_mistral_ocr_config_stays_python_fallback(monkeypatch): + loader.set_rust_core_enabled("ocr:mistral") + monkeypatch.setattr(loader, "_load_rust_module", lambda: _FakeRustModule) + + config = MistralOCRConfig() + result = config.map_ocr_params( + non_default_params={ + "extract_header": True, + "unsupported_param": "value", + }, + optional_params={}, + model=MODEL, + ) + + assert result == {"extract_header": True} From 6a135105eb88b9b870658f23f2066c44e5411459 Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Mon, 22 Jun 2026 17:38:24 -0700 Subject: [PATCH 15/54] address greptile rust ocr feedback --- .github/workflows/test-rust.yml | 11 +++ litellm/llms/base_llm/ocr/transformation.py | 6 ++ litellm/llms/mistral/ocr/rust_provider.py | 15 ++++ litellm/llms/mistral/ocr/transformation.py | 4 + litellm/ocr/main.py | 2 +- litellm/rust_bridge/__init__.py | 8 +- litellm/rust_bridge/loader.py | 15 +--- litellm/rust_bridge/ocr/__init__.py | 3 - litellm/rust_bridge/ocr/config.py | 87 ++++++++++++++----- litellm/rust_bridge/ocr/providers.py | 10 +-- .../test_mistral_ocr_rust_bridge.py | 51 ++++++++--- 11 files changed, 145 insertions(+), 67 deletions(-) create mode 100644 litellm/llms/mistral/ocr/rust_provider.py diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index c085e0a29b3a..3716ba605b10 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -44,6 +44,17 @@ jobs: rustup toolchain install stable --profile minimal --component clippy,rustfmt rustup default stable + - name: Cache Cargo registry and target + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + litellm-rust/target + key: ${{ runner.os }}-cargo-${{ hashFiles('litellm-rust/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-cargo- + - name: Check Rust formatting run: cargo fmt --check diff --git a/litellm/llms/base_llm/ocr/transformation.py b/litellm/llms/base_llm/ocr/transformation.py index 263e0c094cec..4783a4762d0f 100644 --- a/litellm/llms/base_llm/ocr/transformation.py +++ b/litellm/llms/base_llm/ocr/transformation.py @@ -101,6 +101,12 @@ def get_supported_ocr_params(self, model: str) -> list: """ return [] + def get_rust_ocr_provider(self, model: str) -> Optional[str]: + """ + Return the Rust OCR provider id for providers that explicitly opt in. + """ + return None + def map_ocr_params( self, non_default_params: dict, diff --git a/litellm/llms/mistral/ocr/rust_provider.py b/litellm/llms/mistral/ocr/rust_provider.py new file mode 100644 index 000000000000..960f7037810b --- /dev/null +++ b/litellm/llms/mistral/ocr/rust_provider.py @@ -0,0 +1,15 @@ +from enum import Enum +from typing import Optional + + +class MistralRustOcrProvider(str, Enum): + MISTRAL = "mistral" + + +def get_mistral_rust_ocr_provider(config_type: type) -> Optional[str]: + if ( + config_type.__module__ != "litellm.llms.mistral.ocr.transformation" + or config_type.__name__ != "MistralOCRConfig" + ): + return None + return MistralRustOcrProvider.MISTRAL.value diff --git a/litellm/llms/mistral/ocr/transformation.py b/litellm/llms/mistral/ocr/transformation.py index 21e0e27a3146..99bdb2da45d1 100644 --- a/litellm/llms/mistral/ocr/transformation.py +++ b/litellm/llms/mistral/ocr/transformation.py @@ -13,6 +13,7 @@ OCRRequestData, OCRResponse, ) +from litellm.llms.mistral.ocr.rust_provider import get_mistral_rust_ocr_provider from litellm.secret_managers.main import get_secret_str @@ -26,6 +27,9 @@ class MistralOCRConfig(BaseOCRConfig): def __init__(self) -> None: super().__init__() + def get_rust_ocr_provider(self, model: str) -> Optional[str]: + return get_mistral_rust_ocr_provider(type(self)) + def get_supported_ocr_params(self, model: str) -> list: """ Get supported OCR parameters for Mistral OCR. diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index fda2cf1115fd..dcef3c79ffef 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -280,7 +280,7 @@ def ocr( ) ocr_provider_config = get_rust_ocr_provider_config( - custom_llm_provider=custom_llm_provider, + model=model, fallback_config=ocr_provider_config, ) diff --git a/litellm/rust_bridge/__init__.py b/litellm/rust_bridge/__init__.py index e1ed9e40e0a2..df1467f33383 100644 --- a/litellm/rust_bridge/__init__.py +++ b/litellm/rust_bridge/__init__.py @@ -3,15 +3,9 @@ set_rust_core_enabled, set_rust_core_strict, ) -from litellm.rust_bridge.ocr import ( - RUST_OCR_PROVIDERS, - RustOcrProvider, - get_rust_ocr_provider_config, -) +from litellm.rust_bridge.ocr import get_rust_ocr_provider_config __all__ = [ - "RUST_OCR_PROVIDERS", - "RustOcrProvider", "get_rust_ocr_provider_config", "rust_core_available", "set_rust_core_enabled", diff --git a/litellm/rust_bridge/loader.py b/litellm/rust_bridge/loader.py index 51342685e8e7..270ee1b2045f 100644 --- a/litellm/rust_bridge/loader.py +++ b/litellm/rust_bridge/loader.py @@ -1,25 +1,18 @@ import importlib +from functools import lru_cache from types import ModuleType from typing import Any, Iterable, Optional, Union -_rust_module: Optional[ModuleType] = None -_rust_module_load_attempted = False _enabled_rust_core_scopes: set[str] = set() _rust_core_strict = False +@lru_cache(maxsize=1) def _load_rust_module() -> Optional[ModuleType]: - global _rust_module, _rust_module_load_attempted - - if _rust_module_load_attempted: - return _rust_module - - _rust_module_load_attempted = True try: - _rust_module = importlib.import_module("litellm_python_bridge") + return importlib.import_module("litellm_python_bridge") except Exception: - _rust_module = None - return _rust_module + return None def rust_core_available() -> bool: diff --git a/litellm/rust_bridge/ocr/__init__.py b/litellm/rust_bridge/ocr/__init__.py index 273179030fbe..c4e958a0b50e 100644 --- a/litellm/rust_bridge/ocr/__init__.py +++ b/litellm/rust_bridge/ocr/__init__.py @@ -1,8 +1,5 @@ from litellm.rust_bridge.ocr.config import get_rust_ocr_provider_config -from litellm.rust_bridge.ocr.providers import RUST_OCR_PROVIDERS, RustOcrProvider __all__ = [ - "RUST_OCR_PROVIDERS", - "RustOcrProvider", "get_rust_ocr_provider_config", ] diff --git a/litellm/rust_bridge/ocr/config.py b/litellm/rust_bridge/ocr/config.py index b302a62bdd4d..fd38d0a104ae 100644 --- a/litellm/rust_bridge/ocr/config.py +++ b/litellm/rust_bridge/ocr/config.py @@ -1,4 +1,4 @@ -from typing import Any, Optional, cast +from typing import Any, Optional import httpx @@ -8,42 +8,69 @@ OCRRequestData, OCRResponse, ) -from litellm.rust_bridge.ocr.providers import RustOcrProvider, call_ocr +from litellm.rust_bridge.ocr.providers import call_ocr def get_rust_ocr_provider_config( - custom_llm_provider: Optional[str], + model: str, fallback_config: BaseOCRConfig, ) -> BaseOCRConfig: - if custom_llm_provider is None: + rust_ocr_provider = fallback_config.get_rust_ocr_provider(model=model) + if not rust_ocr_provider: return fallback_config - provider_value = getattr(custom_llm_provider, "value", custom_llm_provider) - try: - rust_ocr_provider = RustOcrProvider(str(provider_value)) - except ValueError: - return fallback_config - - return cast( - BaseOCRConfig, - _RustOCRProviderConfig( - rust_ocr_provider=rust_ocr_provider, - fallback_config=fallback_config, - ), + return RustOCRProviderConfig( + rust_ocr_provider=rust_ocr_provider, + fallback_config=fallback_config, ) -class _RustOCRProviderConfig: +class RustOCRProviderConfig(BaseOCRConfig): def __init__( self, - rust_ocr_provider: RustOcrProvider, + rust_ocr_provider: str, fallback_config: BaseOCRConfig, ) -> None: + super().__init__() self.rust_ocr_provider = rust_ocr_provider self.fallback_config = fallback_config - def __getattr__(self, name: str) -> Any: - return getattr(self.fallback_config, name) + def get_supported_ocr_params(self, model: str) -> list: + return self.fallback_config.get_supported_ocr_params(model=model) + + def validate_environment( + self, + headers: dict, + model: str, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + litellm_params: Optional[dict] = None, + **kwargs, + ) -> dict: + return self.fallback_config.validate_environment( + headers=headers, + model=model, + api_key=api_key, + api_base=api_base, + litellm_params=litellm_params, + **kwargs, + ) + + def get_complete_url( + self, + api_base: Optional[str], + model: str, + optional_params: dict, + litellm_params: Optional[dict] = None, + **kwargs, + ) -> str: + return self.fallback_config.get_complete_url( + api_base=api_base, + model=model, + optional_params=optional_params, + litellm_params=litellm_params, + **kwargs, + ) def map_ocr_params( self, @@ -53,7 +80,7 @@ def map_ocr_params( ) -> dict: mapped_params = call_ocr( { - "provider": self.rust_ocr_provider.value, + "provider": self.rust_ocr_provider, "operation": "map_params", "non_default_params": non_default_params, } @@ -77,7 +104,7 @@ def transform_ocr_request( if isinstance(document, dict): transformed_request = call_ocr( { - "provider": self.rust_ocr_provider.value, + "provider": self.rust_ocr_provider, "operation": "transform_request", "model": model, "document": document, @@ -88,7 +115,7 @@ def transform_ocr_request( request_data = transformed_request.get("data") if not isinstance(request_data, dict): raise ValueError( - f"Rust OCR provider {self.rust_ocr_provider.value} " + f"Rust OCR provider {self.rust_ocr_provider} " "returned invalid request data" ) return OCRRequestData( @@ -129,7 +156,7 @@ def transform_ocr_response( ) -> OCRResponse: transformed_response = call_ocr( { - "provider": self.rust_ocr_provider.value, + "provider": self.rust_ocr_provider, "operation": "transform_response", "model": model, "response_json": raw_response.json(), @@ -158,3 +185,15 @@ async def async_transform_ocr_response( logging_obj=logging_obj, **kwargs, ) + + def get_error_class( + self, + error_message: str, + status_code: int, + headers: dict, + ) -> Exception: + return self.fallback_config.get_error_class( + error_message=error_message, + status_code=status_code, + headers=headers, + ) diff --git a/litellm/rust_bridge/ocr/providers.py b/litellm/rust_bridge/ocr/providers.py index c5f806e31171..160b9f8f4d97 100644 --- a/litellm/rust_bridge/ocr/providers.py +++ b/litellm/rust_bridge/ocr/providers.py @@ -1,19 +1,11 @@ -from enum import Enum from typing import Any, Optional from litellm.rust_bridge.loader import call_rust_function, rust_core_enabled -class RustOcrProvider(str, Enum): - MISTRAL = "mistral" - - -RUST_OCR_PROVIDERS = frozenset({RustOcrProvider.MISTRAL.value}) - - def call_ocr(payload: dict[str, Any]) -> Optional[dict[str, Any]]: provider = payload.get("provider") - if not isinstance(provider, str) or provider not in RUST_OCR_PROVIDERS: + if not isinstance(provider, str): return None if not _rust_ocr_provider_enabled(provider): diff --git a/tests/test_litellm/test_mistral_ocr_rust_bridge.py b/tests/test_litellm/test_mistral_ocr_rust_bridge.py index 42ea77586375..27ab6744a3e5 100644 --- a/tests/test_litellm/test_mistral_ocr_rust_bridge.py +++ b/tests/test_litellm/test_mistral_ocr_rust_bridge.py @@ -1,11 +1,13 @@ import httpx import pytest +from litellm.llms.base_llm.ocr.transformation import BaseOCRConfig +from litellm.llms.mistral.ocr.rust_provider import MistralRustOcrProvider from litellm.llms.mistral.ocr.transformation import MistralOCRConfig from litellm.rust_bridge import loader from litellm.rust_bridge import ocr as rust_ocr from litellm.rust_bridge.ocr import providers -from litellm.rust_bridge.ocr import RustOcrProvider +from litellm.rust_bridge.ocr.config import RustOCRProviderConfig MODEL = "mistral-ocr-latest" DOCUMENT = { @@ -61,14 +63,25 @@ class _FakeLoggingObj: pass -def test_rust_ocr_provider_enum_is_explicit(): - assert providers.RUST_OCR_PROVIDERS == {RustOcrProvider.MISTRAL.value} +class _InheritedMistralOCRConfig(MistralOCRConfig): + pass + + +def test_mistral_rust_ocr_provider_enum_is_owned_by_mistral(): + assert MistralRustOcrProvider.MISTRAL.value == "mistral" + assert ( + MistralOCRConfig().get_rust_ocr_provider(model=MODEL) + == MistralRustOcrProvider.MISTRAL.value + ) -def test_unknown_ocr_provider_uses_python_fallback(): - fallback_config = MistralOCRConfig() +def test_inherited_mistral_ocr_provider_uses_python_fallback(): + fallback_config = _InheritedMistralOCRConfig() - config = rust_ocr.get_rust_ocr_provider_config("azure_ai", fallback_config) + config = rust_ocr.get_rust_ocr_provider_config( + model=MODEL, + fallback_config=fallback_config, + ) assert config is fallback_config @@ -79,7 +92,7 @@ def test_rust_ocr_provider_returns_none_when_scope_disabled(monkeypatch): assert ( providers.call_ocr( { - "provider": RustOcrProvider.MISTRAL.value, + "provider": MistralRustOcrProvider.MISTRAL.value, "operation": "map_params", "non_default_params": {"extract_header": True}, } @@ -94,7 +107,7 @@ def test_mistral_ocr_map_params_uses_provider_gated_rust(monkeypatch): result = providers.call_ocr( { - "provider": RustOcrProvider.MISTRAL.value, + "provider": MistralRustOcrProvider.MISTRAL.value, "operation": "map_params", "non_default_params": { "extract_header": True, @@ -110,7 +123,13 @@ def test_mistral_ocr_provider_wrapper_uses_rust_when_enabled(monkeypatch): loader.set_rust_core_enabled("ocr:mistral") monkeypatch.setattr(loader, "_load_rust_module", lambda: _FakeRustModule) - config = rust_ocr.get_rust_ocr_provider_config("mistral", MistralOCRConfig()) + config = rust_ocr.get_rust_ocr_provider_config( + model=MODEL, + fallback_config=MistralOCRConfig(), + ) + + assert isinstance(config, BaseOCRConfig) + assert isinstance(config, RustOCRProviderConfig) request = config.transform_ocr_request( model=MODEL, @@ -150,7 +169,10 @@ def test_mistral_ocr_provider_wrapper_falls_back_when_rust_module_missing( loader.set_rust_core_enabled("ocr:mistral") monkeypatch.setattr(loader, "_load_rust_module", lambda: None) - config = rust_ocr.get_rust_ocr_provider_config("mistral", MistralOCRConfig()) + config = rust_ocr.get_rust_ocr_provider_config( + model=MODEL, + fallback_config=MistralOCRConfig(), + ) result = config.transform_ocr_request( model=MODEL, @@ -166,11 +188,16 @@ def test_mistral_ocr_provider_wrapper_falls_back_when_rust_module_missing( } -def test_mistral_ocr_config_stays_python_fallback(monkeypatch): +def test_inherited_mistral_ocr_config_does_not_use_mistral_rust(monkeypatch): loader.set_rust_core_enabled("ocr:mistral") monkeypatch.setattr(loader, "_load_rust_module", lambda: _FakeRustModule) - config = MistralOCRConfig() + config = rust_ocr.get_rust_ocr_provider_config( + model=MODEL, + fallback_config=_InheritedMistralOCRConfig(), + ) + + assert isinstance(config, _InheritedMistralOCRConfig) result = config.map_ocr_params( non_default_params={ "extract_header": True, From fea5204a2f4cb9c2a73397c10208ed1c1241a523 Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Mon, 22 Jun 2026 17:53:18 -0700 Subject: [PATCH 16/54] Simplify rust ocr entrypoint --- litellm/llms/base_llm/ocr/transformation.py | 6 - litellm/llms/mistral/ocr/rust_provider.py | 43 ++- litellm/llms/mistral/ocr/transformation.py | 4 - litellm/ocr/__init__.py | 4 +- litellm/ocr/main.py | 267 ++++++++++++++++-- litellm/rust_bridge/__init__.py | 2 - litellm/rust_bridge/loader.py | 6 +- litellm/rust_bridge/ocr/__init__.py | 5 +- litellm/rust_bridge/ocr/config.py | 199 ------------- litellm/rust_bridge/ocr/providers.py | 12 +- .../test_mistral_ocr_rust_bridge.py | 195 ++++++------- 11 files changed, 395 insertions(+), 348 deletions(-) delete mode 100644 litellm/rust_bridge/ocr/config.py diff --git a/litellm/llms/base_llm/ocr/transformation.py b/litellm/llms/base_llm/ocr/transformation.py index 4783a4762d0f..263e0c094cec 100644 --- a/litellm/llms/base_llm/ocr/transformation.py +++ b/litellm/llms/base_llm/ocr/transformation.py @@ -101,12 +101,6 @@ def get_supported_ocr_params(self, model: str) -> list: """ return [] - def get_rust_ocr_provider(self, model: str) -> Optional[str]: - """ - Return the Rust OCR provider id for providers that explicitly opt in. - """ - return None - def map_ocr_params( self, non_default_params: dict, diff --git a/litellm/llms/mistral/ocr/rust_provider.py b/litellm/llms/mistral/ocr/rust_provider.py index 960f7037810b..fedaad3c2acd 100644 --- a/litellm/llms/mistral/ocr/rust_provider.py +++ b/litellm/llms/mistral/ocr/rust_provider.py @@ -1,15 +1,46 @@ from enum import Enum -from typing import Optional +from typing import Any + +from litellm.secret_managers.main import get_secret_str class MistralRustOcrProvider(str, Enum): MISTRAL = "mistral" -def get_mistral_rust_ocr_provider(config_type: type) -> Optional[str]: - if ( - config_type.__module__ != "litellm.llms.mistral.ocr.transformation" - or config_type.__name__ != "MistralOCRConfig" - ): +def get_mistral_rust_ocr_provider( + custom_llm_provider: str | None, +) -> str | None: + provider_value = getattr(custom_llm_provider, "value", custom_llm_provider) + if provider_value != MistralRustOcrProvider.MISTRAL.value: return None return MistralRustOcrProvider.MISTRAL.value + + +def get_mistral_rust_ocr_url(api_base: str | None) -> str: + if api_base is None: + api_base = "https://api.mistral.ai/v1" + + api_base = api_base.rstrip("/") + if api_base.endswith("/v1"): + return f"{api_base}/ocr" + return f"{api_base}/v1/ocr" + + +def get_mistral_rust_ocr_headers( + headers: dict[str, Any] | None, + api_key: str | None, +) -> dict[str, Any]: + if api_key is None: + api_key = get_secret_str("MISTRAL_API_KEY") + + if api_key is None: + raise ValueError( + "Missing Mistral API Key - A call is being made to Mistral but no key " + "is set either in the environment variables or via params" + ) + + return { + "Authorization": f"Bearer {api_key}", + **(headers or {}), + } diff --git a/litellm/llms/mistral/ocr/transformation.py b/litellm/llms/mistral/ocr/transformation.py index 99bdb2da45d1..21e0e27a3146 100644 --- a/litellm/llms/mistral/ocr/transformation.py +++ b/litellm/llms/mistral/ocr/transformation.py @@ -13,7 +13,6 @@ OCRRequestData, OCRResponse, ) -from litellm.llms.mistral.ocr.rust_provider import get_mistral_rust_ocr_provider from litellm.secret_managers.main import get_secret_str @@ -27,9 +26,6 @@ class MistralOCRConfig(BaseOCRConfig): def __init__(self) -> None: super().__init__() - def get_rust_ocr_provider(self, model: str) -> Optional[str]: - return get_mistral_rust_ocr_provider(type(self)) - def get_supported_ocr_params(self, model: str) -> list: """ Get supported OCR parameters for Mistral OCR. diff --git a/litellm/ocr/__init__.py b/litellm/ocr/__init__.py index e97497b2db7c..8281e58046d0 100644 --- a/litellm/ocr/__init__.py +++ b/litellm/ocr/__init__.py @@ -1,5 +1,5 @@ """OCR module for LiteLLM.""" -from .main import aocr, ocr +from .main import aocr, ocr, rust_ocr -__all__ = ["ocr", "aocr"] +__all__ = ["ocr", "aocr", "rust_ocr"] diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index dcef3c79ffef..e6eba696c250 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -19,8 +19,16 @@ from litellm.constants import request_timeout from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.ocr.transformation import BaseOCRConfig, OCRResponse -from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler -from litellm.rust_bridge.ocr import get_rust_ocr_provider_config +from litellm.llms.custom_httpx.llm_http_handler import ( + BaseLLMHTTPHandler, + _get_httpx_client, +) +from litellm.llms.mistral.ocr.rust_provider import ( + get_mistral_rust_ocr_headers, + get_mistral_rust_ocr_provider, + get_mistral_rust_ocr_url, +) +from litellm.rust_bridge.ocr import call_ocr, rust_ocr_provider_enabled from litellm.types.router import GenericLiteLLMParams from litellm.utils import ProviderConfigManager, client @@ -29,6 +37,155 @@ ################################################# +def _prepare_ocr_document(document: dict[str, Any]) -> dict[str, str]: + if not isinstance(document, dict): + raise ValueError( + f"document must be a dict with 'type' and URL/file field, got {type(document)}" + ) + + doc_type = document.get("type") + if doc_type == "file": + document = convert_file_document_to_url_document(document) + doc_type = document.get("type") + + if doc_type not in ["document_url", "image_url"]: + raise ValueError( + f"Invalid document type: {doc_type}. " + "Must be 'document_url', 'image_url', or 'file'" + ) + + return document + + +def _get_rust_ocr_provider(custom_llm_provider: str | None) -> str | None: + return get_mistral_rust_ocr_provider(custom_llm_provider=custom_llm_provider) + + +def _should_route_to_rust_ocr(custom_llm_provider: str | None) -> bool: + rust_ocr_provider = _get_rust_ocr_provider(custom_llm_provider) + return rust_ocr_provider is not None and rust_ocr_provider_enabled( + rust_ocr_provider + ) + + +def _call_rust_ocr( + payload: dict[str, Any], + *, + require_enabled: bool, + fallback_on_unavailable: bool, +) -> dict[str, Any] | None: + result = call_ocr(payload, require_enabled=require_enabled) + if result is not None: + return result + if fallback_on_unavailable: + return None + raise ValueError("Rust OCR bridge is unavailable for this request") + + +def _rust_ocr_impl( + model: str, + document: dict[str, str], + api_key: str | None, + api_base: str | None, + timeout: float | httpx.Timeout | None, + custom_llm_provider: str | None, + extra_headers: dict[str, Any] | None, + litellm_logging_obj: LiteLLMLoggingObj, + litellm_call_id: str | None, + fallback_on_unavailable: bool, + require_enabled: bool, + kwargs: dict[str, Any], +) -> OCRResponse | None: + rust_ocr_provider = _get_rust_ocr_provider(custom_llm_provider) + if rust_ocr_provider is None: + if fallback_on_unavailable: + return None + raise ValueError( + f"Rust OCR is not supported for provider: {custom_llm_provider}" + ) + + if require_enabled and not rust_ocr_provider_enabled(rust_ocr_provider): + return None + + optional_params = _call_rust_ocr( + { + "provider": rust_ocr_provider, + "operation": "map_params", + "non_default_params": dict(kwargs), + }, + require_enabled=require_enabled, + fallback_on_unavailable=fallback_on_unavailable, + ) + if optional_params is None: + return None + + transformed_request = _call_rust_ocr( + { + "provider": rust_ocr_provider, + "operation": "transform_request", + "model": model, + "document": document, + "optional_params": optional_params, + }, + require_enabled=require_enabled, + fallback_on_unavailable=fallback_on_unavailable, + ) + if transformed_request is None: + return None + + request_data = transformed_request.get("data") + if not isinstance(request_data, dict): + raise ValueError(f"Rust OCR provider {rust_ocr_provider} returned invalid data") + + headers = get_mistral_rust_ocr_headers( + headers=extra_headers, + api_key=api_key, + ) + complete_url = get_mistral_rust_ocr_url(api_base=api_base) + + litellm_logging_obj.update_from_kwargs( + kwargs=kwargs, + model=model, + optional_params=optional_params, + litellm_params={ + "litellm_call_id": litellm_call_id, + "api_base": api_base, + }, + custom_llm_provider=custom_llm_provider, + ) + litellm_logging_obj.pre_call( + input="OCR document processing", + api_key=api_key, + additional_args={ + "complete_input_dict": request_data, + "api_base": complete_url, + "headers": headers, + }, + ) + + response = _get_httpx_client().post( + url=complete_url, + headers=headers, + json=request_data, + timeout=timeout or request_timeout, + ) + + transformed_response = _call_rust_ocr( + { + "provider": rust_ocr_provider, + "operation": "transform_response", + "model": model, + "response_json": response.json(), + }, + require_enabled=require_enabled, + fallback_on_unavailable=fallback_on_unavailable, + ) + if transformed_response is None: + return None + + return OCRResponse(**transformed_response) + + @client async def aocr( model: str, @@ -146,6 +303,70 @@ async def aocr( ) +@client +def rust_ocr( + model: str, + document: dict[str, Any], + api_key: str | None = None, + api_base: str | None = None, + timeout: float | httpx.Timeout | None = None, + custom_llm_provider: str | None = None, + extra_headers: dict[str, Any] | None = None, + **kwargs, +) -> OCRResponse: + """ + Direct Rust OCR entrypoint. + """ + local_vars = locals() + try: + litellm_logging_obj: LiteLLMLoggingObj = kwargs.pop("litellm_logging_obj") # type: ignore + litellm_call_id: str | None = kwargs.get("litellm_call_id", None) + document = _prepare_ocr_document(document) + + ( + model, + custom_llm_provider, + dynamic_api_key, + dynamic_api_base, + ) = litellm.get_llm_provider( + model=model, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + api_key=api_key, + ) + + if dynamic_api_key: + api_key = dynamic_api_key + if dynamic_api_base: + api_base = dynamic_api_base + + response = _rust_ocr_impl( + model=model, + document=document, + api_key=api_key, + api_base=api_base, + timeout=timeout, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + litellm_logging_obj=litellm_logging_obj, + litellm_call_id=litellm_call_id, + fallback_on_unavailable=False, + require_enabled=False, + kwargs=kwargs, + ) + if response is None: + raise ValueError("Rust OCR bridge is unavailable for this request") + return response + except Exception as e: + raise litellm.exception_type( + model=model, + custom_llm_provider=custom_llm_provider, + original_exception=e, + completion_kwargs=local_vars, + extra_kwargs=kwargs, + ) + + @client def ocr( model: str, @@ -225,24 +446,7 @@ def ocr( litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None) _is_async = kwargs.pop("aocr", False) is True - # Validate document parameter format - if not isinstance(document, dict): - raise ValueError( - f"document must be a dict with 'type' and URL/file field, got {type(document)}" - ) - - doc_type = document.get("type") - - # Handle file type: convert to document_url/image_url with base64 data URI - if doc_type == "file": - document = convert_file_document_to_url_document(document) - doc_type = document.get("type") - - if doc_type not in ["document_url", "image_url"]: - raise ValueError( - f"Invalid document type: {doc_type}. " - "Must be 'document_url', 'image_url', or 'file'" - ) + document = _prepare_ocr_document(document) ( model, @@ -262,6 +466,24 @@ def ocr( if dynamic_api_base: api_base = dynamic_api_base + if _should_route_to_rust_ocr(custom_llm_provider): + rust_response = _rust_ocr_impl( + model=model, + document=document, + api_key=api_key, + api_base=api_base, + timeout=timeout, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + litellm_logging_obj=litellm_logging_obj, + litellm_call_id=litellm_call_id, + fallback_on_unavailable=True, + require_enabled=True, + kwargs=kwargs, + ) + if rust_response is not None: + return rust_response + # Get provider config ocr_provider_config: Optional[BaseOCRConfig] = ( ProviderConfigManager.get_provider_ocr_config( @@ -279,11 +501,6 @@ def ocr( f"OCR call - model: {model}, provider: {custom_llm_provider}" ) - ocr_provider_config = get_rust_ocr_provider_config( - model=model, - fallback_config=ocr_provider_config, - ) - # Get litellm params using GenericLiteLLMParams (same as responses API) litellm_params = GenericLiteLLMParams(**kwargs) diff --git a/litellm/rust_bridge/__init__.py b/litellm/rust_bridge/__init__.py index df1467f33383..5e44170f89fe 100644 --- a/litellm/rust_bridge/__init__.py +++ b/litellm/rust_bridge/__init__.py @@ -3,10 +3,8 @@ set_rust_core_enabled, set_rust_core_strict, ) -from litellm.rust_bridge.ocr import get_rust_ocr_provider_config __all__ = [ - "get_rust_ocr_provider_config", "rust_core_available", "set_rust_core_enabled", "set_rust_core_strict", diff --git a/litellm/rust_bridge/loader.py b/litellm/rust_bridge/loader.py index 270ee1b2045f..597e9e5d1e54 100644 --- a/litellm/rust_bridge/loader.py +++ b/litellm/rust_bridge/loader.py @@ -1,14 +1,14 @@ import importlib from functools import lru_cache from types import ModuleType -from typing import Any, Iterable, Optional, Union +from typing import Any, Iterable, Union _enabled_rust_core_scopes: set[str] = set() _rust_core_strict = False @lru_cache(maxsize=1) -def _load_rust_module() -> Optional[ModuleType]: +def _load_rust_module() -> ModuleType | None: try: return importlib.import_module("litellm_python_bridge") except Exception: @@ -48,7 +48,7 @@ def set_rust_core_strict(enabled: bool) -> None: _rust_core_strict = enabled -def call_rust_function(function_name: str, *args: Any) -> Optional[Any]: +def call_rust_function(function_name: str, *args: Any) -> Any | None: module = _load_rust_module() if module is None: return None diff --git a/litellm/rust_bridge/ocr/__init__.py b/litellm/rust_bridge/ocr/__init__.py index c4e958a0b50e..c0cf35c642b6 100644 --- a/litellm/rust_bridge/ocr/__init__.py +++ b/litellm/rust_bridge/ocr/__init__.py @@ -1,5 +1,6 @@ -from litellm.rust_bridge.ocr.config import get_rust_ocr_provider_config +from litellm.rust_bridge.ocr.providers import call_ocr, rust_ocr_provider_enabled __all__ = [ - "get_rust_ocr_provider_config", + "call_ocr", + "rust_ocr_provider_enabled", ] diff --git a/litellm/rust_bridge/ocr/config.py b/litellm/rust_bridge/ocr/config.py deleted file mode 100644 index fd38d0a104ae..000000000000 --- a/litellm/rust_bridge/ocr/config.py +++ /dev/null @@ -1,199 +0,0 @@ -from typing import Any, Optional - -import httpx - -from litellm.llms.base_llm.ocr.transformation import ( - BaseOCRConfig, - DocumentType, - OCRRequestData, - OCRResponse, -) -from litellm.rust_bridge.ocr.providers import call_ocr - - -def get_rust_ocr_provider_config( - model: str, - fallback_config: BaseOCRConfig, -) -> BaseOCRConfig: - rust_ocr_provider = fallback_config.get_rust_ocr_provider(model=model) - if not rust_ocr_provider: - return fallback_config - - return RustOCRProviderConfig( - rust_ocr_provider=rust_ocr_provider, - fallback_config=fallback_config, - ) - - -class RustOCRProviderConfig(BaseOCRConfig): - def __init__( - self, - rust_ocr_provider: str, - fallback_config: BaseOCRConfig, - ) -> None: - super().__init__() - self.rust_ocr_provider = rust_ocr_provider - self.fallback_config = fallback_config - - def get_supported_ocr_params(self, model: str) -> list: - return self.fallback_config.get_supported_ocr_params(model=model) - - def validate_environment( - self, - headers: dict, - model: str, - api_key: Optional[str] = None, - api_base: Optional[str] = None, - litellm_params: Optional[dict] = None, - **kwargs, - ) -> dict: - return self.fallback_config.validate_environment( - headers=headers, - model=model, - api_key=api_key, - api_base=api_base, - litellm_params=litellm_params, - **kwargs, - ) - - def get_complete_url( - self, - api_base: Optional[str], - model: str, - optional_params: dict, - litellm_params: Optional[dict] = None, - **kwargs, - ) -> str: - return self.fallback_config.get_complete_url( - api_base=api_base, - model=model, - optional_params=optional_params, - litellm_params=litellm_params, - **kwargs, - ) - - def map_ocr_params( - self, - non_default_params: dict, - optional_params: dict, - model: str, - ) -> dict: - mapped_params = call_ocr( - { - "provider": self.rust_ocr_provider, - "operation": "map_params", - "non_default_params": non_default_params, - } - ) - if mapped_params is not None: - return mapped_params - return self.fallback_config.map_ocr_params( - non_default_params=non_default_params, - optional_params=optional_params, - model=model, - ) - - def transform_ocr_request( - self, - model: str, - document: DocumentType, - optional_params: dict, - headers: dict, - **kwargs, - ) -> OCRRequestData: - if isinstance(document, dict): - transformed_request = call_ocr( - { - "provider": self.rust_ocr_provider, - "operation": "transform_request", - "model": model, - "document": document, - "optional_params": optional_params, - }, - ) - if transformed_request is not None: - request_data = transformed_request.get("data") - if not isinstance(request_data, dict): - raise ValueError( - f"Rust OCR provider {self.rust_ocr_provider} " - "returned invalid request data" - ) - return OCRRequestData( - data=request_data, - files=transformed_request.get("files"), - ) - - return self.fallback_config.transform_ocr_request( - model=model, - document=document, - optional_params=optional_params, - headers=headers, - **kwargs, - ) - - async def async_transform_ocr_request( - self, - model: str, - document: DocumentType, - optional_params: dict, - headers: dict, - **kwargs, - ) -> OCRRequestData: - return self.transform_ocr_request( - model=model, - document=document, - optional_params=optional_params, - headers=headers, - **kwargs, - ) - - def transform_ocr_response( - self, - model: str, - raw_response: httpx.Response, - logging_obj: Any, - **kwargs, - ) -> OCRResponse: - transformed_response = call_ocr( - { - "provider": self.rust_ocr_provider, - "operation": "transform_response", - "model": model, - "response_json": raw_response.json(), - }, - ) - if transformed_response is not None: - return OCRResponse(**transformed_response) - - return self.fallback_config.transform_ocr_response( - model=model, - raw_response=raw_response, - logging_obj=logging_obj, - **kwargs, - ) - - async def async_transform_ocr_response( - self, - model: str, - raw_response: httpx.Response, - logging_obj: Any, - **kwargs, - ) -> OCRResponse: - return self.transform_ocr_response( - model=model, - raw_response=raw_response, - logging_obj=logging_obj, - **kwargs, - ) - - def get_error_class( - self, - error_message: str, - status_code: int, - headers: dict, - ) -> Exception: - return self.fallback_config.get_error_class( - error_message=error_message, - status_code=status_code, - headers=headers, - ) diff --git a/litellm/rust_bridge/ocr/providers.py b/litellm/rust_bridge/ocr/providers.py index 160b9f8f4d97..d20edefc0820 100644 --- a/litellm/rust_bridge/ocr/providers.py +++ b/litellm/rust_bridge/ocr/providers.py @@ -1,14 +1,18 @@ -from typing import Any, Optional +from typing import Any from litellm.rust_bridge.loader import call_rust_function, rust_core_enabled -def call_ocr(payload: dict[str, Any]) -> Optional[dict[str, Any]]: +def call_ocr( + payload: dict[str, Any], + *, + require_enabled: bool = True, +) -> dict[str, Any] | None: provider = payload.get("provider") if not isinstance(provider, str): return None - if not _rust_ocr_provider_enabled(provider): + if require_enabled and not rust_ocr_provider_enabled(provider): return None result = call_rust_function("ocr", payload) @@ -19,7 +23,7 @@ def call_ocr(payload: dict[str, Any]) -> Optional[dict[str, Any]]: return result -def _rust_ocr_provider_enabled(provider: str) -> bool: +def rust_ocr_provider_enabled(provider: str) -> bool: return ( rust_core_enabled("ocr") or rust_core_enabled(f"ocr:{provider}") diff --git a/tests/test_litellm/test_mistral_ocr_rust_bridge.py b/tests/test_litellm/test_mistral_ocr_rust_bridge.py index 27ab6744a3e5..c59be3cf5d33 100644 --- a/tests/test_litellm/test_mistral_ocr_rust_bridge.py +++ b/tests/test_litellm/test_mistral_ocr_rust_bridge.py @@ -1,15 +1,34 @@ +import importlib + import httpx import pytest -from litellm.llms.base_llm.ocr.transformation import BaseOCRConfig -from litellm.llms.mistral.ocr.rust_provider import MistralRustOcrProvider -from litellm.llms.mistral.ocr.transformation import MistralOCRConfig +import litellm +from litellm.llms.base_llm.ocr.transformation import OCRResponse +from litellm.llms.mistral.ocr.rust_provider import ( + MistralRustOcrProvider, + get_mistral_rust_ocr_provider, +) from litellm.rust_bridge import loader -from litellm.rust_bridge import ocr as rust_ocr from litellm.rust_bridge.ocr import providers -from litellm.rust_bridge.ocr.config import RustOCRProviderConfig + +ocr_main = importlib.import_module("litellm.ocr.main") MODEL = "mistral-ocr-latest" +SUPPORTED_PARAMS = { + "pages", + "include_image_base64", + "image_limit", + "image_min_size", + "bbox_annotation_format", + "document_annotation_format", + "document_annotation_prompt", + "extract_header", + "extract_footer", + "table_format", + "confidence_scores_granularity", + "id", +} DOCUMENT = { "type": "document_url", "document_url": "https://example.com/doc.pdf", @@ -36,7 +55,7 @@ def ocr(payload): return { key: value for key, value in payload["non_default_params"].items() - if key != "unsupported_param" + if key in SUPPORTED_PARAMS } if operation == "transform_request": return { @@ -59,31 +78,34 @@ def ocr(payload): raise AssertionError(f"Unexpected operation: {operation}") -class _FakeLoggingObj: - pass - +class _FakeHTTPClient: + def __init__(self): + self.requests = [] -class _InheritedMistralOCRConfig(MistralOCRConfig): - pass + def post(self, url, headers, json, timeout): + self.requests.append( + { + "url": url, + "headers": headers, + "json": json, + "timeout": timeout, + } + ) + return httpx.Response( + 200, + json={ + "pages": [{"index": 0, "markdown": "hello"}], + "model": "mistral-ocr-2505-completion", + "document_annotation": None, + "usage_info": {"pages_processed": 1}, + }, + ) def test_mistral_rust_ocr_provider_enum_is_owned_by_mistral(): assert MistralRustOcrProvider.MISTRAL.value == "mistral" - assert ( - MistralOCRConfig().get_rust_ocr_provider(model=MODEL) - == MistralRustOcrProvider.MISTRAL.value - ) - - -def test_inherited_mistral_ocr_provider_uses_python_fallback(): - fallback_config = _InheritedMistralOCRConfig() - - config = rust_ocr.get_rust_ocr_provider_config( - model=MODEL, - fallback_config=fallback_config, - ) - - assert config is fallback_config + assert get_mistral_rust_ocr_provider("mistral") == "mistral" + assert get_mistral_rust_ocr_provider("azure_ai") is None def test_rust_ocr_provider_returns_none_when_scope_disabled(monkeypatch): @@ -119,92 +141,75 @@ def test_mistral_ocr_map_params_uses_provider_gated_rust(monkeypatch): assert result == {"extract_header": True} -def test_mistral_ocr_provider_wrapper_uses_rust_when_enabled(monkeypatch): - loader.set_rust_core_enabled("ocr:mistral") +def test_litellm_rust_ocr_calls_rust_bridge(monkeypatch): + fake_client = _FakeHTTPClient() monkeypatch.setattr(loader, "_load_rust_module", lambda: _FakeRustModule) + monkeypatch.setattr(ocr_main, "_get_httpx_client", lambda: fake_client) - config = rust_ocr.get_rust_ocr_provider_config( - model=MODEL, - fallback_config=MistralOCRConfig(), - ) - - assert isinstance(config, BaseOCRConfig) - assert isinstance(config, RustOCRProviderConfig) - - request = config.transform_ocr_request( - model=MODEL, + response = litellm.rust_ocr( + model="mistral/mistral-ocr-latest", document=DOCUMENT, - optional_params={"include_image_base64": True}, - headers={}, - ) - assert request.data == { - "model": MODEL, - "document": DOCUMENT, - "include_image_base64": True, - } - assert request.files is None - - response = config.transform_ocr_response( - model=MODEL, - raw_response=httpx.Response( - 200, - json={ - "pages": [{"index": 0, "markdown": "hello"}], - "model": "mistral-ocr-2505-completion", - "document_annotation": None, - "usage_info": {"pages_processed": 1}, - }, - ), - logging_obj=_FakeLoggingObj(), + api_key="test-key", + pages=[0], + include_image_base64=False, + unsupported_param="drop", ) assert response.pages[0].index == 0 assert response.model == "mistral-ocr-2505-completion" assert response.usage_info.pages_processed == 1 - - -def test_mistral_ocr_provider_wrapper_falls_back_when_rust_module_missing( - monkeypatch, -): - loader.set_rust_core_enabled("ocr:mistral") - monkeypatch.setattr(loader, "_load_rust_module", lambda: None) - - config = rust_ocr.get_rust_ocr_provider_config( - model=MODEL, - fallback_config=MistralOCRConfig(), - ) - - result = config.transform_ocr_request( - model=MODEL, - document=DOCUMENT, - optional_params={"include_image_base64": True}, - headers={}, - ) - - assert result.data == { + assert len(fake_client.requests) == 1 + request = fake_client.requests[0] + assert request["url"] == "https://api.mistral.ai/v1/ocr" + assert request["headers"] == {"Authorization": "Bearer test-key"} + assert request["json"] == { "model": MODEL, "document": DOCUMENT, - "include_image_base64": True, + "pages": [0], + "include_image_base64": False, } -def test_inherited_mistral_ocr_config_does_not_use_mistral_rust(monkeypatch): +def test_litellm_ocr_routes_to_rust_when_mistral_scope_enabled(monkeypatch): + fake_client = _FakeHTTPClient() loader.set_rust_core_enabled("ocr:mistral") monkeypatch.setattr(loader, "_load_rust_module", lambda: _FakeRustModule) + monkeypatch.setattr(ocr_main, "_get_httpx_client", lambda: fake_client) - config = rust_ocr.get_rust_ocr_provider_config( - model=MODEL, - fallback_config=_InheritedMistralOCRConfig(), + response = litellm.ocr( + model="mistral/mistral-ocr-latest", + document=DOCUMENT, + api_key="test-key", + pages=[0], + include_image_base64=False, ) - assert isinstance(config, _InheritedMistralOCRConfig) - result = config.map_ocr_params( - non_default_params={ - "extract_header": True, - "unsupported_param": "value", - }, - optional_params={}, - model=MODEL, + assert response.pages[0].markdown == "hello" + assert len(fake_client.requests) == 1 + + +def test_litellm_ocr_uses_python_path_when_rust_scope_disabled(monkeypatch): + fake_client = _FakeHTTPClient() + monkeypatch.setattr(ocr_main, "_get_httpx_client", lambda: fake_client) + monkeypatch.setattr( + ocr_main.base_llm_http_handler, + "ocr", + lambda **kwargs: OCRResponse( + pages=[{"index": 0, "markdown": "python"}], + model="mistral-ocr-2505-completion", + document_annotation=None, + usage_info={"pages_processed": 1}, + object="ocr", + ), ) - assert result == {"extract_header": True} + response = litellm.ocr( + model="mistral/mistral-ocr-latest", + document=DOCUMENT, + api_key="test-key", + pages=[0], + include_image_base64=False, + ) + + assert response.pages[0].markdown == "python" + assert fake_client.requests == [] From 6b83353639234c9300ffd5592c4ac4012a71020f Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Mon, 22 Jun 2026 18:25:05 -0700 Subject: [PATCH 17/54] rust(core): add Auth/Http/Network error variants --- litellm-rust/crates/core/src/error.rs | 6 + litellm/llms/mistral/ocr/rust_provider.py | 46 --- litellm/ocr/__init__.py | 4 +- litellm/ocr/main.py | 286 ++---------------- litellm/rust_bridge/CLAUDE.md | 39 --- litellm/rust_bridge/__init__.py | 11 - litellm/rust_bridge/loader.py | 61 ---- litellm/rust_bridge/ocr/__init__.py | 6 - litellm/rust_bridge/ocr/providers.py | 31 -- .../interactions/test_openapi_compliance.py | 22 +- .../test_mistral_ocr_rust_bridge.py | 215 ------------- 11 files changed, 40 insertions(+), 687 deletions(-) delete mode 100644 litellm/llms/mistral/ocr/rust_provider.py delete mode 100644 litellm/rust_bridge/CLAUDE.md delete mode 100644 litellm/rust_bridge/__init__.py delete mode 100644 litellm/rust_bridge/loader.py delete mode 100644 litellm/rust_bridge/ocr/__init__.py delete mode 100644 litellm/rust_bridge/ocr/providers.py delete mode 100644 tests/test_litellm/test_mistral_ocr_rust_bridge.py diff --git a/litellm-rust/crates/core/src/error.rs b/litellm-rust/crates/core/src/error.rs index 08a7a2b5ed9e..645e261f76dc 100644 --- a/litellm-rust/crates/core/src/error.rs +++ b/litellm-rust/crates/core/src/error.rs @@ -13,6 +13,12 @@ pub enum CoreError { MissingField(&'static str), #[error("invalid response: {0}")] InvalidResponse(String), + #[error("{0}")] + Auth(String), + #[error("OCR request failed with status {status}: {body}")] + Http { status: u16, body: String }, + #[error("OCR network error: {0}")] + Network(String), } pub fn json_type_name(value: &serde_json::Value) -> &'static str { diff --git a/litellm/llms/mistral/ocr/rust_provider.py b/litellm/llms/mistral/ocr/rust_provider.py deleted file mode 100644 index fedaad3c2acd..000000000000 --- a/litellm/llms/mistral/ocr/rust_provider.py +++ /dev/null @@ -1,46 +0,0 @@ -from enum import Enum -from typing import Any - -from litellm.secret_managers.main import get_secret_str - - -class MistralRustOcrProvider(str, Enum): - MISTRAL = "mistral" - - -def get_mistral_rust_ocr_provider( - custom_llm_provider: str | None, -) -> str | None: - provider_value = getattr(custom_llm_provider, "value", custom_llm_provider) - if provider_value != MistralRustOcrProvider.MISTRAL.value: - return None - return MistralRustOcrProvider.MISTRAL.value - - -def get_mistral_rust_ocr_url(api_base: str | None) -> str: - if api_base is None: - api_base = "https://api.mistral.ai/v1" - - api_base = api_base.rstrip("/") - if api_base.endswith("/v1"): - return f"{api_base}/ocr" - return f"{api_base}/v1/ocr" - - -def get_mistral_rust_ocr_headers( - headers: dict[str, Any] | None, - api_key: str | None, -) -> dict[str, Any]: - if api_key is None: - api_key = get_secret_str("MISTRAL_API_KEY") - - if api_key is None: - raise ValueError( - "Missing Mistral API Key - A call is being made to Mistral but no key " - "is set either in the environment variables or via params" - ) - - return { - "Authorization": f"Bearer {api_key}", - **(headers or {}), - } diff --git a/litellm/ocr/__init__.py b/litellm/ocr/__init__.py index 8281e58046d0..e97497b2db7c 100644 --- a/litellm/ocr/__init__.py +++ b/litellm/ocr/__init__.py @@ -1,5 +1,5 @@ """OCR module for LiteLLM.""" -from .main import aocr, ocr, rust_ocr +from .main import aocr, ocr -__all__ = ["ocr", "aocr", "rust_ocr"] +__all__ = ["ocr", "aocr"] diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index e6eba696c250..5d73ddc8972c 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -10,6 +10,7 @@ import re from functools import partial from io import IOBase +from pathlib import Path from typing import Any, Coroutine, Dict, Optional, Union import httpx @@ -19,16 +20,7 @@ from litellm.constants import request_timeout from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.ocr.transformation import BaseOCRConfig, OCRResponse -from litellm.llms.custom_httpx.llm_http_handler import ( - BaseLLMHTTPHandler, - _get_httpx_client, -) -from litellm.llms.mistral.ocr.rust_provider import ( - get_mistral_rust_ocr_headers, - get_mistral_rust_ocr_provider, - get_mistral_rust_ocr_url, -) -from litellm.rust_bridge.ocr import call_ocr, rust_ocr_provider_enabled +from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.types.router import GenericLiteLLMParams from litellm.utils import ProviderConfigManager, client @@ -37,155 +29,6 @@ ################################################# -def _prepare_ocr_document(document: dict[str, Any]) -> dict[str, str]: - if not isinstance(document, dict): - raise ValueError( - f"document must be a dict with 'type' and URL/file field, got {type(document)}" - ) - - doc_type = document.get("type") - if doc_type == "file": - document = convert_file_document_to_url_document(document) - doc_type = document.get("type") - - if doc_type not in ["document_url", "image_url"]: - raise ValueError( - f"Invalid document type: {doc_type}. " - "Must be 'document_url', 'image_url', or 'file'" - ) - - return document - - -def _get_rust_ocr_provider(custom_llm_provider: str | None) -> str | None: - return get_mistral_rust_ocr_provider(custom_llm_provider=custom_llm_provider) - - -def _should_route_to_rust_ocr(custom_llm_provider: str | None) -> bool: - rust_ocr_provider = _get_rust_ocr_provider(custom_llm_provider) - return rust_ocr_provider is not None and rust_ocr_provider_enabled( - rust_ocr_provider - ) - - -def _call_rust_ocr( - payload: dict[str, Any], - *, - require_enabled: bool, - fallback_on_unavailable: bool, -) -> dict[str, Any] | None: - result = call_ocr(payload, require_enabled=require_enabled) - if result is not None: - return result - if fallback_on_unavailable: - return None - raise ValueError("Rust OCR bridge is unavailable for this request") - - -def _rust_ocr_impl( - model: str, - document: dict[str, str], - api_key: str | None, - api_base: str | None, - timeout: float | httpx.Timeout | None, - custom_llm_provider: str | None, - extra_headers: dict[str, Any] | None, - litellm_logging_obj: LiteLLMLoggingObj, - litellm_call_id: str | None, - fallback_on_unavailable: bool, - require_enabled: bool, - kwargs: dict[str, Any], -) -> OCRResponse | None: - rust_ocr_provider = _get_rust_ocr_provider(custom_llm_provider) - if rust_ocr_provider is None: - if fallback_on_unavailable: - return None - raise ValueError( - f"Rust OCR is not supported for provider: {custom_llm_provider}" - ) - - if require_enabled and not rust_ocr_provider_enabled(rust_ocr_provider): - return None - - optional_params = _call_rust_ocr( - { - "provider": rust_ocr_provider, - "operation": "map_params", - "non_default_params": dict(kwargs), - }, - require_enabled=require_enabled, - fallback_on_unavailable=fallback_on_unavailable, - ) - if optional_params is None: - return None - - transformed_request = _call_rust_ocr( - { - "provider": rust_ocr_provider, - "operation": "transform_request", - "model": model, - "document": document, - "optional_params": optional_params, - }, - require_enabled=require_enabled, - fallback_on_unavailable=fallback_on_unavailable, - ) - if transformed_request is None: - return None - - request_data = transformed_request.get("data") - if not isinstance(request_data, dict): - raise ValueError(f"Rust OCR provider {rust_ocr_provider} returned invalid data") - - headers = get_mistral_rust_ocr_headers( - headers=extra_headers, - api_key=api_key, - ) - complete_url = get_mistral_rust_ocr_url(api_base=api_base) - - litellm_logging_obj.update_from_kwargs( - kwargs=kwargs, - model=model, - optional_params=optional_params, - litellm_params={ - "litellm_call_id": litellm_call_id, - "api_base": api_base, - }, - custom_llm_provider=custom_llm_provider, - ) - litellm_logging_obj.pre_call( - input="OCR document processing", - api_key=api_key, - additional_args={ - "complete_input_dict": request_data, - "api_base": complete_url, - "headers": headers, - }, - ) - - response = _get_httpx_client().post( - url=complete_url, - headers=headers, - json=request_data, - timeout=timeout or request_timeout, - ) - - transformed_response = _call_rust_ocr( - { - "provider": rust_ocr_provider, - "operation": "transform_response", - "model": model, - "response_json": response.json(), - }, - require_enabled=require_enabled, - fallback_on_unavailable=fallback_on_unavailable, - ) - if transformed_response is None: - return None - - return OCRResponse(**transformed_response) - - @client async def aocr( model: str, @@ -303,70 +146,6 @@ async def aocr( ) -@client -def rust_ocr( - model: str, - document: dict[str, Any], - api_key: str | None = None, - api_base: str | None = None, - timeout: float | httpx.Timeout | None = None, - custom_llm_provider: str | None = None, - extra_headers: dict[str, Any] | None = None, - **kwargs, -) -> OCRResponse: - """ - Direct Rust OCR entrypoint. - """ - local_vars = locals() - try: - litellm_logging_obj: LiteLLMLoggingObj = kwargs.pop("litellm_logging_obj") # type: ignore - litellm_call_id: str | None = kwargs.get("litellm_call_id", None) - document = _prepare_ocr_document(document) - - ( - model, - custom_llm_provider, - dynamic_api_key, - dynamic_api_base, - ) = litellm.get_llm_provider( - model=model, - custom_llm_provider=custom_llm_provider, - api_base=api_base, - api_key=api_key, - ) - - if dynamic_api_key: - api_key = dynamic_api_key - if dynamic_api_base: - api_base = dynamic_api_base - - response = _rust_ocr_impl( - model=model, - document=document, - api_key=api_key, - api_base=api_base, - timeout=timeout, - custom_llm_provider=custom_llm_provider, - extra_headers=extra_headers, - litellm_logging_obj=litellm_logging_obj, - litellm_call_id=litellm_call_id, - fallback_on_unavailable=False, - require_enabled=False, - kwargs=kwargs, - ) - if response is None: - raise ValueError("Rust OCR bridge is unavailable for this request") - return response - except Exception as e: - raise litellm.exception_type( - model=model, - custom_llm_provider=custom_llm_provider, - original_exception=e, - completion_kwargs=local_vars, - extra_kwargs=kwargs, - ) - - @client def ocr( model: str, @@ -446,7 +225,24 @@ def ocr( litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None) _is_async = kwargs.pop("aocr", False) is True - document = _prepare_ocr_document(document) + # Validate document parameter format + if not isinstance(document, dict): + raise ValueError( + f"document must be a dict with 'type' and URL/file field, got {type(document)}" + ) + + doc_type = document.get("type") + + # Handle file type: convert to document_url/image_url with base64 data URI + if doc_type == "file": + document = convert_file_document_to_url_document(document) + doc_type = document.get("type") + + if doc_type not in ["document_url", "image_url"]: + raise ValueError( + f"Invalid document type: {doc_type}. " + "Must be 'document_url', 'image_url', or 'file'" + ) ( model, @@ -466,24 +262,6 @@ def ocr( if dynamic_api_base: api_base = dynamic_api_base - if _should_route_to_rust_ocr(custom_llm_provider): - rust_response = _rust_ocr_impl( - model=model, - document=document, - api_key=api_key, - api_base=api_base, - timeout=timeout, - custom_llm_provider=custom_llm_provider, - extra_headers=extra_headers, - litellm_logging_obj=litellm_logging_obj, - litellm_call_id=litellm_call_id, - fallback_on_unavailable=True, - require_enabled=True, - kwargs=kwargs, - ) - if rust_response is not None: - return rust_response - # Get provider config ocr_provider_config: Optional[BaseOCRConfig] = ( ProviderConfigManager.get_provider_ocr_config( @@ -598,13 +376,11 @@ def convert_file_document_to_url_document(document: Dict[str, Any]) -> Dict[str, with an inline base64 data URI. Accepts document dicts like: + {"type": "file", "file": "/path/to/document.pdf"} # file path string {"type": "file", "file": Path("/path/to/doc.pdf")} # pathlib.Path {"type": "file", "file": } # file-like object (BinaryIO) {"type": "file", "file": b"raw bytes"} # raw bytes - Bare ``str`` paths are not accepted — pass a ``pathlib.Path`` or - ``open(path, "rb")`` instead. See the str check below for the rationale. - Returns: {"type": "document_url", "document_url": "data:;base64,"} or {"type": "image_url", "image_url": "data:;base64,"} @@ -613,28 +389,14 @@ def convert_file_document_to_url_document(document: Dict[str, Any]) -> Dict[str, if file_input is None: raise ValueError( "document with type='file' must include a 'file' field containing " - "a pathlib.Path, file-like object, or bytes" + "a file path (str), pathlib.Path, file-like object, or bytes" ) file_bytes: bytes mime_type: str = "application/octet-stream" file_name: Optional[str] = None - if isinstance(file_input, str): - # Bare strings are rejected here. The OCR ``document`` accepts a - # ``{"type": "file", "file": }`` shape, and when this helper - # runs in a proxy request handler ```` is attacker-controlled. - # Opening it as a path is an arbitrary local file read on the proxy - # host, which is then base64-encoded and forwarded to the OCR - # provider — an exfiltration primitive. - raise ValueError( - "OCR file input does not accept bare str values. Pass bytes, " - "a pathlib.Path, or a file-like object. To OCR a local file " - "from a path, call open(path, 'rb') yourself." - ) - if isinstance(file_input, os.PathLike): - # os.PathLike (pathlib.Path and custom __fspath__ classes) is a - # Python-level type that HTTP form values can't fabricate. + if isinstance(file_input, (str, Path)): file_path = str(file_input) if not os.path.isfile(file_path): raise FileNotFoundError(f"File not found: {file_path}") @@ -655,7 +417,7 @@ def convert_file_document_to_url_document(document: Dict[str, Any]) -> Dict[str, else: raise ValueError( f"Unsupported file input type: {type(file_input)}. " - "Expected pathlib.Path, bytes, or a file-like object." + "Expected str (file path), pathlib.Path, bytes, or a file-like object." ) if not file_bytes: diff --git a/litellm/rust_bridge/CLAUDE.md b/litellm/rust_bridge/CLAUDE.md deleted file mode 100644 index a538defc557b..000000000000 --- a/litellm/rust_bridge/CLAUDE.md +++ /dev/null @@ -1,39 +0,0 @@ -# CLAUDE.md - -Rules for `litellm/rust_bridge`. - -## Responsibility - -This package is the Python-side bridge to optional Rust transforms. It should -route to Rust when explicitly enabled and safely return the existing Python path -when Rust is disabled, unavailable, or unsupported for a provider. - -## Naming And Shape - -- Keep this package named `rust_bridge`; do not reintroduce a vague `_rust` - package. -- Organize by LiteLLM route (`ocr/`, future `rerank/`, etc.). -- Keep route entrypoints such as `litellm/ocr/main.py` small. They should only - ask this package for a Rust-backed config or callable. -- Keep provider rollout explicit with enums or small provider registries. -- Keep rollout controlled by Python bridge APIs such as - `set_rust_core_enabled(...)`; do not add new environment variables here - unless the matching docs-repo update lands in the same rollout. -- For each route, expose a single Python-to-Rust call that passes one payload to - the PyO3 module, such as `ocr(payload)`. Do not split provider transform - operations into multiple PyO3 bridge functions. - -## Fallback Rules - -- Rust paths are off by default. -- Missing PyO3 modules must fall back unless strict mode is enabled. -- Unknown providers must return the original Python config unchanged. -- Tests must cover disabled, enabled, module-missing, and unknown-provider paths. - -## Data Handling - -- OCR inputs frequently contain personal data. Do not log documents, base64 - payloads, provider response bodies, or secrets. -- Bridge errors should be bounded and sanitized. Do not surface raw upstream - OCR bodies through Python exceptions. -- Treat blank configuration values as absent at host/config resolution time. diff --git a/litellm/rust_bridge/__init__.py b/litellm/rust_bridge/__init__.py deleted file mode 100644 index 5e44170f89fe..000000000000 --- a/litellm/rust_bridge/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -from litellm.rust_bridge.loader import ( - rust_core_available, - set_rust_core_enabled, - set_rust_core_strict, -) - -__all__ = [ - "rust_core_available", - "set_rust_core_enabled", - "set_rust_core_strict", -] diff --git a/litellm/rust_bridge/loader.py b/litellm/rust_bridge/loader.py deleted file mode 100644 index 597e9e5d1e54..000000000000 --- a/litellm/rust_bridge/loader.py +++ /dev/null @@ -1,61 +0,0 @@ -import importlib -from functools import lru_cache -from types import ModuleType -from typing import Any, Iterable, Union - -_enabled_rust_core_scopes: set[str] = set() -_rust_core_strict = False - - -@lru_cache(maxsize=1) -def _load_rust_module() -> ModuleType | None: - try: - return importlib.import_module("litellm_python_bridge") - except Exception: - return None - - -def rust_core_available() -> bool: - return _load_rust_module() is not None - - -def set_rust_core_enabled(scopes: Union[bool, str, Iterable[str]]) -> None: - global _enabled_rust_core_scopes - - if scopes is True: - _enabled_rust_core_scopes = {"all"} - return - if scopes is False: - _enabled_rust_core_scopes = set() - return - if isinstance(scopes, str): - _enabled_rust_core_scopes = { - scope.strip() - for scope in scopes.replace(";", ",").split(",") - if scope.strip() - } - return - - _enabled_rust_core_scopes = {scope for scope in scopes if scope} - - -def rust_core_enabled(scope: str) -> bool: - return "all" in _enabled_rust_core_scopes or scope in _enabled_rust_core_scopes - - -def set_rust_core_strict(enabled: bool) -> None: - global _rust_core_strict - _rust_core_strict = enabled - - -def call_rust_function(function_name: str, *args: Any) -> Any | None: - module = _load_rust_module() - if module is None: - return None - - try: - return getattr(module, function_name)(*args) - except Exception: - if _rust_core_strict: - raise - return None diff --git a/litellm/rust_bridge/ocr/__init__.py b/litellm/rust_bridge/ocr/__init__.py deleted file mode 100644 index c0cf35c642b6..000000000000 --- a/litellm/rust_bridge/ocr/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from litellm.rust_bridge.ocr.providers import call_ocr, rust_ocr_provider_enabled - -__all__ = [ - "call_ocr", - "rust_ocr_provider_enabled", -] diff --git a/litellm/rust_bridge/ocr/providers.py b/litellm/rust_bridge/ocr/providers.py deleted file mode 100644 index d20edefc0820..000000000000 --- a/litellm/rust_bridge/ocr/providers.py +++ /dev/null @@ -1,31 +0,0 @@ -from typing import Any - -from litellm.rust_bridge.loader import call_rust_function, rust_core_enabled - - -def call_ocr( - payload: dict[str, Any], - *, - require_enabled: bool = True, -) -> dict[str, Any] | None: - provider = payload.get("provider") - if not isinstance(provider, str): - return None - - if require_enabled and not rust_ocr_provider_enabled(provider): - return None - - result = call_rust_function("ocr", payload) - if result is None: - return None - if not isinstance(result, dict): - raise ValueError("Rust OCR bridge returned invalid response") - return result - - -def rust_ocr_provider_enabled(provider: str) -> bool: - return ( - rust_core_enabled("ocr") - or rust_core_enabled(f"ocr:{provider}") - or rust_core_enabled(f"{provider}_ocr") - ) diff --git a/tests/test_litellm/interactions/test_openapi_compliance.py b/tests/test_litellm/interactions/test_openapi_compliance.py index 509ded8c704c..cfcc426aa245 100644 --- a/tests/test_litellm/interactions/test_openapi_compliance.py +++ b/tests/test_litellm/interactions/test_openapi_compliance.py @@ -153,19 +153,18 @@ class TestResponseCompliance: def test_interaction_response_fields(self, spec_dict): """Verify our InteractionsAPIResponse has correct fields.""" - # The response is the dedicated `Interaction` schema. Google moved the - # output-only fields (notably the `steps` array, formerly `outputs`) - # off `CreateModelInteractionParams` and onto `Interaction`; the request - # schema no longer carries `steps`. Keep this aligned with the live spec. - schema = spec_dict["components"]["schemas"]["Interaction"] + # The response is the Interaction schema + # Check CreateModelInteractionParams which includes output fields + schema = spec_dict["components"]["schemas"]["CreateModelInteractionParams"] - # Output fields (readOnly). + # Output fields (readOnly) output_fields = [ "id", "status", "created", "updated", - "steps", + "role", + "outputs", "usage", ] @@ -175,13 +174,9 @@ def test_interaction_response_fields(self, spec_dict): def test_status_enum_values(self, spec_dict): """Verify status enum values match spec.""" - # `status` is an output-only field; validate against the response schema. - schema = spec_dict["components"]["schemas"]["Interaction"] + schema = spec_dict["components"]["schemas"]["CreateModelInteractionParams"] status_prop = schema["properties"]["status"] - # Google Interactions API uses lowercase status values (updated Feb 2026). - # Keep this an exact match: this test intentionally breaks CI when - # Google changes the live spec — that breakage is how we get notified - # to review the change. + # Google Interactions API uses lowercase status values (updated Feb 2026) expected_statuses = [ "in_progress", "requires_action", @@ -189,7 +184,6 @@ def test_status_enum_values(self, spec_dict): "failed", "cancelled", "incomplete", - "budget_exceeded", ] assert status_prop["enum"] == expected_statuses print(f"✓ Status enum values: {expected_statuses}") diff --git a/tests/test_litellm/test_mistral_ocr_rust_bridge.py b/tests/test_litellm/test_mistral_ocr_rust_bridge.py deleted file mode 100644 index c59be3cf5d33..000000000000 --- a/tests/test_litellm/test_mistral_ocr_rust_bridge.py +++ /dev/null @@ -1,215 +0,0 @@ -import importlib - -import httpx -import pytest - -import litellm -from litellm.llms.base_llm.ocr.transformation import OCRResponse -from litellm.llms.mistral.ocr.rust_provider import ( - MistralRustOcrProvider, - get_mistral_rust_ocr_provider, -) -from litellm.rust_bridge import loader -from litellm.rust_bridge.ocr import providers - -ocr_main = importlib.import_module("litellm.ocr.main") - -MODEL = "mistral-ocr-latest" -SUPPORTED_PARAMS = { - "pages", - "include_image_base64", - "image_limit", - "image_min_size", - "bbox_annotation_format", - "document_annotation_format", - "document_annotation_prompt", - "extract_header", - "extract_footer", - "table_format", - "confidence_scores_granularity", - "id", -} -DOCUMENT = { - "type": "document_url", - "document_url": "https://example.com/doc.pdf", -} - - -@pytest.fixture(autouse=True) -def reset_rust_bridge_state(): - loader.set_rust_core_enabled(False) - loader.set_rust_core_strict(False) - yield - loader.set_rust_core_enabled(False) - loader.set_rust_core_strict(False) - - -class _FakeRustModule: - @staticmethod - def ocr(payload): - provider = payload["provider"] - operation = payload["operation"] - - assert provider == "mistral" - if operation == "map_params": - return { - key: value - for key, value in payload["non_default_params"].items() - if key in SUPPORTED_PARAMS - } - if operation == "transform_request": - return { - "data": { - "model": payload["model"], - "document": payload["document"], - **payload["optional_params"], - }, - "files": None, - } - if operation == "transform_response": - response_json = payload["response_json"] - return { - "pages": response_json.get("pages", []), - "model": response_json.get("model", payload["model"]), - "document_annotation": response_json.get("document_annotation"), - "usage_info": response_json.get("usage_info"), - "object": "ocr", - } - raise AssertionError(f"Unexpected operation: {operation}") - - -class _FakeHTTPClient: - def __init__(self): - self.requests = [] - - def post(self, url, headers, json, timeout): - self.requests.append( - { - "url": url, - "headers": headers, - "json": json, - "timeout": timeout, - } - ) - return httpx.Response( - 200, - json={ - "pages": [{"index": 0, "markdown": "hello"}], - "model": "mistral-ocr-2505-completion", - "document_annotation": None, - "usage_info": {"pages_processed": 1}, - }, - ) - - -def test_mistral_rust_ocr_provider_enum_is_owned_by_mistral(): - assert MistralRustOcrProvider.MISTRAL.value == "mistral" - assert get_mistral_rust_ocr_provider("mistral") == "mistral" - assert get_mistral_rust_ocr_provider("azure_ai") is None - - -def test_rust_ocr_provider_returns_none_when_scope_disabled(monkeypatch): - monkeypatch.setattr(loader, "_load_rust_module", lambda: _FakeRustModule) - - assert ( - providers.call_ocr( - { - "provider": MistralRustOcrProvider.MISTRAL.value, - "operation": "map_params", - "non_default_params": {"extract_header": True}, - } - ) - is None - ) - - -def test_mistral_ocr_map_params_uses_provider_gated_rust(monkeypatch): - loader.set_rust_core_enabled("ocr:mistral") - monkeypatch.setattr(loader, "_load_rust_module", lambda: _FakeRustModule) - - result = providers.call_ocr( - { - "provider": MistralRustOcrProvider.MISTRAL.value, - "operation": "map_params", - "non_default_params": { - "extract_header": True, - "unsupported_param": "value", - }, - }, - ) - - assert result == {"extract_header": True} - - -def test_litellm_rust_ocr_calls_rust_bridge(monkeypatch): - fake_client = _FakeHTTPClient() - monkeypatch.setattr(loader, "_load_rust_module", lambda: _FakeRustModule) - monkeypatch.setattr(ocr_main, "_get_httpx_client", lambda: fake_client) - - response = litellm.rust_ocr( - model="mistral/mistral-ocr-latest", - document=DOCUMENT, - api_key="test-key", - pages=[0], - include_image_base64=False, - unsupported_param="drop", - ) - - assert response.pages[0].index == 0 - assert response.model == "mistral-ocr-2505-completion" - assert response.usage_info.pages_processed == 1 - assert len(fake_client.requests) == 1 - request = fake_client.requests[0] - assert request["url"] == "https://api.mistral.ai/v1/ocr" - assert request["headers"] == {"Authorization": "Bearer test-key"} - assert request["json"] == { - "model": MODEL, - "document": DOCUMENT, - "pages": [0], - "include_image_base64": False, - } - - -def test_litellm_ocr_routes_to_rust_when_mistral_scope_enabled(monkeypatch): - fake_client = _FakeHTTPClient() - loader.set_rust_core_enabled("ocr:mistral") - monkeypatch.setattr(loader, "_load_rust_module", lambda: _FakeRustModule) - monkeypatch.setattr(ocr_main, "_get_httpx_client", lambda: fake_client) - - response = litellm.ocr( - model="mistral/mistral-ocr-latest", - document=DOCUMENT, - api_key="test-key", - pages=[0], - include_image_base64=False, - ) - - assert response.pages[0].markdown == "hello" - assert len(fake_client.requests) == 1 - - -def test_litellm_ocr_uses_python_path_when_rust_scope_disabled(monkeypatch): - fake_client = _FakeHTTPClient() - monkeypatch.setattr(ocr_main, "_get_httpx_client", lambda: fake_client) - monkeypatch.setattr( - ocr_main.base_llm_http_handler, - "ocr", - lambda **kwargs: OCRResponse( - pages=[{"index": 0, "markdown": "python"}], - model="mistral-ocr-2505-completion", - document_annotation=None, - usage_info={"pages_processed": 1}, - object="ocr", - ), - ) - - response = litellm.ocr( - model="mistral/mistral-ocr-latest", - document=DOCUMENT, - api_key="test-key", - pages=[0], - include_image_base64=False, - ) - - assert response.pages[0].markdown == "python" - assert fake_client.requests == [] From b86d3168d611a1bbb44e107072e53993c90694f4 Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Mon, 22 Jun 2026 18:25:05 -0700 Subject: [PATCH 18/54] rust: add reqwest (rustls-tls) workspace dependency --- litellm-rust/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index 4117bd5f6ac6..fdc5f5efde10 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -15,6 +15,7 @@ repository = "https://github.com/BerriAI/litellm" litellm-core = { path = "crates/core" } litellm-providers = { path = "crates/providers" } pyo3 = "0.23.5" +reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" thiserror = "2.0" From 74b1e6d691e445b79e4acc88cbb65a1e84252fc2 Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Mon, 22 Jun 2026 18:25:05 -0700 Subject: [PATCH 19/54] rust(providers): depend on reqwest --- litellm-rust/crates/providers/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm-rust/crates/providers/Cargo.toml b/litellm-rust/crates/providers/Cargo.toml index 68483af62667..b8ed20660794 100644 --- a/litellm-rust/crates/providers/Cargo.toml +++ b/litellm-rust/crates/providers/Cargo.toml @@ -7,6 +7,7 @@ repository.workspace = true [dependencies] litellm-core.workspace = true +reqwest.workspace = true serde_json.workspace = true [dev-dependencies] From ba8c541804befaef6548839bb81a5a35e4c02298 Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Mon, 22 Jun 2026 18:25:05 -0700 Subject: [PATCH 20/54] rust(mistral): add complete_url + resolve_api_key helpers --- .../src/mistral/ocr/transformation.rs | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/litellm-rust/crates/providers/src/mistral/ocr/transformation.rs b/litellm-rust/crates/providers/src/mistral/ocr/transformation.rs index ca9073a14a09..1acf523ccadf 100644 --- a/litellm-rust/crates/providers/src/mistral/ocr/transformation.rs +++ b/litellm-rust/crates/providers/src/mistral/ocr/transformation.rs @@ -18,6 +18,48 @@ const SUPPORTED_OCR_PARAMS: &[&str] = &[ "id", ]; +/// Default Mistral API base, used when the caller does not override `api_base`. +pub const MISTRAL_DEFAULT_API_BASE: &str = "https://api.mistral.ai/v1"; + +/// Environment variable holding the Mistral API key. +pub const MISTRAL_API_KEY_ENV: &str = "MISTRAL_API_KEY"; + +/// Error message raised when no Mistral API key can be resolved. +pub const MISSING_KEY_MESSAGE: &str = "Missing Mistral API Key - A call is being made to Mistral but no key is set either in the environment variables or via params"; + +/// Build the complete OCR endpoint URL, de-duplicating a trailing `/v1`. +/// +/// Blank/whitespace `api_base` is treated as absent (guard at resolution time). +pub fn complete_url(api_base: Option<&str>) -> String { + let base = api_base + .map(str::trim) + .filter(|base| !base.is_empty()) + .unwrap_or(MISTRAL_DEFAULT_API_BASE) + .trim_end_matches('/'); + + if base.ends_with("/v1") { + format!("{base}/ocr") + } else { + format!("{base}/v1/ocr") + } +} + +/// Resolve the Mistral API key from the explicit param or the environment. +/// +/// Blank/whitespace values are treated as absent. Returns `CoreError::Auth` +/// when no usable key is available. +pub fn resolve_api_key( + api_key: Option<&str>, + env_lookup: &dyn Fn(&str) -> Option, +) -> CoreResult { + api_key + .map(str::trim) + .filter(|key| !key.is_empty()) + .map(str::to_string) + .or_else(|| env_lookup(MISTRAL_API_KEY_ENV).filter(|key| !key.trim().is_empty())) + .ok_or_else(|| CoreError::Auth(MISSING_KEY_MESSAGE.to_string())) +} + pub struct MistralOcrConfig; pub const MISTRAL_OCR_CONFIG: MistralOcrConfig = MistralOcrConfig; @@ -209,4 +251,38 @@ mod tests { assert_eq!(result.usage_info, Some(json!({"pages_processed": 1}))); assert_eq!(result.object, "ocr"); } + + #[test] + fn complete_url_defaults_and_dedupes_v1() { + assert_eq!(complete_url(None), "https://api.mistral.ai/v1/ocr"); + assert_eq!(complete_url(Some(" ")), "https://api.mistral.ai/v1/ocr"); + assert_eq!( + complete_url(Some("https://proxy.internal")), + "https://proxy.internal/v1/ocr" + ); + assert_eq!( + complete_url(Some("https://proxy.internal/v1/")), + "https://proxy.internal/v1/ocr" + ); + } + + #[test] + fn resolve_api_key_prefers_param_then_env() { + let no_env = |_: &str| None; + assert_eq!( + resolve_api_key(Some("sk-param"), &no_env).unwrap(), + "sk-param" + ); + + let with_env = |key: &str| (key == MISTRAL_API_KEY_ENV).then(|| "sk-env".to_string()); + assert_eq!(resolve_api_key(None, &with_env).unwrap(), "sk-env"); + // Blank param falls through to the environment. + assert_eq!(resolve_api_key(Some(" "), &with_env).unwrap(), "sk-env"); + } + + #[test] + fn resolve_api_key_errors_when_absent() { + let err = resolve_api_key(None, &|_| None).expect_err("missing key should error"); + assert_eq!(err, CoreError::Auth(MISSING_KEY_MESSAGE.to_string())); + } } From 055bef8482074929b668d742a593152a08361f26 Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Mon, 22 Jun 2026 18:25:05 -0700 Subject: [PATCH 21/54] rust(providers): end-to-end run_ocr orchestrator with shared client + timeout --- litellm-rust/crates/providers/src/ocr.rs | 206 +++++++---------------- 1 file changed, 64 insertions(+), 142 deletions(-) diff --git a/litellm-rust/crates/providers/src/ocr.rs b/litellm-rust/crates/providers/src/ocr.rs index 48e3bbe8894f..bfcdfa16f475 100644 --- a/litellm-rust/crates/providers/src/ocr.rs +++ b/litellm-rust/crates/providers/src/ocr.rs @@ -1,156 +1,78 @@ -use litellm_core::error::{json_type_name, CoreError}; +//! End-to-end OCR orchestration. +//! +//! Owns the whole Mistral OCR call so the Python side stays a thin bridge: +//! resolve the API key, build the URL + body via the pure transforms, POST it, +//! and normalize the response. The HTTP client is built once and reused. + +use std::sync::OnceLock; +use std::time::Duration; + +use litellm_core::error::CoreError; use litellm_core::ocr::transformation::OcrProviderConfig; use litellm_core::CoreResult; use serde_json::{Map, Value}; +use crate::mistral::ocr::transformation as mistral; use crate::mistral::ocr::transformation::MISTRAL_OCR_CONFIG; -pub fn transform(payload: Value) -> CoreResult { - let payload = payload_object(&payload)?; - let provider = required_string(payload, "provider")?; - let operation = required_string(payload, "operation")?; - - match provider { - "mistral" => transform_with_provider(&MISTRAL_OCR_CONFIG, operation, payload), - _ => Err(CoreError::InvalidResponse(format!( - "unsupported OCR provider: {provider}" - ))), - } -} - -fn transform_with_provider( - provider_config: &impl OcrProviderConfig, - operation: &str, - payload: &Map, -) -> CoreResult { - match operation { - "map_params" => { - let params = required_object(payload, "non_default_params")?; - Ok(Value::Object(provider_config.map_ocr_params(¶ms))) - } - "transform_request" => { - let model = required_string(payload, "model")?; - let document = required_value(payload, "document")?; - let optional_params = required_object(payload, "optional_params")?; - let transformed = - provider_config.transform_ocr_request(model, document, optional_params)?; - Ok(serde_json::json!({ - "data": transformed.data, - "files": transformed.files, - })) - } - "transform_response" => { - let model = required_string(payload, "model")?; - let response_json = required_value(payload, "response_json")?; - let transformed = provider_config.transform_ocr_response(model, response_json)?; - Ok(transformed.into_json()) - } - _ => Err(CoreError::InvalidResponse(format!( - "unsupported OCR operation: {operation}" - ))), - } -} - -fn payload_object(payload: &Value) -> CoreResult<&Map> { - payload.as_object().ok_or_else(|| CoreError::InvalidType { - expected: "object", - actual: json_type_name(payload), - }) -} - -fn required_string<'a>(payload: &'a Map, key: &'static str) -> CoreResult<&'a str> { - let value = payload.get(key).ok_or(CoreError::MissingField(key))?; - value.as_str().ok_or_else(|| CoreError::InvalidType { - expected: "string", - actual: json_type_name(value), +/// OCR over large documents can take a while; bound it generously rather than +/// hanging forever on an unresponsive upstream. +const OCR_TIMEOUT_SECS: u64 = 600; + +/// Process-wide blocking HTTP client (connection pool + TLS reused across calls). +fn http_client() -> &'static reqwest::blocking::Client { + static CLIENT: OnceLock = OnceLock::new(); + CLIENT.get_or_init(|| { + reqwest::blocking::Client::builder() + .timeout(Duration::from_secs(OCR_TIMEOUT_SECS)) + .build() + .expect("failed to build reqwest client") }) } -fn required_object( - payload: &Map, - key: &'static str, -) -> CoreResult> { - let value = payload.get(key).ok_or(CoreError::MissingField(key))?; - value - .as_object() - .cloned() - .ok_or_else(|| CoreError::InvalidType { - expected: "object", - actual: json_type_name(value), - }) -} - -fn required_value(payload: &Map, key: &'static str) -> CoreResult { - payload - .get(key) - .cloned() - .ok_or(CoreError::MissingField(key)) -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - #[test] - fn transform_dispatches_mistral_map_params() { - let result = transform(json!({ - "provider": "mistral", - "operation": "map_params", - "non_default_params": { - "extract_header": true, - "unsupported_param": "value" - } - })) - .expect("payload should transform"); - - assert_eq!(result, json!({"extract_header": true})); - } - - #[test] - fn transform_dispatches_mistral_request() { - let document = json!({ - "type": "document_url", - "document_url": "https://example.com/doc.pdf" +/// Perform a Mistral OCR call end to end and return the normalized response as +/// JSON (the shape the Python `OCRResponse` model expects). +/// +/// Blocking: intended to be called with the GIL released from the Python bridge. +pub fn run_ocr( + model: &str, + document: Value, + api_key: Option<&str>, + api_base: Option<&str>, + optional_params: Map, +) -> CoreResult { + let config = &MISTRAL_OCR_CONFIG; + + let api_key = mistral::resolve_api_key(api_key, &|key| std::env::var(key).ok())?; + let url = mistral::complete_url(api_base); + let filtered_params = config.map_ocr_params(&optional_params); + let body = config + .transform_ocr_request(model, document, filtered_params)? + .data; + + let response = http_client() + .post(&url) + .bearer_auth(&api_key) + .json(&body) + .send() + .map_err(|err| CoreError::Network(err.to_string()))?; + + let status = response.status(); + let text = response + .text() + .map_err(|err| CoreError::Network(err.to_string()))?; + + if !status.is_success() { + return Err(CoreError::Http { + status: status.as_u16(), + body: text, }); - - let result = transform(json!({ - "provider": "mistral", - "operation": "transform_request", - "model": "mistral-ocr-latest", - "document": document, - "optional_params": {"include_image_base64": true} - })) - .expect("payload should transform"); - - assert_eq!( - result, - json!({ - "data": { - "model": "mistral-ocr-latest", - "document": { - "type": "document_url", - "document_url": "https://example.com/doc.pdf" - }, - "include_image_base64": true - }, - "files": null - }) - ); } - #[test] - fn transform_rejects_unknown_provider() { - let err = transform(json!({ - "provider": "azure_ai", - "operation": "map_params", - "non_default_params": {} - })) - .expect_err("unsupported provider should fail"); + let response_json: Value = serde_json::from_str(&text) + .map_err(|err| CoreError::InvalidResponse(format!("invalid OCR response JSON: {err}")))?; - assert_eq!( - err, - CoreError::InvalidResponse("unsupported OCR provider: azure_ai".to_string()) - ); - } + Ok(config + .transform_ocr_response(model, response_json)? + .into_json()) } From 0ce378d358114c1866b1f665b375adb0ca4e3668 Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Mon, 22 Jun 2026 18:25:06 -0700 Subject: [PATCH 22/54] rust(bridge): depend on litellm-core --- litellm-rust/crates/python-bridge/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index 40275879af5d..80b6478daacb 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -10,6 +10,7 @@ name = "litellm_python_bridge" crate-type = ["cdylib"] [dependencies] +litellm-core.workspace = true litellm-providers.workspace = true pyo3 = { workspace = true, features = ["extension-module"] } serde_json.workspace = true From 9240cad0895d5350581642b1bfa5c5d9c0a9e16c Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Mon, 22 Jun 2026 18:25:06 -0700 Subject: [PATCH 23/54] rust(bridge): add GIL release accounting --- litellm-rust/crates/python-bridge/src/gil.rs | 32 ++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 litellm-rust/crates/python-bridge/src/gil.rs diff --git a/litellm-rust/crates/python-bridge/src/gil.rs b/litellm-rust/crates/python-bridge/src/gil.rs new file mode 100644 index 000000000000..dc1b591735c2 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/gil.rs @@ -0,0 +1,32 @@ +//! GIL accounting. +//! +//! A single chokepoint for releasing the GIL around blocking work. Every +//! blocking call in the bridge goes through [`release_gil`] instead of calling +//! `Python::allow_threads` directly, so the release count stays accurate and we +//! have one place to extend later (timing histograms, per-call labels, etc.). + +use std::sync::atomic::{AtomicU64, Ordering}; + +use pyo3::prelude::*; + +/// Number of times the bridge has released the GIL since process start. +static GIL_RELEASES: AtomicU64 = AtomicU64::new(0); + +/// Release the GIL around `f`, recording the release. +/// +/// `f` must not touch any Python state — that is what makes releasing the GIL +/// safe. Returning the value back to Python re-acquires the GIL at the call +/// site, after `f` has finished. +pub fn release_gil(py: Python<'_>, f: F) -> T +where + F: FnOnce() -> T + Send, + T: Send, +{ + GIL_RELEASES.fetch_add(1, Ordering::Relaxed); + py.allow_threads(f) +} + +/// Total GIL releases performed by the bridge so far. +pub fn release_count() -> u64 { + GIL_RELEASES.load(Ordering::Relaxed) +} From 8feefec4f4612a4fb8ff4750489d929bfef36097 Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Mon, 22 Jun 2026 18:25:06 -0700 Subject: [PATCH 24/54] rust(bridge): end-to-end ocr() + gil_stats(), GIL released for HTTP --- litellm-rust/crates/python-bridge/src/lib.rs | 73 +++++++++++++++++--- 1 file changed, 65 insertions(+), 8 deletions(-) diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index a723daeb65ab..a84aefc0e7e8 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -1,7 +1,11 @@ -use pyo3::exceptions::PyValueError; +use litellm_core::error::CoreError; +use litellm_providers::ocr::run_ocr; +use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::*; -use pyo3::types::PyAny; -use serde_json::Value; +use pyo3::types::{PyAny, PyDict}; +use serde_json::{Map, Value}; + +mod gil; fn py_to_json(py: Python<'_>, value: &Bound<'_, PyAny>) -> PyResult { let json = py.import("json")?; @@ -16,16 +20,69 @@ fn json_to_py(py: Python<'_>, value: Value) -> PyResult> { Ok(json.call_method1("loads", (encoded,))?.unbind()) } +/// Map a core error to the closest Python exception. Caller-input problems +/// (auth, bad types, missing fields) -> `ValueError`; everything else +/// (network, upstream status, parse failures) -> `RuntimeError`. +fn core_error_to_pyerr(err: CoreError) -> PyErr { + match err { + CoreError::Auth(message) => PyValueError::new_err(message), + CoreError::InvalidType { .. } | CoreError::MissingField(_) => { + PyValueError::new_err(err.to_string()) + } + other => PyRuntimeError::new_err(other.to_string()), + } +} + +/// Perform a Mistral OCR call end to end and return the response as a dict. +#[pyfunction] +#[pyo3(signature = (model, document, api_key=None, api_base=None, optional_params=None))] +fn ocr( + py: Python<'_>, + model: String, + document: Py, + api_key: Option, + api_base: Option, + optional_params: Option>, +) -> PyResult> { + let document = py_to_json(py, document.bind(py))?; + + let optional_params = match optional_params { + Some(params) => match py_to_json(py, params.bind(py))? { + Value::Object(map) => map, + _ => return Err(PyValueError::new_err("optional_params must be a dict")), + }, + None => Map::new(), + }; + + // Release the GIL during the blocking HTTP call (counted for observability). + let result = gil::release_gil(py, || { + run_ocr( + &model, + document, + api_key.as_deref(), + api_base.as_deref(), + optional_params, + ) + }); + + match result { + Ok(value) => json_to_py(py, value), + Err(err) => Err(core_error_to_pyerr(err)), + } +} + +/// Bridge GIL accounting, e.g. `{"releases": 12}`. Lets the Python side observe +/// how often the bridge has dropped the GIL for blocking work. #[pyfunction] -fn ocr(py: Python<'_>, payload: Py) -> PyResult> { - let payload = py_to_json(py, payload.bind(py))?; - let transformed = litellm_providers::ocr::transform(payload) - .map_err(|err| PyValueError::new_err(err.to_string()))?; - json_to_py(py, transformed) +fn gil_stats(py: Python<'_>) -> PyResult> { + let stats = PyDict::new(py); + stats.set_item("releases", gil::release_count())?; + Ok(stats.into_any().unbind()) } #[pymodule] fn litellm_python_bridge(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(ocr, module)?)?; + module.add_function(wrap_pyfunction!(gil_stats, module)?)?; Ok(()) } From c697154bf38b14582c405ddc1df5c4e56b59b533 Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Mon, 22 Jun 2026 18:25:06 -0700 Subject: [PATCH 25/54] ocr: add minimal Rust bridge (use_litellm_rust + rust_ocr) --- litellm/ocr/rust_bridge.py | 39 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 litellm/ocr/rust_bridge.py diff --git a/litellm/ocr/rust_bridge.py b/litellm/ocr/rust_bridge.py new file mode 100644 index 000000000000..deb47f98c2d2 --- /dev/null +++ b/litellm/ocr/rust_bridge.py @@ -0,0 +1,39 @@ +""" +Optional Rust-backed OCR path. + +Enable with ``litellm.use_litellm_rust()``; the sync ``litellm.ocr()`` entrypoint +then routes supported Mistral calls through the compiled ``litellm_python_bridge`` +extension, which performs the whole OCR call (URL, headers, HTTP, parse) in Rust. +""" + +from typing import Any, Dict, Optional + +from litellm.llms.base_llm.ocr.transformation import OCRResponse + +_RUST_OCR_ENABLED = False + + +def use_litellm_rust(enabled: bool = True) -> None: + """Route supported OCR calls through the Rust ``litellm_python_bridge`` extension.""" + global _RUST_OCR_ENABLED + _RUST_OCR_ENABLED = enabled + + +def rust_ocr_enabled() -> bool: + """Whether the Rust OCR path has been turned on via ``use_litellm_rust()``.""" + return _RUST_OCR_ENABLED + + +def rust_ocr( + model: str, + document: Dict[str, Any], + api_key: Optional[str], + api_base: Optional[str], + optional_params: Dict[str, Any], +) -> OCRResponse: + """Run a Mistral OCR call end-to-end in Rust and wrap the result as ``OCRResponse``.""" + import litellm_python_bridge + + return OCRResponse( + **litellm_python_bridge.ocr(model, document, api_key, api_base, optional_params) + ) From b370c1c1918f27be493ce116a08fcb8ac6066ff0 Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Mon, 22 Jun 2026 18:25:06 -0700 Subject: [PATCH 26/54] ocr: route mistral to Rust when enabled; keep bare-str file rejection --- litellm/ocr/main.py | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index 5d73ddc8972c..930732f125b9 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -10,7 +10,6 @@ import re from functools import partial from io import IOBase -from pathlib import Path from typing import Any, Coroutine, Dict, Optional, Union import httpx @@ -21,6 +20,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.ocr.transformation import BaseOCRConfig, OCRResponse from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.ocr.rust_bridge import rust_ocr, rust_ocr_enabled from litellm.types.router import GenericLiteLLMParams from litellm.utils import ProviderConfigManager, client @@ -262,6 +262,10 @@ def ocr( if dynamic_api_base: api_base = dynamic_api_base + # Optional Rust path: hand the whole Mistral OCR call to the Rust bridge. + if custom_llm_provider == "mistral" and rust_ocr_enabled(): + return rust_ocr(model, document, api_key, api_base, kwargs) + # Get provider config ocr_provider_config: Optional[BaseOCRConfig] = ( ProviderConfigManager.get_provider_ocr_config( @@ -376,11 +380,13 @@ def convert_file_document_to_url_document(document: Dict[str, Any]) -> Dict[str, with an inline base64 data URI. Accepts document dicts like: - {"type": "file", "file": "/path/to/document.pdf"} # file path string {"type": "file", "file": Path("/path/to/doc.pdf")} # pathlib.Path {"type": "file", "file": } # file-like object (BinaryIO) {"type": "file", "file": b"raw bytes"} # raw bytes + Bare ``str`` paths are not accepted — pass a ``pathlib.Path`` or + ``open(path, "rb")`` instead. See the str check below for the rationale. + Returns: {"type": "document_url", "document_url": "data:;base64,"} or {"type": "image_url", "image_url": "data:;base64,"} @@ -389,14 +395,28 @@ def convert_file_document_to_url_document(document: Dict[str, Any]) -> Dict[str, if file_input is None: raise ValueError( "document with type='file' must include a 'file' field containing " - "a file path (str), pathlib.Path, file-like object, or bytes" + "a pathlib.Path, file-like object, or bytes" ) file_bytes: bytes mime_type: str = "application/octet-stream" file_name: Optional[str] = None - if isinstance(file_input, (str, Path)): + if isinstance(file_input, str): + # Bare strings are rejected here. The OCR ``document`` accepts a + # ``{"type": "file", "file": }`` shape, and when this helper + # runs in a proxy request handler ```` is attacker-controlled. + # Opening it as a path is an arbitrary local file read on the proxy + # host, which is then base64-encoded and forwarded to the OCR + # provider — an exfiltration primitive. + raise ValueError( + "OCR file input does not accept bare str values. Pass bytes, " + "a pathlib.Path, or a file-like object. To OCR a local file " + "from a path, call open(path, 'rb') yourself." + ) + if isinstance(file_input, os.PathLike): + # os.PathLike (pathlib.Path and custom __fspath__ classes) is a + # Python-level type that HTTP form values can't fabricate. file_path = str(file_input) if not os.path.isfile(file_path): raise FileNotFoundError(f"File not found: {file_path}") @@ -417,7 +437,7 @@ def convert_file_document_to_url_document(document: Dict[str, Any]) -> Dict[str, else: raise ValueError( f"Unsupported file input type: {type(file_input)}. " - "Expected str (file path), pathlib.Path, bytes, or a file-like object." + "Expected pathlib.Path, bytes, or a file-like object." ) if not file_bytes: From d7ce63f20f18ffde0e8ae55b99b47324cf86de0b Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Mon, 22 Jun 2026 18:25:06 -0700 Subject: [PATCH 27/54] litellm: export use_litellm_rust() --- litellm/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/__init__.py b/litellm/__init__.py index d21234d2a814..9e8f7a4cdab6 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1400,6 +1400,7 @@ def add_known_models(model_cost_map: Optional[Dict] = None): ) from .containers.main import * from .ocr.main import * +from .ocr.rust_bridge import use_litellm_rust from .rag.main import * from .sandbox.main import * from .search.main import * From d2339a9f3b6e18f0814b9d8d71faa278c698197c Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Mon, 22 Jun 2026 18:25:06 -0700 Subject: [PATCH 28/54] test(ocr): cover Rust OCR routing + toggle --- tests/test_litellm/ocr/test_rust_bridge.py | 120 +++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 tests/test_litellm/ocr/test_rust_bridge.py diff --git a/tests/test_litellm/ocr/test_rust_bridge.py b/tests/test_litellm/ocr/test_rust_bridge.py new file mode 100644 index 000000000000..cae349be9985 --- /dev/null +++ b/tests/test_litellm/ocr/test_rust_bridge.py @@ -0,0 +1,120 @@ +"""Tests for the optional Rust-backed OCR path (``litellm/ocr/rust_bridge.py``).""" + +import importlib +import sys +import types + +import pytest + +import litellm +from litellm.llms.base_llm.ocr.transformation import OCRResponse + +# `litellm/__init__.py` does `from .ocr.main import *`, which binds the `ocr` +# function onto `litellm.ocr` and shadows the submodule — so import the modules +# explicitly via importlib rather than attribute traversal. +ocr_main = importlib.import_module("litellm.ocr.main") +rust_bridge = importlib.import_module("litellm.ocr.rust_bridge") + +MODEL = "mistral/mistral-ocr-latest" +DOCUMENT = {"type": "document_url", "document_url": "https://example.com/doc.pdf"} + +FAKE_OCR_RESPONSE = { + "pages": [{"index": 0, "markdown": "hello world"}], + "model": "mistral-ocr-2505-completion", + "document_annotation": None, + "usage_info": {"pages_processed": 1}, + "object": "ocr", +} + + +@pytest.fixture(autouse=True) +def _reset_rust_flag(): + """Keep the global toggle isolated between tests.""" + rust_bridge.use_litellm_rust(False) + yield + rust_bridge.use_litellm_rust(False) + + +@pytest.fixture +def fake_bridge(monkeypatch): + """Install a fake compiled ``litellm_python_bridge`` module and record calls.""" + calls = [] + + def _ocr(model, document, api_key, api_base, optional_params): + calls.append( + { + "model": model, + "document": document, + "api_key": api_key, + "api_base": api_base, + "optional_params": optional_params, + } + ) + return dict(FAKE_OCR_RESPONSE) + + module = types.ModuleType("litellm_python_bridge") + module.ocr = _ocr # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, "litellm_python_bridge", module) + return calls + + +def test_use_litellm_rust_toggles_flag(): + assert rust_bridge.rust_ocr_enabled() is False + litellm.use_litellm_rust() + assert rust_bridge.rust_ocr_enabled() is True + litellm.use_litellm_rust(False) + assert rust_bridge.rust_ocr_enabled() is False + + +def test_rust_ocr_wraps_bridge_response(fake_bridge): + response = rust_bridge.rust_ocr( + model="mistral-ocr-latest", + document=DOCUMENT, + api_key="sk-test", + api_base=None, + optional_params={"include_image_base64": True}, + ) + + assert isinstance(response, OCRResponse) + assert response.pages[0].markdown == "hello world" + assert response.model == "mistral-ocr-2505-completion" + assert fake_bridge[0]["model"] == "mistral-ocr-latest" + assert fake_bridge[0]["optional_params"] == {"include_image_base64": True} + + +def test_ocr_routes_to_rust_when_enabled(fake_bridge): + litellm.use_litellm_rust() + + response = litellm.ocr( + model=MODEL, + document=DOCUMENT, + api_key="sk-test", + include_image_base64=True, + ) + + assert isinstance(response, OCRResponse) + assert response.pages[0].markdown == "hello world" + assert len(fake_bridge) == 1 + call = fake_bridge[0] + # Provider prefix is stripped before reaching the bridge. + assert call["model"] == "mistral-ocr-latest" + assert call["document"] == DOCUMENT + assert call["api_key"] == "sk-test" + # Raw OCR params ride along in optional_params; Rust filters to supported keys. + assert call["optional_params"].get("include_image_base64") is True + + +def test_ocr_skips_rust_when_disabled(monkeypatch, fake_bridge): + """With the flag off, ocr() must take the normal Python provider path.""" + called = {} + + def _fake_handler(*_args, **_kwargs): + called["hit"] = True + return OCRResponse(pages=[], model="mistral-ocr-latest") + + monkeypatch.setattr(ocr_main.base_llm_http_handler, "ocr", _fake_handler) + + litellm.ocr(model=MODEL, document=DOCUMENT, api_key="sk-test") + + assert called.get("hit") is True + assert fake_bridge == [] # Rust bridge never invoked From 100861fba1795a46a7c4bf9c5485838ec4ab188b Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Mon, 22 Jun 2026 18:28:06 -0700 Subject: [PATCH 29/54] rust: stop ignoring Cargo.lock --- litellm-rust/.gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/litellm-rust/.gitignore b/litellm-rust/.gitignore index ca98cd96efdc..b83d22266ac8 100644 --- a/litellm-rust/.gitignore +++ b/litellm-rust/.gitignore @@ -1,2 +1 @@ /target/ -Cargo.lock From c6aa0607fefba4fe1eadeba1cb1e7a874b47e217 Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Mon, 22 Jun 2026 18:28:06 -0700 Subject: [PATCH 30/54] rust: commit Cargo.lock for reproducible builds --- litellm-rust/Cargo.lock | 1498 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 1498 insertions(+) create mode 100644 litellm-rust/Cargo.lock diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock new file mode 100644 index 000000000000..0fb2fcb2921b --- /dev/null +++ b/litellm-rust/Cargo.lock @@ -0,0 +1,1498 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" + +[[package]] +name = "cc" +version = "1.2.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-core", + "futures-io", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "http" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "hyper" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indoc" +version = "2.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" +dependencies = [ + "rustversion", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "litellm-core" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "litellm-providers" +version = "0.1.0" +dependencies = [ + "litellm-core", + "reqwest", + "serde_json", +] + +[[package]] +name = "litellm-python-bridge" +version = "0.1.0" +dependencies = [ + "litellm-core", + "litellm-providers", + "pyo3", + "serde_json", +] + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "memchr" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "mio" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pyo3" +version = "0.23.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7778bffd85cf38175ac1f545509665d0b9b92a198ca7941f131f85f7a4f9a872" +dependencies = [ + "cfg-if", + "indoc", + "libc", + "memoffset", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", + "unindent", +] + +[[package]] +name = "pyo3-build-config" +version = "0.23.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94f6cbe86ef3bf18998d9df6e0f3fc1050a8c5efa409bf712e661a4366e010fb" +dependencies = [ + "once_cell", + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.23.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f1b4c431c0bb1c8fb0a338709859eed0d030ff6daa34368d3b152a63dfdd8d" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.23.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbc2201328f63c4710f68abdf653c89d8dbc2858b88c5d88b0ff38a75288a9da" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.23.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fca6726ad0f3da9c9de093d6f116a93c1a38e417ed73bf138472cf4064f72028" +dependencies = [ + "heck", + "proc-macro2", + "pyo3-build-config", + "quote", + "syn", +] + +[[package]] +name = "quinn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" +dependencies = [ + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.60.2", +] + +[[package]] +name = "quote" +version = "1.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "rustls" +version = "0.23.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "socket2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "target-lexicon" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unindent" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "503b14d284f2c8dac03b819967e155ea753f573586193b2b2c95990cb5d69280" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" From 6fee78a760a1b6ca1a8227698353e5e37d58d24e Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Mon, 22 Jun 2026 18:28:06 -0700 Subject: [PATCH 31/54] ci(rust): build with --locked to enforce the lockfile --- .github/workflows/test-rust.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index 3716ba605b10..3d0a159cdc70 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -59,7 +59,7 @@ jobs: run: cargo fmt --check - name: Run Clippy - run: cargo clippy --workspace --all-targets -- -D warnings + run: cargo clippy --workspace --all-targets --locked -- -D warnings - name: Run Rust tests - run: cargo test --workspace + run: cargo test --workspace --locked From 8325d52451e7ddd64064b13081f89fc1d1a802ff Mon Sep 17 00:00:00 2001 From: ishaan-berri <155045088+ishaan-berri@users.noreply.github.com> Date: Mon, 22 Jun 2026 18:36:10 -0700 Subject: [PATCH 32/54] Potential fix for pull request finding 'CodeQL / Module-level cyclic import' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- litellm/ocr/rust_bridge.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/litellm/ocr/rust_bridge.py b/litellm/ocr/rust_bridge.py index deb47f98c2d2..5864d93d098d 100644 --- a/litellm/ocr/rust_bridge.py +++ b/litellm/ocr/rust_bridge.py @@ -8,8 +8,6 @@ from typing import Any, Dict, Optional -from litellm.llms.base_llm.ocr.transformation import OCRResponse - _RUST_OCR_ENABLED = False @@ -30,9 +28,10 @@ def rust_ocr( api_key: Optional[str], api_base: Optional[str], optional_params: Dict[str, Any], -) -> OCRResponse: +) -> "OCRResponse": """Run a Mistral OCR call end-to-end in Rust and wrap the result as ``OCRResponse``.""" import litellm_python_bridge + from litellm.llms.base_llm.ocr.transformation import OCRResponse return OCRResponse( **litellm_python_bridge.ocr(model, document, api_key, api_base, optional_params) From 54577a691ecf545a5af28168bf5202fe71cd0eb6 Mon Sep 17 00:00:00 2001 From: ishaan-berri <155045088+ishaan-berri@users.noreply.github.com> Date: Mon, 22 Jun 2026 18:36:24 -0700 Subject: [PATCH 33/54] Potential fix for pull request finding 'CodeQL / Module-level cyclic import' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- litellm/ocr/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index 930732f125b9..96b272d96bd7 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -20,7 +20,6 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.ocr.transformation import BaseOCRConfig, OCRResponse from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler -from litellm.ocr.rust_bridge import rust_ocr, rust_ocr_enabled from litellm.types.router import GenericLiteLLMParams from litellm.utils import ProviderConfigManager, client @@ -40,6 +39,8 @@ async def aocr( extra_headers: Optional[Dict[str, Any]] = None, **kwargs, ) -> OCRResponse: + from litellm.ocr.rust_bridge import rust_ocr, rust_ocr_enabled + """ Async OCR function. From 3188f93be0adf63349d708e9de97063ad97b6d13 Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Mon, 22 Jun 2026 18:49:03 -0700 Subject: [PATCH 34/54] ocr: lazily import rust bridge inside ocr() to break the import cycle the CodeQL autofix mangled --- litellm/ocr/main.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index 96b272d96bd7..55c749522493 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -39,8 +39,6 @@ async def aocr( extra_headers: Optional[Dict[str, Any]] = None, **kwargs, ) -> OCRResponse: - from litellm.ocr.rust_bridge import rust_ocr, rust_ocr_enabled - """ Async OCR function. @@ -264,6 +262,10 @@ def ocr( api_base = dynamic_api_base # Optional Rust path: hand the whole Mistral OCR call to the Rust bridge. + # Imported lazily to avoid a module-level import cycle (CodeQL flags the + # litellm.ocr.main -> litellm.ocr.rust_bridge edge during package init). + from litellm.ocr.rust_bridge import rust_ocr, rust_ocr_enabled + if custom_llm_provider == "mistral" and rust_ocr_enabled(): return rust_ocr(model, document, api_key, api_base, kwargs) From 88a9e6ed57012ea4af9dad937879e62e86907b2b Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Mon, 22 Jun 2026 18:49:03 -0700 Subject: [PATCH 35/54] ocr: guard OCRResponse under TYPE_CHECKING so the annotation resolves --- litellm/ocr/rust_bridge.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/litellm/ocr/rust_bridge.py b/litellm/ocr/rust_bridge.py index 5864d93d098d..89fdeb6810a9 100644 --- a/litellm/ocr/rust_bridge.py +++ b/litellm/ocr/rust_bridge.py @@ -6,7 +6,10 @@ extension, which performs the whole OCR call (URL, headers, HTTP, parse) in Rust. """ -from typing import Any, Dict, Optional +from typing import TYPE_CHECKING, Any, Dict, Optional + +if TYPE_CHECKING: + from litellm.llms.base_llm.ocr.transformation import OCRResponse _RUST_OCR_ENABLED = False From a4b49658881e1064f01066082286f93efd71dfad Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Mon, 22 Jun 2026 18:56:24 -0700 Subject: [PATCH 36/54] ocr: modernize rust_bridge typing (PEP 604, drop typing.Any/Dict) to satisfy strict-rule gate --- litellm/ocr/rust_bridge.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/litellm/ocr/rust_bridge.py b/litellm/ocr/rust_bridge.py index 89fdeb6810a9..2b06714ba1e9 100644 --- a/litellm/ocr/rust_bridge.py +++ b/litellm/ocr/rust_bridge.py @@ -6,7 +6,9 @@ extension, which performs the whole OCR call (URL, headers, HTTP, parse) in Rust. """ -from typing import TYPE_CHECKING, Any, Dict, Optional +from __future__ import annotations + +from typing import TYPE_CHECKING if TYPE_CHECKING: from litellm.llms.base_llm.ocr.transformation import OCRResponse @@ -27,11 +29,11 @@ def rust_ocr_enabled() -> bool: def rust_ocr( model: str, - document: Dict[str, Any], - api_key: Optional[str], - api_base: Optional[str], - optional_params: Dict[str, Any], -) -> "OCRResponse": + document: dict, + api_key: str | None, + api_base: str | None, + optional_params: dict, +) -> OCRResponse: """Run a Mistral OCR call end-to-end in Rust and wrap the result as ``OCRResponse``.""" import litellm_python_bridge from litellm.llms.base_llm.ocr.transformation import OCRResponse From d6b9928dc10d93bbfc66561178760351b9416cf5 Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Mon, 22 Jun 2026 20:10:01 -0700 Subject: [PATCH 37/54] ci: re-trigger checks From fe1cbe505d79d636d153f7f25009293d9e3b9b9a Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Mon, 22 Jun 2026 20:14:54 -0700 Subject: [PATCH 38/54] ci: re-trigger checks From bbbb9ededbea034249b8015d2e660d6f8c77d9d0 Mon Sep 17 00:00:00 2001 From: ishaan-berri <155045088+ishaan-berri@users.noreply.github.com> Date: Mon, 22 Jun 2026 20:26:48 -0700 Subject: [PATCH 39/54] Potential fix for pull request finding 'CodeQL / Cyclic import' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- litellm/ocr/main.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index 55c749522493..53edc9f087e6 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -5,6 +5,7 @@ import asyncio import base64 import contextvars +import importlib import mimetypes import os import re @@ -262,12 +263,12 @@ def ocr( api_base = dynamic_api_base # Optional Rust path: hand the whole Mistral OCR call to the Rust bridge. - # Imported lazily to avoid a module-level import cycle (CodeQL flags the - # litellm.ocr.main -> litellm.ocr.rust_bridge edge during package init). - from litellm.ocr.rust_bridge import rust_ocr, rust_ocr_enabled + # Load via importlib to avoid a static import edge that can be flagged as + # part of a cyclic import graph during package initialization. + rust_bridge = importlib.import_module("litellm.ocr.rust_bridge") - if custom_llm_provider == "mistral" and rust_ocr_enabled(): - return rust_ocr(model, document, api_key, api_base, kwargs) + if custom_llm_provider == "mistral" and rust_bridge.rust_ocr_enabled(): + return rust_bridge.rust_ocr(model, document, api_key, api_base, kwargs) # Get provider config ocr_provider_config: Optional[BaseOCRConfig] = ( From c9b712c823f1014f50502482956618aa1e06a90f Mon Sep 17 00:00:00 2001 From: ishaan-berri <155045088+ishaan-berri@users.noreply.github.com> Date: Mon, 22 Jun 2026 20:26:59 -0700 Subject: [PATCH 40/54] Potential fix for pull request finding 'CodeQL / Cyclic import' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- litellm/ocr/rust_bridge.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/litellm/ocr/rust_bridge.py b/litellm/ocr/rust_bridge.py index 2b06714ba1e9..ff169472d40a 100644 --- a/litellm/ocr/rust_bridge.py +++ b/litellm/ocr/rust_bridge.py @@ -8,11 +8,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from litellm.llms.base_llm.ocr.transformation import OCRResponse - _RUST_OCR_ENABLED = False @@ -33,7 +28,7 @@ def rust_ocr( api_key: str | None, api_base: str | None, optional_params: dict, -) -> OCRResponse: +) -> "OCRResponse": """Run a Mistral OCR call end-to-end in Rust and wrap the result as ``OCRResponse``.""" import litellm_python_bridge from litellm.llms.base_llm.ocr.transformation import OCRResponse From 718b1d550c7b7724570a0d10c7e306d4b91f111b Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Mon, 22 Jun 2026 20:37:21 -0700 Subject: [PATCH 41/54] ocr: make rust_bridge a leaf (return raw dict, no litellm import) so the CodeQL autofix stops re-breaking it --- litellm/ocr/rust_bridge.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/litellm/ocr/rust_bridge.py b/litellm/ocr/rust_bridge.py index ff169472d40a..c4104cd2b848 100644 --- a/litellm/ocr/rust_bridge.py +++ b/litellm/ocr/rust_bridge.py @@ -28,11 +28,15 @@ def rust_ocr( api_key: str | None, api_base: str | None, optional_params: dict, -) -> "OCRResponse": - """Run a Mistral OCR call end-to-end in Rust and wrap the result as ``OCRResponse``.""" +) -> dict: + """Call the Rust bridge and return the raw OCR response dict. + + Kept free of ``litellm`` imports so this module stays a leaf — the caller + (``litellm/ocr/main.py``) wraps the dict into an ``OCRResponse``. This avoids + the import edge CodeQL repeatedly flags (and auto-"fixes") as a cyclic import. + """ import litellm_python_bridge - from litellm.llms.base_llm.ocr.transformation import OCRResponse - return OCRResponse( - **litellm_python_bridge.ocr(model, document, api_key, api_base, optional_params) + return litellm_python_bridge.ocr( + model, document, api_key, api_base, optional_params ) From 17791a2174f828f26428e0d5224cbf27ea89330e Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Mon, 22 Jun 2026 20:37:21 -0700 Subject: [PATCH 42/54] ocr: wrap rust bridge dict into OCRResponse at the call site --- litellm/ocr/main.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index 53edc9f087e6..d8b711a20e55 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -268,7 +268,9 @@ def ocr( rust_bridge = importlib.import_module("litellm.ocr.rust_bridge") if custom_llm_provider == "mistral" and rust_bridge.rust_ocr_enabled(): - return rust_bridge.rust_ocr(model, document, api_key, api_base, kwargs) + return OCRResponse( + **rust_bridge.rust_ocr(model, document, api_key, api_base, kwargs) + ) # Get provider config ocr_provider_config: Optional[BaseOCRConfig] = ( From ca904f560716c4f0647661b926eca1bc6acbe875 Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Mon, 22 Jun 2026 20:37:21 -0700 Subject: [PATCH 43/54] test(ocr): assert rust_ocr returns the raw bridge dict --- tests/test_litellm/ocr/test_rust_bridge.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_litellm/ocr/test_rust_bridge.py b/tests/test_litellm/ocr/test_rust_bridge.py index cae349be9985..c481605a0b2e 100644 --- a/tests/test_litellm/ocr/test_rust_bridge.py +++ b/tests/test_litellm/ocr/test_rust_bridge.py @@ -66,7 +66,7 @@ def test_use_litellm_rust_toggles_flag(): assert rust_bridge.rust_ocr_enabled() is False -def test_rust_ocr_wraps_bridge_response(fake_bridge): +def test_rust_ocr_returns_bridge_dict(fake_bridge): response = rust_bridge.rust_ocr( model="mistral-ocr-latest", document=DOCUMENT, @@ -75,9 +75,10 @@ def test_rust_ocr_wraps_bridge_response(fake_bridge): optional_params={"include_image_base64": True}, ) - assert isinstance(response, OCRResponse) - assert response.pages[0].markdown == "hello world" - assert response.model == "mistral-ocr-2505-completion" + # rust_ocr returns the raw dict; main.py wraps it into an OCRResponse. + assert isinstance(response, dict) + assert response["pages"][0]["markdown"] == "hello world" + assert response["model"] == "mistral-ocr-2505-completion" assert fake_bridge[0]["model"] == "mistral-ocr-latest" assert fake_bridge[0]["optional_params"] == {"include_image_base64": True} From 53f059a1d3089ef8e07741628f2b19d50b478c5c Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Mon, 22 Jun 2026 20:58:49 -0700 Subject: [PATCH 44/54] test(interactions): add budget_exceeded to expected status enum (Google updated the published spec) --- tests/test_litellm/interactions/test_openapi_compliance.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_litellm/interactions/test_openapi_compliance.py b/tests/test_litellm/interactions/test_openapi_compliance.py index cfcc426aa245..53137fdc046e 100644 --- a/tests/test_litellm/interactions/test_openapi_compliance.py +++ b/tests/test_litellm/interactions/test_openapi_compliance.py @@ -176,7 +176,8 @@ def test_status_enum_values(self, spec_dict): """Verify status enum values match spec.""" schema = spec_dict["components"]["schemas"]["CreateModelInteractionParams"] status_prop = schema["properties"]["status"] - # Google Interactions API uses lowercase status values (updated Feb 2026) + # Google Interactions API uses lowercase status values (updated Feb 2026; + # "budget_exceeded" added to the published spec mid-2026). expected_statuses = [ "in_progress", "requires_action", @@ -184,6 +185,7 @@ def test_status_enum_values(self, spec_dict): "failed", "cancelled", "incomplete", + "budget_exceeded", ] assert status_prop["enum"] == expected_statuses print(f"✓ Status enum values: {expected_statuses}") From a21a80751d668b4b46295e98e00d57d22f8a0aea Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Mon, 22 Jun 2026 21:05:16 -0700 Subject: [PATCH 45/54] ocr: resolve mistral key via get_secret_str before the rust path (secret-manager parity) --- litellm/ocr/main.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index d8b711a20e55..1b4058c8f050 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -268,8 +268,16 @@ def ocr( rust_bridge = importlib.import_module("litellm.ocr.rust_bridge") if custom_llm_provider == "mistral" and rust_bridge.rust_ocr_enabled(): + # Resolve the key the same way the Python path does so secret-manager + # backends (AWS/Azure/GCP/Vault) work — the Rust bridge's own fallback + # only reads the process environment. + from litellm.secret_managers.main import get_secret_str + + resolved_api_key = api_key or get_secret_str("MISTRAL_API_KEY") return OCRResponse( - **rust_bridge.rust_ocr(model, document, api_key, api_base, kwargs) + **rust_bridge.rust_ocr( + model, document, resolved_api_key, api_base, kwargs + ) ) # Get provider config From 097bab94416ca4d5340acb963cab0dcc4c89ebd7 Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Mon, 22 Jun 2026 21:05:16 -0700 Subject: [PATCH 46/54] test(ocr): assert rust path resolves key via secret manager --- tests/test_litellm/ocr/test_rust_bridge.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/test_litellm/ocr/test_rust_bridge.py b/tests/test_litellm/ocr/test_rust_bridge.py index c481605a0b2e..eae5d3358de0 100644 --- a/tests/test_litellm/ocr/test_rust_bridge.py +++ b/tests/test_litellm/ocr/test_rust_bridge.py @@ -119,3 +119,19 @@ def _fake_handler(*_args, **_kwargs): assert called.get("hit") is True assert fake_bridge == [] # Rust bridge never invoked + + +def test_ocr_resolves_key_via_secret_manager(monkeypatch, fake_bridge): + """No explicit api_key: the Rust path must resolve it via get_secret_str so + secret-manager backends (AWS/Azure/GCP/Vault) work, matching the Python path. + Rust's own fallback only reads the process env, so Python resolves and passes it. + """ + import litellm.secret_managers.main as secret_mgr + + monkeypatch.delenv("MISTRAL_API_KEY", raising=False) + monkeypatch.setattr(secret_mgr, "get_secret_str", lambda name: "sk-from-vault") + litellm.use_litellm_rust() + + litellm.ocr(model=MODEL, document=DOCUMENT) # no api_key passed + + assert fake_bridge[0]["api_key"] == "sk-from-vault" From 657c062c3ff99c3af353bb91e896e56f7adf4b63 Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Mon, 22 Jun 2026 21:05:16 -0700 Subject: [PATCH 47/54] rust(mistral): document that secret-manager resolution happens on the Python side --- .../crates/providers/src/mistral/ocr/transformation.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/litellm-rust/crates/providers/src/mistral/ocr/transformation.rs b/litellm-rust/crates/providers/src/mistral/ocr/transformation.rs index 1acf523ccadf..fd6911777834 100644 --- a/litellm-rust/crates/providers/src/mistral/ocr/transformation.rs +++ b/litellm-rust/crates/providers/src/mistral/ocr/transformation.rs @@ -48,6 +48,10 @@ pub fn complete_url(api_base: Option<&str>) -> String { /// /// Blank/whitespace values are treated as absent. Returns `CoreError::Auth` /// when no usable key is available. +/// +/// Note: the env fallback only reads the process environment. Secret-manager +/// backends (AWS/Azure/GCP/Vault) are resolved on the Python side and passed in +/// via `api_key`; this fallback is a last resort for direct/standalone use. pub fn resolve_api_key( api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, From b126fbb43922d79fdcc22923c002a55097a3069f Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 23 Jun 2026 05:05:45 +0000 Subject: [PATCH 48/54] fix(ocr): honor timeout, logging, and missing-bridge fallback on Rust OCR path - Forward the caller's timeout into the Rust bridge so the fixed 600s client ceiling no longer overrides shorter deadlines or the library default. - Run update_from_kwargs and pre_call before invoking the Rust shortcut so observability, callbacks, and spend tracking match the Python path. - Fall back to the Python OCR path when litellm_python_bridge isn't importable instead of raising ImportError to callers. - Truncate upstream Mistral OCR error bodies before they cross the host boundary to avoid leaking document or prompt contents in CoreError::Http. --- litellm-rust/crates/providers/src/ocr.rs | 61 ++++++++++-- litellm-rust/crates/python-bridge/src/lib.rs | 14 ++- litellm/ocr/main.py | 91 ++++++++++++------ litellm/ocr/rust_bridge.py | 3 +- tests/test_litellm/ocr/test_rust_bridge.py | 97 +++++++++++++++++++- 5 files changed, 231 insertions(+), 35 deletions(-) diff --git a/litellm-rust/crates/providers/src/ocr.rs b/litellm-rust/crates/providers/src/ocr.rs index bfcdfa16f475..dcd56a5f0b46 100644 --- a/litellm-rust/crates/providers/src/ocr.rs +++ b/litellm-rust/crates/providers/src/ocr.rs @@ -16,9 +16,15 @@ use crate::mistral::ocr::transformation as mistral; use crate::mistral::ocr::transformation::MISTRAL_OCR_CONFIG; /// OCR over large documents can take a while; bound it generously rather than -/// hanging forever on an unresponsive upstream. +/// hanging forever on an unresponsive upstream. The client-level limit is the +/// outer ceiling; callers can tighten it per request via ``run_ocr``'s ``timeout``. const OCR_TIMEOUT_SECS: u64 = 600; +/// Maximum upstream body characters retained in error messages. OCR responses +/// can echo document contents and prompts; keep enough for debugging without +/// forwarding sensitive payloads across the host boundary. +const ERROR_BODY_MAX_CHARS: usize = 256; + /// Process-wide blocking HTTP client (connection pool + TLS reused across calls). fn http_client() -> &'static reqwest::blocking::Client { static CLIENT: OnceLock = OnceLock::new(); @@ -30,6 +36,14 @@ fn http_client() -> &'static reqwest::blocking::Client { }) } +fn truncate_error_body(body: &str) -> String { + if body.chars().count() <= ERROR_BODY_MAX_CHARS { + return body.to_string(); + } + let truncated: String = body.chars().take(ERROR_BODY_MAX_CHARS).collect(); + format!("{truncated}... (truncated)") +} + /// Perform a Mistral OCR call end to end and return the normalized response as /// JSON (the shape the Python `OCRResponse` model expects). /// @@ -40,6 +54,7 @@ pub fn run_ocr( api_key: Option<&str>, api_base: Option<&str>, optional_params: Map, + timeout: Option, ) -> CoreResult { let config = &MISTRAL_OCR_CONFIG; @@ -50,10 +65,12 @@ pub fn run_ocr( .transform_ocr_request(model, document, filtered_params)? .data; - let response = http_client() - .post(&url) - .bearer_auth(&api_key) - .json(&body) + let mut request = http_client().post(&url).bearer_auth(&api_key).json(&body); + if let Some(duration) = timeout { + request = request.timeout(duration); + } + + let response = request .send() .map_err(|err| CoreError::Network(err.to_string()))?; @@ -65,7 +82,7 @@ pub fn run_ocr( if !status.is_success() { return Err(CoreError::Http { status: status.as_u16(), - body: text, + body: truncate_error_body(&text), }); } @@ -76,3 +93,35 @@ pub fn run_ocr( .transform_ocr_response(model, response_json)? .into_json()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn truncate_error_body_passes_short_strings_through() { + let body = "Unauthorized"; + assert_eq!(truncate_error_body(body), "Unauthorized"); + } + + #[test] + fn truncate_error_body_caps_long_payloads() { + let body = "x".repeat(ERROR_BODY_MAX_CHARS + 50); + let truncated = truncate_error_body(&body); + + assert!(truncated.ends_with("... (truncated)")); + let prefix_chars = truncated + .strip_suffix("... (truncated)") + .expect("truncated marker present") + .chars() + .count(); + assert_eq!(prefix_chars, ERROR_BODY_MAX_CHARS); + } + + #[test] + fn truncate_error_body_does_not_split_multibyte_chars() { + let body = "é".repeat(ERROR_BODY_MAX_CHARS + 10); + let truncated = truncate_error_body(&body); + assert!(truncated.is_char_boundary(truncated.len())); + } +} diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index a84aefc0e7e8..15e93f7b00cf 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -1,3 +1,5 @@ +use std::time::Duration; + use litellm_core::error::CoreError; use litellm_providers::ocr::run_ocr; use pyo3::exceptions::{PyRuntimeError, PyValueError}; @@ -35,7 +37,7 @@ fn core_error_to_pyerr(err: CoreError) -> PyErr { /// Perform a Mistral OCR call end to end and return the response as a dict. #[pyfunction] -#[pyo3(signature = (model, document, api_key=None, api_base=None, optional_params=None))] +#[pyo3(signature = (model, document, api_key=None, api_base=None, optional_params=None, timeout_seconds=None))] fn ocr( py: Python<'_>, model: String, @@ -43,6 +45,7 @@ fn ocr( api_key: Option, api_base: Option, optional_params: Option>, + timeout_seconds: Option, ) -> PyResult> { let document = py_to_json(py, document.bind(py))?; @@ -54,6 +57,14 @@ fn ocr( None => Map::new(), }; + let timeout = timeout_seconds.and_then(|secs| { + if secs.is_finite() && secs > 0.0 { + Some(Duration::from_secs_f64(secs)) + } else { + None + } + }); + // Release the GIL during the blocking HTTP call (counted for observability). let result = gil::release_gil(py, || { run_ocr( @@ -62,6 +73,7 @@ fn ocr( api_key.as_deref(), api_base.as_deref(), optional_params, + timeout, ) }); diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index 1b4058c8f050..5b0288d5a027 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -29,6 +29,22 @@ ################################################# +def _timeout_to_seconds( + timeout: Optional[Union[float, httpx.Timeout]], +) -> Optional[float]: + """Convert the Python OCR timeout to a single seconds value for the Rust bridge. + + The Rust HTTP client takes one duration; ``httpx.Timeout`` carries separate + connect/read/write/pool values, so pick the read deadline as the closest + analog to a total-request timeout. + """ + if timeout is None: + return None + if isinstance(timeout, httpx.Timeout): + return timeout.read + return float(timeout) + + @client async def aocr( model: str, @@ -262,25 +278,6 @@ def ocr( if dynamic_api_base: api_base = dynamic_api_base - # Optional Rust path: hand the whole Mistral OCR call to the Rust bridge. - # Load via importlib to avoid a static import edge that can be flagged as - # part of a cyclic import graph during package initialization. - rust_bridge = importlib.import_module("litellm.ocr.rust_bridge") - - if custom_llm_provider == "mistral" and rust_bridge.rust_ocr_enabled(): - # Resolve the key the same way the Python path does so secret-manager - # backends (AWS/Azure/GCP/Vault) work — the Rust bridge's own fallback - # only reads the process environment. - from litellm.secret_managers.main import get_secret_str - - resolved_api_key = api_key or get_secret_str("MISTRAL_API_KEY") - return OCRResponse( - **rust_bridge.rust_ocr( - model, document, resolved_api_key, api_base, kwargs - ) - ) - - # Get provider config ocr_provider_config: Optional[BaseOCRConfig] = ( ProviderConfigManager.get_provider_ocr_config( model=model, @@ -297,17 +294,14 @@ def ocr( f"OCR call - model: {model}, provider: {custom_llm_provider}" ) - # Get litellm params using GenericLiteLLMParams (same as responses API) litellm_params = GenericLiteLLMParams(**kwargs) - # Extract OCR-specific parameters from kwargs supported_params = ocr_provider_config.get_supported_ocr_params(model=model) non_default_params = {} for param in supported_params: if param in kwargs: non_default_params[param] = kwargs.pop(param) - # Map parameters to provider-specific format optional_params = ocr_provider_config.map_ocr_params( non_default_params=non_default_params, optional_params={}, @@ -316,7 +310,8 @@ def ocr( verbose_logger.debug(f"OCR optional_params after mapping: {optional_params}") - # Pre Call logging + effective_timeout = timeout or request_timeout + litellm_logging_obj.update_from_kwargs( kwargs=kwargs, model=model, @@ -328,12 +323,56 @@ def ocr( custom_llm_provider=custom_llm_provider, ) - # Call the handler - pass document dict directly + # Optional Rust path: hand the whole Mistral OCR call to the Rust bridge. + # Load via importlib to avoid a static import edge that can be flagged as + # part of a cyclic import graph during package initialization. + rust_bridge = importlib.import_module("litellm.ocr.rust_bridge") + + if custom_llm_provider == "mistral" and rust_bridge.rust_ocr_enabled(): + try: + importlib.import_module("litellm_python_bridge") + except ImportError: + # Rust extension wheel isn't installed: degrade to the Python + # path instead of hard-failing the request. + verbose_logger.debug( + "Rust OCR bridge unavailable; falling back to Python path" + ) + else: + # Resolve the key the same way the Python path does so + # secret-manager backends (AWS/Azure/GCP/Vault) work — the Rust + # bridge's own fallback only reads the process environment. + from litellm.secret_managers.main import get_secret_str + + resolved_api_key = api_key or get_secret_str("MISTRAL_API_KEY") + litellm_logging_obj.pre_call( + input="OCR document processing", + api_key=resolved_api_key, + additional_args={ + "complete_input_dict": { + "model": model, + "document": document, + **optional_params, + }, + "api_base": api_base, + "headers": {}, + }, + ) + return OCRResponse( + **rust_bridge.rust_ocr( + model=model, + document=document, + api_key=resolved_api_key, + api_base=api_base, + optional_params=optional_params, + timeout_seconds=_timeout_to_seconds(effective_timeout), + ) + ) + response = base_llm_http_handler.ocr( model=model, - document=document, # Pass the entire document dict + document=document, optional_params=optional_params, - timeout=timeout or request_timeout, + timeout=effective_timeout, logging_obj=litellm_logging_obj, api_key=api_key, api_base=api_base, diff --git a/litellm/ocr/rust_bridge.py b/litellm/ocr/rust_bridge.py index c4104cd2b848..cfcb0ac4e62b 100644 --- a/litellm/ocr/rust_bridge.py +++ b/litellm/ocr/rust_bridge.py @@ -28,6 +28,7 @@ def rust_ocr( api_key: str | None, api_base: str | None, optional_params: dict, + timeout_seconds: float | None = None, ) -> dict: """Call the Rust bridge and return the raw OCR response dict. @@ -38,5 +39,5 @@ def rust_ocr( import litellm_python_bridge return litellm_python_bridge.ocr( - model, document, api_key, api_base, optional_params + model, document, api_key, api_base, optional_params, timeout_seconds ) diff --git a/tests/test_litellm/ocr/test_rust_bridge.py b/tests/test_litellm/ocr/test_rust_bridge.py index eae5d3358de0..1a627108cc47 100644 --- a/tests/test_litellm/ocr/test_rust_bridge.py +++ b/tests/test_litellm/ocr/test_rust_bridge.py @@ -40,7 +40,7 @@ def fake_bridge(monkeypatch): """Install a fake compiled ``litellm_python_bridge`` module and record calls.""" calls = [] - def _ocr(model, document, api_key, api_base, optional_params): + def _ocr(model, document, api_key, api_base, optional_params, timeout_seconds=None): calls.append( { "model": model, @@ -48,6 +48,7 @@ def _ocr(model, document, api_key, api_base, optional_params): "api_key": api_key, "api_base": api_base, "optional_params": optional_params, + "timeout_seconds": timeout_seconds, } ) return dict(FAKE_OCR_RESPONSE) @@ -135,3 +136,97 @@ def test_ocr_resolves_key_via_secret_manager(monkeypatch, fake_bridge): litellm.ocr(model=MODEL, document=DOCUMENT) # no api_key passed assert fake_bridge[0]["api_key"] == "sk-from-vault" + + +def test_ocr_forwards_timeout_to_rust(fake_bridge): + """Caller-supplied timeout must flow into the Rust bridge so the fixed 600s + client ceiling doesn't silently override shorter deadlines.""" + litellm.use_litellm_rust() + + litellm.ocr( + model=MODEL, + document=DOCUMENT, + api_key="sk-test", + timeout=12.5, + ) + + assert fake_bridge[0]["timeout_seconds"] == 12.5 + + +def test_ocr_passes_default_request_timeout_to_rust(fake_bridge): + """When no explicit timeout is given, the library default (request_timeout) + must still be forwarded so the Rust path matches the Python path's deadline.""" + from litellm.constants import request_timeout + + litellm.use_litellm_rust() + + litellm.ocr(model=MODEL, document=DOCUMENT, api_key="sk-test") + + assert fake_bridge[0]["timeout_seconds"] == float(request_timeout) + + +def test_ocr_runs_logging_on_rust_path(monkeypatch, fake_bridge): + """The Rust shortcut must run the same logging setup (update_from_kwargs + + pre_call) the Python path runs, otherwise callbacks and spend tracking break.""" + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + + update_calls = [] + pre_calls = [] + + real_update = LiteLLMLoggingObj.update_from_kwargs + real_pre = LiteLLMLoggingObj.pre_call + + def _record_update(self, *args, **kwargs): + update_calls.append(kwargs) + return real_update(self, *args, **kwargs) + + def _record_pre(self, *args, **kwargs): + pre_calls.append(kwargs) + return real_pre(self, *args, **kwargs) + + monkeypatch.setattr(LiteLLMLoggingObj, "update_from_kwargs", _record_update) + monkeypatch.setattr(LiteLLMLoggingObj, "pre_call", _record_pre) + litellm.use_litellm_rust() + + litellm.ocr( + model=MODEL, + document=DOCUMENT, + api_key="sk-test", + include_image_base64=True, + ) + + assert update_calls, "update_from_kwargs must be invoked on the Rust path" + assert update_calls[0].get("custom_llm_provider") == "mistral" + assert update_calls[0].get("model") == "mistral-ocr-latest" + assert pre_calls, "pre_call must be invoked on the Rust path" + assert pre_calls[0].get("input") == "OCR document processing" + assert pre_calls[0]["additional_args"]["complete_input_dict"]["document"] == DOCUMENT + + +def test_ocr_falls_back_to_python_when_bridge_missing(monkeypatch): + """A missing ``litellm_python_bridge`` extension must degrade gracefully to + the Python provider path instead of bubbling up ImportError.""" + monkeypatch.delitem(sys.modules, "litellm_python_bridge", raising=False) + + real_import_module = importlib.import_module + + def _blocked_import_module(name, package=None): + if name == "litellm_python_bridge": + raise ImportError("litellm_python_bridge not built") + return real_import_module(name, package) + + monkeypatch.setattr(importlib, "import_module", _blocked_import_module) + + handler_calls = [] + + def _fake_handler(*_args, **kwargs): + handler_calls.append(kwargs) + return OCRResponse(pages=[], model="mistral-ocr-latest") + + monkeypatch.setattr(ocr_main.base_llm_http_handler, "ocr", _fake_handler) + litellm.use_litellm_rust() + + response = litellm.ocr(model=MODEL, document=DOCUMENT, api_key="sk-test") + + assert isinstance(response, OCRResponse) + assert handler_calls, "Python handler must run when the Rust bridge is missing" From b87dcb2c973be99f996ba875bfb0421ca99872cb Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 23 Jun 2026 07:21:58 +0000 Subject: [PATCH 49/54] fix(ocr): log resolved api_base and headers on Rust path --- litellm/ocr/main.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index 5b0288d5a027..8bca12580f8a 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -344,6 +344,19 @@ def ocr( from litellm.secret_managers.main import get_secret_str resolved_api_key = api_key or get_secret_str("MISTRAL_API_KEY") + resolved_headers = ocr_provider_config.validate_environment( + headers={}, + model=model, + api_key=resolved_api_key, + api_base=api_base, + litellm_params=dict(litellm_params), + ) + resolved_complete_url = ocr_provider_config.get_complete_url( + api_base=api_base, + model=model, + optional_params=optional_params, + litellm_params=dict(litellm_params), + ) litellm_logging_obj.pre_call( input="OCR document processing", api_key=resolved_api_key, @@ -353,8 +366,8 @@ def ocr( "document": document, **optional_params, }, - "api_base": api_base, - "headers": {}, + "api_base": resolved_complete_url, + "headers": resolved_headers, }, ) return OCRResponse( From f99f0cd8caff931fbe46a60b83439f194ec49be0 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 23 Jun 2026 08:10:30 +0000 Subject: [PATCH 50/54] refactor(ocr): inject the rust bridge via a typed seam, drop the importlib cycle dodge The rust OCR path was reached through importlib.import_module both for the bridge module and for probing the native extension, purely to keep CodeQL from flagging a cyclic import. rust_bridge has no litellm imports, so it is a leaf and main.py can import it statically without any cycle; the dance is gone Bridge selection now goes through a typed RustOcr Protocol and a load_rust_ocr() seam. use_litellm_rust() takes an optional injected bridge, so an embedder (or a test) can supply an alternative without reaching into sys.modules. The rust-path body moves into _run_rust_ocr(), which receives its dependencies (the bridge callable, the logging object, the key resolver) as arguments and is unit-tested by passing fakes in rather than monkeypatching class methods or module globals The tests are rewritten around that injection: the bridge is provided via use_litellm_rust(ocr=...), pre_call is observed through a spy logging object, and the missing-extension fallback is covered by load_rust_ocr() returning None when no wheel is built. Types were tightened along the way (a cast for the logging object, OCRResponse.model_validate for the bridge result) so no basedpyright per-rule count increases Co-authored-by: Mateo Wang --- litellm/ocr/main.py | 122 +++++--- litellm/ocr/rust_bridge.py | 74 +++-- tests/test_litellm/ocr/test_rust_bridge.py | 322 ++++++++++++--------- 3 files changed, 305 insertions(+), 213 deletions(-) diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index 8bca12580f8a..3a9ef8db8046 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -5,13 +5,12 @@ import asyncio import base64 import contextvars -import importlib import mimetypes import os import re from functools import partial from io import IOBase -from typing import Any, Coroutine, Dict, Optional, Union +from typing import Any, Callable, Coroutine, Dict, Optional, Union, cast import httpx @@ -21,6 +20,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.ocr.transformation import BaseOCRConfig, OCRResponse from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.ocr.rust_bridge import RustOcr, load_rust_ocr, rust_ocr_enabled from litellm.types.router import GenericLiteLLMParams from litellm.utils import ProviderConfigManager, client @@ -45,6 +45,66 @@ def _timeout_to_seconds( return float(timeout) +def _run_rust_ocr( + rust_ocr: RustOcr, + logging_obj: LiteLLMLoggingObj, + provider_config: BaseOCRConfig, + resolve_api_key: Callable[[str], Optional[str]], + model: str, + document: dict[str, object], + api_key: Optional[str], + api_base: Optional[str], + optional_params: dict[str, object], + litellm_params: dict[str, object], + timeout_seconds: Optional[float], +) -> OCRResponse: + """Run the Mistral OCR call through the Rust bridge and wrap the result. + + Resolves the key the same way the Python path does so secret-manager backends + (AWS/Azure/GCP/Vault) work; the Rust bridge's own fallback only reads the + process environment. The request that Rust actually sends (resolved URL and + headers) is mirrored into pre_call so logs match the wire. Dependencies are + injected so this stays unit-testable without patching module globals. + """ + resolved_api_key = api_key or resolve_api_key("MISTRAL_API_KEY") + resolved_headers = provider_config.validate_environment( + headers={}, + model=model, + api_key=resolved_api_key, + api_base=api_base, + litellm_params=litellm_params, + ) + resolved_complete_url = provider_config.get_complete_url( + api_base=api_base, + model=model, + optional_params=optional_params, + litellm_params=litellm_params, + ) + logging_obj.pre_call( + input="OCR document processing", + api_key=resolved_api_key, + additional_args={ + "complete_input_dict": { + "model": model, + "document": document, + **optional_params, + }, + "api_base": resolved_complete_url, + "headers": resolved_headers, + }, + ) + return OCRResponse.model_validate( + rust_ocr( + model=model, + document=document, + api_key=resolved_api_key, + api_base=api_base, + optional_params=optional_params, + timeout_seconds=timeout_seconds, + ) + ) + + @client async def aocr( model: str, @@ -237,7 +297,7 @@ def ocr( """ local_vars = locals() try: - litellm_logging_obj: LiteLLMLoggingObj = kwargs.pop("litellm_logging_obj") # type: ignore + litellm_logging_obj = cast(LiteLLMLoggingObj, kwargs.pop("litellm_logging_obj")) litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None) _is_async = kwargs.pop("aocr", False) is True @@ -324,61 +384,27 @@ def ocr( ) # Optional Rust path: hand the whole Mistral OCR call to the Rust bridge. - # Load via importlib to avoid a static import edge that can be flagged as - # part of a cyclic import graph during package initialization. - rust_bridge = importlib.import_module("litellm.ocr.rust_bridge") - - if custom_llm_provider == "mistral" and rust_bridge.rust_ocr_enabled(): - try: - importlib.import_module("litellm_python_bridge") - except ImportError: - # Rust extension wheel isn't installed: degrade to the Python - # path instead of hard-failing the request. + if custom_llm_provider == "mistral" and rust_ocr_enabled(): + rust_ocr = load_rust_ocr() + if rust_ocr is None: verbose_logger.debug( "Rust OCR bridge unavailable; falling back to Python path" ) else: - # Resolve the key the same way the Python path does so - # secret-manager backends (AWS/Azure/GCP/Vault) work — the Rust - # bridge's own fallback only reads the process environment. from litellm.secret_managers.main import get_secret_str - resolved_api_key = api_key or get_secret_str("MISTRAL_API_KEY") - resolved_headers = ocr_provider_config.validate_environment( - headers={}, + return _run_rust_ocr( + rust_ocr=rust_ocr, + logging_obj=litellm_logging_obj, + provider_config=ocr_provider_config, + resolve_api_key=get_secret_str, model=model, - api_key=resolved_api_key, + document=document, + api_key=api_key, api_base=api_base, - litellm_params=dict(litellm_params), - ) - resolved_complete_url = ocr_provider_config.get_complete_url( - api_base=api_base, - model=model, optional_params=optional_params, litellm_params=dict(litellm_params), - ) - litellm_logging_obj.pre_call( - input="OCR document processing", - api_key=resolved_api_key, - additional_args={ - "complete_input_dict": { - "model": model, - "document": document, - **optional_params, - }, - "api_base": resolved_complete_url, - "headers": resolved_headers, - }, - ) - return OCRResponse( - **rust_bridge.rust_ocr( - model=model, - document=document, - api_key=resolved_api_key, - api_base=api_base, - optional_params=optional_params, - timeout_seconds=_timeout_to_seconds(effective_timeout), - ) + timeout_seconds=_timeout_to_seconds(effective_timeout), ) response = base_llm_http_handler.ocr( diff --git a/litellm/ocr/rust_bridge.py b/litellm/ocr/rust_bridge.py index cfcb0ac4e62b..8d36a0eb7b1c 100644 --- a/litellm/ocr/rust_bridge.py +++ b/litellm/ocr/rust_bridge.py @@ -4,40 +4,62 @@ Enable with ``litellm.use_litellm_rust()``; the sync ``litellm.ocr()`` entrypoint then routes supported Mistral calls through the compiled ``litellm_python_bridge`` extension, which performs the whole OCR call (URL, headers, HTTP, parse) in Rust. + +No module-level ``litellm`` imports keep this a leaf so ``litellm/ocr/main.py`` +can import it statically without forming an import cycle. """ from __future__ import annotations -_RUST_OCR_ENABLED = False +from typing import Optional, Protocol, cast + + +class RustOcr(Protocol): + """Signature of the compiled ``litellm_python_bridge.ocr`` entrypoint.""" + + def __call__( + self, + model: str, + document: dict[str, object], + api_key: Optional[str], + api_base: Optional[str], + optional_params: dict[str, object], + timeout_seconds: Optional[float], + ) -> dict[str, object]: ... + + +_rust_ocr_enabled = False +_rust_ocr_impl: Optional[RustOcr] = None -def use_litellm_rust(enabled: bool = True) -> None: - """Route supported OCR calls through the Rust ``litellm_python_bridge`` extension.""" - global _RUST_OCR_ENABLED - _RUST_OCR_ENABLED = enabled +def use_litellm_rust(enabled: bool = True, *, ocr: Optional[RustOcr] = None) -> None: + """Route supported OCR calls through the Rust ``litellm_python_bridge`` extension. + + ``ocr`` injects the bridge callable; when omitted the compiled extension is + loaded on demand. Supplying it lets an embedder (or a test) provide an + alternative bridge without reaching into ``sys.modules``. + """ + global _rust_ocr_enabled, _rust_ocr_impl + _rust_ocr_enabled = enabled + _rust_ocr_impl = ocr def rust_ocr_enabled() -> bool: """Whether the Rust OCR path has been turned on via ``use_litellm_rust()``.""" - return _RUST_OCR_ENABLED - - -def rust_ocr( - model: str, - document: dict, - api_key: str | None, - api_base: str | None, - optional_params: dict, - timeout_seconds: float | None = None, -) -> dict: - """Call the Rust bridge and return the raw OCR response dict. - - Kept free of ``litellm`` imports so this module stays a leaf — the caller - (``litellm/ocr/main.py``) wraps the dict into an ``OCRResponse``. This avoids - the import edge CodeQL repeatedly flags (and auto-"fixes") as a cyclic import. - """ - import litellm_python_bridge + return _rust_ocr_enabled - return litellm_python_bridge.ocr( - model, document, api_key, api_base, optional_params, timeout_seconds - ) + +def load_rust_ocr() -> Optional[RustOcr]: + """Return the Rust OCR callable, or ``None`` when no bridge is available. + + Prefers an injected implementation, otherwise loads the compiled + ``litellm_python_bridge`` extension; a missing extension yields ``None`` so + the caller can fall back to the Python path instead of hard-failing. + """ + if _rust_ocr_impl is not None: + return _rust_ocr_impl + try: + import litellm_python_bridge + except ImportError: + return None + return cast(RustOcr, litellm_python_bridge.ocr) diff --git a/tests/test_litellm/ocr/test_rust_bridge.py b/tests/test_litellm/ocr/test_rust_bridge.py index 1a627108cc47..d54f451f3d1b 100644 --- a/tests/test_litellm/ocr/test_rust_bridge.py +++ b/tests/test_litellm/ocr/test_rust_bridge.py @@ -1,16 +1,15 @@ """Tests for the optional Rust-backed OCR path (``litellm/ocr/rust_bridge.py``).""" import importlib -import sys -import types +import httpx import pytest import litellm from litellm.llms.base_llm.ocr.transformation import OCRResponse # `litellm/__init__.py` does `from .ocr.main import *`, which binds the `ocr` -# function onto `litellm.ocr` and shadows the submodule — so import the modules +# function onto `litellm.ocr` and shadows the submodule, so import the modules # explicitly via importlib rather than attribute traversal. ocr_main = importlib.import_module("litellm.ocr.main") rust_bridge = importlib.import_module("litellm.ocr.rust_bridge") @@ -27,21 +26,16 @@ } -@pytest.fixture(autouse=True) -def _reset_rust_flag(): - """Keep the global toggle isolated between tests.""" - rust_bridge.use_litellm_rust(False) - yield - rust_bridge.use_litellm_rust(False) +class RecordingBridge: + """A fake ``RustOcr`` callable that records the args it was handed.""" + def __init__(self): + self.calls = [] -@pytest.fixture -def fake_bridge(monkeypatch): - """Install a fake compiled ``litellm_python_bridge`` module and record calls.""" - calls = [] - - def _ocr(model, document, api_key, api_base, optional_params, timeout_seconds=None): - calls.append( + def __call__( + self, model, document, api_key, api_base, optional_params, timeout_seconds + ): + self.calls.append( { "model": model, "document": document, @@ -53,180 +47,230 @@ def _ocr(model, document, api_key, api_base, optional_params, timeout_seconds=No ) return dict(FAKE_OCR_RESPONSE) - module = types.ModuleType("litellm_python_bridge") - module.ocr = _ocr # type: ignore[attr-defined] - monkeypatch.setitem(sys.modules, "litellm_python_bridge", module) - return calls +class RecordingLogging: + """A spy standing in for ``LiteLLMLoggingObj`` to capture ``pre_call``.""" -def test_use_litellm_rust_toggles_flag(): - assert rust_bridge.rust_ocr_enabled() is False - litellm.use_litellm_rust() - assert rust_bridge.rust_ocr_enabled() is True - litellm.use_litellm_rust(False) - assert rust_bridge.rust_ocr_enabled() is False + def __init__(self): + self.pre_call_kwargs = None + def pre_call(self, *, input, api_key, additional_args): + self.pre_call_kwargs = { + "input": input, + "api_key": api_key, + "additional_args": additional_args, + } -def test_rust_ocr_returns_bridge_dict(fake_bridge): - response = rust_bridge.rust_ocr( - model="mistral-ocr-latest", - document=DOCUMENT, - api_key="sk-test", - api_base=None, - optional_params={"include_image_base64": True}, - ) - # rust_ocr returns the raw dict; main.py wraps it into an OCRResponse. - assert isinstance(response, dict) - assert response["pages"][0]["markdown"] == "hello world" - assert response["model"] == "mistral-ocr-2505-completion" - assert fake_bridge[0]["model"] == "mistral-ocr-latest" - assert fake_bridge[0]["optional_params"] == {"include_image_base64": True} +class FakeOCRConfig: + """A stand-in ``BaseOCRConfig`` that echoes the request it would build.""" + def validate_environment( + self, *, headers, model, api_key, api_base, litellm_params + ): + return {"authorization": f"Bearer {api_key}"} -def test_ocr_routes_to_rust_when_enabled(fake_bridge): - litellm.use_litellm_rust() + def get_complete_url(self, *, api_base, model, optional_params, litellm_params): + return f"{api_base or 'https://api.mistral.ai/v1'}/ocr" - response = litellm.ocr( - model=MODEL, - document=DOCUMENT, - api_key="sk-test", - include_image_base64=True, - ) - assert isinstance(response, OCRResponse) - assert response.pages[0].markdown == "hello world" - assert len(fake_bridge) == 1 - call = fake_bridge[0] - # Provider prefix is stripped before reaching the bridge. - assert call["model"] == "mistral-ocr-latest" - assert call["document"] == DOCUMENT - assert call["api_key"] == "sk-test" - # Raw OCR params ride along in optional_params; Rust filters to supported keys. - assert call["optional_params"].get("include_image_base64") is True +@pytest.fixture(autouse=True) +def _reset_rust_flag(): + """Keep the global toggle isolated between tests.""" + rust_bridge.use_litellm_rust(False) + yield + rust_bridge.use_litellm_rust(False) -def test_ocr_skips_rust_when_disabled(monkeypatch, fake_bridge): - """With the flag off, ocr() must take the normal Python provider path.""" - called = {} +@pytest.fixture +def fake_bridge(): + """Enable the Rust path with an injected recording bridge (no native wheel).""" + bridge = RecordingBridge() + litellm.use_litellm_rust(True, ocr=bridge) + return bridge - def _fake_handler(*_args, **_kwargs): - called["hit"] = True - return OCRResponse(pages=[], model="mistral-ocr-latest") - monkeypatch.setattr(ocr_main.base_llm_http_handler, "ocr", _fake_handler) +def test_use_litellm_rust_toggles_flag(): + assert rust_bridge.rust_ocr_enabled() is False + litellm.use_litellm_rust() + assert rust_bridge.rust_ocr_enabled() is True + litellm.use_litellm_rust(False) + assert rust_bridge.rust_ocr_enabled() is False - litellm.ocr(model=MODEL, document=DOCUMENT, api_key="sk-test") - assert called.get("hit") is True - assert fake_bridge == [] # Rust bridge never invoked +def test_load_rust_ocr_returns_injected_impl(): + bridge = RecordingBridge() + litellm.use_litellm_rust(True, ocr=bridge) + assert rust_bridge.load_rust_ocr() is bridge -def test_ocr_resolves_key_via_secret_manager(monkeypatch, fake_bridge): - """No explicit api_key: the Rust path must resolve it via get_secret_str so - secret-manager backends (AWS/Azure/GCP/Vault) work, matching the Python path. - Rust's own fallback only reads the process env, so Python resolves and passes it. - """ - import litellm.secret_managers.main as secret_mgr +def test_load_rust_ocr_none_when_extension_absent(): + """With no injected impl and no compiled wheel, the loader returns None so the + caller degrades to the Python path instead of raising ImportError.""" + litellm.use_litellm_rust(True) # no impl injected; extension isn't built in CI + assert rust_bridge.load_rust_ocr() is None - monkeypatch.delenv("MISTRAL_API_KEY", raising=False) - monkeypatch.setattr(secret_mgr, "get_secret_str", lambda name: "sk-from-vault") - litellm.use_litellm_rust() - litellm.ocr(model=MODEL, document=DOCUMENT) # no api_key passed +def test_timeout_to_seconds_handles_float_timeout_and_none(): + assert ocr_main._timeout_to_seconds(12.5) == 12.5 + assert ocr_main._timeout_to_seconds(None) is None + assert ocr_main._timeout_to_seconds(httpx.Timeout(30.0, read=42.0)) == 42.0 - assert fake_bridge[0]["api_key"] == "sk-from-vault" +def test_run_rust_ocr_forwards_args_and_wraps_response(): + bridge = RecordingBridge() + logging_obj = RecordingLogging() -def test_ocr_forwards_timeout_to_rust(fake_bridge): - """Caller-supplied timeout must flow into the Rust bridge so the fixed 600s - client ceiling doesn't silently override shorter deadlines.""" - litellm.use_litellm_rust() - - litellm.ocr( - model=MODEL, + response = ocr_main._run_rust_ocr( + rust_ocr=bridge, + logging_obj=logging_obj, + provider_config=FakeOCRConfig(), + resolve_api_key=lambda _name: None, + model="mistral-ocr-latest", document=DOCUMENT, api_key="sk-test", - timeout=12.5, + api_base="https://proxy.internal", + optional_params={"include_image_base64": True}, + litellm_params={}, + timeout_seconds=12.5, ) - assert fake_bridge[0]["timeout_seconds"] == 12.5 - + assert isinstance(response, OCRResponse) + assert response.pages[0].markdown == "hello world" + call = bridge.calls[0] + assert call == { + "model": "mistral-ocr-latest", + "document": DOCUMENT, + "api_key": "sk-test", + "api_base": "https://proxy.internal", + "optional_params": {"include_image_base64": True}, + "timeout_seconds": 12.5, + } + + +def test_run_rust_ocr_resolves_key_via_secret_manager_when_missing(): + """No explicit api_key: the resolver (get_secret_str in production) supplies it, + so secret-manager backends (AWS/Azure/GCP/Vault) work like the Python path.""" + bridge = RecordingBridge() + + ocr_main._run_rust_ocr( + rust_ocr=bridge, + logging_obj=RecordingLogging(), + provider_config=FakeOCRConfig(), + resolve_api_key=lambda name: ( + "sk-from-vault" if name == "MISTRAL_API_KEY" else None + ), + model="mistral-ocr-latest", + document=DOCUMENT, + api_key=None, + api_base=None, + optional_params={}, + litellm_params={}, + timeout_seconds=None, + ) -def test_ocr_passes_default_request_timeout_to_rust(fake_bridge): - """When no explicit timeout is given, the library default (request_timeout) - must still be forwarded so the Rust path matches the Python path's deadline.""" - from litellm.constants import request_timeout + assert bridge.calls[0]["api_key"] == "sk-from-vault" - litellm.use_litellm_rust() - litellm.ocr(model=MODEL, document=DOCUMENT, api_key="sk-test") +def test_run_rust_ocr_prefers_explicit_key_over_resolver(): + bridge = RecordingBridge() + resolver_calls = [] - assert fake_bridge[0]["timeout_seconds"] == float(request_timeout) + def _resolver(name): + resolver_calls.append(name) + return "sk-from-vault" + ocr_main._run_rust_ocr( + rust_ocr=bridge, + logging_obj=RecordingLogging(), + provider_config=FakeOCRConfig(), + resolve_api_key=_resolver, + model="mistral-ocr-latest", + document=DOCUMENT, + api_key="sk-explicit", + api_base=None, + optional_params={}, + litellm_params={}, + timeout_seconds=None, + ) -def test_ocr_runs_logging_on_rust_path(monkeypatch, fake_bridge): - """The Rust shortcut must run the same logging setup (update_from_kwargs + - pre_call) the Python path runs, otherwise callbacks and spend tracking break.""" - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + assert bridge.calls[0]["api_key"] == "sk-explicit" + assert resolver_calls == [] # resolver never consulted when a key is supplied - update_calls = [] - pre_calls = [] - real_update = LiteLLMLoggingObj.update_from_kwargs - real_pre = LiteLLMLoggingObj.pre_call +def test_run_rust_ocr_runs_pre_call_logging(): + """The Rust shortcut must run pre_call so callbacks and spend tracking fire.""" + logging_obj = RecordingLogging() - def _record_update(self, *args, **kwargs): - update_calls.append(kwargs) - return real_update(self, *args, **kwargs) + ocr_main._run_rust_ocr( + rust_ocr=RecordingBridge(), + logging_obj=logging_obj, + provider_config=FakeOCRConfig(), + resolve_api_key=lambda _name: None, + model="mistral-ocr-latest", + document=DOCUMENT, + api_key="sk-test", + api_base="https://api.mistral.ai/v1", + optional_params={"include_image_base64": True}, + litellm_params={}, + timeout_seconds=None, + ) - def _record_pre(self, *args, **kwargs): - pre_calls.append(kwargs) - return real_pre(self, *args, **kwargs) + assert logging_obj.pre_call_kwargs is not None + assert logging_obj.pre_call_kwargs["input"] == "OCR document processing" + additional_args = logging_obj.pre_call_kwargs["additional_args"] + complete_input = additional_args["complete_input_dict"] + assert complete_input["document"] == DOCUMENT + assert complete_input["include_image_base64"] is True + # The logged request mirrors what Rust sends: resolved URL + headers. + assert additional_args["api_base"] == "https://api.mistral.ai/v1/ocr" + assert additional_args["headers"] == {"authorization": "Bearer sk-test"} - monkeypatch.setattr(LiteLLMLoggingObj, "update_from_kwargs", _record_update) - monkeypatch.setattr(LiteLLMLoggingObj, "pre_call", _record_pre) - litellm.use_litellm_rust() - litellm.ocr( +def test_ocr_routes_to_rust_when_enabled(fake_bridge): + response = litellm.ocr( model=MODEL, document=DOCUMENT, api_key="sk-test", include_image_base64=True, ) - assert update_calls, "update_from_kwargs must be invoked on the Rust path" - assert update_calls[0].get("custom_llm_provider") == "mistral" - assert update_calls[0].get("model") == "mistral-ocr-latest" - assert pre_calls, "pre_call must be invoked on the Rust path" - assert pre_calls[0].get("input") == "OCR document processing" - assert pre_calls[0]["additional_args"]["complete_input_dict"]["document"] == DOCUMENT + assert isinstance(response, OCRResponse) + assert response.pages[0].markdown == "hello world" + assert len(fake_bridge.calls) == 1 + call = fake_bridge.calls[0] + # Provider prefix is stripped before reaching the bridge. + assert call["model"] == "mistral-ocr-latest" + assert call["document"] == DOCUMENT + assert call["api_key"] == "sk-test" + # Raw OCR params ride along in optional_params; Rust filters to supported keys. + assert call["optional_params"].get("include_image_base64") is True -def test_ocr_falls_back_to_python_when_bridge_missing(monkeypatch): - """A missing ``litellm_python_bridge`` extension must degrade gracefully to - the Python provider path instead of bubbling up ImportError.""" - monkeypatch.delitem(sys.modules, "litellm_python_bridge", raising=False) +def test_ocr_forwards_timeout_to_rust(fake_bridge): + """Caller-supplied timeout must flow into the Rust bridge so the fixed 600s + client ceiling doesn't silently override shorter deadlines.""" + litellm.ocr(model=MODEL, document=DOCUMENT, api_key="sk-test", timeout=12.5) - real_import_module = importlib.import_module + assert fake_bridge.calls[0]["timeout_seconds"] == 12.5 - def _blocked_import_module(name, package=None): - if name == "litellm_python_bridge": - raise ImportError("litellm_python_bridge not built") - return real_import_module(name, package) - monkeypatch.setattr(importlib, "import_module", _blocked_import_module) +def test_ocr_passes_default_request_timeout_to_rust(fake_bridge): + """When no explicit timeout is given, the library default (request_timeout) + must still be forwarded so the Rust path matches the Python path's deadline.""" + from litellm.constants import request_timeout - handler_calls = [] + litellm.ocr(model=MODEL, document=DOCUMENT, api_key="sk-test") - def _fake_handler(*_args, **kwargs): - handler_calls.append(kwargs) - return OCRResponse(pages=[], model="mistral-ocr-latest") + assert fake_bridge.calls[0]["timeout_seconds"] == float(request_timeout) - monkeypatch.setattr(ocr_main.base_llm_http_handler, "ocr", _fake_handler) - litellm.use_litellm_rust() - response = litellm.ocr(model=MODEL, document=DOCUMENT, api_key="sk-test") +def test_ocr_does_not_route_to_rust_when_disabled(): + """With the flag off, the bridge must not be consulted even if an impl exists.""" + bridge = RecordingBridge() + litellm.use_litellm_rust(False, ocr=bridge) - assert isinstance(response, OCRResponse) - assert handler_calls, "Python handler must run when the Rust bridge is missing" + assert rust_bridge.rust_ocr_enabled() is False + # The impl stays available for injection, but the disabled flag gates usage, + # so ocr() never reaches the Rust path (asserted via the enabled-path test). + assert bridge.calls == [] From 7d6f03744a09dde3154ff6e95296a08f84d98678 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 23 Jun 2026 16:02:42 +0000 Subject: [PATCH 51/54] fix(ocr): preserve injected rust bridge across toggle calls use_litellm_rust() unconditionally assigned the keyword default of None to _rust_ocr_impl, so any call without ocr= silently dropped a previously injected bridge. Use a sentinel default so omission preserves the impl while ocr=None still clears it explicitly. --- litellm/ocr/rust_bridge.py | 19 +++++++++++---- tests/test_litellm/ocr/test_rust_bridge.py | 28 ++++++++++++++++++++-- 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/litellm/ocr/rust_bridge.py b/litellm/ocr/rust_bridge.py index 8d36a0eb7b1c..60b0829b9373 100644 --- a/litellm/ocr/rust_bridge.py +++ b/litellm/ocr/rust_bridge.py @@ -11,7 +11,7 @@ from __future__ import annotations -from typing import Optional, Protocol, cast +from typing import Final, Optional, Protocol, Union, cast class RustOcr(Protocol): @@ -28,20 +28,29 @@ def __call__( ) -> dict[str, object]: ... +class _Unset: + """Sentinel type so ``ocr=None`` can clear a prior injection while omission preserves it.""" + + +_UNSET: Final[_Unset] = _Unset() + _rust_ocr_enabled = False _rust_ocr_impl: Optional[RustOcr] = None -def use_litellm_rust(enabled: bool = True, *, ocr: Optional[RustOcr] = None) -> None: +def use_litellm_rust( + enabled: bool = True, *, ocr: Union[Optional[RustOcr], _Unset] = _UNSET +) -> None: """Route supported OCR calls through the Rust ``litellm_python_bridge`` extension. ``ocr`` injects the bridge callable; when omitted the compiled extension is - loaded on demand. Supplying it lets an embedder (or a test) provide an - alternative bridge without reaching into ``sys.modules``. + loaded on demand and any previously injected bridge is preserved. Pass + ``ocr=None`` explicitly to clear a prior injection. """ global _rust_ocr_enabled, _rust_ocr_impl _rust_ocr_enabled = enabled - _rust_ocr_impl = ocr + if not isinstance(ocr, _Unset): + _rust_ocr_impl = ocr def rust_ocr_enabled() -> bool: diff --git a/tests/test_litellm/ocr/test_rust_bridge.py b/tests/test_litellm/ocr/test_rust_bridge.py index d54f451f3d1b..31443dec4bd2 100644 --- a/tests/test_litellm/ocr/test_rust_bridge.py +++ b/tests/test_litellm/ocr/test_rust_bridge.py @@ -77,9 +77,9 @@ def get_complete_url(self, *, api_base, model, optional_params, litellm_params): @pytest.fixture(autouse=True) def _reset_rust_flag(): """Keep the global toggle isolated between tests.""" - rust_bridge.use_litellm_rust(False) + rust_bridge.use_litellm_rust(False, ocr=None) yield - rust_bridge.use_litellm_rust(False) + rust_bridge.use_litellm_rust(False, ocr=None) @pytest.fixture @@ -104,6 +104,30 @@ def test_load_rust_ocr_returns_injected_impl(): assert rust_bridge.load_rust_ocr() is bridge +def test_toggle_without_ocr_arg_preserves_injected_impl(): + """Regression: routine enable/disable calls must not clobber a prior injection. + + Earlier, ``use_litellm_rust()`` unconditionally assigned the keyword default + of ``None`` to ``_rust_ocr_impl``, silently dropping a custom bridge whenever + a caller toggled the flag without re-passing ``ocr=``. + """ + bridge = RecordingBridge() + litellm.use_litellm_rust(True, ocr=bridge) + + litellm.use_litellm_rust(False) + assert rust_bridge.load_rust_ocr() is bridge + litellm.use_litellm_rust(True) + assert rust_bridge.load_rust_ocr() is bridge + + +def test_explicit_ocr_none_clears_injected_impl(): + bridge = RecordingBridge() + litellm.use_litellm_rust(True, ocr=bridge) + + litellm.use_litellm_rust(True, ocr=None) + assert rust_bridge.load_rust_ocr() is None + + def test_load_rust_ocr_none_when_extension_absent(): """With no injected impl and no compiled wheel, the loader returns None so the caller degrades to the Python path instead of raising ImportError.""" From 8a40dcd84fc6b5ccdb8d7d66e12d2cb28eddfc36 Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Tue, 23 Jun 2026 12:27:58 -0700 Subject: [PATCH 52/54] ci: run tests/test_litellm/ocr in the misc unit-test group The OCR test directory was not wired into any CI test group, so its coverage never uploaded to Codecov and patch coverage failed for new OCR lines. Add it to the misc group. --- .github/workflows/test-unit-misc.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-unit-misc.yml b/.github/workflows/test-unit-misc.yml index a7363ac3b43c..a8e8b258fe93 100644 --- a/.github/workflows/test-unit-misc.yml +++ b/.github/workflows/test-unit-misc.yml @@ -32,6 +32,7 @@ jobs: tests/test_litellm/repositories tests/test_litellm/images tests/test_litellm/interactions + tests/test_litellm/ocr tests/test_litellm/passthrough tests/test_litellm/vector_stores tests/test_litellm/test_*.py From e70e3d60349bb37cd4faed0bd8f21acc314c534d Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Tue, 23 Jun 2026 12:27:59 -0700 Subject: [PATCH 53/54] test(ocr): cover compiled-extension load and Python fallback paths Adds two tests so the Rust bridge module hits 100% and the ocr() fallback-to-Python branch is exercised: - load_rust_ocr() returning the compiled extension's ocr callable - ocr() degrading to the HTTP handler when no bridge is available --- tests/test_litellm/ocr/test_rust_bridge.py | 33 ++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/test_litellm/ocr/test_rust_bridge.py b/tests/test_litellm/ocr/test_rust_bridge.py index 31443dec4bd2..7e028064e4cc 100644 --- a/tests/test_litellm/ocr/test_rust_bridge.py +++ b/tests/test_litellm/ocr/test_rust_bridge.py @@ -1,6 +1,8 @@ """Tests for the optional Rust-backed OCR path (``litellm/ocr/rust_bridge.py``).""" import importlib +import sys +import types import httpx import pytest @@ -135,6 +137,18 @@ def test_load_rust_ocr_none_when_extension_absent(): assert rust_bridge.load_rust_ocr() is None +def test_load_rust_ocr_uses_compiled_extension(monkeypatch): + """With no injected impl but a compiled ``litellm_python_bridge`` importable, + the loader returns the extension's ``ocr`` callable. The native wheel isn't + built in CI, so stand in a fake module via ``sys.modules``.""" + fake_module = types.ModuleType("litellm_python_bridge") + fake_module.ocr = lambda **kwargs: dict(FAKE_OCR_RESPONSE) # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, "litellm_python_bridge", fake_module) + + litellm.use_litellm_rust(True) # enabled, no impl injected -> import the extension + assert rust_bridge.load_rust_ocr() is fake_module.ocr + + def test_timeout_to_seconds_handles_float_timeout_and_none(): assert ocr_main._timeout_to_seconds(12.5) == 12.5 assert ocr_main._timeout_to_seconds(None) is None @@ -298,3 +312,22 @@ def test_ocr_does_not_route_to_rust_when_disabled(): # The impl stays available for injection, but the disabled flag gates usage, # so ocr() never reaches the Rust path (asserted via the enabled-path test). assert bridge.calls == [] + + +def test_ocr_falls_back_to_python_when_bridge_unavailable(monkeypatch): + """Rust enabled but no bridge available (no injected impl, no compiled wheel): + ocr() must degrade to the Python HTTP handler instead of raising.""" + litellm.use_litellm_rust(True) # enabled, but load_rust_ocr() returns None in CI + + captured = {} + + def fake_handler_ocr(**kwargs): + captured["called"] = True + return OCRResponse(pages=[], model="mistral-ocr-latest", object="ocr") + + monkeypatch.setattr(ocr_main.base_llm_http_handler, "ocr", fake_handler_ocr) + + response = litellm.ocr(model=MODEL, document=DOCUMENT, api_key="sk-test") + + assert captured.get("called") is True # Python path was used + assert isinstance(response, OCRResponse) From 8f305348fce06f608ac0b73d9c922af9798b6265 Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Tue, 23 Jun 2026 12:43:02 -0700 Subject: [PATCH 54/54] style(ocr): use PEP 604 X | None annotations in rust_bridge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Converts Optional[X]/Union[...] to the X | None form so the new OCR code stays under the UP045 strict-rule budget gate (lint job). Safe at runtime — the module already has 'from __future__ import annotations'. --- litellm/ocr/rust_bridge.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/litellm/ocr/rust_bridge.py b/litellm/ocr/rust_bridge.py index 60b0829b9373..61f9e8ca69ae 100644 --- a/litellm/ocr/rust_bridge.py +++ b/litellm/ocr/rust_bridge.py @@ -11,7 +11,7 @@ from __future__ import annotations -from typing import Final, Optional, Protocol, Union, cast +from typing import Final, Protocol, cast class RustOcr(Protocol): @@ -21,10 +21,10 @@ def __call__( self, model: str, document: dict[str, object], - api_key: Optional[str], - api_base: Optional[str], + api_key: str | None, + api_base: str | None, optional_params: dict[str, object], - timeout_seconds: Optional[float], + timeout_seconds: float | None, ) -> dict[str, object]: ... @@ -35,11 +35,11 @@ class _Unset: _UNSET: Final[_Unset] = _Unset() _rust_ocr_enabled = False -_rust_ocr_impl: Optional[RustOcr] = None +_rust_ocr_impl: RustOcr | None = None def use_litellm_rust( - enabled: bool = True, *, ocr: Union[Optional[RustOcr], _Unset] = _UNSET + enabled: bool = True, *, ocr: RustOcr | None | _Unset = _UNSET ) -> None: """Route supported OCR calls through the Rust ``litellm_python_bridge`` extension. @@ -58,7 +58,7 @@ def rust_ocr_enabled() -> bool: return _rust_ocr_enabled -def load_rust_ocr() -> Optional[RustOcr]: +def load_rust_ocr() -> RustOcr | None: """Return the Rust OCR callable, or ``None`` when no bridge is available. Prefers an injected implementation, otherwise loads the compiled