From a23214f6c69c8c49de401eaa53412d3a5841859d Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 18 Jul 2026 20:54:11 +0000 Subject: [PATCH 1/3] feat(messages): route native Anthropic /messages through Rust behind RUST env var Co-Authored-By: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> --- .../ai-gateway/src/messages/common_utils.rs | 2 + .../crates/ai-gateway/src/messages/tests.rs | 54 +++++++++++++++++-- litellm/llms/custom_httpx/llm_http_handler.py | 9 +++- .../test_rust_bridge_messages.py | 53 +++++++++++++++++- 4 files changed, 111 insertions(+), 7 deletions(-) diff --git a/litellm-rust/crates/ai-gateway/src/messages/common_utils.rs b/litellm-rust/crates/ai-gateway/src/messages/common_utils.rs index fe4ac4cf26f0..bc4b13bfe3d3 100644 --- a/litellm-rust/crates/ai-gateway/src/messages/common_utils.rs +++ b/litellm-rust/crates/ai-gateway/src/messages/common_utils.rs @@ -1,5 +1,6 @@ use litellm_core::error::{json_type_name, CoreError}; use litellm_core::messages::transformation::AnthropicMessagesProviderConfig; +use litellm_core::providers::anthropic::messages::transformation::ANTHROPIC_MESSAGES_CONFIG; use litellm_core::providers::azure_ai::messages::transformation::AZURE_ANTHROPIC_MESSAGES_CONFIG; use litellm_core::CoreResult; use serde_json::{Map, Value}; @@ -19,6 +20,7 @@ pub(super) fn messages_provider_config( ) -> Option<&'static dyn AnthropicMessagesProviderConfig> { match provider { "azure_ai" => Some(&AZURE_ANTHROPIC_MESSAGES_CONFIG), + "anthropic" => Some(&ANTHROPIC_MESSAGES_CONFIG), _ => None, } } diff --git a/litellm-rust/crates/ai-gateway/src/messages/tests.rs b/litellm-rust/crates/ai-gateway/src/messages/tests.rs index 30f6642400ea..1175d800e1c7 100644 --- a/litellm-rust/crates/ai-gateway/src/messages/tests.rs +++ b/litellm-rust/crates/ai-gateway/src/messages/tests.rs @@ -52,9 +52,9 @@ fn write_response(body: &str) -> String { } #[test] -fn provider_config_only_resolves_azure_ai() { +fn provider_config_resolves_anthropic_and_azure_ai() { assert!(messages_provider_config("azure_ai").is_some()); - assert!(messages_provider_config("anthropic").is_none()); + assert!(messages_provider_config("anthropic").is_some()); assert!(messages_provider_config("openai").is_none()); } @@ -148,6 +148,52 @@ async fn messages_round_trip_builds_azure_request_and_passes_response_through() ); } +#[tokio::test] +async fn messages_round_trip_builds_native_anthropic_request() { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds"); + let addr = listener.local_addr().expect("addr"); + + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.expect("accepts request"); + let request = read_http_request(&mut socket).await; + let response_body = r#"{"id":"msg_1","type":"message","role":"assistant","content":[{"type":"text","text":"hi"}],"model":"claude-sonnet-4-5","stop_reason":"end_turn","usage":{"input_tokens":1,"output_tokens":2}}"#; + socket + .write_all(write_response(response_body).as_bytes()) + .await + .expect("writes response"); + request + }); + + let response = messages(MessagesRequest { + model: "claude-sonnet-4-5", + body: json!({ + "model": "claude-sonnet-4-5", + "max_tokens": 1024, + "messages": [{"role": "user", "content": "hi"}] + }), + api_key: Some("sk-ant"), + api_base: Some(&format!("http://{addr}")), + custom_llm_provider: Some("anthropic"), + extra_headers: None, + timeout: Some(Duration::from_secs(5)), + }) + .await + .expect("messages request succeeds"); + + assert_eq!(response["content"][0]["text"], "hi"); + assert_eq!(response["stop_reason"], "end_turn"); + + let request = server.await.expect("server task completes"); + let (head, _) = request.split_once("\r\n\r\n").expect("has body"); + assert!(head.starts_with("POST /v1/messages "), "{head}"); + let head_lower = head.to_ascii_lowercase(); + assert!(head_lower.contains("x-api-key: sk-ant"), "{head}"); + assert!( + head_lower.contains("anthropic-version: 2023-06-01"), + "{head}" + ); +} + #[tokio::test] async fn messages_does_not_duplicate_auth_when_x_api_key_supplied() { let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds"); @@ -248,12 +294,12 @@ async fn messages_rejects_unsupported_provider() { body: json!({"model": "claude-3-5-sonnet", "max_tokens": 8, "messages": []}), api_key: Some("sk"), api_base: Some("http://127.0.0.1:1"), - custom_llm_provider: Some("anthropic"), + custom_llm_provider: Some("openai"), extra_headers: None, timeout: Some(Duration::from_millis(50)), }) .await .expect_err("unsupported provider errors"); - assert!(matches!(err, CoreError::InvalidProvider(provider) if provider == "anthropic")); + assert!(matches!(err, CoreError::InvalidProvider(provider) if provider == "openai")); } diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 3e6f9ee08eeb..125f2c38da62 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -1,5 +1,6 @@ import asyncio import json +import os import ssl from functools import lru_cache from typing import ( @@ -2247,6 +2248,10 @@ async def _finalize_anthropic_messages_response( "anthropic_messages", ) + @staticmethod + def _rust_env_enabled() -> bool: + return os.getenv("RUST", "").strip().lower() in {"1", "true", "yes", "on"} + @staticmethod async def _maybe_rust_anthropic_messages( *, @@ -2261,7 +2266,9 @@ async def _maybe_rust_anthropic_messages( request_body: dict, timeout: float | httpx.Timeout | None, ) -> AnthropicMessagesResponse | None: - if custom_llm_provider != "azure_ai" or litellm_params.get("rust") is not True: + if custom_llm_provider not in ("azure_ai", "anthropic"): + return None + if litellm_params.get("rust") is not True and not BaseLLMHTTPHandler._rust_env_enabled(): return None if stream and not rust_stream_eligible: return None diff --git a/tests/test_litellm/anthropic_interface/test_rust_bridge_messages.py b/tests/test_litellm/anthropic_interface/test_rust_bridge_messages.py index 26ca7d272105..e5fa76cb3796 100644 --- a/tests/test_litellm/anthropic_interface/test_rust_bridge_messages.py +++ b/tests/test_litellm/anthropic_interface/test_rust_bridge_messages.py @@ -285,11 +285,60 @@ async def test_gate_skips_rust_when_flag_false(): @pytest.mark.asyncio -async def test_gate_skips_rust_for_non_azure_provider(): +async def test_gate_invokes_rust_for_native_anthropic_provider(): + bridge = RecordingAsyncMessages() + litellm.use_litellm_rust(True, amessages=bridge) + + response = await _gate( + custom_llm_provider="anthropic", + litellm_params=GenericLiteLLMParams(api_key="sk-ant", rust=True), + api_key="sk-ant", + api_base="https://api.anthropic.com", + headers={"x-api-key": "sk-ant", "anthropic-version": "2023-06-01"}, + ) + + assert response is not None + assert response["_hidden_params"]["additional_headers"] == {"x-litellm-rust": "true"} + assert bridge.calls[0]["custom_llm_provider"] == "anthropic" + assert bridge.calls[0]["api_key"] == "sk-ant" + + +@pytest.mark.asyncio +async def test_gate_invokes_rust_when_env_var_set(monkeypatch): + bridge = RecordingAsyncMessages() + litellm.use_litellm_rust(True, amessages=bridge) + monkeypatch.setenv("RUST", "1") + + response = await _gate( + custom_llm_provider="anthropic", + litellm_params=GenericLiteLLMParams(api_key="sk-ant"), + ) + + assert response is not None + assert bridge.calls[0]["custom_llm_provider"] == "anthropic" + + +@pytest.mark.asyncio +async def test_gate_env_var_falsey_does_not_enable(monkeypatch): + bridge = ExplodingAsyncMessages() + litellm.use_litellm_rust(True, amessages=bridge) + monkeypatch.setenv("RUST", "0") + + response = await _gate( + custom_llm_provider="anthropic", + litellm_params=GenericLiteLLMParams(api_key="sk-ant"), + ) + + assert response is None + assert bridge.calls == 0 + + +@pytest.mark.asyncio +async def test_gate_skips_rust_for_unsupported_provider(): bridge = ExplodingAsyncMessages() litellm.use_litellm_rust(True, amessages=bridge) - response = await _gate(custom_llm_provider="anthropic") + response = await _gate(custom_llm_provider="openai") assert response is None assert bridge.calls == 0 From a496ecade2e5ea217c8707e0ced84e1d9c6f2110 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 18 Jul 2026 21:19:43 +0000 Subject: [PATCH 2/3] test(docs): exclude RUST rollout flag from env-key documentation check Co-Authored-By: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> --- tests/documentation_tests/test_env_keys.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/documentation_tests/test_env_keys.py b/tests/documentation_tests/test_env_keys.py index 60fbd505d674..23ed1db54362 100644 --- a/tests/documentation_tests/test_env_keys.py +++ b/tests/documentation_tests/test_env_keys.py @@ -28,6 +28,7 @@ # environment settings docs until the feature is ready for broad use. EXCLUDED_ROLLOUT_FLAGS = { "LITELLM_USE_RUST_OCR", + "RUST", } EXCLUDED_TERMINAL_VARS = { From a461146af8b2ce7c0fe42cc4a930ea44c0a04e85 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 18 Jul 2026 22:12:51 +0000 Subject: [PATCH 3/3] refactor(messages): rename RUST rollout env var to LITELLM_RUST Co-Authored-By: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> --- litellm/llms/custom_httpx/llm_http_handler.py | 2 +- tests/documentation_tests/test_env_keys.py | 2 +- .../anthropic_interface/test_rust_bridge_messages.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 125f2c38da62..116ab905b88c 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -2250,7 +2250,7 @@ async def _finalize_anthropic_messages_response( @staticmethod def _rust_env_enabled() -> bool: - return os.getenv("RUST", "").strip().lower() in {"1", "true", "yes", "on"} + return os.getenv("LITELLM_RUST", "").strip().lower() in {"1", "true", "yes", "on"} @staticmethod async def _maybe_rust_anthropic_messages( diff --git a/tests/documentation_tests/test_env_keys.py b/tests/documentation_tests/test_env_keys.py index 23ed1db54362..3bf2c88a848a 100644 --- a/tests/documentation_tests/test_env_keys.py +++ b/tests/documentation_tests/test_env_keys.py @@ -28,7 +28,7 @@ # environment settings docs until the feature is ready for broad use. EXCLUDED_ROLLOUT_FLAGS = { "LITELLM_USE_RUST_OCR", - "RUST", + "LITELLM_RUST", } EXCLUDED_TERMINAL_VARS = { diff --git a/tests/test_litellm/anthropic_interface/test_rust_bridge_messages.py b/tests/test_litellm/anthropic_interface/test_rust_bridge_messages.py index e5fa76cb3796..b745ca8eadf5 100644 --- a/tests/test_litellm/anthropic_interface/test_rust_bridge_messages.py +++ b/tests/test_litellm/anthropic_interface/test_rust_bridge_messages.py @@ -307,7 +307,7 @@ async def test_gate_invokes_rust_for_native_anthropic_provider(): async def test_gate_invokes_rust_when_env_var_set(monkeypatch): bridge = RecordingAsyncMessages() litellm.use_litellm_rust(True, amessages=bridge) - monkeypatch.setenv("RUST", "1") + monkeypatch.setenv("LITELLM_RUST", "1") response = await _gate( custom_llm_provider="anthropic", @@ -322,7 +322,7 @@ async def test_gate_invokes_rust_when_env_var_set(monkeypatch): async def test_gate_env_var_falsey_does_not_enable(monkeypatch): bridge = ExplodingAsyncMessages() litellm.use_litellm_rust(True, amessages=bridge) - monkeypatch.setenv("RUST", "0") + monkeypatch.setenv("LITELLM_RUST", "0") response = await _gate( custom_llm_provider="anthropic",