Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions litellm-rust/crates/ai-gateway/src/messages/common_utils.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
use litellm_core::error::{json_type_name, CoreError};
use litellm_core::messages::transformation::AnthropicMessagesProviderConfig;
use litellm_core::providers::anthropic::messages::transformation::ANTHROPIC_MESSAGES_CONFIG;
use litellm_core::providers::azure_ai::messages::transformation::AZURE_ANTHROPIC_MESSAGES_CONFIG;
use litellm_core::CoreResult;
use serde_json::{Map, Value};
Expand All @@ -19,6 +20,7 @@ pub(super) fn messages_provider_config(
) -> Option<&'static dyn AnthropicMessagesProviderConfig> {
match provider {
"azure_ai" => Some(&AZURE_ANTHROPIC_MESSAGES_CONFIG),
"anthropic" => Some(&ANTHROPIC_MESSAGES_CONFIG),
_ => None,
}
}
Expand Down
54 changes: 50 additions & 4 deletions litellm-rust/crates/ai-gateway/src/messages/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -52,9 +52,9 @@ fn write_response(body: &str) -> String {
}

#[test]
fn provider_config_only_resolves_azure_ai() {
fn provider_config_resolves_anthropic_and_azure_ai() {
assert!(messages_provider_config("azure_ai").is_some());
assert!(messages_provider_config("anthropic").is_none());
assert!(messages_provider_config("anthropic").is_some());
assert!(messages_provider_config("openai").is_none());
}

Expand Down Expand Up @@ -148,6 +148,52 @@ async fn messages_round_trip_builds_azure_request_and_passes_response_through()
);
}

#[tokio::test]
async fn messages_round_trip_builds_native_anthropic_request() {
let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds");
let addr = listener.local_addr().expect("addr");

let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.expect("accepts request");
let request = read_http_request(&mut socket).await;
let response_body = r#"{"id":"msg_1","type":"message","role":"assistant","content":[{"type":"text","text":"hi"}],"model":"claude-sonnet-4-5","stop_reason":"end_turn","usage":{"input_tokens":1,"output_tokens":2}}"#;
socket
.write_all(write_response(response_body).as_bytes())
.await
.expect("writes response");
request
});

let response = messages(MessagesRequest {
model: "claude-sonnet-4-5",
body: json!({
"model": "claude-sonnet-4-5",
"max_tokens": 1024,
"messages": [{"role": "user", "content": "hi"}]
}),
api_key: Some("sk-ant"),
api_base: Some(&format!("http://{addr}")),
custom_llm_provider: Some("anthropic"),
extra_headers: None,
timeout: Some(Duration::from_secs(5)),
})
.await
.expect("messages request succeeds");

assert_eq!(response["content"][0]["text"], "hi");
assert_eq!(response["stop_reason"], "end_turn");

let request = server.await.expect("server task completes");
let (head, _) = request.split_once("\r\n\r\n").expect("has body");
assert!(head.starts_with("POST /v1/messages "), "{head}");
let head_lower = head.to_ascii_lowercase();
assert!(head_lower.contains("x-api-key: sk-ant"), "{head}");
assert!(
head_lower.contains("anthropic-version: 2023-06-01"),
"{head}"
);
}

#[tokio::test]
async fn messages_does_not_duplicate_auth_when_x_api_key_supplied() {
let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds");
Expand Down Expand Up @@ -248,12 +294,12 @@ async fn messages_rejects_unsupported_provider() {
body: json!({"model": "claude-3-5-sonnet", "max_tokens": 8, "messages": []}),
api_key: Some("sk"),
api_base: Some("http://127.0.0.1:1"),
custom_llm_provider: Some("anthropic"),
custom_llm_provider: Some("openai"),
extra_headers: None,
timeout: Some(Duration::from_millis(50)),
})
.await
.expect_err("unsupported provider errors");

assert!(matches!(err, CoreError::InvalidProvider(provider) if provider == "anthropic"));
assert!(matches!(err, CoreError::InvalidProvider(provider) if provider == "openai"));
}
9 changes: 8 additions & 1 deletion litellm/llms/custom_httpx/llm_http_handler.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import asyncio
import json
import os
import ssl
from functools import lru_cache
from typing import (
Expand Down Expand Up @@ -2247,6 +2248,10 @@ async def _finalize_anthropic_messages_response(
"anthropic_messages",
)

@staticmethod
def _rust_env_enabled() -> bool:
return os.getenv("LITELLM_RUST", "").strip().lower() in {"1", "true", "yes", "on"}

@staticmethod
async def _maybe_rust_anthropic_messages(
*,
Expand All @@ -2261,7 +2266,9 @@ async def _maybe_rust_anthropic_messages(
request_body: dict,
timeout: float | httpx.Timeout | None,
) -> AnthropicMessagesResponse | None:
if custom_llm_provider != "azure_ai" or litellm_params.get("rust") is not True:
if custom_llm_provider not in ("azure_ai", "anthropic"):
return None
if litellm_params.get("rust") is not True and not BaseLLMHTTPHandler._rust_env_enabled():
return None
if stream and not rust_stream_eligible:
return None
Expand Down
1 change: 1 addition & 0 deletions tests/documentation_tests/test_env_keys.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
# environment settings docs until the feature is ready for broad use.
EXCLUDED_ROLLOUT_FLAGS = {
"LITELLM_USE_RUST_OCR",
"LITELLM_RUST",
}

EXCLUDED_TERMINAL_VARS = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -285,11 +285,60 @@ async def test_gate_skips_rust_when_flag_false():


@pytest.mark.asyncio
async def test_gate_skips_rust_for_non_azure_provider():
async def test_gate_invokes_rust_for_native_anthropic_provider():
bridge = RecordingAsyncMessages()
litellm.use_litellm_rust(True, amessages=bridge)

response = await _gate(
custom_llm_provider="anthropic",
litellm_params=GenericLiteLLMParams(api_key="sk-ant", rust=True),
api_key="sk-ant",
api_base="https://api.anthropic.com",
headers={"x-api-key": "sk-ant", "anthropic-version": "2023-06-01"},
)

assert response is not None
assert response["_hidden_params"]["additional_headers"] == {"x-litellm-rust": "true"}
assert bridge.calls[0]["custom_llm_provider"] == "anthropic"
assert bridge.calls[0]["api_key"] == "sk-ant"


@pytest.mark.asyncio
async def test_gate_invokes_rust_when_env_var_set(monkeypatch):
bridge = RecordingAsyncMessages()
litellm.use_litellm_rust(True, amessages=bridge)
monkeypatch.setenv("LITELLM_RUST", "1")

response = await _gate(
custom_llm_provider="anthropic",
litellm_params=GenericLiteLLMParams(api_key="sk-ant"),
)

assert response is not None
assert bridge.calls[0]["custom_llm_provider"] == "anthropic"


@pytest.mark.asyncio
async def test_gate_env_var_falsey_does_not_enable(monkeypatch):
bridge = ExplodingAsyncMessages()
litellm.use_litellm_rust(True, amessages=bridge)
monkeypatch.setenv("LITELLM_RUST", "0")

response = await _gate(
custom_llm_provider="anthropic",
litellm_params=GenericLiteLLMParams(api_key="sk-ant"),
)

assert response is None
assert bridge.calls == 0


@pytest.mark.asyncio
async def test_gate_skips_rust_for_unsupported_provider():
bridge = ExplodingAsyncMessages()
litellm.use_litellm_rust(True, amessages=bridge)

response = await _gate(custom_llm_provider="anthropic")
response = await _gate(custom_llm_provider="openai")

assert response is None
assert bridge.calls == 0
Expand Down
Loading