fix(proxy): classify /v1/messages pass-through streams as Anthropic on any host

litellm/llms/anthropic/common_utils.py

@@ -9,7 +9,7 @@
 from datetime import datetime, timezone
 from types import MappingProxyType
 from typing import Any, ClassVar, Final, Literal, TypeVar
-from urllib.parse import quote
+from urllib.parse import quote, urlparse
 
 import httpx
 from pydantic import ConfigDict, Field, StrictBool, TypeAdapter, ValidationError
@@ -113,6 +113,12 @@ def anthropic_error_frame_exception(error_type: str, message: str, status_code:
 _CLAUDE_CODE_USER_AGENT_PREFIXES: Final = ("claude-cli/", "claude-code/")
 
 
+def is_anthropic_messages_url(url: str) -> bool:
+    """Check whether a URL addresses Anthropic's Messages API."""
+    parsed_url: Final = urlparse(url)
+    return parsed_url.hostname == "api.anthropic.com" or parsed_url.path.removesuffix("/").endswith("/v1/messages")
+
+
 def requires_native_compaction_beta(
     custom_llm_provider: str,
     optional_params: Mapping[str, object],

litellm/proxy/pass_through_endpoints/pass_through_endpoints.py

@@ -66,6 +66,7 @@
 from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
 from litellm.litellm_core_utils.redact_messages import should_redact_message_logging
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
+from litellm.llms.anthropic.common_utils import is_anthropic_messages_url
 from litellm.llms.base_llm.managed_resources.utils import (
     resolve_passthrough_managed_id_provider,
 )
@@ -407,7 +408,7 @@ def get_endpoint_type(url: str, custom_llm_provider: str | None = None) -> Endpo
             or ("streamRawPredict") in url
         ):
             return EndpointType.VERTEX_AI
-        elif parsed_url.hostname == "api.anthropic.com":
+        elif is_anthropic_messages_url(url):
             return EndpointType.ANTHROPIC
         elif (
             parsed_url.hostname == "api.openai.com"

tests/integration/observability/test_arize_otel_v2_openinference_spans.py

@@ -2,6 +2,7 @@
 
 import asyncio
 import json
+import re
 import uuid
 from itertools import chain
 from pathlib import Path
@@ -898,10 +899,7 @@ def upstream(request: Request) -> Reply:
             request for request in rig.provider.drain() if request.method == "POST" and marker.encode() in request.body
         )
         assert len(forwarded) == 1, forwarded
-        if surface == "messages":
-            assert not second[2].get("x-litellm-cache-key"), second[2]
-        else:
-            assert second[2].get("x-litellm-cache-key"), second[2]
+        assert re.fullmatch(r"[0-9a-f]{64}", second[2].get("x-litellm-cache-key", "")), second[2]
         forwarded_body: Final = _json_object(forwarded[0].body)
         if surface in ("chat", "chat-stream"):
             assert "metadata" not in forwarded_body, forwarded[0]

tests/unit/llms/anthropic/test_anthropic_common_utils.py

@@ -32,6 +32,23 @@
 FAKE_AUTH_TOKEN = "sk-ant-aut01-fake-auth-token-for-testing-123456789"
 
 
+@pytest.mark.parametrize(
+    ("url", "expected"),
+    [
+        pytest.param("https://api.anthropic.com/v1/messages", True, id="anthropic-host"),
+        pytest.param("http://127.0.0.1:8190/v1/messages", True, id="local-messages-route"),
+        pytest.param("https://gateway.example.com/anthropic/v1/messages/", True, id="alternate-host-messages-route"),
+        pytest.param("https://gateway.example.com/v1/messages/batches", False, id="generic-batches-route"),
+        pytest.param("https://gateway.example.com/v1/messages/count_tokens", False, id="generic-count-tokens-route"),
+        pytest.param("https://gateway.example.com/v1/other", False, id="generic-route"),
+    ],
+)
+def test_is_anthropic_messages_url(url: str, expected: bool) -> None:
+    from litellm.llms.anthropic.common_utils import is_anthropic_messages_url
+
+    assert is_anthropic_messages_url(url) is expected
+
+
 @pytest.mark.parametrize(
     "messages,system,expected",
     [

tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py

@@ -104,6 +104,40 @@ def test_is_multipart():
     assert HttpPassThroughEndpointHelpers.is_multipart(request) is False
 
 
+@pytest.mark.parametrize(
+    ("url", "expected_endpoint_type"),
+    [
+        pytest.param("https://api.anthropic.com/v1/messages", EndpointType.ANTHROPIC, id="anthropic-host"),
+        pytest.param("http://127.0.0.1:8190/v1/messages", EndpointType.ANTHROPIC, id="local-messages-route"),
+        pytest.param(
+            "https://gateway.example.com/anthropic/v1/messages/",
+            EndpointType.ANTHROPIC,
+            id="alternate-host-messages-route",
+        ),
+        pytest.param(
+            "https://gateway.example.com/v1/messages/batches",
+            EndpointType.GENERIC,
+            id="generic-batches-route",
+        ),
+        pytest.param(
+            "https://gateway.example.com/v1/messages/count_tokens",
+            EndpointType.GENERIC,
+            id="generic-count-tokens-route",
+        ),
+        pytest.param("https://gateway.example.com/v1/other", EndpointType.GENERIC, id="generic-route"),
+        pytest.param(
+            "https://vertex.example.com/v1/models/x:streamRawPredict",
+            EndpointType.VERTEX_AI,
+            id="vertex-stream-raw-predict",
+        ),
+    ],
+)
+def test_get_endpoint_type_classifies_anthropic_messages_routes(
+    url: str, expected_endpoint_type: EndpointType
+) -> None:
+    assert HttpPassThroughEndpointHelpers.get_endpoint_type(url) == expected_endpoint_type
+
+
 # Test _build_request_files_from_upload_file
 @pytest.mark.asyncio
 async def test_build_request_files_from_upload_file():