fix(proxy): classify /v1/messages pass-through streams as Anthropic on any host

litellm/proxy/pass_through_endpoints/pass_through_endpoints.py

@@ -407,7 +407,7 @@ def get_endpoint_type(url: str, custom_llm_provider: str | None = None) -> Endpo
             or ("streamRawPredict") in url
         ):
             return EndpointType.VERTEX_AI
-        elif parsed_url.hostname == "api.anthropic.com":
+        elif parsed_url.hostname == "api.anthropic.com" or parsed_url.path.removesuffix("/").endswith("/v1/messages"):
             return EndpointType.ANTHROPIC
         elif (
             parsed_url.hostname == "api.openai.com"

tests/integration/observability/test_arize_otel_v2_openinference_spans.py

@@ -2,6 +2,7 @@
 
 import asyncio
 import json
+import re
 import uuid
 from itertools import chain
 from pathlib import Path
@@ -898,10 +899,7 @@ def upstream(request: Request) -> Reply:
             request for request in rig.provider.drain() if request.method == "POST" and marker.encode() in request.body
         )
         assert len(forwarded) == 1, forwarded
-        if surface == "messages":
-            assert not second[2].get("x-litellm-cache-key"), second[2]
-        else:
-            assert second[2].get("x-litellm-cache-key"), second[2]
+        assert re.fullmatch(r"[0-9a-f]{64}", second[2].get("x-litellm-cache-key", "")), second[2]
         forwarded_body: Final = _json_object(forwarded[0].body)
         if surface in ("chat", "chat-stream"):
             assert "metadata" not in forwarded_body, forwarded[0]

tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py

@@ -104,6 +104,40 @@ def test_is_multipart():
     assert HttpPassThroughEndpointHelpers.is_multipart(request) is False
 
 
+@pytest.mark.parametrize(
+    ("url", "expected_endpoint_type"),
+    [
+        pytest.param("https://api.anthropic.com/v1/messages", EndpointType.ANTHROPIC, id="anthropic-host"),
+        pytest.param("http://127.0.0.1:8190/v1/messages", EndpointType.ANTHROPIC, id="local-messages-route"),
+        pytest.param(
+            "https://gateway.example.com/anthropic/v1/messages/",
+            EndpointType.ANTHROPIC,
+            id="alternate-host-messages-route",
+        ),
+        pytest.param(
+            "https://gateway.example.com/v1/messages/batches",
+            EndpointType.GENERIC,
+            id="generic-batches-route",
+        ),
+        pytest.param(
+            "https://gateway.example.com/v1/messages/count_tokens",
+            EndpointType.GENERIC,
+            id="generic-count-tokens-route",
+        ),
+        pytest.param("https://gateway.example.com/v1/other", EndpointType.GENERIC, id="generic-route"),
+        pytest.param(
+            "https://vertex.example.com/v1/models/x:streamRawPredict",
+            EndpointType.VERTEX_AI,
+            id="vertex-stream-raw-predict",
+        ),
+    ],
+)
+def test_get_endpoint_type_classifies_anthropic_messages_routes(
+    url: str, expected_endpoint_type: EndpointType
+) -> None:
+    assert HttpPassThroughEndpointHelpers.get_endpoint_type(url) == expected_endpoint_type
+
+
 # Test _build_request_files_from_upload_file
 @pytest.mark.asyncio
 async def test_build_request_files_from_upload_file():