Skip to content
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -369,6 +369,11 @@ def get_endpoint_type(url: str) -> EndpointType:
or (parsed_url.hostname and "openai.com" in parsed_url.hostname)
):
return EndpointType.OPENAI
route: Final = parsed_url.path.rstrip("/")
if route.endswith("/v1/messages"):
return EndpointType.ANTHROPIC
elif route.endswith("/v1/chat/completions"):
return EndpointType.OPENAI
return EndpointType.GENERIC

@staticmethod
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5837,3 +5837,95 @@ def test_passthrough_client_cannot_forge_session_id_omission(client_metadata_key
)
== "per-call-random-trace-id"
)


ANTHROPIC_SSE_STREAM = (
"event: message_start",
'data: {"type":"message_start","message":{"id":"msg_01","type":"message","role":"assistant",'
'"model":"claude-sonnet-4-5-20250929","content":[],"stop_reason":null,"stop_sequence":null,'
'"usage":{"input_tokens":17,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,'
'"output_tokens":5}}}',
"event: content_block_start",
'data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}',
"event: content_block_delta",
'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hello there."}}',
"event: content_block_stop",
'data: {"type":"content_block_stop","index":0}',
"event: message_delta",
'data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},'
'"usage":{"output_tokens":40}}',
"event: message_stop",
'data: {"type":"message_stop"}',
)


def _log_passthrough_stream(url: str):
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
from datetime import datetime

from litellm.proxy.pass_through_endpoints.streaming_handler import (
PassThroughStreamingHandler,
)

logging_obj = MagicMock(spec=LiteLLMLoggingObj)
logging_obj.model_call_details = {}
logging_obj.optional_params = {}
logging_obj.litellm_call_id = "test-call-id"

return PassThroughStreamingHandler._build_passthrough_logging_result(
litellm_logging_obj=logging_obj,
passthrough_success_handler_obj=PassThroughEndpointLogging(),
url_route=url,
request_body={"model": "claude-sonnet-4-5-20250929"},
endpoint_type=HttpPassThroughEndpointHelpers.get_endpoint_type(url),
start_time=datetime.now(),
raw_bytes=[(line + "\n").encode("utf-8") for line in ANTHROPIC_SSE_STREAM],
end_time=datetime.now(),
model="claude-sonnet-4-5-20250929",
)


def test_anthropic_compatible_passthrough_stream_is_billed_off_api_anthropic_com():
"""
Regression for #40117: the endpoint type was resolved from the hostname alone, so a
stream from a provider that speaks the Anthropic Messages API on any other host fell
through to GENERIC, matched no branch in the streaming logging dispatch, and was
recorded as "cannot parse chunks to standard response object" with no usage and no
cost. The same bytes must produce the same accounting whatever host served them.
"""
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
anthropic_result, anthropic_kwargs = _log_passthrough_stream("https://api.anthropic.com/v1/messages")
custom_result, custom_kwargs = _log_passthrough_stream("https://my-provider.example.com/v1/messages")

assert custom_kwargs["response_cost"] == anthropic_kwargs["response_cost"]
assert custom_kwargs["response_cost"] > 0
assert custom_result.usage.prompt_tokens == anthropic_result.usage.prompt_tokens == 17
assert custom_result.usage.completion_tokens == anthropic_result.usage.completion_tokens == 40


def test_openai_compatible_passthrough_route_resolves_off_api_openai_com():
from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType

assert (
HttpPassThroughEndpointHelpers.get_endpoint_type("https://my-provider.example.com/v1/chat/completions")
== EndpointType.OPENAI
)
assert (
HttpPassThroughEndpointHelpers.get_endpoint_type("https://my-provider.example.com/openai/v1/chat/completions/")
== EndpointType.OPENAI
)


def test_passthrough_routes_outside_the_known_shapes_stay_generic():
"""
The path fallback must only claim the two canonical routes. Sending a body the
Anthropic or OpenAI handler cannot parse into that handler drops the log row
entirely, which is the failure mode LIT-4527 fixed for Vertex.
"""
from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType

for url in (
"https://my-provider.example.com/v1/messages/count_tokens",
"https://my-provider.example.com/v1/messages/batches",
"https://my-provider.example.com/v1/embeddings",
"https://upstream.example.com/ml/api/v1/time-series-forecast/predict",
):
assert HttpPassThroughEndpointHelpers.get_endpoint_type(url) == EndpointType.GENERIC
Loading