diff --git a/litellm/constants.py b/litellm/constants.py index 1014b472c615..78bfc6501e82 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1297,6 +1297,7 @@ LITELLM_METADATA_FIELD = "litellm_metadata" OLD_LITELLM_METADATA_FIELD = "metadata" RETURN_RAW_MODEL_NAME_METADATA_KEY = "_complexity_router_return_raw_model_name" +INTERNAL_CALL_ORIGIN_METADATA_KEY = "internal_call_origin" LITELLM_TRUNCATED_PAYLOAD_FIELD = "litellm_truncated" LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE = ( "Truncation is a DB storage safeguard. " diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 9e98cb46b9a4..d85ad1734344 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -44,6 +44,7 @@ EmbeddingResponse, GenericBudgetConfigType, ImageResponse, + InternalCallOrigin, LiteLLMPydanticObjectBase, ModelResponse, ProviderField, @@ -3304,6 +3305,7 @@ class SpendLogsMetadata(TypedDict): mcp_tool_call_metadata: Optional[StandardLoggingMCPToolCall] vector_store_request_metadata: Optional[List[StandardLoggingVectorStoreRequest]] routing_decision: StandardLoggingRoutingDecision | None + internal_call_origin: InternalCallOrigin | None guardrail_information: Optional[List[StandardLoggingGuardrailInformation]] eval_information: Optional[Any] status: StandardLoggingPayloadStatus diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 673e73f72fbe..1fad1954dc42 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -13,7 +13,11 @@ import litellm from litellm._logging import verbose_logger, verbose_proxy_logger from litellm._service_logger import ServiceLogging -from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS, PRE_CALL_EXECUTED_GUARDRAILS_KEY +from litellm.constants import ( + INTERNAL_CALL_ORIGIN_METADATA_KEY, + LITELLM_PROXY_MASTER_KEY_ALIAS, + PRE_CALL_EXECUTED_GUARDRAILS_KEY, +) from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( iter_client_callback_metadata_dicts, @@ -199,6 +203,7 @@ def parse_cache_control(cache_control): "applied_policies", "policy_sources", "routing_decision", + INTERNAL_CALL_ORIGIN_METADATA_KEY, "standard_logging_object", "proxy_server_request", "secret_fields", diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index a6105b6dff9f..a6a67d575829 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -109,6 +109,7 @@ def _get_spend_logs_metadata( model_map_information=None, usage_object=None, guardrail_information=None, + internal_call_origin=None, eval_information=None, cold_storage_object_key=cold_storage_object_key, litellm_overhead_time_ms=None, diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 933c6d170cfc..72d80f968034 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -24,10 +24,11 @@ from pydantic import BaseModel from litellm._logging import verbose_router_logger -from litellm.constants import RETURN_RAW_MODEL_NAME_METADATA_KEY +from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY, RETURN_RAW_MODEL_NAME_METADATA_KEY from litellm.integrations.custom_logger import CustomLogger from litellm.llms.base_llm.base_utils import type_to_response_format_param from litellm.types.utils import ( + AUTOROUTER_CLASSIFIER_CALL_ORIGIN, ModelResponse, RoutingDecisionCause, StandardLoggingRoutingDecision, @@ -116,7 +117,12 @@ def _classifier_call_metadata(metadata: dict[str, Any] | None) -> dict[str, Any] k: _sanitize_user_api_key_auth(v) if k == "user_api_key_auth" else v for k, v in metadata.items() if k not in _BUDGET_RESERVATION_METADATA_KEYS - } + } | {INTERNAL_CALL_ORIGIN_METADATA_KEY: AUTOROUTER_CLASSIFIER_CALL_ORIGIN} + + +def _parent_session_kwargs(request_kwargs: Mapping[str, Any] | None) -> Mapping[str, Any]: + kwargs = request_kwargs or {} + return {k: kwargs[k] for k in ("litellm_session_id", "litellm_trace_id") if kwargs.get(k) is not None} def _effective_turn_off_message_logging(request_kwargs: Mapping[str, Any] | None) -> bool | None: @@ -569,6 +575,7 @@ async def _classify_with_llm( metadata=metadata, proxy_server_request=proxy_server_request, turn_off_message_logging=turn_off_message_logging, + **_parent_session_kwargs(request_kwargs), ) content = response.choices[0].message.content if not content: @@ -967,6 +974,7 @@ async def _semantic_tier_override(self, user_message: str, request_kwargs: dict) litellm_metadata=litellm_metadata, proxy_server_request=proxy_server_request, turn_off_message_logging=turn_off_message_logging, + **_parent_session_kwargs(request_kwargs), ) )[0] route_choice = await routelayer.acall(vector=query_vector) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 9df44c6202ce..eb6695866310 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2703,6 +2703,13 @@ class StandardLoggingRoutingDecisionTierBoundaries(TypedDict): ] +InternalCallOrigin = Literal["autorouter_classifier"] +"""Which internal litellm feature originated a billed sub-call, so a spend log row +records that it is not traffic the caller sent.""" + +AUTOROUTER_CLASSIFIER_CALL_ORIGIN: InternalCallOrigin = "autorouter_classifier" + + class StandardLoggingRoutingDecision(TypedDict, total=False): """Per-request provenance for a pre-routing strategy (auto-router) decision.""" diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index 795a99ec2667..aa20c3f6ed46 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -2396,7 +2396,7 @@ async def test_spend_logs_payload_e2e(self): "model": "gpt-4o", "user": "", "team_id": "", - "metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}', + "metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}', "cache_key": "Cache OFF", "spend": 0.00022500000000000002, "total_tokens": 30, @@ -2492,7 +2492,7 @@ async def test_spend_logs_payload_success_log_with_api_base(self, monkeypatch): "model": "claude-4-sonnet-20250514", "user": "", "team_id": "", - "metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}', + "metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}', "cache_key": "Cache OFF", "spend": 0.01383, "total_tokens": 2598, @@ -2586,7 +2586,7 @@ async def test_spend_logs_payload_success_log_with_router(self, monkeypatch): "model": "claude-4-sonnet-20250514", "user": "", "team_id": "", - "metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}', + "metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}', "cache_key": "Cache OFF", "spend": 0.01383, "total_tokens": 2598, diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index c6f2a6f1792c..9eb45c399dbf 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -2916,3 +2916,46 @@ def test_no_routing_decision_key_defaults_to_none_in_spend_log_metadata(): ) metadata = json.loads(payload["metadata"]) assert metadata["routing_decision"] is None + + +@pytest.mark.parametrize("bucket", ["metadata", "litellm_metadata"]) +def test_internal_call_origin_survives_into_spend_log_metadata(bucket): + """The origin is only useful if it reaches the row the Logs UI reads. + + _get_spend_logs_metadata projects onto SpendLogsMetadata.__annotations__, so an + undeclared key is dropped silently. Both buckets are covered because the resolver + returns litellm_metadata when present and metadata otherwise, and the classifier + sub-call populates whichever the parent route used. + """ + payload = get_logging_payload( + kwargs={ + "model": "gpt-4o-mini", + "litellm_params": { + bucket: { + "user_api_key": "test-key", + "internal_call_origin": "autorouter_classifier", + } + }, + }, + response_obj=litellm.ModelResponse(id="chatcmpl-classifier", choices=[], usage=litellm.Usage()), + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + metadata = json.loads(payload["metadata"]) + assert metadata["internal_call_origin"] == "autorouter_classifier" + + +def test_user_traffic_carries_no_internal_call_origin(): + """The negative class the badge depends on: an ordinary request must be + distinguishable from a classifier call, not merely unlabelled by accident.""" + payload = get_logging_payload( + kwargs={ + "model": "gpt-4o-mini", + "litellm_params": {"metadata": {"user_api_key": "test-key"}}, + }, + response_obj=litellm.ModelResponse(id="chatcmpl-user-traffic", choices=[], usage=litellm.Usage()), + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + metadata = json.loads(payload["metadata"]) + assert metadata["internal_call_origin"] is None diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index e8acd7e6b75f..bceefae3a9f2 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -672,6 +672,7 @@ async def test_add_litellm_data_to_request_strips_user_control_fields(): "applied_policies": ["spoofed-policy"], "policy_sources": {"spoofed-policy": "request"}, "routing_decision": {"cause": "forged", "routed_model": "spoofed"}, + "internal_call_origin": "autorouter_classifier", "_guardrail_pipelines": [{"name": "spoofed"}], "_pipeline_managed_guardrails": ["evaded"], "safe_user_metadata": "kept", @@ -714,6 +715,7 @@ async def test_add_litellm_data_to_request_strips_user_control_fields(): "applied_policies", "policy_sources", "routing_decision", + "internal_call_origin", "_guardrail_pipelines", "_pipeline_managed_guardrails", } diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index e734d8ec8766..ae4db76903df 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -1422,7 +1422,7 @@ async def test_aclassify_forwards_request_metadata_for_spend_tracking( request_metadata = {"user_api_key": "sk-abc", "user_api_key_team_id": "team-1"} await llm_complexity_router.aclassify("hi", request_kwargs={"litellm_metadata": request_metadata}) call_kwargs = mock_router_instance.acompletion.call_args.kwargs - assert call_kwargs["metadata"] == request_metadata + assert call_kwargs["metadata"] == {**request_metadata, "internal_call_origin": "autorouter_classifier"} @pytest.mark.asyncio async def test_aclassify_forwards_metadata_key_used_by_chat_completions( @@ -1440,7 +1440,7 @@ async def test_aclassify_forwards_metadata_key_used_by_chat_completions( request_metadata = {"user_api_key": "sk-abc", "user_api_key_team_id": "team-1"} await llm_complexity_router.aclassify("hi", request_kwargs={"metadata": request_metadata}) call_kwargs = mock_router_instance.acompletion.call_args.kwargs - assert call_kwargs["metadata"] == request_metadata + assert call_kwargs["metadata"] == {**request_metadata, "internal_call_origin": "autorouter_classifier"} @pytest.mark.asyncio async def test_aclassify_captures_request_body_in_proxy_server_request( @@ -1551,12 +1551,38 @@ async def test_aclassify_strips_budget_reservation_from_classifier_metadata( "user_api_key": "sk-abc", "user_api_key_team_id": "team-1", "user_api_key_auth": {"models": ["gpt-4o"]}, + "internal_call_origin": "autorouter_classifier", } assert request_metadata["user_api_key_auth"] == { "models": ["gpt-4o"], "budget_reservation": {"reserved_cost": 1.0}, } + @pytest.mark.asyncio + @pytest.mark.parametrize( + "parent_kwargs, expected", + [ + ({"litellm_trace_id": "trace-1"}, {"litellm_trace_id": "trace-1"}), + ({"litellm_session_id": "sess-1"}, {"litellm_session_id": "sess-1"}), + ( + {"litellm_session_id": "sess-1", "litellm_trace_id": "trace-1"}, + {"litellm_session_id": "sess-1", "litellm_trace_id": "trace-1"}, + ), + ({}, {}), + ], + ) + async def test_aclassify_chains_classifier_call_into_parent_session( + self, llm_complexity_router, mock_router_instance, parent_kwargs, expected + ): + """Without the parent's session identity the router mints a fresh trace id for the + sub-call, so the classifier's spend row lands in a session of its own and never + appears in the trace of the request that triggered it.""" + mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "SIMPLE"}')) + await llm_complexity_router.aclassify("hi", request_kwargs={"metadata": {}, **parent_kwargs}) + call_kwargs = mock_router_instance.acompletion.call_args.kwargs + for key in ("litellm_session_id", "litellm_trace_id"): + assert call_kwargs.get(key) == expected.get(key) + @pytest.mark.asyncio async def test_aclassify_falls_back_to_heuristic_on_llm_exception( self, llm_complexity_router, mock_router_instance @@ -1604,7 +1630,7 @@ async def test_pre_routing_hook_uses_llm_classifier_end_to_end(self, llm_complex assert result is not None assert result.model == "o1-preview" # REASONING tier model call_kwargs = mock_router_instance.acompletion.call_args.kwargs - assert call_kwargs["metadata"] == request_metadata + assert call_kwargs["metadata"] == {**request_metadata, "internal_call_origin": "autorouter_classifier"} class TestRouterPreRoutingAliasOverrides: @@ -2281,8 +2307,9 @@ async def test_semantic_embedding_call_carries_caller_metadata(self, basic_confi ) assert result is not None assert fake_router.async_embedding_kwargs, "expected an embedding call for the prompt" - assert fake_router.async_embedding_kwargs[0]["metadata"] == caller_metadata - assert fake_router.async_embedding_kwargs[0]["litellm_metadata"] == caller_litellm_metadata + origin = {"internal_call_origin": "autorouter_classifier"} + assert fake_router.async_embedding_kwargs[0]["metadata"] == {**caller_metadata, **origin} + assert fake_router.async_embedding_kwargs[0]["litellm_metadata"] == {**caller_litellm_metadata, **origin} @pytest.mark.asyncio async def test_semantic_embedding_call_captures_request_body_in_proxy_server_request(self, basic_config): @@ -2391,6 +2418,7 @@ async def test_semantic_embedding_call_strips_budget_reservation(self, basic_con "user_api_key_hash": "hash-abc", "user_api_key_team_id": "team-1", "user_api_key_auth": {"models": ["voyage-3-5"]}, + "internal_call_origin": "autorouter_classifier", } assert fake_router.async_embedding_kwargs[0]["metadata"] == expected assert fake_router.async_embedding_kwargs[0]["litellm_metadata"] == expected @@ -2726,15 +2754,46 @@ def test_cost_callback_cannot_recover_reservation_from_sanitized_metadata(self): assert sanitized["user_api_key_auth"] is not None assert _get_budget_reservation_from_metadata(sanitized) is None - def test_returns_empty_dict_for_missing_metadata(self): + def test_absent_parent_bucket_stays_empty(self): + """An absent bucket must not be materialized just to carry the origin. + + The embedding path passes both buckets, and get_litellm_metadata_from_kwargs + prefers litellm_metadata whenever it is truthy, backfilling only user_api_key* + keys from metadata. Returning an origin-only dict here would make a chat + completions parent's empty litellm_metadata win and silently drop + requester_ip_address, tags and spend_logs_metadata from the classifier's row.""" from litellm.router_strategy.complexity_router.complexity_router import ( _classifier_call_metadata, ) for absent in (None, {}): - result = _classifier_call_metadata(absent) - assert result == {} - assert isinstance(result, dict) + assert _classifier_call_metadata(absent) == {} + + def test_classifier_buckets_keep_non_spend_fields_on_a_chat_completions_parent(self): + """Drives the real resolver over the buckets the embedding classifier builds.""" + from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs + from litellm.router_strategy.complexity_router.complexity_router import ( + _classifier_call_metadata, + ) + + parent = { + "user_api_key": "sk-abc", + "requester_ip_address": "10.0.0.1", + "spend_logs_metadata": {"team_note": "keep me"}, + "tags": ["prod"], + } + resolved = get_litellm_metadata_from_kwargs( + { + "litellm_params": { + "metadata": _classifier_call_metadata(parent), + "litellm_metadata": _classifier_call_metadata(None), + } + } + ) + assert resolved["internal_call_origin"] == "autorouter_classifier" + assert resolved["requester_ip_address"] == "10.0.0.1" + assert resolved["spend_logs_metadata"] == {"team_note": "keep me"} + assert resolved["tags"] == ["prod"] def test_sanitized_auth_keeps_access_group_fields_and_leaves_original_untouched(self): from litellm.proxy._types import UserAPIKeyAuth