From a6c67c1fc8d5b2c60330d76d7ccfeca99b0a1f39 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Fri, 21 Aug 2026 15:05:55 -0700 Subject: [PATCH] feat(spend): surface per-request auto-router savings to logging callbacks The auto-router savings figure was computed only inside the spend-update writer, downstream of where logging callbacks consume the standard logging payload, so Datadog-style callbacks never received it. Compute it once in the payload builder, stamp it as a top-level payload field beside cost_breakdown, thread it into the spend log metadata, and have both spend-writer call sites read the recorded value with recomputation as the fallback for rows written before the field shipped. Internal sub-calls (classifier, shadow eval) are never stamped, and a caller-forged metadata value is discarded by the unconditional overwrite. Resolves LIT-5973 --- litellm/litellm_core_utils/litellm_logging.py | 45 +++++- litellm/proxy/_types.py | 3 +- litellm/proxy/db/db_spend_update_writer.py | 2 + litellm/proxy/spend_tracking/savings.py | 140 ++++++++++++++---- .../spend_tracking/spend_tracking_utils.py | 6 + litellm/types/utils.py | 1 + .../test_gcs_pub_sub.py | 1 + .../proxy/spend_tracking/test_savings.py | 119 +++++++++++++++ .../test_spend_management_endpoints.py | 1 + .../test_spend_tracking_utils.py | 35 +++++ 10 files changed, 326 insertions(+), 27 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 9b7707eabe1c..c14dd6c3d8b6 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -5615,6 +5615,37 @@ def _extract_response_obj_and_hidden_params( return response_obj, hidden_params +def _autorouter_savings_for_payload( + request_metadata: Mapping[str, object], + model: str | None, + custom_llm_provider: str | None, + model_id: str | None, + usage_object: Mapping[str, object] | None, + cost_breakdown: Mapping[str, object] | None, +) -> float | None: + """The auto-router savings figure for the payload, or ``None`` when there is none. + + Lazy proxy import: the savings module lives with the spend trackers that own the + math, and SDK-only installs have no proxy package to import. + """ + try: + from litellm.proxy.spend_tracking.savings import autorouter_savings_for_logging_payload + except Exception: # noqa: BLE001 # SDK-only install: no savings driver to run + return None + try: + return autorouter_savings_for_logging_payload( + request_metadata=request_metadata, + model=model, + custom_llm_provider=custom_llm_provider, + model_id=model_id, + usage_object=usage_object, + cost_breakdown=cost_breakdown, + ) + except Exception as e: # noqa: BLE001 # a savings figure must never fail request logging + verbose_logger.debug("autorouter savings skipped on logging payload: %s", e) + return None + + def get_standard_logging_object_payload( kwargs: dict | None, init_response_obj: Any | BaseModel | dict, @@ -5772,6 +5803,16 @@ def get_standard_logging_object_payload( ): model_name = response_model_name + request_cost_breakdown: Final = cost_breakdown_with_guardrail(logging_obj.cost_breakdown, guardrail_cost) + autorouter_savings: Final = _autorouter_savings_for_payload( + request_metadata=metadata, + model=model_name, + custom_llm_provider=custom_llm_provider, + model_id=_model_id, + usage_object=usage_dict, + cost_breakdown=request_cost_breakdown, + ) + payload: Final[StandardLoggingPayload] = StandardLoggingPayload( id=str(id), litellm_call_id=kwargs.get("litellm_call_id") or litellm_params.get("litellm_call_id"), @@ -5802,7 +5843,8 @@ def get_standard_logging_object_payload( metadata=clean_metadata, cache_key=clean_hidden_params["cache_key"], response_cost=response_cost, - cost_breakdown=cost_breakdown_with_guardrail(logging_obj.cost_breakdown, guardrail_cost), + cost_breakdown=request_cost_breakdown, + autorouter_savings=autorouter_savings, total_tokens=usage_dict.get("total_tokens", 0), prompt_tokens=usage_dict.get("prompt_tokens", 0), completion_tokens=usage_dict.get("completion_tokens", 0), @@ -5998,6 +6040,7 @@ def create_dummy_standard_logging_payload() -> StandardLoggingPayload: call_type="completion", stream=False, response_cost=response_cost, + autorouter_savings=None, response_cost_failure_debug_info=None, status="success", total_tokens=int(DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT + DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT), diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index de5e4628f54b..0840d37ffa11 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -15,7 +15,7 @@ field_validator, model_validator, ) -from typing_extensions import NotRequired, Required, TypedDict +from typing_extensions import NotRequired, ReadOnly, Required, TypedDict from litellm._uuid import uuid from litellm.constants import DEFAULT_STAGGER_WINDOW_SECONDS, MCP_STDIO_ALLOWED_COMMANDS @@ -3537,6 +3537,7 @@ class SpendLogsMetadata(TypedDict): max_retries: int | None # Max retries configured for this request cost_breakdown: CostBreakdown | None # Detailed cost breakdown (input_cost, output_cost, margin, discount, etc.) compression_savings: CompressionSavingsMetadata | None + autorouter_savings: ReadOnly[float | None] # stamped by the logging payload; None = not auto-routed class SpendLogsPayload(TypedDict): diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 65a271d40290..283194bad7cb 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -316,6 +316,7 @@ async def _enqueue_autorouter_turn_transaction( model_id=payload.get("model_id"), llm_router=_get_llm_router, cost_breakdown=metadata.get("cost_breakdown"), + recorded_autorouter_savings=metadata.get("autorouter_savings"), ) transaction: Final = build_autorouter_turn_transaction( payload=payload, @@ -1877,6 +1878,7 @@ async def _common_add_spend_log_transaction_to_daily_transaction( llm_router=_get_llm_router, usage_object=usage_obj, cost_breakdown=_metadata.get("cost_breakdown"), + recorded_autorouter_savings=_metadata.get("autorouter_savings"), ) daily_transaction: Final = BaseDailySpendTransaction( diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index 997180efdde9..b0f1546e15e0 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -13,6 +13,7 @@ import litellm from litellm._logging import verbose_proxy_logger +from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY from litellm.litellm_core_utils.llm_cost_calc.utils import _get_cost_per_unit, generic_cost_per_token if TYPE_CHECKING: @@ -437,6 +438,97 @@ def extract_cache_creation_tokens(usage_object: Mapping[str, object] | None) -> return int(written) +def _proxy_llm_router() -> "Router | None": + """The running proxy's router, or ``None`` outside a proxy (public rates only).""" + try: + from litellm.proxy.proxy_server import llm_router + except Exception: # noqa: BLE001 # SDK-only usage has no proxy module to import + return None + return llm_router + + +def _numeric_savings(value: object) -> float | None: + """``value`` as a recorded savings figure, or ``None`` when it is not one.""" + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + return float(value) + + +def autorouter_savings_for_request( + model: str | None, + custom_llm_provider: str | None, + routing_decision: Mapping[str, object] | None, + usage_object: Mapping[str, object] | None, + model_id: str | None = None, + llm_router: "Callable[[], Router | None] | None" = None, + cost_breakdown: Mapping[str, object] | None = None, +) -> float | None: + """Auto-router savings for one request, or ``None`` when the driver is off. + + ``None`` and ``0.0`` are different facts: ``None`` means this request cannot carry a + figure at all (no routing decision, no baseline, unusable usage), while ``0.0`` is a + real figure for a routed request whose baseline resolved to the served deployment. + Never raises: pricing failures inside degrade to zero, and the driver-off cases + return ``None``, so this is safe on the logging path where a raise would fail the + request's logging. + """ + usage: Final = _usage_from_spend_log(usage_object) + if usage is None or not model: + return None + # The configured `autorouter_savings_baseline_model` wins; otherwise the baseline + # the deciding router recorded on its decision; neither means the driver is off. + decision: Final = routing_decision if isinstance(routing_decision, Mapping) else {} + recorded: Final = decision.get("savings_baseline_model") + recorded_id: Final = decision.get("savings_baseline_deployment_id") + configured: Final = litellm.autorouter_savings_baseline_model + baseline_model: Final = configured or (recorded if isinstance(recorded, str) else None) + baseline_id: Final = recorded_id if configured is None and isinstance(recorded_id, str) else None + if not decision or not baseline_model: + return None + router_instance: Final = llm_router() if llm_router else None + return compute_autorouter_savings( + baseline_model=baseline_model, + selected_model=model, + selected_provider=custom_llm_provider, + usage=usage, + # Absent means the router never recorded a shape, which is the conservative + # reading: charge the cache write rather than claim a first turn's saving. + conversation_continuing=decision.get("conversation_continuing") is not False, + selected_info=_effective_model_info(router_instance, model_id, model or ""), + baseline_info=_effective_model_info(router_instance, baseline_id, baseline_model or ""), + cost_breakdown=cost_breakdown, + ) + + +def autorouter_savings_for_logging_payload( + request_metadata: Mapping[str, object], + model: str | None, + custom_llm_provider: str | None, + model_id: str | None, + usage_object: Mapping[str, object] | None, + cost_breakdown: Mapping[str, object] | None, +) -> float | None: + """The figure the logging payload records for a request, or ``None`` when none should be. + + Internal sub-calls (the auto-router classifier, shadow eval's shadow and judge legs) + are excluded here for the same reason the spend writer zeroes them: they can carry a + real routing decision, but they are not requests the caller made, so a figure stamped + on them would report savings for traffic no user sent. + """ + if request_metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY): + return None + routing_decision: Final = request_metadata.get("routing_decision") + return autorouter_savings_for_request( + model=model, + custom_llm_provider=custom_llm_provider, + routing_decision=routing_decision if isinstance(routing_decision, Mapping) else None, + usage_object=usage_object, + model_id=model_id, + llm_router=_proxy_llm_router, + cost_breakdown=cost_breakdown, + ) + + def compute_savings_spend( model: str | None, custom_llm_provider: str | None, @@ -446,6 +538,7 @@ def compute_savings_spend( model_id: str | None = None, llm_router: "Callable[[], Router | None] | None" = None, cost_breakdown: Mapping[str, object] | None = None, + recorded_autorouter_savings: object = None, ) -> SavingsSpend: """ Dollar savings for one request, split by optimization driver. @@ -488,6 +581,11 @@ def compute_savings_spend( hypothetical token delta off flat rate keys, so they are blind to tiered pricing in the same way; that is pre-existing behaviour on two shipped drivers rather than something introduced here, and moving those numbers is its own change. + + ``recorded_autorouter_savings`` is the figure the logging path stamped on the spend + log's metadata, honoured over recomputation so the rollup, the turn table and the + per-request record cannot disagree; rows written before the field shipped carry + nothing and recompute, mirroring ``_recorded_token_cost``. """ # Deployment rates when the request came through one, public rates otherwise -- # `_effective_model_info` merges a deployment's configured prices over the built-in @@ -505,32 +603,24 @@ def compute_savings_spend( write_premium: Final = max(cache_creation_input_tokens, 0) * (cache_write_cost - input_cost) prompt_caching: Final = read_discount - write_premium - usage: Final = _usage_from_spend_log(usage_object) - if usage is None or not model: - return SavingsSpend(compression=compression, prompt_caching=prompt_caching) - - # The configured `autorouter_savings_baseline_model` wins; otherwise the baseline - # the deciding router recorded on its decision; neither means the driver is off. - decision: Final = routing_decision if isinstance(routing_decision, Mapping) else {} - recorded: Final = decision.get("savings_baseline_model") - recorded_id: Final = decision.get("savings_baseline_deployment_id") - configured: Final = litellm.autorouter_savings_baseline_model - baseline_model: Final = configured or (recorded if isinstance(recorded, str) else None) - baseline_id: Final = recorded_id if configured is None and isinstance(recorded_id, str) else None + # The figure the logging path recorded wins, before the usage gate on purpose: a row + # whose usage no longer parses still carries the number computed when it did. + recorded_savings: Final = _numeric_savings(recorded_autorouter_savings) autorouter: Final = ( - compute_autorouter_savings( - baseline_model=baseline_model, - selected_model=model, - selected_provider=custom_llm_provider, - usage=usage, - # Absent means the router never recorded a shape, which is the conservative - # reading: charge the cache write rather than claim a first turn's saving. - conversation_continuing=decision.get("conversation_continuing") is not False, - selected_info=_effective_model_info(router_instance, model_id, model or ""), - baseline_info=_effective_model_info(router_instance, baseline_id, baseline_model or ""), + recorded_savings + if recorded_savings is not None + else autorouter_savings_for_request( + model=model, + custom_llm_provider=custom_llm_provider, + routing_decision=routing_decision, + usage_object=usage_object, + model_id=model_id, + llm_router=llm_router, cost_breakdown=cost_breakdown, ) - if decision and baseline_model - else 0.0 ) - return SavingsSpend(compression=compression, prompt_caching=prompt_caching, autorouter=autorouter) + return SavingsSpend( + compression=compression, + prompt_caching=prompt_caching, + autorouter=0.0 if autorouter is None else autorouter, + ) diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index cba1f9069d38..b6f695db5122 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -100,6 +100,7 @@ def _get_spend_logs_metadata( litellm_overhead_time_ms: float | None = None, cost_breakdown: CostBreakdown | None = None, litellm_call_id: str | None = None, + autorouter_savings: float | None = None, ) -> SpendLogsMetadata: if metadata is None: return SpendLogsMetadata( @@ -132,6 +133,7 @@ def _get_spend_logs_metadata( max_retries=None, cost_breakdown=None, compression_savings=None, + autorouter_savings=autorouter_savings, litellm_call_id=litellm_call_id, ) verbose_proxy_logger.debug( @@ -158,6 +160,7 @@ def _get_spend_logs_metadata( clean_metadata["cold_storage_object_key"] = cold_storage_object_key clean_metadata["litellm_overhead_time_ms"] = litellm_overhead_time_ms clean_metadata["cost_breakdown"] = cost_breakdown + clean_metadata["autorouter_savings"] = autorouter_savings clean_metadata["litellm_call_id"] = litellm_call_id return clean_metadata @@ -385,6 +388,9 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs cost_breakdown=( standard_logging_payload.get("cost_breakdown", None) if standard_logging_payload is not None else None ), + autorouter_savings=( + standard_logging_payload.get("autorouter_savings", None) if standard_logging_payload is not None else None + ), litellm_call_id=cast( str | None, kwargs.get("litellm_call_id") or litellm_params.get("litellm_call_id"), diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 8a71d209618f..c9d1f51e2463 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3193,6 +3193,7 @@ class StandardLoggingPayload(TypedDict): stream: bool | None response_cost: float cost_breakdown: CostBreakdown | None # Detailed cost breakdown + autorouter_savings: ReadOnly[float | None] # None = not an auto-routed caller request; 0.0 is a real figure response_cost_failure_debug_info: StandardLoggingModelCostFailureDebugInformation | None status: StandardLoggingPayloadStatus status_fields: StandardLoggingPayloadStatusFields diff --git a/tests/logging_callback_tests/test_gcs_pub_sub.py b/tests/logging_callback_tests/test_gcs_pub_sub.py index 2f6cdb631921..17322b965a73 100644 --- a/tests/logging_callback_tests/test_gcs_pub_sub.py +++ b/tests/logging_callback_tests/test_gcs_pub_sub.py @@ -42,6 +42,7 @@ "metadata.cold_storage_object_key", "metadata.litellm_overhead_time_ms", "metadata.cost_breakdown", + "metadata.autorouter_savings", "metadata.eval_information", ] diff --git a/tests/test_litellm/proxy/spend_tracking/test_savings.py b/tests/test_litellm/proxy/spend_tracking/test_savings.py index 9006288bdae9..227c824afd72 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_savings.py +++ b/tests/test_litellm/proxy/spend_tracking/test_savings.py @@ -1011,3 +1011,122 @@ def test_a_recorded_baseline_deployment_prices_at_its_configured_rate(): llm_router=lambda: router, ) assert with_deployment_rate.autorouter > at_public_rate.autorouter + + +def _routed_decision() -> dict: + return {"savings_baseline_model": "anthropic/claude-opus-5", "conversation_continuing": True} + + +def test_recorded_savings_win_over_recomputation(): + """The figure the logging path stamped is the one the rollup keeps, so the + per-request record and the daily rollup cannot disagree.""" + result = compute_savings_spend( + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + compression_saved_tokens=0, + routing_decision=_routed_decision(), + usage_object=_cached_usage_object(), + recorded_autorouter_savings=0.5, + ) + assert result.autorouter == 0.5 + + +def test_recorded_savings_survive_an_unusable_usage_object(): + """A recorded figure was computed when the usage still parsed; a later row whose + usage_object no longer does must keep the number, not zero it.""" + result = compute_savings_spend( + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + compression_saved_tokens=0, + routing_decision=_routed_decision(), + usage_object={"prompt_tokens": ["not", "a", "number"]}, + recorded_autorouter_savings=0.25, + ) + assert result.autorouter == 0.25 + + +def test_a_boolean_is_not_a_recorded_savings_figure(): + result = compute_savings_spend( + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + compression_saved_tokens=0, + routing_decision=None, + usage_object=_cached_usage_object(), + recorded_autorouter_savings=True, + ) + assert result.autorouter == 0.0 + + +def test_rows_written_before_the_field_shipped_recompute(): + """No recorded figure means the row predates the logging-path stamp; the writer + recomputes exactly what the one shared helper would have recorded.""" + from litellm.proxy.spend_tracking.savings import autorouter_savings_for_request + + recomputed = compute_savings_spend( + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + compression_saved_tokens=0, + routing_decision=_routed_decision(), + usage_object=_cached_usage_object(), + ) + direct = autorouter_savings_for_request( + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + routing_decision=_routed_decision(), + usage_object=_cached_usage_object(), + ) + assert direct is not None and direct != 0.0 + assert recomputed.autorouter == direct + + +def test_driver_off_is_none_not_zero_for_the_request_helper(): + """None and 0.0 are different facts on the logging payload: absence means the + request was never auto-routed, zero is a real figure for a routed request.""" + from litellm.proxy.spend_tracking.savings import autorouter_savings_for_request + + assert ( + autorouter_savings_for_request( + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + routing_decision=None, + usage_object=_cached_usage_object(), + ) + is None + ) + assert ( + autorouter_savings_for_request( + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + routing_decision={"conversation_continuing": True}, + usage_object=_cached_usage_object(), + ) + is None + ) + + +def test_logging_payload_never_stamps_internal_calls(): + """Shadow eval and classifier sub-calls carry a real routing decision but are not + requests the caller made; a stamped figure would report savings for traffic no + user sent, which the spend writer deliberately zeroes.""" + from litellm.proxy.spend_tracking.savings import autorouter_savings_for_logging_payload + + routed_metadata = {"routing_decision": _routed_decision()} + stamped = autorouter_savings_for_logging_payload( + request_metadata=routed_metadata, + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + model_id=None, + usage_object=_cached_usage_object(), + cost_breakdown=None, + ) + assert stamped is not None and stamped != 0.0 + + internal = autorouter_savings_for_logging_payload( + request_metadata={**routed_metadata, "internal_call_origin": "shadow_eval_shadow"}, + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + model_id=None, + usage_object=_cached_usage_object(), + cost_breakdown=None, + ) + assert internal is None diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index b2ec500d0458..24e209a25374 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -468,6 +468,7 @@ def test_ui_view_request_response_forbids_non_admin_without_db(client, monkeypat "metadata.additional_usage_values.iterations", "metadata.litellm_overhead_time_ms", "metadata.cost_breakdown", + "metadata.autorouter_savings", "metadata.user_api_key", "metadata.user_api_key_alias", "metadata.user_api_key_team_id", diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index f2dd66ee6775..e2c1a8357504 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -3569,3 +3569,38 @@ def test_redact_logged_api_key_bearer_sha256_without_flag_is_hashed(): assert result is not None assert result != already_hashed assert result == hash_token(already_hashed) + + +def test_autorouter_savings_flow_from_logging_payload_into_spend_log_metadata(): + """The figure the logging path computed is what the spend writer reads back, so it + is threaded from the StandardLoggingPayload like cost_breakdown, never re-derived.""" + payload = get_logging_payload( + kwargs={ + "model": "gpt-4o-mini", + "litellm_params": {"metadata": {"user_api_key": "test-key"}}, + "standard_logging_object": {"autorouter_savings": 0.42, "metadata": {}, "model_map_information": None}, + }, + response_obj=litellm.ModelResponse(id="chatcmpl-ar-savings", choices=[], usage=litellm.Usage()), + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + metadata = json.loads(payload["metadata"]) + assert metadata["autorouter_savings"] == 0.42 + + +@pytest.mark.parametrize("bucket", ["metadata", "litellm_metadata"]) +def test_caller_forged_autorouter_savings_is_discarded(bucket): + """The raw request bucket is client-writable and _get_spend_logs_metadata projects + every SpendLogsMetadata key from it, so the logging payload's value must overwrite + unconditionally or a caller could report savings the router never produced.""" + payload = get_logging_payload( + kwargs={ + "model": "gpt-4o-mini", + "litellm_params": {bucket: {"user_api_key": "test-key", "autorouter_savings": 999.0}}, + }, + response_obj=litellm.ModelResponse(id="chatcmpl-forged-savings", choices=[], usage=litellm.Usage()), + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + metadata = json.loads(payload["metadata"]) + assert metadata["autorouter_savings"] is None