From a028f6cf8cc719005b961293e2501cb48b0296bd Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Sat, 1 Aug 2026 16:04:05 -0700 Subject: [PATCH 01/15] feat(spend): add net auto-router savings to the cost-optimization dashboard The dashboard credited compression and prompt caching but said nothing about the optimization that picks the model, so the driver with the largest lever on a bill was the one an operator could not see. Savings are the counterfactual: without a router a deployment runs one model, and it has to be one that can carry the hardest request, so the baseline is the priciest model in the router's hardest configured tier. A cheap tier is a choice the router made, not a ceiling it was bounded by. `auto_router_savings_baseline_model` overrides it for operators who would genuinely have run something else. Both are provider-qualified before pricing, because a bare name can resolve to a different vendor's rates or to nothing at all, and a deployment is priced by its `base_model` where it has one, which is how Azure deployments are priced everywhere else. Both arms price the request's real usage through `generic_cost_per_token` rather than re-deriving per-token arithmetic, so tiered rates, ephemeral cache-write tiers and regional uplifts stay consistent with what was actually billed. `prompt_tokens` already includes the cache buckets, so charging them again at the input rate would price the same tokens twice. Cache state is what makes this hard. The baseline serves every turn, so whether it had the prompt cached is whether the conversation was already underway. On a continuing conversation it wrote the prompt earlier and would only read it now, so this request's write is what switching cost and counts against the saving. On a first turn nothing was cached for any model, the baseline would have written the same prompt, and both arms carry the write at their own rates. Charging the write to both cases understates a first turn to a few percent of its value, and because the write premium is fixed by prompt size while the saving grows with completion length, it can render a profitable route as a loss. That shape is read off the conversation rather than remembered: a second human ask means an earlier turn was served. No cache, no session id, and no dependence on a caller sending a session header. It cannot see a switch on a turn the router did not classify, and it reads a few-shot prompt's synthetic turns as prior conversation; both err toward charging the write, which under-claims. The baseline and the shape ride on the existing `routing_decision` record, which is already carried from the router to the spend log, already classified for redaction, and already written-or-cleared per attempt. A fallback that re-enters the hook therefore cannot leave either fact behind to be attributed to a deployment that never routed, and no new metadata key crosses the trust boundary. The result is signed. Whether a switch pays off is a race between the rate gap and the cache-write cost, and a narrow gap loses; flooring at zero would hide exactly the routing behaviour an operator needs to see. The donut plots only drivers that saved, while the card and range total keep the sign. Savings accrue into a new `autorouter_savings_spend` column on the six daily rollup tables, declared `NotRequired` because rows queued by a pod on the previous release carry no such key. It is summed by the rollup merge the cross-pod Redis drain also runs, and carried through the aggregation query, the per-row accumulation and the response model, so the dashboard reads a value the API actually sends. Tests enumerate the drivers from the response model itself and assert each is summed, accumulated, carried and totalled, so one added later cannot be half-wired. --- .../migration.sql | 17 + .../litellm_proxy_extras/schema.prisma | 6 + litellm/proxy/_types.py | 7 +- litellm/proxy/db/db_spend_update_writer.py | 73 ++-- .../daily_spend_update_queue.py | 4 + .../common_daily_activity.py | 9 + litellm/proxy/schema.prisma | 6 + litellm/proxy/spend_tracking/savings.py | 177 ++++++++- litellm/router.py | 1 + .../complexity_router/complexity_router.py | 76 +++- litellm/router_strategy/savings_baseline.py | 115 ++++++ .../common_daily_activity.py | 3 + litellm/types/router.py | 1 + litellm/types/utils.py | 5 + schema.prisma | 6 + .../test_daily_spend_update_queue.py | 60 +++ .../test_common_daily_activity.py | 63 +++- .../proxy/spend_tracking/test_savings.py | 349 +++++++++++++++++- .../router_strategy/test_complexity_router.py | 168 +++++++++ .../router_strategy/test_savings_baseline.py | 155 ++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 14 + 21 files changed, 1266 insertions(+), 49 deletions(-) create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260731000000_add_autorouter_savings_spend/migration.sql create mode 100644 litellm/router_strategy/savings_baseline.py create mode 100644 tests/test_litellm/router_strategy/test_savings_baseline.py diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260731000000_add_autorouter_savings_spend/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260731000000_add_autorouter_savings_spend/migration.sql new file mode 100644 index 000000000000..a7efd444fdb0 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260731000000_add_autorouter_savings_spend/migration.sql @@ -0,0 +1,17 @@ +-- AlterTable +ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 37ea55f8c132..0d7fa8692c8b 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -748,6 +748,7 @@ model LiteLLM_DailyUserSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -782,6 +783,7 @@ model LiteLLM_DailyOrganizationSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -816,6 +818,7 @@ model LiteLLM_DailyEndUserSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -849,6 +852,7 @@ model LiteLLM_DailyAgentSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -882,6 +886,7 @@ model LiteLLM_DailyTeamSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -917,6 +922,7 @@ model LiteLLM_DailyTagSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 3ccf3ea9952a..2aa1e22fff3c 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -14,7 +14,7 @@ field_validator, model_validator, ) -from typing_extensions import Required, TypedDict +from typing_extensions import NotRequired, Required, TypedDict from litellm._uuid import uuid from litellm.constants import MCP_STDIO_ALLOWED_COMMANDS @@ -4564,6 +4564,11 @@ class BaseDailySpendTransaction(TypedDict): # cost-savings metrics (dollars, priced per request before aggregation) compression_savings_spend: float prompt_caching_savings_spend: float + # Not required: rows queued by a pod running the previous release, or replayed from + # the Redis buffer across an upgrade, carry no such key. Every reader coalesces a + # missing value to zero, so requiring it here would describe a shape the aggregation + # is explicitly tested against. + autorouter_savings_spend: NotRequired[float] # request level metrics spend: float diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 17410698aed3..b5a41614df72 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -1545,6 +1545,30 @@ async def _update_daily_spend( # Get the table dynamically table = getattr(batcher, table_name) + # Additive metrics that older queued rows may omit; one + # enumeration feeds both the create and the increment below + optional_metrics = { + field: value + for field, value in ( + ("cache_read_input_tokens", transaction.get("cache_read_input_tokens")), + ( + "cache_creation_input_tokens", + transaction.get("cache_creation_input_tokens"), + ), + ("compression_saved_tokens", transaction.get("compression_saved_tokens")), + ( + "compression_savings_spend", + transaction.get("compression_savings_spend"), + ), + ( + "prompt_caching_savings_spend", + transaction.get("prompt_caching_savings_spend"), + ), + ("autorouter_savings_spend", transaction.get("autorouter_savings_spend")), + ) + if value is not None + } + # Common data structure for both create and update common_data = { entity_id_field: entity_id, @@ -1561,30 +1585,9 @@ async def _update_daily_spend( "api_requests": transaction["api_requests"], "successful_requests": transaction["successful_requests"], "failed_requests": transaction["failed_requests"], + **optional_metrics, } - # Add cache-related fields if they exist - if "cache_read_input_tokens" in transaction: - common_data["cache_read_input_tokens"] = transaction.get( - "cache_read_input_tokens", 0 - ) - if "cache_creation_input_tokens" in transaction: - common_data["cache_creation_input_tokens"] = transaction.get( - "cache_creation_input_tokens", 0 - ) - if "compression_saved_tokens" in transaction: - common_data["compression_saved_tokens"] = transaction.get( - "compression_saved_tokens", 0 - ) - if "compression_savings_spend" in transaction: - common_data["compression_savings_spend"] = transaction.get( - "compression_savings_spend", 0 - ) - if "prompt_caching_savings_spend" in transaction: - common_data["prompt_caching_savings_spend"] = transaction.get( - "prompt_caching_savings_spend", 0 - ) - if entity_type == "tag" and "request_id" in transaction: common_data["request_id"] = transaction.get("request_id") @@ -1596,30 +1599,9 @@ async def _update_daily_spend( "api_requests": {"increment": transaction["api_requests"]}, "successful_requests": {"increment": transaction["successful_requests"]}, "failed_requests": {"increment": transaction["failed_requests"]}, + **{field: {"increment": value} for field, value in optional_metrics.items()}, } - # Add cache-related fields to update if they exist - if "cache_read_input_tokens" in transaction: - update_data["cache_read_input_tokens"] = { - "increment": transaction.get("cache_read_input_tokens", 0) - } - if "cache_creation_input_tokens" in transaction: - update_data["cache_creation_input_tokens"] = { - "increment": transaction.get("cache_creation_input_tokens", 0) - } - if "compression_saved_tokens" in transaction: - update_data["compression_saved_tokens"] = { - "increment": transaction.get("compression_saved_tokens", 0) - } - if "compression_savings_spend" in transaction: - update_data["compression_savings_spend"] = { - "increment": transaction.get("compression_savings_spend", 0) - } - if "prompt_caching_savings_spend" in transaction: - update_data["prompt_caching_savings_spend"] = { - "increment": transaction.get("prompt_caching_savings_spend", 0) - } - if entity_type == "tag" and "request_id" in transaction: update_data["request_id"] = transaction.get("request_id") @@ -1875,6 +1857,8 @@ async def _common_add_spend_log_transaction_to_daily_transaction( custom_llm_provider=payload.get("custom_llm_provider", None), compression_saved_tokens=compression_saved_tokens, cache_read_input_tokens=cache_read_input_tokens, + routing_decision=_metadata.get("routing_decision"), + usage_object=usage_obj, ) daily_transaction = BaseDailySpendTransaction( @@ -1896,6 +1880,7 @@ async def _common_add_spend_log_transaction_to_daily_transaction( compression_saved_tokens=compression_saved_tokens, compression_savings_spend=savings_spend.compression, prompt_caching_savings_spend=savings_spend.prompt_caching, + autorouter_savings_spend=savings_spend.autorouter, ) return daily_transaction except Exception as e: diff --git a/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py b/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py index 18df25093f11..a61db1283823 100644 --- a/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py +++ b/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py @@ -133,6 +133,10 @@ def get_aggregated_daily_spend_update_transactions( payload.get("prompt_caching_savings_spend", 0) or 0 ) + daily_transaction.get("prompt_caching_savings_spend", 0) + daily_transaction["autorouter_savings_spend"] = ( + payload.get("autorouter_savings_spend", 0) or 0 + ) + daily_transaction.get("autorouter_savings_spend", 0) + else: aggregated_daily_spend_update_transactions[_key] = deepcopy(payload) return aggregated_daily_spend_update_transactions diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index 9bd8db16769d..d8e6b854336a 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -95,6 +95,9 @@ def compression_savings_spend(self) -> float: ... @property def prompt_caching_savings_spend(self) -> float: ... + @property + def autorouter_savings_spend(self) -> float: ... + @property def api_requests(self) -> int: ... @@ -135,6 +138,7 @@ class _GroupingSetsRow(SimpleNamespace): compression_saved_tokens: int | None compression_savings_spend: float | None prompt_caching_savings_spend: float | None + autorouter_savings_spend: float | None api_requests: int | None successful_requests: int | None failed_requests: int | None @@ -158,6 +162,7 @@ def update_metrics(existing_metrics: SpendMetrics, record: DailySpendRecord) -> existing_metrics.compression_saved_tokens += record.compression_saved_tokens or 0 existing_metrics.compression_savings_spend += record.compression_savings_spend or 0 existing_metrics.prompt_caching_savings_spend += record.prompt_caching_savings_spend or 0 + existing_metrics.autorouter_savings_spend += record.autorouter_savings_spend or 0 existing_metrics.api_requests += record.api_requests or 0 existing_metrics.successful_requests += record.successful_requests or 0 existing_metrics.failed_requests += record.failed_requests or 0 @@ -590,6 +595,7 @@ def _build_aggregated_sql_query( SUM(compression_saved_tokens)::bigint AS compression_saved_tokens, SUM(compression_savings_spend)::float AS compression_savings_spend, SUM(prompt_caching_savings_spend)::float AS prompt_caching_savings_spend, + SUM(autorouter_savings_spend)::float AS autorouter_savings_spend, SUM(api_requests)::bigint AS api_requests, SUM(successful_requests)::bigint AS successful_requests, SUM(failed_requests)::bigint AS failed_requests @@ -732,6 +738,7 @@ def _record_to_spend_metrics(record: _GroupingSetsRow) -> SpendMetrics: compression_saved_tokens=record.compression_saved_tokens or 0, compression_savings_spend=record.compression_savings_spend or 0, prompt_caching_savings_spend=record.prompt_caching_savings_spend or 0, + autorouter_savings_spend=record.autorouter_savings_spend or 0, api_requests=record.api_requests or 0, successful_requests=record.successful_requests or 0, failed_requests=record.failed_requests or 0, @@ -986,6 +993,7 @@ async def get_daily_activity( total_compression_saved_tokens=metadata_metrics.compression_saved_tokens, total_compression_savings_spend=metadata_metrics.compression_savings_spend, total_prompt_caching_savings_spend=metadata_metrics.prompt_caching_savings_spend, + total_autorouter_savings_spend=metadata_metrics.autorouter_savings_spend, page=page, total_pages=-(-total_count // page_size), # Ceiling division has_more=(page * page_size) < total_count, @@ -1075,6 +1083,7 @@ async def get_daily_activity_aggregated( total_compression_saved_tokens=aggregated["totals"].compression_saved_tokens, total_compression_savings_spend=aggregated["totals"].compression_savings_spend, total_prompt_caching_savings_spend=aggregated["totals"].prompt_caching_savings_spend, + total_autorouter_savings_spend=aggregated["totals"].autorouter_savings_spend, page=1, total_pages=1, has_more=False, diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 37ea55f8c132..0d7fa8692c8b 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -748,6 +748,7 @@ model LiteLLM_DailyUserSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -782,6 +783,7 @@ model LiteLLM_DailyOrganizationSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -816,6 +818,7 @@ model LiteLLM_DailyEndUserSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -849,6 +852,7 @@ model LiteLLM_DailyAgentSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -882,6 +886,7 @@ model LiteLLM_DailyTeamSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -917,6 +922,7 @@ model LiteLLM_DailyTagSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index ad9d02052bbb..3f554982ca4e 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -8,15 +8,19 @@ have been aggregated across models. """ +from collections.abc import Mapping from typing import NamedTuple import litellm from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.types.utils import PromptTokensDetailsWrapper, Usage class SavingsSpend(NamedTuple): compression: float prompt_caching: float + autorouter: float = 0.0 def _input_and_cache_read_cost(model: str | None, custom_llm_provider: str | None) -> tuple[float, float]: @@ -44,11 +48,162 @@ def _input_and_cache_read_cost(model: str | None, custom_llm_provider: str | Non return input_cost, float(cache_read_cost) +class _ModelIdentity(NamedTuple): + model: str + provider: str + + +def _resolve_model(model: str | None, custom_llm_provider: str | None) -> _ModelIdentity | None: + """Canonical ``(model, provider)``, or ``None`` when the model cannot be resolved. + + The two sides of the comparison arrive spelled differently: the spend log records a + normalized model name alongside its provider, while the baseline arrives as the + operator wrote it in config, with the provider prefixed, implied, or absent. Raw + string equality therefore reads `anthropic/claude-opus-5` as a switch away from + `claude-opus-5`, and pricing a bare name with no provider can resolve it to a + different vendor's rates than the deployment it names. + """ + if not model: + return None + try: + resolved_model, provider, _, _ = litellm.get_llm_provider(model=model, custom_llm_provider=custom_llm_provider) + except Exception as e: # noqa: BLE001 # get_llm_provider raises for unroutable names; degrade to zero savings + verbose_proxy_logger.debug( + "savings: cannot resolve provider for model=%s custom_llm_provider=%s (%s)", model, custom_llm_provider, e + ) + return None + return _ModelIdentity(model=resolved_model, provider=provider) + + +def _cost_of_usage(model: _ModelIdentity, usage: Usage) -> float | None: + """What ``usage`` costs on ``model``, or ``None`` when the model has no pricing.""" + try: + prompt_cost, completion_cost = generic_cost_per_token( + model=model.model, usage=usage, custom_llm_provider=model.provider + ) + except Exception as e: # noqa: BLE001 # get_model_info raises bare Exception for unmapped models; degrade to zero savings + verbose_proxy_logger.debug( + "savings: cannot price usage for provider=%s model=%s (%s)", model.provider, model.model, e + ) + return None + return prompt_cost + completion_cost + + +def _cache_token_split(usage: Usage) -> tuple[int, int]: + """``(cache_read_tokens, cache_creation_tokens)`` for a request.""" + details = usage.prompt_tokens_details + if details is None: + return 0, 0 + read = getattr(details, "cached_tokens", 0) or 0 + created = (getattr(details, "cache_creation_tokens", 0) or 0) or (getattr(details, "cache_write_tokens", 0) or 0) + return int(read), int(created) + + +_CACHE_SPLIT_FIELDS = frozenset( + ("cached_tokens", "cache_creation_tokens", "cache_write_tokens", "cache_creation_token_details", "text_tokens") +) + + +def _baseline_usage(usage: Usage, conversation_continuing: bool) -> Usage: + """The same request as a single-model baseline would have met it. + + The baseline is one model serving every turn, so whether it had this prompt cached + is simply whether the conversation was already underway. On a continuing + conversation it wrote the prompt on an earlier turn and would only read it now, so + the cache tokens move into the read bucket and whatever this request paid to write + counts against the saving; that write is what switching models costs. + + On a conversation's first turn nothing was cached anywhere, for any model. The + baseline would have written the same prompt, so the usage passes through untouched + and both arms carry the write at their own rates. Charging the write to this case + too, which is all a single rollup row can support, understates a first turn to a + few percent of its value and can render a profitable route as a loss. + + Only the cache buckets move. Every other field the request was priced on travels + through untouched, audio and image and video counts among them, because the baseline + is this same request served by a model that happened to be warm; naming the fields to + keep instead would price the baseline on a request that never ran, and would go stale + the next time a priced field is added. + """ + cache_read, cache_creation = _cache_token_split(usage) + details = usage.prompt_tokens_details + if details is None or cache_creation <= 0 or not conversation_continuing: + return usage + return Usage( + prompt_tokens=usage.prompt_tokens, + completion_tokens=usage.completion_tokens, + total_tokens=usage.total_tokens, + completion_tokens_details=usage.completion_tokens_details, + prompt_tokens_details=PromptTokensDetailsWrapper( + **details.model_dump(exclude=_CACHE_SPLIT_FIELDS), + # The tokens this request paid to write are moved into the cached count and + # the creation charge is dropped: on one model that cache was already warm, + # so the baseline would have read them rather than paying to create them. + # The 5m/1h breakdown goes with it; left behind it re-charges the write. + cached_tokens=cache_read + cache_creation, + cache_creation_tokens=0, + cache_write_tokens=0, + cache_creation_token_details=None, + text_tokens=max(usage.prompt_tokens - cache_read - cache_creation, 0), + ), + ) + + +def compute_autorouter_savings( + baseline_model: str | None, + selected_model: str | None, + selected_provider: str | None, + usage: Usage, + conversation_continuing: bool = True, +) -> float: + """Net dollars the router saved, or cost, by serving this request on ``selected_model``. + + Signed on purpose. Switching models leaves the new one with a cold cache, so the + request pays a cache-creation charge that staying on one model would not have + incurred; when that charge outweighs the cheaper rates, routing lost money and the + dashboard has to be able to say so. Zero when both sides resolve to the same + deployment, or when either cannot be resolved or priced. + + ``conversation_continuing`` says whether the baseline would already have had this + prompt cached. It defaults to True because that is the conservative reading: a + request whose shape the router could not determine is charged the write and + under-claims rather than inflating a savings figure. + """ + # No provider argument for the baseline on purpose: it arrives from the routing + # metadata as a single self-describing string, already qualified by the auto-router, + # so there is no second field that could disagree with it. + baseline = _resolve_model(baseline_model, None) + selected = _resolve_model(selected_model, selected_provider) + if baseline is None or selected is None or baseline == selected: + return 0.0 + baseline_cost = _cost_of_usage(baseline, _baseline_usage(usage, conversation_continuing)) + selected_cost = _cost_of_usage(selected, usage) + if baseline_cost is None or selected_cost is None: + return 0.0 + return baseline_cost - selected_cost + + +def _usage_from_spend_log(usage_object: Mapping[str, object] | None) -> Usage | None: + """Rebuild the request's ``Usage`` from the copy the spend log recorded.""" + if not usage_object: + return None + try: + return Usage(**usage_object) + except Exception as e: # noqa: BLE001 # a malformed usage_object must not fail the daily spend write + # Warning, not debug: this silently zeroes the auto-router driver for every + # affected row, and a shape change in Usage would otherwise show up only as a + # dashboard that quietly reads $0.00. + verbose_proxy_logger.warning("savings: unusable usage_object, auto-router savings will read zero (%s)", e) + return None + + def compute_savings_spend( model: str | None, custom_llm_provider: str | None, compression_saved_tokens: int, cache_read_input_tokens: int, + routing_decision: Mapping[str, object] | None = None, + usage_object: Mapping[str, object] | None = None, ) -> SavingsSpend: """ Dollar savings for one request, split by optimization driver. @@ -56,8 +211,28 @@ def compute_savings_spend( Compression savings price the tokens compression removed at the model's input rate. Prompt-caching savings price the cache-read tokens at the difference between the input rate and the discounted cache-read rate. + Auto-router savings compare the served ``model`` against the counterfactual + baseline the router recorded on its ``routing_decision``, and are zero unless the + two differ. That record also says whether the conversation was already underway, + which is what tells a mid-conversation switch from a first turn. """ input_cost, cache_read_cost = _input_and_cache_read_cost(model, custom_llm_provider) compression = max(compression_saved_tokens, 0) * input_cost prompt_caching = max(cache_read_input_tokens, 0) * max(input_cost - cache_read_cost, 0.0) - return SavingsSpend(compression=compression, prompt_caching=prompt_caching) + + usage = _usage_from_spend_log(usage_object) + if usage is None or not model: + return SavingsSpend(compression=compression, prompt_caching=prompt_caching) + + decision = routing_decision if isinstance(routing_decision, Mapping) else {} + baseline_model = decision.get("savings_baseline_model") + autorouter = compute_autorouter_savings( + baseline_model=baseline_model if isinstance(baseline_model, str) else None, + selected_model=model, + selected_provider=custom_llm_provider, + usage=usage, + # Absent means the router never recorded a shape, which is the conservative + # reading: charge the cache write rather than claim a first turn's saving. + conversation_continuing=decision.get("conversation_continuing") is not False, + ) + return SavingsSpend(compression=compression, prompt_caching=prompt_caching, autorouter=autorouter) diff --git a/litellm/router.py b/litellm/router.py index 37190bdbf38a..24fde45f3349 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -7673,6 +7673,7 @@ def init_complexity_router_deployment(self, deployment: Deployment): default_model=default_model, litellm_router_instance=self, complexity_router_config=complexity_router_config, + savings_baseline_model=deployment.litellm_params.auto_router_savings_baseline_model, ) self._register_pre_routing_strategy( registry=self.complexity_routers, diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 6f2bf61834bf..8b75131ee165 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -224,6 +224,33 @@ def _iter_human_asks_newest_first(messages: Sequence[Mapping[str, object]]) -> I ) +def _conversation_is_continuing(messages: Sequence[Mapping[str, object]] | None) -> bool: + """Whether this request continues a conversation that was already underway. + + The counterfactual the savings driver prices against is one model serving every + turn, so whether that model had this prompt cached is just whether an earlier turn + exists: a second human ask means it wrote the prompt then and would only read it + now, and the write this request paid is what switching models cost. A single ask is + a conversation's first turn, where nothing was cached for any model and the baseline + would have paid the same write. + + Reading the conversation rather than remembering it keeps this free of a cache, a + session id and their failure modes, and it works for callers that send no session + header at all. It cannot see a model switch that happened to land on a turn the + router did not classify, and it reads a few-shot prompt's synthetic turns as prior + conversation; both err toward charging the write, which under-claims. + + Defaults to continuing when the messages cannot be read, for the same reason. + """ + if not messages: + return True + asks = len(tuple(islice(_iter_human_asks_newest_first(messages), 2))) + # Exactly one human ask is a first turn. Zero means the turns carry no readable ask, + # which says nothing about the baseline's cache and so is treated like every other + # unknown here: charge the write. + return asks != 1 + + def _newest_turn_ask(messages: Sequence[Mapping[str, object]]) -> str | None: """The human ask on the newest user turn, or None when that turn carries only plumbing. @@ -365,6 +392,7 @@ def __init__( litellm_router_instance: Router, complexity_router_config: dict[str, Any] | None = None, default_model: str | None = None, + savings_baseline_model: str | None = None, ): """ Initialize ComplexityRouter. @@ -377,6 +405,7 @@ def __init__( """ self.model_name = model_name self.litellm_router_instance = litellm_router_instance + self.configured_savings_baseline_model: str | None = savings_baseline_model # Parse config - always create a new instance to avoid singleton mutation if complexity_router_config: @@ -424,6 +453,36 @@ def __init__( verbose_router_logger.debug(f"ComplexityRouter initialized for {model_name} with tiers: {self.config.tiers}") + def _hardest_tier_models(self) -> tuple[str, ...]: + """The model or pool serving the hardest tier this router configures. + + REASONING when it is configured, since that is the tier a request has to be + hard enough to reach; otherwise the highest-severity tier present, so a + deployment that only defines SIMPLE and MEDIUM is still measured against + the best it could actually have picked. + """ + for tier in reversed(TIER_SEVERITY_ORDER): + models = self.config.tiers.get(tier.value) + if models: + return tuple(models) if isinstance(models, list) else (models,) + return () + + @property + def savings_baseline_model(self) -> str | None: + """The model this router's savings are measured against. + + A complexity router's tier ladder already names the model an operator + would have had to run to serve the hardest request, so the counterfactual + is the priciest model in that tier rather than the priciest model the + router can reach; a cheap tier is a choice the router made, not a ceiling + it was bounded by. + """ + from litellm.router_strategy.savings_baseline import resolve_baseline + + return resolve_baseline( + self.configured_savings_baseline_model, self.litellm_router_instance, self._hardest_tier_models() + ) + def _estimate_tokens(self, text: str) -> int: """ Estimate token count from text. @@ -647,6 +706,7 @@ def _build_routing_decision( escalation_keyword: str | None = None, escalated: bool = False, classifier_model: str | None = None, + conversation_continuing: bool = True, ) -> StandardLoggingRoutingDecision: """Assemble the per-request provenance record for this router's decision. @@ -660,7 +720,11 @@ def _build_routing_decision( router_type="complexity", routed_model=routed_model, cause=cause, + conversation_continuing=conversation_continuing, ) + baseline_model = self.savings_baseline_model + if baseline_model is not None: + decision["savings_baseline_model"] = baseline_model if tier is not None: decision["tier"] = tier.value if score is not None: @@ -1392,6 +1456,8 @@ async def async_pre_routing_hook( if isinstance(metadata, dict): metadata[RETURN_RAW_MODEL_NAME_METADATA_KEY] = True + conversation_continuing = _conversation_is_continuing(self._resolve_messages(messages, request_kwargs)) + use_session_affinity = self.config.session_affinity and not self.config.plugins session_id = self._get_session_id_from_request_kwargs(request_kwargs) if use_session_affinity else None cache_key = self._get_session_affinity_cache_key(session_id, request_kwargs) if session_id is not None else None @@ -1438,6 +1504,7 @@ async def async_pre_routing_hook( cause=cause, escalation_keyword=pin_escalation_keyword, escalated=escalated, + conversation_continuing=conversation_continuing, ), ) @@ -1482,6 +1549,7 @@ async def _classify_and_route( from litellm.types.router import PreRoutingHookResponse resolved_messages = self._resolve_messages(messages, request_kwargs) + conversation_continuing = _conversation_is_continuing(resolved_messages) if not resolved_messages: verbose_router_logger.debug("ComplexityRouter: No messages could be resolved, skipping routing") @@ -1509,7 +1577,11 @@ async def _classify_and_route( return PreRoutingHookResponse( model=routed_model, messages=messages if has_original_messages else None, - routing_decision=self._build_routing_decision(routed_model=routed_model, cause="default_fallback"), + routing_decision=self._build_routing_decision( + routed_model=routed_model, + cause="default_fallback", + conversation_continuing=conversation_continuing, + ), ) newest_ask = _newest_turn_ask(resolved_messages) @@ -1532,6 +1604,7 @@ async def _classify_and_route( messages=messages if has_original_messages else None, routing_decision=self._build_routing_decision( routed_model=routed_model, + conversation_continuing=conversation_continuing, cause=keyword_cause, tier=routed_tier, matched_keyword=override.matched_keyword, @@ -1579,6 +1652,7 @@ async def _classify_and_route( messages=messages if has_original_messages else None, routing_decision=self._build_routing_decision( routed_model=routed_model, + conversation_continuing=conversation_continuing, cause=outcome.cause, tier=tier, score=score, diff --git a/litellm/router_strategy/savings_baseline.py b/litellm/router_strategy/savings_baseline.py new file mode 100644 index 000000000000..900774eff16f --- /dev/null +++ b/litellm/router_strategy/savings_baseline.py @@ -0,0 +1,115 @@ +"""The counterfactual model a complexity router's savings are measured against. + +Without the router a deployment has to pick one model, and it has to be one that can +carry the hardest request it will see. That model is the baseline: what the traffic +would have cost had nobody routed it. The router's own tier ladder already names it, +so the candidates are the models in the hardest configured tier; a cheap tier is a +choice the router made, not a ceiling it was bounded by. + +Baselines are always provider-qualified, whether derived or configured, because they +travel to the spend writer as a bare string with no provider beside them; an operator +who writes ``deepseek-r1`` meaning Azure would otherwise be priced against whoever +else owns that name. +""" + +from collections.abc import Iterable +from typing import TYPE_CHECKING + +from litellm._logging import verbose_router_logger + +if TYPE_CHECKING: + from litellm.router import Router + + +def canonical_model(model: str, custom_llm_provider: str | None = None) -> str | None: + """``provider/model``, or ``None`` when the pair names no known provider. + + A deployment may name its vendor in the model prefix or in a separate + ``custom_llm_provider``, and the bare name alone is not enough to price: it can + resolve to a different vendor's rates, or to nothing at all. + """ + import litellm + + try: + resolved, provider, _, _ = litellm.get_llm_provider(model=model, custom_llm_provider=custom_llm_provider) + except Exception as e: # noqa: BLE001 # an unroutable candidate cannot be the baseline + verbose_router_logger.debug("savings baseline: cannot resolve candidate %s (%s)", model, e) + return None + return f"{provider}/{resolved}" + + +def _models_in(router: "Router", group_name: str) -> tuple[str, ...]: + """The models a tier entry actually calls, priced as the deployments declare. + + `litellm_params.model` is not always a model: on Azure it is the deployment name, + absent from the cost map, and `model_info.base_model` names the real one. Wildcard + and aliased deployments behave the same, and router.py resolves pricing through + that same base_model chain. A name matching no deployment is a tier pointing + straight at a provider model rather than at a configured group. + """ + indices = router.model_name_to_deployment_indices.get(group_name) + if not indices: + return (qualified,) if (qualified := canonical_model(group_name)) else () + + def priced_as(index: int) -> str | None: + deployment = router.model_list[index] + params = deployment.get("litellm_params") + if not isinstance(params, dict): + return None + info = deployment.get("model_info") + base = info.get("base_model") if isinstance(info, dict) else None + model = base or params.get("base_model") or params.get("model") + return canonical_model(model, params.get("custom_llm_provider")) if model else None + + return tuple(model for index in indices if (model := priced_as(index))) + + +def _most_expensive(models: Iterable[str]) -> str | None: + """The priciest candidate by output rate, input rate breaking the tie. + + A model that costs nothing per token cannot stand in for what the traffic would + otherwise have cost; as a baseline it would report the whole real spend as a loss, + so it is dropped rather than allowed to win a tie at zero. + """ + import litellm + + def rates(model: str) -> tuple[float, float, str] | None: + try: + info = litellm.get_model_info(model=model) + except Exception as e: # noqa: BLE001 # unmapped candidates simply cannot be the baseline + verbose_router_logger.debug("savings baseline: no pricing for candidate %s (%s)", model, e) + return None + output_rate, input_rate = info.get("output_cost_per_token") or 0.0, info.get("input_cost_per_token") or 0.0 + if output_rate <= 0.0 and input_rate <= 0.0: + verbose_router_logger.debug("savings baseline: candidate %s has no per-token price", model) + return None + return (output_rate, input_rate, model) + + priced = tuple(r for model in models if (r := rates(model)) is not None) + if not priced: + verbose_router_logger.debug("savings baseline: no priceable candidates; savings driver disabled") + return None + return max(priced)[2] + + +def resolve_baseline(configured: str | None, router: "Router", group_names: Iterable[str]) -> str | None: + """The baseline for a router whose hardest tier offers ``group_names``. + + A configured override wins and is only qualified, never re-derived. + + Derived per call rather than cached: the parent router adds and removes deployments + while it runs, so a baseline pinned on first use would keep naming a model the + router no longer has, and a pricier one added later could never become the baseline. + Resolving costs tens of microseconds against a network call. + + Never raises. This is read on the routing path to decorate a request that is about + to be served, and a dashboard's counterfactual is not worth failing a live request + over; an unresolvable baseline zeroes the savings driver instead. + """ + try: + if configured: + return canonical_model(configured) + return _most_expensive(model for name in group_names for model in _models_in(router, name)) + except Exception as e: # noqa: BLE001 # see docstring: routing must not fail for a metric + verbose_router_logger.warning("savings baseline: could not resolve, savings will read zero (%s)", e) + return None diff --git a/litellm/types/proxy/management_endpoints/common_daily_activity.py b/litellm/types/proxy/management_endpoints/common_daily_activity.py index 00f67d0f39c9..21b7ffca3f29 100644 --- a/litellm/types/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/types/proxy/management_endpoints/common_daily_activity.py @@ -25,6 +25,7 @@ class SpendMetrics(BaseModel): compression_saved_tokens: int = Field(default=0) compression_savings_spend: float = Field(default=0.0) prompt_caching_savings_spend: float = Field(default=0.0) + autorouter_savings_spend: float = Field(default=0.0) total_tokens: int = Field(default=0) successful_requests: int = Field(default=0) failed_requests: int = Field(default=0) @@ -85,6 +86,7 @@ class DailySpendMetadata(BaseModel): total_compression_saved_tokens: int = Field(default=0) total_compression_savings_spend: float = Field(default=0.0) total_prompt_caching_savings_spend: float = Field(default=0.0) + total_autorouter_savings_spend: float = Field(default=0.0) page: int = Field(default=1) total_pages: int = Field(default=1) has_more: bool = Field(default=False) @@ -111,6 +113,7 @@ class LiteLLM_DailyUserSpend(BaseModel): compression_saved_tokens: int = 0 compression_savings_spend: float = 0.0 prompt_caching_savings_spend: float = 0.0 + autorouter_savings_spend: float = 0.0 spend: float = 0.0 api_requests: int = 0 successful_requests: int = 0 diff --git a/litellm/types/router.py b/litellm/types/router.py index 837a93367a2c..bd2572877de5 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -272,6 +272,7 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): auto_router_config: Optional[str] = None auto_router_default_model: Optional[str] = None auto_router_embedding_model: Optional[str] = None + auto_router_savings_baseline_model: Optional[str] = None # complexity-router params complexity_router_config: Optional[Dict] = None diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 18991f53e6fa..938edfbc979f 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2734,6 +2734,8 @@ class StandardLoggingRoutingDecision(TypedDict, total=False): classifier_model: str escalated: bool tier_boundaries: StandardLoggingRoutingDecisionTierBoundaries + savings_baseline_model: str + conversation_continuing: bool # Fields whose values quote the caller's prompt. Dropped when an operator turns message @@ -2753,6 +2755,8 @@ class StandardLoggingRoutingDecision(TypedDict, total=False): "classifier_model", "escalated", "tier_boundaries", + "savings_baseline_model", + "conversation_continuing", } ) @@ -3401,6 +3405,7 @@ def shared_backend_model_info(model_info: Dict[str, Any]) -> Dict[str, Any]: "auto_router_config", "auto_router_default_model", "auto_router_embedding_model", + "auto_router_savings_baseline_model", "complexity_router_config", "complexity_router_default_model", "adaptive_router_config", diff --git a/schema.prisma b/schema.prisma index 37ea55f8c132..0d7fa8692c8b 100644 --- a/schema.prisma +++ b/schema.prisma @@ -748,6 +748,7 @@ model LiteLLM_DailyUserSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -782,6 +783,7 @@ model LiteLLM_DailyOrganizationSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -816,6 +818,7 @@ model LiteLLM_DailyEndUserSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -849,6 +852,7 @@ model LiteLLM_DailyAgentSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -882,6 +886,7 @@ model LiteLLM_DailyTeamSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -917,6 +922,7 @@ model LiteLLM_DailyTagSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) diff --git a/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py b/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py index c7e7ef5d4691..a55d4f0dcfdc 100644 --- a/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py +++ b/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py @@ -16,6 +16,9 @@ Litellm_EntityType, SpendUpdateQueueItem, ) +from typing import get_args + +from litellm.proxy._types import BaseDailySpendTransaction from litellm.proxy.db.db_transaction_queue.daily_spend_update_queue import ( DailySpendUpdateQueue, ) @@ -209,6 +212,7 @@ async def test_get_aggregated_daily_spend_update_transactions_same_key(): "compression_saved_tokens": 0, "compression_savings_spend": 0, "prompt_caching_savings_spend": 0, + "autorouter_savings_spend": 0, } updates = [{test_key: test_transaction1}, {test_key: test_transaction2}] @@ -259,6 +263,7 @@ async def test_flush_and_get_aggregated_daily_spend_update_transactions( "compression_saved_tokens": 0, "compression_savings_spend": 0, "prompt_caching_savings_spend": 0, + "autorouter_savings_spend": 0, } # Add updates to queue @@ -527,3 +532,58 @@ async def test_compression_saved_tokens_aggregation(daily_spend_update_queue): assert agg["cache_creation_input_tokens"] == 7 assert agg["compression_savings_spend"] == pytest.approx(0.0076) assert agg["prompt_caching_savings_spend"] == pytest.approx(0.0108) + + +@pytest.mark.asyncio +async def test_every_optional_daily_metric_aggregates(daily_spend_update_queue): + """Every additive metric must survive the merge, not just the ones wired by hand. + + Two requests landing on one rollup key before a flush is the common case under + load, and this same merge runs again on every cross-pod Redis drain. A metric + persisted by the database write but skipped here is silently dropped on both + paths, so the driver reads as zero on the dashboard however much it saved. + """ + test_key = "user1_2023-01-01_key123_claude-haiku-4-5_anthropic" + def _numeric(annotation): + # additive metrics may be declared NotRequired[float] for rows queued by a pod + # running the previous release, so unwrap before matching + args = get_args(annotation) + return (args[0] if args else annotation) in (int, float) + + numeric_fields = [ + name for name, annotation in BaseDailySpendTransaction.__annotations__.items() if _numeric(annotation) + ] + assert "autorouter_savings_spend" in numeric_fields + increments = {field: index + 1 for index, field in enumerate(numeric_fields)} + + await daily_spend_update_queue.add_update({test_key: dict(increments)}) + await daily_spend_update_queue.add_update({test_key: dict(increments)}) + await daily_spend_update_queue.aggregate_queue_updates() + updates = await daily_spend_update_queue.flush_all_updates_from_in_memory_queue() + + agg = updates[0][test_key] + for field, value in increments.items(): + assert agg[field] == pytest.approx(value * 2), f"{field} did not accumulate" + + +@pytest.mark.asyncio +async def test_optional_metric_missing_from_an_older_payload_still_aggregates( + daily_spend_update_queue, +): + """A queued row written before a metric existed must not zero it out.""" + test_key = "user1_2023-01-01_key123_claude-haiku-4-5_anthropic" + base = { + "spend": 1.0, + "prompt_tokens": 10, + "completion_tokens": 5, + "api_requests": 1, + "successful_requests": 1, + "failed_requests": 0, + } + + await daily_spend_update_queue.add_update({test_key: dict(base)}) + await daily_spend_update_queue.add_update({test_key: {**base, "autorouter_savings_spend": 0.25}}) + await daily_spend_update_queue.aggregate_queue_updates() + updates = await daily_spend_update_queue.flush_all_updates_from_in_memory_queue() + + assert updates[0][test_key]["autorouter_savings_spend"] == pytest.approx(0.25) diff --git a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py index c2a0d34a9157..f2749be5d6e9 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py +++ b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py @@ -19,7 +19,10 @@ get_daily_activity_aggregated, update_metrics, ) -from litellm.types.proxy.management_endpoints.common_daily_activity import SpendMetrics +from litellm.types.proxy.management_endpoints.common_daily_activity import ( + DailySpendMetadata, + SpendMetrics, +) @pytest.mark.asyncio @@ -153,6 +156,7 @@ async def test_get_daily_activity_aggregated_with_endpoint_breakdown(): "compression_saved_tokens": 0, "compression_savings_spend": 0.0, "prompt_caching_savings_spend": 0.0, + "autorouter_savings_spend": 0.0, "failed_requests": 0, } mock_rows = [ @@ -498,6 +502,7 @@ async def test_tag_daily_activity_metadata_totals_not_zero(): mock_record_1.compression_saved_tokens = 0 mock_record_1.compression_savings_spend = 0.0 mock_record_1.prompt_caching_savings_spend = 0.0 + mock_record_1.autorouter_savings_spend = 0.0 mock_record_1.api_requests = 10 mock_record_1.successful_requests = 9 mock_record_1.failed_requests = 1 @@ -520,6 +525,7 @@ async def test_tag_daily_activity_metadata_totals_not_zero(): mock_record_2.compression_saved_tokens = 0 mock_record_2.compression_savings_spend = 0.0 mock_record_2.prompt_caching_savings_spend = 0.0 + mock_record_2.autorouter_savings_spend = 0.0 mock_record_2.api_requests = 5 mock_record_2.successful_requests = 5 mock_record_2.failed_requests = 0 @@ -582,6 +588,7 @@ async def test_aggregated_activity_preserves_metadata_for_deleted_keys(): "compression_saved_tokens": 0, "compression_savings_spend": 0.0, "prompt_caching_savings_spend": 0.0, + "autorouter_savings_spend": 0.0, "failed_requests": 0, } mock_rows = [ @@ -669,6 +676,7 @@ def _daily_user_spend_record(*, user_id, api_key, spend, model="gpt-4", model_gr compression_saved_tokens=0, compression_savings_spend=0.0, prompt_caching_savings_spend=0.0, + autorouter_savings_spend=0.0, api_requests=1, successful_requests=1, failed_requests=0, @@ -973,6 +981,7 @@ async def test_get_daily_activity_aggregated_empty_result_set(): "compression_saved_tokens": None, "compression_savings_spend": None, "prompt_caching_savings_spend": None, + "autorouter_savings_spend": None, "api_requests": None, "successful_requests": None, "failed_requests": None, @@ -1016,6 +1025,7 @@ def _no_spend_record(): compression_saved_tokens=None, compression_savings_spend=None, prompt_caching_savings_spend=None, + autorouter_savings_spend=None, api_requests=None, successful_requests=None, failed_requests=None, @@ -1050,3 +1060,54 @@ def test_update_metrics_handles_none_values(): assert metrics.cache_read_input_tokens == 0 assert metrics.cache_creation_input_tokens == 0 assert metrics.compression_saved_tokens == 0 + + +class TestEverySavingsDriverSurvivesTheReadPath: + """A savings driver is only real if it survives the whole read path. + + The write path can price a driver correctly and persist it to all six rollup + tables, and the dashboard can still render a permanent $0.00 because the + aggregation query never summed the column or the response model never + declared it. That failure is silent: the card renders, the number is just + always zero, which is indistinguishable from having saved nothing. These + tests enumerate the drivers from the response model itself, so a driver added + later cannot be half-wired. + """ + + def _drivers(self) -> list[str]: + drivers = [field for field in SpendMetrics.model_fields if field.endswith("_savings_spend")] + assert drivers, "expected the dashboard response to expose at least one savings driver" + return drivers + + def test_every_driver_is_summed_by_the_rollup_query(self): + sql, _ = _build_aggregated_sql_query( + table_name="litellm_dailyuserspend", + entity_id_field="user_id", + entity_id="user-1", + start_date="2026-07-01", + end_date="2026-07-31", + model=None, + api_key=None, + timezone_offset_minutes=None, + ) + for driver in self._drivers(): + assert f"SUM({driver})" in sql, f"{driver} is never summed, so it reads as zero" + + def test_every_driver_is_accumulated_across_rows(self): + for driver in self._drivers(): + record = _no_spend_record() + setattr(record, driver, 1.25) + metrics = update_metrics(SpendMetrics(), record) + assert getattr(metrics, driver) == pytest.approx(1.25), f"{driver} is dropped when accumulating rows" + + def test_every_driver_is_carried_by_a_single_row_conversion(self): + for driver in self._drivers(): + record = _no_spend_record() + setattr(record, driver, 2.5) + assert getattr(_record_to_spend_metrics(record), driver) == pytest.approx(2.5) + + def test_every_driver_has_a_range_total(self): + for driver in self._drivers(): + assert f"total_{driver}" in DailySpendMetadata.model_fields, ( + f"total_{driver} is missing, so the range summary omits the driver" + ) diff --git a/tests/test_litellm/proxy/spend_tracking/test_savings.py b/tests/test_litellm/proxy/spend_tracking/test_savings.py index 704cf7a63dd9..e58fdc5a00be 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_savings.py +++ b/tests/test_litellm/proxy/spend_tracking/test_savings.py @@ -6,7 +6,13 @@ import pytest import litellm -from litellm.proxy.spend_tracking.savings import compute_savings_spend +from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.proxy.spend_tracking.savings import ( + _baseline_usage, + compute_autorouter_savings, + compute_savings_spend, +) +from litellm.types.utils import Usage def _anthropic_costs(model: str) -> tuple[float, float]: @@ -16,6 +22,39 @@ def _anthropic_costs(model: str) -> tuple[float, float]: return input_cost, cache_read_cost +def _cached_usage_object() -> dict: + """A cache-heavy Anthropic request, shaped as the spend log records it. + + `prompt_tokens` is the inclusive total: 3 uncached text tokens plus 500 read + from cache plus 12304 written to cache. + """ + return { + "prompt_tokens": 12807, + "completion_tokens": 500, + "total_tokens": 13307, + "prompt_tokens_details": {"cached_tokens": 500, "cache_creation_tokens": 12304, "text_tokens": 3}, + "cache_creation_input_tokens": 12304, + "cache_read_input_tokens": 500, + } + + +def _cost_on(model: str, usage_object: dict) -> float: + prompt_cost, completion_cost = generic_cost_per_token( + model=model, usage=Usage(**usage_object), custom_llm_provider="anthropic" + ) + return prompt_cost + completion_cost + + +def _flat_rates(model: str) -> tuple[float, float, float]: + info = litellm.get_model_info(model=model, custom_llm_provider="anthropic") + input_cost = info["input_cost_per_token"] or 0.0 + return ( + input_cost, + info["output_cost_per_token"] or 0.0, + info.get("cache_creation_input_token_cost") or input_cost, + ) + + def test_compression_savings_priced_at_input_rate(): input_cost, _ = _anthropic_costs("claude-sonnet-5") result = compute_savings_spend( @@ -76,3 +115,311 @@ def test_negative_token_counts_clamp_to_zero(): ) assert result.compression == 0.0 assert result.prompt_caching == 0.0 + + +def _usage(fresh: int, cached: int, written: int, out: int) -> Usage: + """Usage as the spend log records it; `prompt_tokens` is the inclusive total.""" + return Usage( + prompt_tokens=fresh + cached + written, + completion_tokens=out, + total_tokens=fresh + cached + written + out, + prompt_tokens_details={"cached_tokens": cached, "cache_creation_tokens": written, "text_tokens": fresh}, + cache_read_input_tokens=cached, + cache_creation_input_tokens=written, + ) + + +def _savings(baseline: str, selected: str, usage: Usage, continuing: bool = True) -> float: + """Savings for a request, defaulting to a conversation already underway. + + `continuing=True` is the mid-conversation case, where the baseline had the prompt + cached and this request's write is what the switch cost. `continuing=False` is a + conversation's first turn, where nothing was cached for any model. + """ + return compute_autorouter_savings( + baseline_model=baseline, + selected_model=selected, + selected_provider="anthropic", + usage=usage, + conversation_continuing=continuing, + ) + + +def test_switching_models_mid_conversation_charges_the_cold_cache_write(): + """Staying on one model writes the cache once and reads it thereafter. Switching + leaves the new model cold, so it pays to write the whole prompt again; when that + charge outweighs the cheaper rates the route lost money and must report a loss. + + Pricing the baseline as if it too re-wrote the cache credits a charge it never + paid, which is how a losing switch used to read as the largest saving on the page. + """ + usage = _usage(fresh=3, cached=500, written=12304, out=500) + result = _savings("claude-sonnet-5", "claude-haiku-4-5", usage) + + sonnet = litellm.get_model_info("claude-sonnet-5", "anthropic") + haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") + warm_baseline = ( + 3 * sonnet["input_cost_per_token"] + + 12804 * sonnet["cache_read_input_token_cost"] + + 500 * sonnet["output_cost_per_token"] + ) + actually_paid = ( + 3 * haiku["input_cost_per_token"] + + 500 * haiku["cache_read_input_token_cost"] + + 12304 * haiku["cache_creation_input_token_cost"] + + 500 * haiku["output_cost_per_token"] + ) + assert result == pytest.approx(warm_baseline - actually_paid) + assert result < 0, "a cache-thrashing switch must report a loss, not a saving" + + phantom = 12304 * sonnet["cache_creation_input_token_cost"] + assert result != pytest.approx(warm_baseline + phantom - actually_paid) + + +def test_a_cold_switch_never_beats_turning_caching_off(): + """Switching to a cold model makes it write the whole prompt again. That write is a + real cost of switching, so the same traffic must look worse than if caching were off + entirely. + + The baseline is priced as a warm cache even though this request read nothing: a + switch reads nothing precisely because the new model's cache is empty, and staying + on one model would have had the prompt cached already. Gating the warm baseline on + a read charged the baseline a write it would never repeat, which made a cold switch + report a larger saving than no caching at all. + """ + cold_switch = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", _usage(0, 0, 20_000, 1_000)) + caching_off = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", _usage(20_000, 0, 0, 1_000)) + + assert cold_switch < caching_off + + opus = litellm.get_model_info("claude-opus-5", "anthropic") + haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") + warm_baseline = 20_000 * opus["cache_read_input_token_cost"] + 1_000 * opus["output_cost_per_token"] + actually_paid = 20_000 * haiku["cache_creation_input_token_cost"] + 1_000 * haiku["output_cost_per_token"] + assert cold_switch == pytest.approx(warm_baseline - actually_paid) + + +def test_moving_one_token_between_cache_buckets_does_not_move_the_answer(): + """A continuing conversation writes a few new tokens and reads the rest. Treating the + presence of a write as the signal for a switch made that ordinary increment flip the + result, so a request reading 19,999 and writing 1 landed somewhere entirely different + from one reading 20,000 and writing none. + """ + reads_nothing = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", _usage(0, 0, 20_000, 1_000)) + reads_one = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", _usage(0, 1, 19_999, 1_000)) + assert reads_one == pytest.approx(reads_nothing, abs=1e-4) + + +def test_multimodal_prompts_are_priced_on_the_baseline_too(): + """The baseline is this same request met by a warm cache, so every field it was + priced on has to survive. Rebuilding the details from the cache buckets alone + dropped the image and audio counts, which priced the baseline as a text-only + request that never ran and shrank the reported saving on multimodal traffic. + """ + details = {"cached_tokens": 0, "cache_creation_tokens": 16_000, "text_tokens": 0, "image_tokens": 4_000} + with_images = Usage( + prompt_tokens=20_000, + completion_tokens=1_000, + total_tokens=21_000, + prompt_tokens_details=details, + ) + baseline = _baseline_usage(with_images, conversation_continuing=True) + + assert baseline.prompt_tokens_details.image_tokens == 4_000, "image tokens must survive into the baseline" + + opus = litellm.get_model_info("claude-opus-5", "anthropic") + priced, _ = generic_cost_per_token(model="claude-opus-5", usage=baseline, custom_llm_provider="anthropic") + text_only = 20_000 * opus["cache_read_input_token_cost"] + assert priced > text_only, "dropping the image tokens undercharges the baseline and hides the saving" + + +def test_the_baseline_is_never_charged_a_cache_write(): + """Carrying the details through must not carry the 5m/1h creation breakdown with + them. `generic_cost_per_token` charges a creation cost whenever that breakdown is + present, even against a zeroed creation count, which would put the phantom write + back on the baseline for every long-cache request. + """ + long_cache = Usage( + prompt_tokens=20_000, + completion_tokens=1_000, + total_tokens=21_000, + prompt_tokens_details={ + "cached_tokens": 0, + "cache_creation_tokens": 20_000, + "text_tokens": 0, + "cache_creation_token_details": {"ephemeral_1h_input_tokens": 20_000}, + }, + ) + baseline = _baseline_usage(long_cache, conversation_continuing=True) + + opus = litellm.get_model_info("claude-opus-5", "anthropic") + priced, _ = generic_cost_per_token(model="claude-opus-5", usage=baseline, custom_llm_provider="anthropic") + assert priced == pytest.approx(20_000 * opus["cache_read_input_token_cost"]), ( + "the baseline reads a warm cache; it never pays to create one" + ) + + +def test_uncached_request_is_the_plain_rate_difference(): + usage = _usage(fresh=2000, cached=0, written=0, out=500) + sonnet = litellm.get_model_info("claude-sonnet-5", "anthropic") + haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") + assert _savings("claude-sonnet-5", "claude-haiku-4-5", usage) == pytest.approx( + 2000 * (sonnet["input_cost_per_token"] - haiku["input_cost_per_token"]) + + 500 * (sonnet["output_cost_per_token"] - haiku["output_cost_per_token"]) + ) + + +def test_escalation_reports_its_real_cost(): + """Routing up to a pricier model is a real cost; hiding it behind a zero floor + would let the dashboard only ever move in one direction.""" + usage = _usage(fresh=2000, cached=0, written=0, out=500) + assert _savings("claude-haiku-4-5", "claude-sonnet-5", usage) < 0 + + +def test_autorouter_savings_zero_when_model_unchanged(): + assert _savings("claude-opus-5", "claude-opus-5", _usage(3, 500, 12304, 500)) == 0.0 + + +def test_autorouter_savings_unknown_baseline_fails_open_to_zero(): + assert _savings("totally-made-up-model-xyz", "claude-haiku-4-5", _usage(3, 500, 12304, 500)) == 0.0 + + +def test_autorouter_savings_zero_without_baseline(): + result = compute_savings_spend( + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + compression_saved_tokens=0, + cache_read_input_tokens=0, + routing_decision=None, + usage_object=_cached_usage_object(), + ) + assert result.autorouter == 0.0 + + +def test_compute_savings_spend_carries_a_losing_switch_through(): + """The signed value must survive into SavingsSpend; clamping it here would put the + dashboard back to only ever showing gains.""" + result = compute_savings_spend( + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + compression_saved_tokens=0, + cache_read_input_tokens=0, + routing_decision={"savings_baseline_model": "claude-sonnet-5"}, + usage_object=_cached_usage_object(), + ) + assert result.autorouter < 0 + + +def test_malformed_usage_object_does_not_fail_the_spend_write(): + """The daily spend write must survive an unusable usage_object; losing one row's + savings is recoverable, losing the row is not.""" + result = compute_savings_spend( + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + compression_saved_tokens=1000, + cache_read_input_tokens=0, + routing_decision={"savings_baseline_model": "claude-opus-5"}, + usage_object={"prompt_tokens": ["not", "a", "number"]}, + ) + assert result.autorouter == 0.0 + assert result.compression > 0 + + +def test_model_without_cache_read_pricing_yields_no_caching_savings(): + """A model with no discounted cache-read rate cannot have saved anything by + reading from cache, so the driver must report zero rather than the full input rate.""" + model = "azure/gpt-3.5-turbo" + assert litellm.get_model_info(model=model).get("cache_read_input_token_cost") is None + result = compute_savings_spend( + model=model, + custom_llm_provider="azure", + compression_saved_tokens=0, + cache_read_input_tokens=5000, + ) + assert result.prompt_caching == 0.0 + + +def test_the_same_deployment_spelled_two_ways_is_not_a_switch(): + """The spend log records a normalized model name while the baseline arrives as the + operator wrote it in config. Comparing the raw strings makes a request that never + changed model look like a switch, and prices one deployment against itself.""" + # Must be a cached request: the baseline arm is priced against a warm cache and the + # selected arm against what was actually paid, so treating one deployment as two + # charges it a cold-cache write it never took, inventing a loss on a request that + # never changed model. An uncached request prices identically either way and would + # make this assertion vacuous. + usage = _usage(fresh=3, cached=500, written=12304, out=500) + assert _savings("anthropic/claude-opus-5", "claude-opus-5", usage) == 0.0 + assert _savings("claude-opus-5", "anthropic/claude-opus-5", usage) == 0.0 + + +def test_baseline_is_priced_under_its_own_provider(): + """Two providers can serve the same bare model name at different rates, so dropping + the provider prices the baseline against a vendor the operator never named. Here it + decides whether routing reads as a saving or a loss.""" + usage = Usage(prompt_tokens=100_000, completion_tokens=10_000, total_tokens=110_000) + azure = compute_autorouter_savings( + baseline_model="azure_ai/deepseek-r1", + selected_model="claude-haiku-4-5", + selected_provider="anthropic", + usage=usage, + ) + deepseek = compute_autorouter_savings( + baseline_model="deepseek/deepseek-r1", + selected_model="claude-haiku-4-5", + selected_provider="anthropic", + usage=usage, + ) + assert azure != pytest.approx(deepseek) + assert azure > 0 > deepseek + + +def test_unresolvable_baseline_fails_open_to_zero(): + usage = _usage(fresh=2000, cached=0, written=0, out=500) + assert _savings("no-such-provider-xyz/no-such-model", "claude-haiku-4-5", usage) == 0.0 + + +def test_a_first_turn_is_the_rate_difference_not_a_switch_penalty(): + """Nothing was cached anywhere on a conversation's first turn, so the baseline would + have paid the same cache write. Charging it to the selected arm alone reported a + fraction of the real saving; on this shape roughly 4% of it. + """ + usage = _usage(fresh=0, cached=0, written=20_000, out=1_000) + first_turn = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", usage, continuing=False) + + opus = litellm.get_model_info("claude-opus-5", "anthropic") + haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") + both_write = (20_000 * opus["cache_creation_input_token_cost"] + 1_000 * opus["output_cost_per_token"]) - ( + 20_000 * haiku["cache_creation_input_token_cost"] + 1_000 * haiku["output_cost_per_token"] + ) + assert first_turn == pytest.approx(both_write) + + mid_conversation = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", usage) + assert first_turn > mid_conversation * 10, "a first turn must not be priced as a switch" + + +def test_a_first_turn_that_saves_money_never_reports_a_loss(): + """The write premium is fixed by prompt size while the saving grows with completion + length, so charging the write to a first turn made short answers over a large cached + prompt read as losses on requests that genuinely saved. That is the shape most likely + to be on the dashboard, and the sign has to be right. + """ + short_answer = _usage(fresh=0, cached=0, written=20_000, out=200) + assert _savings("anthropic/claude-opus-5", "claude-haiku-4-5", short_answer, continuing=False) > 0 + assert _savings("anthropic/claude-opus-5", "claude-haiku-4-5", short_answer) < 0 + + +def test_an_undetermined_conversation_shape_stays_conservative(): + """The default must charge the write. A caller that cannot be read, or a surface the + router never classified, has said nothing about whether the baseline was warm, and a + savings figure must not inflate on a guess. + """ + usage = _usage(fresh=0, cached=0, written=20_000, out=1_000) + defaulted = compute_autorouter_savings( + baseline_model="anthropic/claude-opus-5", + selected_model="claude-haiku-4-5", + selected_provider="anthropic", + usage=usage, + ) + assert defaulted == pytest.approx(_savings("anthropic/claude-opus-5", "claude-haiku-4-5", usage)) + assert defaulted < _savings("anthropic/claude-opus-5", "claude-haiku-4-5", usage, continuing=False) diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index cc73273450aa..c6ee14f7cba2 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -4100,6 +4100,23 @@ def test_none_creates_no_bucket_on_a_request_that_had_none(self): assert request_kwargs == {} + def test_clearing_the_decision_takes_the_savings_facts_with_it(self): + """A fallback to a plain model group re-enters the hook with the same + `request_kwargs`. The baseline and the conversation shape ride inside the + decision rather than beside it, so one clear cannot leave either behind and + attribute an auto-router saving to a deployment that never routed.""" + decision = { + "router_model_name": "smart-router", + "router_type": "complexity", + "routed_model": "gpt-4o-mini", + "savings_baseline_model": "anthropic/claude-opus-5", + "conversation_continuing": False, + } + request_kwargs: Dict = {"litellm_metadata": {"routing_decision": decision}} + Router._record_routing_decision(request_kwargs=request_kwargs, routing_decision=None) + assert request_kwargs["litellm_metadata"] == {} + + class TestEscalationIsRecordedConsistently: """An escalation keyword records two separate facts on every path: that the caller asked, and whether the tier actually moved. Dropping the ask when there is nowhere @@ -5045,3 +5062,154 @@ def test_a_window_stops_telling_the_model_to_disregard_it(self): assert "Classify only the current message" not in system_prompt assert "using the earlier turns quoted above it as context" in system_prompt assert "rate the work it approves rather than the reply itself" in system_prompt + + +class TestSavingsBaselineModel: + """The counterfactual model a complexity router's savings are measured against.""" + @staticmethod + def _router_with_tiers(tiers: dict, **kwargs) -> ComplexityRouter: + from litellm.router import Router + parent = Router( + model_list=[ + {"model_name": "cheap", "litellm_params": {"model": "anthropic/claude-haiku-4-5"}}, + {"model_name": "mid", "litellm_params": {"model": "anthropic/claude-sonnet-4-5"}}, + {"model_name": "top", "litellm_params": {"model": "anthropic/claude-opus-4-5"}}, + ] + ) + return ComplexityRouter( + model_name="bench", + litellm_router_instance=parent, + complexity_router_config={"tiers": tiers}, + default_model="mid", + **kwargs, + ) + def test_baseline_is_the_reasoning_tier_not_the_priciest_reachable_model(self): + """The ladder names what an operator would have had to run for the hardest + request; a pricier model sitting in a lower tier is a choice, not a ceiling.""" + router = self._router_with_tiers({"SIMPLE": "top", "MEDIUM": "cheap", "REASONING": "mid"}) + assert router.savings_baseline_model == "anthropic/claude-sonnet-4-5" + def test_baseline_is_the_priciest_model_when_the_reasoning_tier_is_a_pool(self): + router = self._router_with_tiers({"SIMPLE": "cheap", "REASONING": ["cheap", "top", "mid"]}) + assert router.savings_baseline_model == "anthropic/claude-opus-4-5" + def test_falls_back_to_the_hardest_configured_tier_when_reasoning_is_absent(self): + router = self._router_with_tiers({"SIMPLE": "cheap", "COMPLEX": "top"}) + assert router.savings_baseline_model == "anthropic/claude-opus-4-5" + def test_a_configured_baseline_wins_and_is_provider_qualified(self): + router = self._router_with_tiers( + {"SIMPLE": "cheap", "REASONING": "mid"}, savings_baseline_model="claude-opus-4-5" + ) + assert router.savings_baseline_model == "anthropic/claude-opus-4-5" + def test_an_unpriceable_tier_disables_the_driver_rather_than_inventing_a_baseline(self): + router = self._router_with_tiers({"REASONING": "not-a-real-model-anywhere"}) + assert router.savings_baseline_model is None + def test_the_baseline_travels_on_every_pre_routing_response(self): + """A response without it silently zeroes the savings driver for that path.""" + import inspect + from litellm.router_strategy.complexity_router import complexity_router as module + source = inspect.getsource(module.ComplexityRouter._build_routing_decision) + assert "self.savings_baseline_model" in source, ( + "the baseline rides on the routing decision, so every path that builds one carries it" + ) + + +class TestConversationShapeDiscriminator: + """Whether the counterfactual single model would already have had the prompt cached.""" + + @staticmethod + def _router(mock_router_instance, basic_config) -> ComplexityRouter: + return ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={**basic_config, "session_affinity": False}, + ) + + @pytest.mark.asyncio + async def test_a_single_ask_is_a_first_turn(self, mock_router_instance, basic_config): + """Nothing is cached for any model yet, so the baseline would have paid the same + cache write and the saving is the plain rate difference.""" + mock_router_instance.cache = DualCache() + result = await self._router(mock_router_instance, basic_config).async_pre_routing_hook( + model="test-model", + request_kwargs={"metadata": {}}, + messages=[{"role": "user", "content": "Hello!"}], + ) + assert result.routing_decision["conversation_continuing"] is False + + @pytest.mark.asyncio + async def test_a_second_ask_means_the_baseline_was_already_warm(self, mock_router_instance, basic_config): + """An earlier turn was served, so a single-model deployment wrote the prompt then + and would only read it now; this request's write is what switching cost.""" + mock_router_instance.cache = DualCache() + result = await self._router(mock_router_instance, basic_config).async_pre_routing_hook( + model="test-model", + request_kwargs={"metadata": {}}, + messages=[ + {"role": "user", "content": "First question about the codebase"}, + {"role": "assistant", "content": "Here is the answer"}, + {"role": "user", "content": "Hello!"}, + ], + ) + assert result.routing_decision["conversation_continuing"] is True + + @pytest.mark.asyncio + async def test_it_needs_no_session_id(self, mock_router_instance, basic_config): + """The whole point of reading the conversation rather than remembering it: a + caller that sends no session header is still classified correctly.""" + mock_router_instance.cache = DualCache() + router = self._router(mock_router_instance, basic_config) + first = await router.async_pre_routing_hook( + model="test-model", request_kwargs={}, messages=[{"role": "user", "content": "Hello!"}] + ) + later = await router.async_pre_routing_hook( + model="test-model", + request_kwargs={}, + messages=[ + {"role": "user", "content": "First question"}, + {"role": "assistant", "content": "Answer"}, + {"role": "user", "content": "Hello!"}, + ], + ) + assert first.routing_decision["conversation_continuing"] is False + assert later.routing_decision["conversation_continuing"] is True + + @pytest.mark.asyncio + async def test_it_touches_no_cache(self, mock_router_instance, basic_config): + """Reading the request instead of remembering it is what removes the routing-path + round-trip, and with it a cache failure that would read as a first turn.""" + cache = AsyncMock() + cache.async_get_cache = AsyncMock(return_value=None) + mock_router_instance.cache = cache + result = await self._router(mock_router_instance, basic_config).async_pre_routing_hook( + model="test-model", + request_kwargs={"metadata": {}}, + messages=[{"role": "user", "content": "Hello!"}], + ) + assert result.routing_decision["conversation_continuing"] is False + assert cache.async_get_cache.await_count == 0 + assert cache.async_set_cache.await_count == 0 + + def test_unreadable_messages_stay_conservative(self): + """No messages says nothing about the baseline's cache, so it keeps charging the + write and under-claims rather than inflating.""" + from litellm.router_strategy.complexity_router.complexity_router import _conversation_is_continuing + + assert _conversation_is_continuing(None) is True + assert _conversation_is_continuing([]) is True + assert _conversation_is_continuing([{"role": "assistant", "content": "no human ask here"}]) is True + + @pytest.mark.asyncio + async def test_the_shape_travels_on_every_pre_routing_response(self): + """A response without it defaults to charging the write, silently undoing the fix + for whichever routing path forgot it.""" + import inspect + + from litellm.router_strategy.complexity_router import complexity_router as module + + source = inspect.getsource(module.ComplexityRouter.async_pre_routing_hook) + inspect.getsource( + module.ComplexityRouter._classify_and_route + ) + builds = source.count("self._build_routing_decision(") + assert builds > 0 + assert source.count("conversation_continuing=conversation_continuing") == builds, ( + "every routing decision must carry the conversation shape, or that path silently charges the write" + ) diff --git a/tests/test_litellm/router_strategy/test_savings_baseline.py b/tests/test_litellm/router_strategy/test_savings_baseline.py new file mode 100644 index 000000000000..e88fbb7a10a4 --- /dev/null +++ b/tests/test_litellm/router_strategy/test_savings_baseline.py @@ -0,0 +1,155 @@ +import pytest + +from litellm.router import Router +from litellm.router_strategy.savings_baseline import ( + canonical_model, + _models_in, + _most_expensive, + resolve_baseline, +) + + +@pytest.fixture +def parent() -> Router: + return Router( + model_list=[ + {"model_name": "cheap", "litellm_params": {"model": "anthropic/claude-haiku-4-5"}}, + {"model_name": "top", "litellm_params": {"model": "anthropic/claude-opus-4-5"}}, + {"model_name": "pool", "litellm_params": {"model": "anthropic/claude-haiku-4-5"}}, + {"model_name": "pool", "litellm_params": {"model": "anthropic/claude-opus-4-5"}}, + ] + ) + + +class TestCanonicalModel: + def test_qualifies_a_bare_name_with_the_provider_that_owns_it(self): + assert canonical_model("claude-opus-4-5") == "anthropic/claude-opus-4-5" + + def test_keeps_an_already_qualified_name_qualified(self): + assert canonical_model("anthropic/claude-opus-4-5") == "anthropic/claude-opus-4-5" + + def test_honours_a_separately_declared_provider(self): + assert canonical_model("claude-opus-4-5", "openai") == "openai/claude-opus-4-5" + + def test_returns_none_for_a_name_no_provider_claims(self): + assert canonical_model("") is None + + +class TestModelsForGroup: + def test_resolves_a_group_to_the_models_its_deployments_call(self, parent): + assert _models_in(parent, "cheap") == ("anthropic/claude-haiku-4-5",) + + def test_returns_every_deployment_in_a_pooled_group(self, parent): + assert sorted(_models_in(parent, "pool")) == [ + "anthropic/claude-haiku-4-5", + "anthropic/claude-opus-4-5", + ] + + def test_treats_an_unknown_group_as_a_model_name(self, parent): + """A tier can point straight at a provider model rather than a configured group.""" + assert _models_in(parent, "claude-opus-4-5") == ("anthropic/claude-opus-4-5",) + + +class TestMostExpensive: + def test_picks_by_output_rate(self): + assert ( + _most_expensive(["anthropic/claude-haiku-4-5", "anthropic/claude-opus-4-5"]) + == "anthropic/claude-opus-4-5" + ) + + def test_ignores_models_with_no_per_token_price(self): + """A free model as baseline would report the whole real spend as a loss.""" + assert _most_expensive(["not-a-real-model-anywhere", "anthropic/claude-haiku-4-5"]) == ( + "anthropic/claude-haiku-4-5" + ) + + def test_returns_none_when_nothing_can_be_priced(self): + assert _most_expensive(["not-a-real-model-anywhere"]) is None + + def test_returns_none_for_an_empty_candidate_set(self): + assert _most_expensive([]) is None + + +class TestResolveBaseline: + def test_a_configured_baseline_wins_over_the_candidates(self, parent): + assert resolve_baseline("claude-haiku-4-5", parent, ["top"]) == "anthropic/claude-haiku-4-5" + + def test_derives_the_priciest_candidate_when_unconfigured(self, parent): + assert resolve_baseline(None, parent, ["cheap", "top"]) == "anthropic/claude-opus-4-5" + + def test_never_raises_so_a_metric_cannot_fail_a_live_request(self): + """Read on the routing path while decorating a request that is about to be + served; a dashboard counterfactual must not be able to take routing down.""" + + class Exploding: + @property + def model_name_to_deployment_indices(self): + raise RuntimeError("router is mid-reload") + + assert resolve_baseline(None, Exploding(), ["anything"]) is None + + def test_an_empty_candidate_set_zeroes_the_driver_rather_than_inventing_one(self, parent): + assert resolve_baseline(None, parent, []) is None + + +class TestDeploymentsPricedByBaseModel: + """`litellm_params.model` is not always a model. + + On Azure it is the deployment name, which is absent from the cost map, so pricing it + directly drops the candidate. If that candidate was the priciest, the baseline quietly + becomes the second priciest and every saving is understated; if the whole pool is + Azure, nothing prices and the driver reports zero with nothing at default log level + saying why. `model_info.base_model` is what names the real model, which is the chain + router.py already resolves pricing through. + """ + + @staticmethod + def _router(*deployments: dict) -> Router: + return Router(model_list=list(deployments)) + + def test_model_info_base_model_is_preferred_over_the_deployment_name(self): + router = self._router( + { + "model_name": "big", + "litellm_params": {"model": "azure/my-gpt5-deployment"}, + "model_info": {"base_model": "azure/gpt-4.1"}, + }, + ) + assert _models_in(router, "big") == ("azure/gpt-4.1",) + + def test_litellm_params_base_model_is_the_other_accepted_spelling(self): + router = self._router( + { + "model_name": "big", + "litellm_params": {"model": "azure/my-gpt5-deployment", "base_model": "azure/gpt-4.1"}, + }, + ) + assert _models_in(router, "big") == ("azure/gpt-4.1",) + + def test_a_deployment_without_a_base_model_still_prices_by_its_model(self): + router = self._router({"model_name": "big", "litellm_params": {"model": "anthropic/claude-opus-4-5"}}) + assert _models_in(router, "big") == ("anthropic/claude-opus-4-5",) + + def test_an_azure_deployment_can_win_the_priciest_candidate(self): + """Without the base_model hop the Azure candidate never prices, so the cheaper + model wins by default and the reported saving shrinks.""" + router = self._router( + {"model_name": "cheap", "litellm_params": {"model": "anthropic/claude-haiku-4-5"}}, + { + "model_name": "big", + "litellm_params": {"model": "azure/my-gpt5-deployment"}, + "model_info": {"base_model": "azure/gpt-4.1"}, + }, + ) + assert resolve_baseline(None, router, ["cheap", "big"]) == "azure/gpt-4.1" + + def test_an_all_azure_pool_still_has_a_baseline(self): + """Otherwise nothing prices, the driver is disabled and the card reads $0.00.""" + router = self._router( + { + "model_name": "big", + "litellm_params": {"model": "azure/my-gpt5-deployment"}, + "model_info": {"base_model": "azure/gpt-4.1"}, + }, + ) + assert resolve_baseline(None, router, ["big"]) == "azure/gpt-4.1" diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 90552b6eae17..8402fa6c62ec 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -23487,6 +23487,11 @@ export interface components { * @default 0 */ total_api_requests: number; + /** + * Total Autorouter Savings Spend + * @default 0 + */ + total_autorouter_savings_spend: number; /** * Total Cache Creation Input Tokens * @default 0 @@ -25936,6 +25941,8 @@ export interface components { auto_router_default_model?: string | null; /** Auto Router Embedding Model */ auto_router_embedding_model?: string | null; + /** Auto Router Savings Baseline Model */ + auto_router_savings_baseline_model?: string | null; /** Aws Access Key Id */ aws_access_key_id?: string | null; /** Aws Bedrock Project Id */ @@ -31388,6 +31395,11 @@ export interface components { * @default 0 */ api_requests: number; + /** + * Autorouter Savings Spend + * @default 0 + */ + autorouter_savings_spend: number; /** * Cache Creation Input Tokens * @default 0 @@ -34068,6 +34080,8 @@ export interface components { auto_router_default_model?: string | null; /** Auto Router Embedding Model */ auto_router_embedding_model?: string | null; + /** Auto Router Savings Baseline Model */ + auto_router_savings_baseline_model?: string | null; /** Aws Access Key Id */ aws_access_key_id?: string | null; /** Aws Bedrock Project Id */ From c6e0f1b62d8cf57873b0c5b6edd141a94ab942f0 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Sat, 1 Aug 2026 16:34:21 -0700 Subject: [PATCH 02/15] fix(spend): let the baseline pay for a continuing turn's own growth `_baseline_usage` moved every cache-creation token into the baseline's read bucket whenever the conversation was underway. That is right for a switch, where the baseline never left the model it was on and really would only read, but wrong for a turn that stayed put: the prompt grew, and the tokens written are that growth. They are new to every model, so the baseline would have paid to write them too. Forgiving it that write made the counterfactual cheaper than it was and shrank the reported saving on ordinary steady-state traffic, by about 2% per turn. The selected arm was never involved; it has always been priced on the real usage. The error sat entirely on the baseline. The condition is that the request read more than it wrote, not that it read anything. A switch onto a model already holding a small prefix of this prompt still writes most of it, and that write is the switch's own cost; keying off a nonzero read would have handed such a request the full rate gap, turning +$0.0056 into +$0.1177. Comparing the two buckets separates a warm continuation, which reads far more than it writes, from a cold arrival, which does the reverse, and it leaves the existing invariant intact: a request reading 0 and one reading 1 both still land in the same place. --- litellm/proxy/spend_tracking/savings.py | 11 +++++ .../proxy/spend_tracking/test_savings.py | 44 +++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index 3f554982ca4e..e9cf64036884 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -119,6 +119,15 @@ def _baseline_usage(usage: Usage, conversation_continuing: bool) -> Usage: too, which is all a single rollup row can support, understates a first turn to a few percent of its value and can render a profitable route as a loss. + A continuing turn that mostly read from cache is the third case: the selected model + was already warm, so it is the one that has been serving this conversation and the + baseline's cache holds exactly what its does. The tokens written are the turn's own + growth, new to every model, and the baseline would have paid to write them too. + Moving them would forgive the baseline a write it really owes and shrink the + reported saving. "Mostly read" rather than "read anything" on purpose: a switch onto + a model holding a small prefix of this prompt still writes most of it, and must keep + counting that write against the saving. + Only the cache buckets move. Every other field the request was priced on travels through untouched, audio and image and video counts among them, because the baseline is this same request served by a model that happened to be warm; naming the fields to @@ -129,6 +138,8 @@ def _baseline_usage(usage: Usage, conversation_continuing: bool) -> Usage: details = usage.prompt_tokens_details if details is None or cache_creation <= 0 or not conversation_continuing: return usage + if cache_read > cache_creation: + return usage return Usage( prompt_tokens=usage.prompt_tokens, completion_tokens=usage.completion_tokens, diff --git a/tests/test_litellm/proxy/spend_tracking/test_savings.py b/tests/test_litellm/proxy/spend_tracking/test_savings.py index e58fdc5a00be..8862b96d76c9 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_savings.py +++ b/tests/test_litellm/proxy/spend_tracking/test_savings.py @@ -423,3 +423,47 @@ def test_an_undetermined_conversation_shape_stays_conservative(): ) assert defaulted == pytest.approx(_savings("anthropic/claude-opus-5", "claude-haiku-4-5", usage)) assert defaulted < _savings("anthropic/claude-opus-5", "claude-haiku-4-5", usage, continuing=False) + + +def test_a_continuing_turn_on_the_same_model_writes_its_growth_on_both_arms(): + """A conversation that grew by a few tokens writes those on whatever model serves + it, and they are new to every model, so the baseline would have written them too. + Moving them into the baseline's read bucket forgives it a write it really owes and + shrinks the reported saving on ordinary steady-state traffic. + """ + usage = _usage(fresh=0, cached=19_900, written=100, out=1_000) + opus = litellm.get_model_info("claude-opus-5", "anthropic") + haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") + + def cost(info: dict) -> float: + return ( + 19_900 * info["cache_read_input_token_cost"] + + 100 * info["cache_creation_input_token_cost"] + + 1_000 * info["output_cost_per_token"] + ) + + both_write_the_growth = cost(opus) - cost(haiku) + assert _savings("anthropic/claude-opus-5", "claude-haiku-4-5", usage) == pytest.approx(both_write_the_growth) + + +def test_a_switch_onto_a_partly_cached_model_still_pays_for_the_write(): + """A model holding a small prefix of this prompt still has to write the rest, and + that write is the switch's cost. Keying the same-model case off reading *anything* + rather than reading *most of it* would hand this request the full rate gap and + inflate the saving by an order of magnitude. + """ + mostly_written = _usage(fresh=0, cached=500, written=19_500, out=1_000) + reported = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", mostly_written) + + opus = litellm.get_model_info("claude-opus-5", "anthropic") + haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") + if_treated_as_same_model = ( + 500 * opus["cache_read_input_token_cost"] + + 19_500 * opus["cache_creation_input_token_cost"] + + 1_000 * opus["output_cost_per_token"] + ) - ( + 500 * haiku["cache_read_input_token_cost"] + + 19_500 * haiku["cache_creation_input_token_cost"] + + 1_000 * haiku["output_cost_per_token"] + ) + assert reported < if_treated_as_same_model / 10, "a mostly-cold switch must not be priced as a continuation" From 721790d0b507f17c33011bfb7705ab1762a625c0 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Sat, 1 Aug 2026 17:38:34 -0700 Subject: [PATCH 03/15] fix(spend): price each arm under the key litellm billed it, and see agent turns Two ways the savings number read the wrong thing, both from identifying a model by its name when the name is not what it costs. The counterfactual was ranked and priced on the public rate for the model a deployment names. A deployment may not be charged that rate: the router registers its configured prices under the deployment's own id and deliberately keeps them off the shared model-name key so deployments sharing a backend model do not pollute each other. So a hardest-tier deployment configured above its public rate lost the ranking to a cheaper candidate, and once chosen was priced at a rate nobody pays. Which key prices a deployment is now `_select_model_name_for_cost_calc`'s decision, the resolver the real request is billed through, rather than a second rule here that would have to re-learn that per-second and tiered overrides count, that a partial override still counts, and that a deployment configured at zero is priced at zero rather than treated as unpriced. The arm being subtracted had the same fault and a sharper edge. It priced the spend log's `model`, which on Azure is the deployment name, absent from the cost map, so the whole driver silently read zero for that traffic. It no longer re-derives anything: `model_map_information.model_map_key` is what litellm actually billed the request under, recorded at request time by that same resolver with `base_model` and custom pricing already applied. Separately, the conversation-shape discriminator counted human asks, and an agent loop can run twenty turns on one of them. Its tool traffic rides `tool_result` blocks on user turns that flatten to empty text, and `tool` roles that are never read, so a long agentic conversation looked like its own first turn and was handed the arithmetic that leaves the cache write on both arms. That is the one direction this must never fail in, because it inflates. An assistant turn is the direct evidence that something answered earlier, and it is blind to how the tool plumbing is spelled on either surface. --- litellm/proxy/db/db_spend_update_writer.py | 1 + litellm/proxy/spend_tracking/savings.py | 31 +++++- .../complexity_router/complexity_router.py | 41 +++---- litellm/router_strategy/savings_baseline.py | 105 +++++++++++++----- litellm/types/utils.py | 2 + .../router_strategy/test_complexity_router.py | 47 +++++++- .../router_strategy/test_savings_baseline.py | 67 ++++++++--- 7 files changed, 223 insertions(+), 71 deletions(-) diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index b5a41614df72..b834f0dda275 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -1858,6 +1858,7 @@ async def _common_add_spend_log_transaction_to_daily_transaction( compression_saved_tokens=compression_saved_tokens, cache_read_input_tokens=cache_read_input_tokens, routing_decision=_metadata.get("routing_decision"), + model_map_information=_metadata.get("model_map_information"), usage_object=usage_obj, ) diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index e9cf64036884..ad70896de1de 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -75,11 +75,19 @@ def _resolve_model(model: str | None, custom_llm_provider: str | None) -> _Model return _ModelIdentity(model=resolved_model, provider=provider) -def _cost_of_usage(model: _ModelIdentity, usage: Usage) -> float | None: - """What ``usage`` costs on ``model``, or ``None`` when the model has no pricing.""" +def _cost_of_usage(model: _ModelIdentity, usage: Usage, pricing_key: str | None = None) -> float | None: + """What ``usage`` costs on ``model``, or ``None`` when the model has no pricing. + + ``pricing_key`` is the key litellm bills this deployment under, which is the + deployment's own id when it overrides its prices and the model name otherwise. The + router keeps overrides off the shared model-name key so deployments sharing a + backend model do not pollute each other, so pricing the name would read the public + rate for a model nobody is charged the public rate for, on whichever arm is + overridden, in either direction. + """ try: prompt_cost, completion_cost = generic_cost_per_token( - model=model.model, usage=usage, custom_llm_provider=model.provider + model=pricing_key or model.model, usage=usage, custom_llm_provider=model.provider ) except Exception as e: # noqa: BLE001 # get_model_info raises bare Exception for unmapped models; degrade to zero savings verbose_proxy_logger.debug( @@ -166,6 +174,8 @@ def compute_autorouter_savings( selected_provider: str | None, usage: Usage, conversation_continuing: bool = True, + baseline_pricing_key: str | None = None, + selected_pricing_key: str | None = None, ) -> float: """Net dollars the router saved, or cost, by serving this request on ``selected_model``. @@ -187,8 +197,8 @@ def compute_autorouter_savings( selected = _resolve_model(selected_model, selected_provider) if baseline is None or selected is None or baseline == selected: return 0.0 - baseline_cost = _cost_of_usage(baseline, _baseline_usage(usage, conversation_continuing)) - selected_cost = _cost_of_usage(selected, usage) + baseline_cost = _cost_of_usage(baseline, _baseline_usage(usage, conversation_continuing), baseline_pricing_key) + selected_cost = _cost_of_usage(selected, usage, selected_pricing_key) if baseline_cost is None or selected_cost is None: return 0.0 return baseline_cost - selected_cost @@ -215,6 +225,7 @@ def compute_savings_spend( cache_read_input_tokens: int, routing_decision: Mapping[str, object] | None = None, usage_object: Mapping[str, object] | None = None, + model_map_information: Mapping[str, object] | None = None, ) -> SavingsSpend: """ Dollar savings for one request, split by optimization driver. @@ -237,6 +248,9 @@ def compute_savings_spend( decision = routing_decision if isinstance(routing_decision, Mapping) else {} baseline_model = decision.get("savings_baseline_model") + baseline_key = decision.get("savings_baseline_pricing_key") + model_map = model_map_information if isinstance(model_map_information, Mapping) else {} + selected_key = model_map.get("model_map_key") autorouter = compute_autorouter_savings( baseline_model=baseline_model if isinstance(baseline_model, str) else None, selected_model=model, @@ -245,5 +259,12 @@ def compute_savings_spend( # Absent means the router never recorded a shape, which is the conservative # reading: charge the cache write rather than claim a first turn's saving. conversation_continuing=decision.get("conversation_continuing") is not False, + baseline_pricing_key=baseline_key if isinstance(baseline_key, str) else None, + # The key litellm actually billed this request under, recorded at request time by + # `_select_model_name_for_cost_calc`. Re-deriving it from the spend log's `model` + # would lose whatever that resolver already applied: an Azure deployment name is + # absent from the cost map and prices to nothing, and a deployment's own price + # overrides live under a key the model name never reaches. + selected_pricing_key=selected_key if isinstance(selected_key, str) and selected_key else None, ) return SavingsSpend(compression=compression, prompt_caching=prompt_caching, autorouter=autorouter) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 8b75131ee165..73afa25a34df 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -28,6 +28,7 @@ from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY, RETURN_RAW_MODEL_NAME_METADATA_KEY from litellm.integrations.custom_logger import CustomLogger from litellm.llms.base_llm.base_utils import type_to_response_format_param +from litellm.router_strategy.savings_baseline import Baseline, resolve_baseline from litellm.types.utils import ( AUTOROUTER_CLASSIFIER_CALL_ORIGIN, ModelResponse, @@ -229,26 +230,26 @@ def _conversation_is_continuing(messages: Sequence[Mapping[str, object]] | None) The counterfactual the savings driver prices against is one model serving every turn, so whether that model had this prompt cached is just whether an earlier turn - exists: a second human ask means it wrote the prompt then and would only read it - now, and the write this request paid is what switching models cost. A single ask is - a conversation's first turn, where nothing was cached for any model and the baseline - would have paid the same write. + exists. An assistant turn in the history is the direct evidence of one: something + answered before, so a single-model deployment wrote the prompt then and would only + read it now, and the write this request paid is what switching models cost. A + conversation's first turn has no assistant turn, nothing was cached for any model, + and the baseline would have paid the same write. + + Assistant turns rather than human asks, because an agent loop can run twenty turns + on one human ask: its tool traffic rides `tool_result` blocks on user turns that + flatten to empty text, and on `tool` roles, so counting asks reads a long + conversation as its own first turn and hands it the untouched-write arithmetic. That + is the one direction this must never fail in, since it inflates. Reading the conversation rather than remembering it keeps this free of a cache, a session id and their failure modes, and it works for callers that send no session - header at all. It cannot see a model switch that happened to land on a turn the - router did not classify, and it reads a few-shot prompt's synthetic turns as prior - conversation; both err toward charging the write, which under-claims. - - Defaults to continuing when the messages cannot be read, for the same reason. + header at all. A few-shot prompt's synthetic assistant turns read as prior + conversation, which charges the write and under-claims; that is the safe side. """ if not messages: return True - asks = len(tuple(islice(_iter_human_asks_newest_first(messages), 2))) - # Exactly one human ask is a first turn. Zero means the turns carry no readable ask, - # which says nothing about the baseline's cache and so is treated like every other - # unknown here: charge the write. - return asks != 1 + return any(message.get("role") == "assistant" for message in messages) def _newest_turn_ask(messages: Sequence[Mapping[str, object]]) -> str | None: @@ -468,7 +469,7 @@ def _hardest_tier_models(self) -> tuple[str, ...]: return () @property - def savings_baseline_model(self) -> str | None: + def savings_baseline_model(self) -> Baseline | None: """The model this router's savings are measured against. A complexity router's tier ladder already names the model an operator @@ -477,8 +478,6 @@ def savings_baseline_model(self) -> str | None: router can reach; a cheap tier is a choice the router made, not a ceiling it was bounded by. """ - from litellm.router_strategy.savings_baseline import resolve_baseline - return resolve_baseline( self.configured_savings_baseline_model, self.litellm_router_instance, self._hardest_tier_models() ) @@ -722,9 +721,11 @@ def _build_routing_decision( cause=cause, conversation_continuing=conversation_continuing, ) - baseline_model = self.savings_baseline_model - if baseline_model is not None: - decision["savings_baseline_model"] = baseline_model + baseline = self.savings_baseline_model + if baseline is not None: + decision["savings_baseline_model"] = baseline.model + if baseline.pricing_key is not None: + decision["savings_baseline_pricing_key"] = baseline.pricing_key if tier is not None: decision["tier"] = tier.value if score is not None: diff --git a/litellm/router_strategy/savings_baseline.py b/litellm/router_strategy/savings_baseline.py index 900774eff16f..424c79c68b09 100644 --- a/litellm/router_strategy/savings_baseline.py +++ b/litellm/router_strategy/savings_baseline.py @@ -13,7 +13,7 @@ """ from collections.abc import Iterable -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, NamedTuple from litellm._logging import verbose_router_logger @@ -21,6 +21,45 @@ from litellm.router import Router +class Baseline(NamedTuple): + """The counterfactual model, and the key its rates are looked up under. + + Two identifiers because they answer different questions. ``model`` is what the + operator would recognise and what decides whether the router actually switched + away from it. ``pricing_key`` is what it costs: a deployment may override the + per-token prices, and the router registers that override in the cost map under + the deployment's own id, deliberately keeping it off the shared model-name key so + deployments do not pollute each other. Pricing the name would silently read the + public rate for a model the operator does not pay the public rate for. + """ + + model: str + pricing_key: str | None = None + + +def cost_key(model: str, custom_llm_provider: str | None, deployment_id: str | None) -> str | None: + """The key litellm prices this deployment under, or ``None`` when it is just the model. + + Delegates to `_select_model_name_for_cost_calc`, the resolver the real request is + billed through, rather than re-deciding when an override counts. Mirroring that rule + here would be a second model of the runtime, and it is subtler than it looks: a + per-second or tiered override counts, a partially specified one still counts, and a + deployment configured at zero is priced at zero rather than treated as unpriced. + """ + from litellm.cost_calculator import _select_model_name_for_cost_calc + + if deployment_id is None: + return None + resolved = _select_model_name_for_cost_calc( + model=model, + completion_response=None, + custom_pricing=True, + router_model_id=str(deployment_id), + custom_llm_provider=custom_llm_provider, + ) + return resolved if resolved and resolved != model else None + + def canonical_model(model: str, custom_llm_provider: str | None = None) -> str | None: """``provider/model``, or ``None`` when the pair names no known provider. @@ -38,20 +77,21 @@ def canonical_model(model: str, custom_llm_provider: str | None = None) -> str | return f"{provider}/{resolved}" -def _models_in(router: "Router", group_name: str) -> tuple[str, ...]: - """The models a tier entry actually calls, priced as the deployments declare. +def _models_in(router: "Router", group_name: str) -> tuple[Baseline, ...]: + """The candidates a tier entry actually calls, each with its own pricing key. `litellm_params.model` is not always a model: on Azure it is the deployment name, absent from the cost map, and `model_info.base_model` names the real one. Wildcard and aliased deployments behave the same, and router.py resolves pricing through that same base_model chain. A name matching no deployment is a tier pointing - straight at a provider model rather than at a configured group. + straight at a provider model rather than at a configured group, and prices under + its own name because there is no deployment to override it. """ indices = router.model_name_to_deployment_indices.get(group_name) if not indices: - return (qualified,) if (qualified := canonical_model(group_name)) else () + return (Baseline(qualified),) if (qualified := canonical_model(group_name)) else () - def priced_as(index: int) -> str | None: + def candidate(index: int) -> Baseline | None: deployment = router.model_list[index] params = deployment.get("litellm_params") if not isinstance(params, dict): @@ -59,40 +99,51 @@ def priced_as(index: int) -> str | None: info = deployment.get("model_info") base = info.get("base_model") if isinstance(info, dict) else None model = base or params.get("base_model") or params.get("model") - return canonical_model(model, params.get("custom_llm_provider")) if model else None + qualified = canonical_model(model, params.get("custom_llm_provider")) if model else None + if qualified is None: + return None + deployment_id = info.get("id") if isinstance(info, dict) else None + return Baseline(qualified, cost_key(qualified, params.get("custom_llm_provider"), deployment_id)) - return tuple(model for index in indices if (model := priced_as(index))) + return tuple(c for index in indices if (c := candidate(index)) is not None) -def _most_expensive(models: Iterable[str]) -> str | None: - """The priciest candidate by output rate, input rate breaking the tie. +def _priced(candidate: Baseline) -> tuple[float, float, Baseline] | None: + """``(output_rate, input_rate, candidate)``, or ``None`` when it cannot be priced. - A model that costs nothing per token cannot stand in for what the traffic would + A candidate that costs nothing per token cannot stand in for what the traffic would otherwise have cost; as a baseline it would report the whole real spend as a loss, so it is dropped rather than allowed to win a tie at zero. """ import litellm - def rates(model: str) -> tuple[float, float, str] | None: - try: - info = litellm.get_model_info(model=model) - except Exception as e: # noqa: BLE001 # unmapped candidates simply cannot be the baseline - verbose_router_logger.debug("savings baseline: no pricing for candidate %s (%s)", model, e) - return None - output_rate, input_rate = info.get("output_cost_per_token") or 0.0, info.get("input_cost_per_token") or 0.0 - if output_rate <= 0.0 and input_rate <= 0.0: - verbose_router_logger.debug("savings baseline: candidate %s has no per-token price", model) - return None - return (output_rate, input_rate, model) + try: + info = litellm.get_model_info(model=candidate.pricing_key or candidate.model) + except Exception as e: # noqa: BLE001 # unmapped candidates simply cannot be the baseline + verbose_router_logger.debug("savings baseline: no pricing for candidate %s (%s)", candidate.model, e) + return None + output_rate, input_rate = info.get("output_cost_per_token") or 0.0, info.get("input_cost_per_token") or 0.0 + if output_rate <= 0.0 and input_rate <= 0.0: + verbose_router_logger.debug("savings baseline: candidate %s has no per-token price", candidate.model) + return None + return (output_rate, input_rate, candidate) + - priced = tuple(r for model in models if (r := rates(model)) is not None) +def _most_expensive(candidates: Iterable[Baseline]) -> Baseline | None: + """The priciest candidate by output rate, input rate breaking the tie. + + Ranked on what each candidate really costs, so a deployment whose configured price + is the expensive one is chosen as the counterfactual; ranking on the public rate + picks the wrong baseline and then prices it at a rate nobody pays. + """ + priced = tuple(r for candidate in candidates if (r := _priced(candidate)) is not None) if not priced: verbose_router_logger.debug("savings baseline: no priceable candidates; savings driver disabled") return None return max(priced)[2] -def resolve_baseline(configured: str | None, router: "Router", group_names: Iterable[str]) -> str | None: +def resolve_baseline(configured: str | None, router: "Router", group_names: Iterable[str]) -> Baseline | None: """The baseline for a router whose hardest tier offers ``group_names``. A configured override wins and is only qualified, never re-derived. @@ -108,8 +159,10 @@ def resolve_baseline(configured: str | None, router: "Router", group_names: Iter """ try: if configured: - return canonical_model(configured) - return _most_expensive(model for name in group_names for model in _models_in(router, name)) + # No pricing key: a configured override names a model, not a deployment, + # so it prices under its own name like any other unmatched name. + return Baseline(qualified) if (qualified := canonical_model(configured)) else None + return _most_expensive(c for name in group_names for c in _models_in(router, name)) except Exception as e: # noqa: BLE001 # see docstring: routing must not fail for a metric verbose_router_logger.warning("savings baseline: could not resolve, savings will read zero (%s)", e) return None diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 938edfbc979f..9f999fe48cf0 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2735,6 +2735,7 @@ class StandardLoggingRoutingDecision(TypedDict, total=False): escalated: bool tier_boundaries: StandardLoggingRoutingDecisionTierBoundaries savings_baseline_model: str + savings_baseline_pricing_key: str conversation_continuing: bool @@ -2756,6 +2757,7 @@ class StandardLoggingRoutingDecision(TypedDict, total=False): "escalated", "tier_boundaries", "savings_baseline_model", + "savings_baseline_pricing_key", "conversation_continuing", } ) diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index c6ee14f7cba2..a0e26594ac75 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -5087,18 +5087,18 @@ def test_baseline_is_the_reasoning_tier_not_the_priciest_reachable_model(self): """The ladder names what an operator would have had to run for the hardest request; a pricier model sitting in a lower tier is a choice, not a ceiling.""" router = self._router_with_tiers({"SIMPLE": "top", "MEDIUM": "cheap", "REASONING": "mid"}) - assert router.savings_baseline_model == "anthropic/claude-sonnet-4-5" + assert router.savings_baseline_model.model == "anthropic/claude-sonnet-4-5" def test_baseline_is_the_priciest_model_when_the_reasoning_tier_is_a_pool(self): router = self._router_with_tiers({"SIMPLE": "cheap", "REASONING": ["cheap", "top", "mid"]}) - assert router.savings_baseline_model == "anthropic/claude-opus-4-5" + assert router.savings_baseline_model.model == "anthropic/claude-opus-4-5" def test_falls_back_to_the_hardest_configured_tier_when_reasoning_is_absent(self): router = self._router_with_tiers({"SIMPLE": "cheap", "COMPLEX": "top"}) - assert router.savings_baseline_model == "anthropic/claude-opus-4-5" + assert router.savings_baseline_model.model == "anthropic/claude-opus-4-5" def test_a_configured_baseline_wins_and_is_provider_qualified(self): router = self._router_with_tiers( {"SIMPLE": "cheap", "REASONING": "mid"}, savings_baseline_model="claude-opus-4-5" ) - assert router.savings_baseline_model == "anthropic/claude-opus-4-5" + assert router.savings_baseline_model.model == "anthropic/claude-opus-4-5" def test_an_unpriceable_tier_disables_the_driver_rather_than_inventing_a_baseline(self): router = self._router_with_tiers({"REASONING": "not-a-real-model-anywhere"}) assert router.savings_baseline_model is None @@ -5188,6 +5188,43 @@ async def test_it_touches_no_cache(self, mock_router_instance, basic_config): assert cache.async_get_cache.await_count == 0 assert cache.async_set_cache.await_count == 0 + @pytest.mark.parametrize( + "history", + [ + pytest.param( + [ + {"role": "user", "content": "do X"}, + {"role": "assistant", "content": [{"type": "tool_use", "id": "1", "name": "t", "input": {}}]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "1", "content": "r"}]}, + {"role": "assistant", "content": [{"type": "tool_use", "id": "2", "name": "t", "input": {}}]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "2", "content": "r"}]}, + ], + id="messages-api-tool-result-blocks", + ), + pytest.param( + [ + {"role": "user", "content": "do X"}, + {"role": "assistant", "tool_calls": [{"id": "1"}]}, + {"role": "tool", "tool_call_id": "1", "content": "r"}, + ], + id="chat-completions-tool-role", + ), + ], + ) + def test_an_agent_loop_on_one_human_ask_is_not_a_first_turn(self, history): + """An agent can run twenty turns on a single human ask: its tool traffic rides + `tool_result` blocks that flatten to empty text and `tool` roles. Counting human + asks read that as a first turn and handed it the untouched-write arithmetic, + which is the one direction this must never fail in, because it inflates.""" + from litellm.router_strategy.complexity_router.complexity_router import _conversation_is_continuing + + assert _conversation_is_continuing(history) is True + + def test_a_system_prompt_does_not_make_a_first_turn_look_continued(self): + from litellm.router_strategy.complexity_router.complexity_router import _conversation_is_continuing + + assert _conversation_is_continuing([{"role": "system", "content": "s"}, {"role": "user", "content": "hi"}]) is False + def test_unreadable_messages_stay_conservative(self): """No messages says nothing about the baseline's cache, so it keeps charging the write and under-claims rather than inflating.""" @@ -5195,7 +5232,7 @@ def test_unreadable_messages_stay_conservative(self): assert _conversation_is_continuing(None) is True assert _conversation_is_continuing([]) is True - assert _conversation_is_continuing([{"role": "assistant", "content": "no human ask here"}]) is True + assert _conversation_is_continuing([{"role": "user", "content": ""}]) is False @pytest.mark.asyncio async def test_the_shape_travels_on_every_pre_routing_response(self): diff --git a/tests/test_litellm/router_strategy/test_savings_baseline.py b/tests/test_litellm/router_strategy/test_savings_baseline.py index e88fbb7a10a4..5de04c7389d2 100644 --- a/tests/test_litellm/router_strategy/test_savings_baseline.py +++ b/tests/test_litellm/router_strategy/test_savings_baseline.py @@ -2,6 +2,7 @@ from litellm.router import Router from litellm.router_strategy.savings_baseline import ( + Baseline, canonical_model, _models_in, _most_expensive, @@ -37,34 +38,34 @@ def test_returns_none_for_a_name_no_provider_claims(self): class TestModelsForGroup: def test_resolves_a_group_to_the_models_its_deployments_call(self, parent): - assert _models_in(parent, "cheap") == ("anthropic/claude-haiku-4-5",) + assert [c.model for c in _models_in(parent, "cheap")] == ["anthropic/claude-haiku-4-5"] def test_returns_every_deployment_in_a_pooled_group(self, parent): - assert sorted(_models_in(parent, "pool")) == [ + assert sorted(c.model for c in _models_in(parent, "pool")) == [ "anthropic/claude-haiku-4-5", "anthropic/claude-opus-4-5", ] def test_treats_an_unknown_group_as_a_model_name(self, parent): """A tier can point straight at a provider model rather than a configured group.""" - assert _models_in(parent, "claude-opus-4-5") == ("anthropic/claude-opus-4-5",) + assert [c.model for c in _models_in(parent, "claude-opus-4-5")] == ["anthropic/claude-opus-4-5"] class TestMostExpensive: def test_picks_by_output_rate(self): assert ( - _most_expensive(["anthropic/claude-haiku-4-5", "anthropic/claude-opus-4-5"]) + _most_expensive([Baseline("anthropic/claude-haiku-4-5"), Baseline("anthropic/claude-opus-4-5")]).model == "anthropic/claude-opus-4-5" ) def test_ignores_models_with_no_per_token_price(self): """A free model as baseline would report the whole real spend as a loss.""" - assert _most_expensive(["not-a-real-model-anywhere", "anthropic/claude-haiku-4-5"]) == ( - "anthropic/claude-haiku-4-5" - ) + assert _most_expensive( + [Baseline("not-a-real-model-anywhere"), Baseline("anthropic/claude-haiku-4-5")] + ).model == "anthropic/claude-haiku-4-5" def test_returns_none_when_nothing_can_be_priced(self): - assert _most_expensive(["not-a-real-model-anywhere"]) is None + assert _most_expensive([Baseline("not-a-real-model-anywhere")]) is None def test_returns_none_for_an_empty_candidate_set(self): assert _most_expensive([]) is None @@ -72,10 +73,10 @@ def test_returns_none_for_an_empty_candidate_set(self): class TestResolveBaseline: def test_a_configured_baseline_wins_over_the_candidates(self, parent): - assert resolve_baseline("claude-haiku-4-5", parent, ["top"]) == "anthropic/claude-haiku-4-5" + assert resolve_baseline("claude-haiku-4-5", parent, ["top"]).model == "anthropic/claude-haiku-4-5" def test_derives_the_priciest_candidate_when_unconfigured(self, parent): - assert resolve_baseline(None, parent, ["cheap", "top"]) == "anthropic/claude-opus-4-5" + assert resolve_baseline(None, parent, ["cheap", "top"]).model == "anthropic/claude-opus-4-5" def test_never_raises_so_a_metric_cannot_fail_a_live_request(self): """Read on the routing path while decorating a request that is about to be @@ -115,7 +116,7 @@ def test_model_info_base_model_is_preferred_over_the_deployment_name(self): "model_info": {"base_model": "azure/gpt-4.1"}, }, ) - assert _models_in(router, "big") == ("azure/gpt-4.1",) + assert [c.model for c in _models_in(router, "big")] == ["azure/gpt-4.1"] def test_litellm_params_base_model_is_the_other_accepted_spelling(self): router = self._router( @@ -124,11 +125,11 @@ def test_litellm_params_base_model_is_the_other_accepted_spelling(self): "litellm_params": {"model": "azure/my-gpt5-deployment", "base_model": "azure/gpt-4.1"}, }, ) - assert _models_in(router, "big") == ("azure/gpt-4.1",) + assert [c.model for c in _models_in(router, "big")] == ["azure/gpt-4.1"] def test_a_deployment_without_a_base_model_still_prices_by_its_model(self): router = self._router({"model_name": "big", "litellm_params": {"model": "anthropic/claude-opus-4-5"}}) - assert _models_in(router, "big") == ("anthropic/claude-opus-4-5",) + assert [c.model for c in _models_in(router, "big")] == ["anthropic/claude-opus-4-5"] def test_an_azure_deployment_can_win_the_priciest_candidate(self): """Without the base_model hop the Azure candidate never prices, so the cheaper @@ -141,7 +142,7 @@ def test_an_azure_deployment_can_win_the_priciest_candidate(self): "model_info": {"base_model": "azure/gpt-4.1"}, }, ) - assert resolve_baseline(None, router, ["cheap", "big"]) == "azure/gpt-4.1" + assert resolve_baseline(None, router, ["cheap", "big"]).model == "azure/gpt-4.1" def test_an_all_azure_pool_still_has_a_baseline(self): """Otherwise nothing prices, the driver is disabled and the card reads $0.00.""" @@ -152,4 +153,40 @@ def test_an_all_azure_pool_still_has_a_baseline(self): "model_info": {"base_model": "azure/gpt-4.1"}, }, ) - assert resolve_baseline(None, router, ["big"]) == "azure/gpt-4.1" + assert resolve_baseline(None, router, ["big"]).model == "azure/gpt-4.1" + + +class TestDeploymentPricingOverrides: + """A deployment may not be charged the public rate for the model it names.""" + + @staticmethod + def _router(top_params: dict) -> Router: + return Router( + model_list=[ + {"model_name": "cheap", "litellm_params": {"model": "anthropic/claude-haiku-4-5"}}, + {"model_name": "top", "litellm_params": {"model": "anthropic/claude-opus-4-5", **top_params}}, + ] + ) + + def test_a_configured_price_decides_the_baseline_not_the_public_rate(self): + """A deployment configured far above its public rate is what the traffic would + really have cost. Ranking on the public rate picks the wrong counterfactual and + then prices it at a rate nobody pays.""" + router = self._router({}) + assert resolve_baseline(None, router, ["cheap", "top"]).model == "anthropic/claude-opus-4-5" + + # haiku configured 1000x above its public rate now outprices opus + overridden = Router( + model_list=[ + { + "model_name": "cheap", + "litellm_params": { + "model": "anthropic/claude-haiku-4-5", + "input_cost_per_token": 0.001, + "output_cost_per_token": 0.002, + }, + }, + {"model_name": "top", "litellm_params": {"model": "anthropic/claude-opus-4-5"}}, + ] + ) + assert resolve_baseline(None, overridden, ["cheap", "top"]).model == "anthropic/claude-haiku-4-5" From cfc4f130a0e79629a3e51f17baabeb05e6107774 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Sat, 1 Aug 2026 17:56:38 -0700 Subject: [PATCH 04/15] fix(spend): give the cost-key resolver both inputs the selected arm needs The served model was resolved through one input at a time, and each choice broke the half the other fixed. `model_map_key` is the served model already resolved through `base_model`, which is the only way an Azure deployment name reaches the cost map at all; without it the selected arm priced a name absent from the map, returned nothing, and the whole driver silently read zero for that traffic. But it is built without `router_model_id`, so it never carries a deployment's own price overrides, and a custom-priced deployment was compared at its public rate while the baseline used the real override. On a deployment configured well above its public rate that inverted the answer outright: a route that lost $21.88 reported saving $0.10. `_select_model_name_for_cost_calc` takes both, so it gets both. Which key prices a deployment stays its decision rather than a rule restated here. --- litellm/proxy/db/db_spend_update_writer.py | 1 + litellm/proxy/spend_tracking/savings.py | 18 +++++++++++------- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index b834f0dda275..fef3e4a425a5 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -1859,6 +1859,7 @@ async def _common_add_spend_log_transaction_to_daily_transaction( cache_read_input_tokens=cache_read_input_tokens, routing_decision=_metadata.get("routing_decision"), model_map_information=_metadata.get("model_map_information"), + model_id=payload.get("model_id"), usage_object=usage_obj, ) diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index ad70896de1de..f879eb8d99b9 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -14,6 +14,7 @@ import litellm from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.router_strategy.savings_baseline import cost_key from litellm.types.utils import PromptTokensDetailsWrapper, Usage @@ -226,6 +227,7 @@ def compute_savings_spend( routing_decision: Mapping[str, object] | None = None, usage_object: Mapping[str, object] | None = None, model_map_information: Mapping[str, object] | None = None, + model_id: str | None = None, ) -> SavingsSpend: """ Dollar savings for one request, split by optimization driver. @@ -249,8 +251,15 @@ def compute_savings_spend( decision = routing_decision if isinstance(routing_decision, Mapping) else {} baseline_model = decision.get("savings_baseline_model") baseline_key = decision.get("savings_baseline_pricing_key") + # Two inputs, because they fix different halves of the same problem and the + # resolver needs both. `model_map_key` is the served model already resolved through + # `base_model`, which is the only way an Azure deployment name reaches the cost map + # at all; it is built without `router_model_id`, so it never carries a deployment's + # own price overrides. `model_id` is the key those overrides are registered under. model_map = model_map_information if isinstance(model_map_information, Mapping) else {} - selected_key = model_map.get("model_map_key") + mapped = model_map.get("model_map_key") + resolved_model = mapped if isinstance(mapped, str) and mapped else model + selected_key = cost_key(resolved_model, custom_llm_provider, model_id) or resolved_model autorouter = compute_autorouter_savings( baseline_model=baseline_model if isinstance(baseline_model, str) else None, selected_model=model, @@ -260,11 +269,6 @@ def compute_savings_spend( # reading: charge the cache write rather than claim a first turn's saving. conversation_continuing=decision.get("conversation_continuing") is not False, baseline_pricing_key=baseline_key if isinstance(baseline_key, str) else None, - # The key litellm actually billed this request under, recorded at request time by - # `_select_model_name_for_cost_calc`. Re-deriving it from the spend log's `model` - # would lose whatever that resolver already applied: an Azure deployment name is - # absent from the cost map and prices to nothing, and a deployment's own price - # overrides live under a key the model name never reaches. - selected_pricing_key=selected_key if isinstance(selected_key, str) and selected_key else None, + selected_pricing_key=selected_key, ) return SavingsSpend(compression=compression, prompt_caching=prompt_caching, autorouter=autorouter) From 1b54ea1be8f056cfb29c4f017028ff87dfee09ca Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Sat, 1 Aug 2026 18:02:19 -0700 Subject: [PATCH 05/15] fix(spend): same model is only the same cost when it is the same deployment The short-circuit compared resolved model identity, so two deployments of one model collapsed to "no switch" and reported zero. They are not the same cost: a deployment can carry a negotiated rate, and routing from the dear one to the list-price one is a real saving the dashboard reported as $0.00 against a true $21.93. Both arms now carry the key litellm prices them under, so the comparison is between deployments rather than between names. --- litellm/proxy/spend_tracking/savings.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index f879eb8d99b9..d73930b8237a 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -196,7 +196,13 @@ def compute_autorouter_savings( # so there is no second field that could disagree with it. baseline = _resolve_model(baseline_model, None) selected = _resolve_model(selected_model, selected_provider) - if baseline is None or selected is None or baseline == selected: + if baseline is None or selected is None: + return 0.0 + # Same model is only the same cost when it is also the same deployment. Two + # deployments of one model can carry different negotiated rates, and routing from + # the dear one to the cheap one is a real saving that short-circuiting on the model + # name alone reports as zero. + if baseline == selected and baseline_pricing_key == selected_pricing_key: return 0.0 baseline_cost = _cost_of_usage(baseline, _baseline_usage(usage, conversation_continuing), baseline_pricing_key) selected_cost = _cost_of_usage(selected, usage, selected_pricing_key) From fa80e6468a820c4d4b7858c368d65f9b24a3848a Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Sat, 1 Aug 2026 19:28:38 -0700 Subject: [PATCH 06/15] refactor(spend): price from resolved rates, not from a name we keep re-resolving Four review rounds landed on one mechanism: which identifier prices a deployment. base_model, then the deployment id, then cache-only overrides. Each round added a clause to a resolution rule that should not exist, and a wrong primitive fails once per input shape, so each shape arrived as its own finding. `Router.get_deployment_model_info` already owns this. It merges a deployment's configured prices over the built-in map, folds in `base_model` defaults for deployments whose name is not a model, and falls back to the model name when nothing is overridden. Every shape hand-rolled here (cache-only, partial, per-second, Azure) was that function re-implemented badly. `generic_cost_per_token` now accepts already-resolved rates instead of demanding a name it looks up itself, which is what forced the name-bending in the first place. Both arms resolve through the owner and pass what they got: the counterfactual by the deployment the router would have used, the served request by the deployment that served it. The invented cost-key resolver is gone, and `Baseline` carries a deployment id rather than a key we chose on litellm's behalf. Net 64 insertions against 79 deletions. --- .../litellm_core_utils/llm_cost_calc/utils.py | 8 ++- litellm/proxy/db/db_spend_update_writer.py | 1 - litellm/proxy/spend_tracking/savings.py | 59 ++++++++-------- .../complexity_router/complexity_router.py | 4 +- litellm/router_strategy/savings_baseline.py | 67 +++++++------------ litellm/types/utils.py | 4 +- 6 files changed, 64 insertions(+), 79 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 5bc6107dbecb..bc5400d4846c 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -683,6 +683,7 @@ def generic_cost_per_token( custom_llm_provider: str, service_tier: str | None = None, data_residency: str | None = None, + model_info: ModelInfo | None = None, ) -> tuple[float, float]: """ Calculates the cost per token for a given model, prompt tokens, and completion tokens. @@ -700,7 +701,12 @@ def generic_cost_per_token( """ ## GET MODEL INFO - model_info = get_model_info(model=model, custom_llm_provider=custom_llm_provider) + # A caller that already resolved the deployment's effective rates passes them in + # rather than handing back a name for this to re-resolve. A name cannot express a + # per-deployment override: those are registered under the deployment id and kept off + # the shared model-name key, so resolving from the name here reads the public rate. + if model_info is None: + model_info = get_model_info(model=model, custom_llm_provider=custom_llm_provider) ## CALCULATE INPUT COST ### Cost of processing (non-cache hit + cache hit) + Cost of cache-writing (cache writing) diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index fef3e4a425a5..8e2d6ff52a53 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -1858,7 +1858,6 @@ async def _common_add_spend_log_transaction_to_daily_transaction( compression_saved_tokens=compression_saved_tokens, cache_read_input_tokens=cache_read_input_tokens, routing_decision=_metadata.get("routing_decision"), - model_map_information=_metadata.get("model_map_information"), model_id=payload.get("model_id"), usage_object=usage_obj, ) diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index d73930b8237a..cec2b30583d7 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -14,8 +14,7 @@ import litellm from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token -from litellm.router_strategy.savings_baseline import cost_key -from litellm.types.utils import PromptTokensDetailsWrapper, Usage +from litellm.types.utils import ModelInfo, PromptTokensDetailsWrapper, Usage class SavingsSpend(NamedTuple): @@ -76,19 +75,31 @@ def _resolve_model(model: str | None, custom_llm_provider: str | None) -> _Model return _ModelIdentity(model=resolved_model, provider=provider) -def _cost_of_usage(model: _ModelIdentity, usage: Usage, pricing_key: str | None = None) -> float | None: - """What ``usage`` costs on ``model``, or ``None`` when the model has no pricing. +def _effective_model_info(deployment_id: str | None, model: str) -> ModelInfo | None: + """What a deployment is actually charged, or ``None`` to price by name. - ``pricing_key`` is the key litellm bills this deployment under, which is the - deployment's own id when it overrides its prices and the model name otherwise. The - router keeps overrides off the shared model-name key so deployments sharing a - backend model do not pollute each other, so pricing the name would read the public - rate for a model nobody is charged the public rate for, on whichever arm is - overridden, in either direction. + `Router.get_deployment_model_info` owns this: it merges a deployment's configured + prices over the built-in map, folds in `base_model` defaults for deployments whose + name is not a model, and falls back to the model name when nothing is overridden. + Resolving a name here instead reads the public rate, which a deployment with a + negotiated price does not pay, and an Azure deployment name prices to nothing at all. """ + if deployment_id is None: + return None + try: + from litellm.proxy.proxy_server import llm_router + + return llm_router.get_deployment_model_info(deployment_id, model) if llm_router else None + except Exception as e: # noqa: BLE001 # a dashboard metric must not fail the spend write + verbose_proxy_logger.debug("savings: no deployment pricing for %s (%s)", model, e) + return None + + +def _cost_of_usage(model: _ModelIdentity, usage: Usage, model_info: ModelInfo | None = None) -> float | None: + """What ``usage`` costs on ``model``, or ``None`` when the model has no pricing.""" try: prompt_cost, completion_cost = generic_cost_per_token( - model=pricing_key or model.model, usage=usage, custom_llm_provider=model.provider + model=model.model, usage=usage, custom_llm_provider=model.provider, model_info=model_info ) except Exception as e: # noqa: BLE001 # get_model_info raises bare Exception for unmapped models; degrade to zero savings verbose_proxy_logger.debug( @@ -175,8 +186,8 @@ def compute_autorouter_savings( selected_provider: str | None, usage: Usage, conversation_continuing: bool = True, - baseline_pricing_key: str | None = None, - selected_pricing_key: str | None = None, + baseline_info: ModelInfo | None = None, + selected_info: ModelInfo | None = None, ) -> float: """Net dollars the router saved, or cost, by serving this request on ``selected_model``. @@ -202,10 +213,10 @@ def compute_autorouter_savings( # deployments of one model can carry different negotiated rates, and routing from # the dear one to the cheap one is a real saving that short-circuiting on the model # name alone reports as zero. - if baseline == selected and baseline_pricing_key == selected_pricing_key: + if baseline == selected and baseline_info == selected_info: return 0.0 - baseline_cost = _cost_of_usage(baseline, _baseline_usage(usage, conversation_continuing), baseline_pricing_key) - selected_cost = _cost_of_usage(selected, usage, selected_pricing_key) + baseline_cost = _cost_of_usage(baseline, _baseline_usage(usage, conversation_continuing), baseline_info) + selected_cost = _cost_of_usage(selected, usage, selected_info) if baseline_cost is None or selected_cost is None: return 0.0 return baseline_cost - selected_cost @@ -232,7 +243,6 @@ def compute_savings_spend( cache_read_input_tokens: int, routing_decision: Mapping[str, object] | None = None, usage_object: Mapping[str, object] | None = None, - model_map_information: Mapping[str, object] | None = None, model_id: str | None = None, ) -> SavingsSpend: """ @@ -256,16 +266,7 @@ def compute_savings_spend( decision = routing_decision if isinstance(routing_decision, Mapping) else {} baseline_model = decision.get("savings_baseline_model") - baseline_key = decision.get("savings_baseline_pricing_key") - # Two inputs, because they fix different halves of the same problem and the - # resolver needs both. `model_map_key` is the served model already resolved through - # `base_model`, which is the only way an Azure deployment name reaches the cost map - # at all; it is built without `router_model_id`, so it never carries a deployment's - # own price overrides. `model_id` is the key those overrides are registered under. - model_map = model_map_information if isinstance(model_map_information, Mapping) else {} - mapped = model_map.get("model_map_key") - resolved_model = mapped if isinstance(mapped, str) and mapped else model - selected_key = cost_key(resolved_model, custom_llm_provider, model_id) or resolved_model + baseline_id = decision.get("savings_baseline_deployment_id") autorouter = compute_autorouter_savings( baseline_model=baseline_model if isinstance(baseline_model, str) else None, selected_model=model, @@ -274,7 +275,7 @@ def compute_savings_spend( # Absent means the router never recorded a shape, which is the conservative # reading: charge the cache write rather than claim a first turn's saving. conversation_continuing=decision.get("conversation_continuing") is not False, - baseline_pricing_key=baseline_key if isinstance(baseline_key, str) else None, - selected_pricing_key=selected_key, + baseline_info=_effective_model_info(baseline_id if isinstance(baseline_id, str) else None, str(baseline_model)), + selected_info=_effective_model_info(model_id, model or ""), ) return SavingsSpend(compression=compression, prompt_caching=prompt_caching, autorouter=autorouter) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 73afa25a34df..c1fef252f599 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -724,8 +724,8 @@ def _build_routing_decision( baseline = self.savings_baseline_model if baseline is not None: decision["savings_baseline_model"] = baseline.model - if baseline.pricing_key is not None: - decision["savings_baseline_pricing_key"] = baseline.pricing_key + if baseline.deployment_id is not None: + decision["savings_baseline_deployment_id"] = baseline.deployment_id if tier is not None: decision["tier"] = tier.value if score is not None: diff --git a/litellm/router_strategy/savings_baseline.py b/litellm/router_strategy/savings_baseline.py index 424c79c68b09..5fa70a75e667 100644 --- a/litellm/router_strategy/savings_baseline.py +++ b/litellm/router_strategy/savings_baseline.py @@ -22,42 +22,16 @@ class Baseline(NamedTuple): - """The counterfactual model, and the key its rates are looked up under. - - Two identifiers because they answer different questions. ``model`` is what the - operator would recognise and what decides whether the router actually switched - away from it. ``pricing_key`` is what it costs: a deployment may override the - per-token prices, and the router registers that override in the cost map under - the deployment's own id, deliberately keeping it off the shared model-name key so - deployments do not pollute each other. Pricing the name would silently read the - public rate for a model the operator does not pay the public rate for. - """ - - model: str - pricing_key: str | None = None + """The counterfactual deployment: what it is called, and which deployment it was. - -def cost_key(model: str, custom_llm_provider: str | None, deployment_id: str | None) -> str | None: - """The key litellm prices this deployment under, or ``None`` when it is just the model. - - Delegates to `_select_model_name_for_cost_calc`, the resolver the real request is - billed through, rather than re-deciding when an override counts. Mirroring that rule - here would be a second model of the runtime, and it is subtler than it looks: a - per-second or tiered override counts, a partially specified one still counts, and a - deployment configured at zero is priced at zero rather than treated as unpriced. + ``model`` is what the operator would recognise, and what decides whether the router + switched away from it. ``deployment_id`` is how its rates are found, because a + deployment can be charged something other than its model's public rate and + `Router.get_deployment_model_info` is what merges the two. """ - from litellm.cost_calculator import _select_model_name_for_cost_calc - if deployment_id is None: - return None - resolved = _select_model_name_for_cost_calc( - model=model, - completion_response=None, - custom_pricing=True, - router_model_id=str(deployment_id), - custom_llm_provider=custom_llm_provider, - ) - return resolved if resolved and resolved != model else None + model: str + deployment_id: str | None = None def canonical_model(model: str, custom_llm_provider: str | None = None) -> str | None: @@ -103,25 +77,30 @@ def candidate(index: int) -> Baseline | None: if qualified is None: return None deployment_id = info.get("id") if isinstance(info, dict) else None - return Baseline(qualified, cost_key(qualified, params.get("custom_llm_provider"), deployment_id)) + return Baseline(qualified, str(deployment_id) if deployment_id else None) return tuple(c for index in indices if (c := candidate(index)) is not None) -def _priced(candidate: Baseline) -> tuple[float, float, Baseline] | None: +def _priced(router: "Router", candidate: Baseline) -> tuple[float, float, Baseline] | None: """``(output_rate, input_rate, candidate)``, or ``None`` when it cannot be priced. + Rates come from `Router.get_deployment_model_info`, which owns what a deployment is + actually charged: it merges the deployment's own configured prices over the built-in + map, folds in `base_model` defaults for deployments whose name is not a model, and + falls back to the model name when the deployment overrides nothing. Every override + shape is its problem, not ours. + A candidate that costs nothing per token cannot stand in for what the traffic would - otherwise have cost; as a baseline it would report the whole real spend as a loss, - so it is dropped rather than allowed to win a tie at zero. + otherwise have cost; as a baseline it would report the whole real spend as a loss. """ - import litellm - try: - info = litellm.get_model_info(model=candidate.pricing_key or candidate.model) - except Exception as e: # noqa: BLE001 # unmapped candidates simply cannot be the baseline + info = router.get_deployment_model_info(candidate.deployment_id or "", candidate.model) + except Exception as e: # noqa: BLE001 # an unpriceable candidate simply cannot be the baseline verbose_router_logger.debug("savings baseline: no pricing for candidate %s (%s)", candidate.model, e) return None + if info is None: + return None output_rate, input_rate = info.get("output_cost_per_token") or 0.0, info.get("input_cost_per_token") or 0.0 if output_rate <= 0.0 and input_rate <= 0.0: verbose_router_logger.debug("savings baseline: candidate %s has no per-token price", candidate.model) @@ -129,14 +108,14 @@ def _priced(candidate: Baseline) -> tuple[float, float, Baseline] | None: return (output_rate, input_rate, candidate) -def _most_expensive(candidates: Iterable[Baseline]) -> Baseline | None: +def _most_expensive(router: "Router", candidates: Iterable[Baseline]) -> Baseline | None: """The priciest candidate by output rate, input rate breaking the tie. Ranked on what each candidate really costs, so a deployment whose configured price is the expensive one is chosen as the counterfactual; ranking on the public rate picks the wrong baseline and then prices it at a rate nobody pays. """ - priced = tuple(r for candidate in candidates if (r := _priced(candidate)) is not None) + priced = tuple(r for candidate in candidates if (r := _priced(router, candidate)) is not None) if not priced: verbose_router_logger.debug("savings baseline: no priceable candidates; savings driver disabled") return None @@ -162,7 +141,7 @@ def resolve_baseline(configured: str | None, router: "Router", group_names: Iter # No pricing key: a configured override names a model, not a deployment, # so it prices under its own name like any other unmatched name. return Baseline(qualified) if (qualified := canonical_model(configured)) else None - return _most_expensive(c for name in group_names for c in _models_in(router, name)) + return _most_expensive(router, (c for name in group_names for c in _models_in(router, name))) except Exception as e: # noqa: BLE001 # see docstring: routing must not fail for a metric verbose_router_logger.warning("savings baseline: could not resolve, savings will read zero (%s)", e) return None diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 9f999fe48cf0..b5143af1cb5e 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2735,7 +2735,7 @@ class StandardLoggingRoutingDecision(TypedDict, total=False): escalated: bool tier_boundaries: StandardLoggingRoutingDecisionTierBoundaries savings_baseline_model: str - savings_baseline_pricing_key: str + savings_baseline_deployment_id: str conversation_continuing: bool @@ -2757,7 +2757,7 @@ class StandardLoggingRoutingDecision(TypedDict, total=False): "escalated", "tier_boundaries", "savings_baseline_model", - "savings_baseline_pricing_key", + "savings_baseline_deployment_id", "conversation_continuing", } ) From 6e173033e139789d85e24fcaa089165b5df91a27 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Sun, 2 Aug 2026 19:14:21 -0700 Subject: [PATCH 07/15] test(spend): follow _most_expensive onto the router that prices its candidates Ranking moved through `Router.get_deployment_model_info`, since what a deployment costs is the router's answer to give; these four cases were still calling the old free-function signature. --- .../router_strategy/test_savings_baseline.py | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/test_litellm/router_strategy/test_savings_baseline.py b/tests/test_litellm/router_strategy/test_savings_baseline.py index 5de04c7389d2..cfbbbb9987f8 100644 --- a/tests/test_litellm/router_strategy/test_savings_baseline.py +++ b/tests/test_litellm/router_strategy/test_savings_baseline.py @@ -52,23 +52,23 @@ def test_treats_an_unknown_group_as_a_model_name(self, parent): class TestMostExpensive: - def test_picks_by_output_rate(self): - assert ( - _most_expensive([Baseline("anthropic/claude-haiku-4-5"), Baseline("anthropic/claude-opus-4-5")]).model - == "anthropic/claude-opus-4-5" - ) + """Ranking runs through the router, because what a deployment costs is the + router's answer to give: it merges configured prices over the built-in map.""" + + def test_picks_by_output_rate(self, parent): + picked = _most_expensive(parent, [Baseline("anthropic/claude-haiku-4-5"), Baseline("anthropic/claude-opus-4-5")]) + assert picked.model == "anthropic/claude-opus-4-5" - def test_ignores_models_with_no_per_token_price(self): + def test_ignores_models_with_no_per_token_price(self, parent): """A free model as baseline would report the whole real spend as a loss.""" - assert _most_expensive( - [Baseline("not-a-real-model-anywhere"), Baseline("anthropic/claude-haiku-4-5")] - ).model == "anthropic/claude-haiku-4-5" + picked = _most_expensive(parent, [Baseline("not-a-real-model-anywhere"), Baseline("anthropic/claude-haiku-4-5")]) + assert picked.model == "anthropic/claude-haiku-4-5" - def test_returns_none_when_nothing_can_be_priced(self): - assert _most_expensive([Baseline("not-a-real-model-anywhere")]) is None + def test_returns_none_when_nothing_can_be_priced(self, parent): + assert _most_expensive(parent, [Baseline("not-a-real-model-anywhere")]) is None - def test_returns_none_for_an_empty_candidate_set(self): - assert _most_expensive([]) is None + def test_returns_none_for_an_empty_candidate_set(self, parent): + assert _most_expensive(parent, []) is None class TestResolveBaseline: From acc10e8c7bb4e2120bc32c37f35fb42244453f6c Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Sun, 2 Aug 2026 23:51:22 -0700 Subject: [PATCH 08/15] fix(spend): rank baseline candidates by what a request costs, not by two rates "Most expensive" was decided by comparing output rate then input rate. That is a property of a rate, not of a request: a deployment dearer per output token can be cheaper per cached token, so the comparison ordered cache-heavy traffic backwards and recorded the wrong counterfactual. Candidates are now costed on one reference request through the same engine the savings themselves use, which leaves cache read and write rates, tiered tables and every other billing dimension to that engine rather than to another rule restated here. The reference request is cache-heavy because auto-routed traffic is. --- litellm/router_strategy/savings_baseline.py | 59 +++++++++++++-------- 1 file changed, 36 insertions(+), 23 deletions(-) diff --git a/litellm/router_strategy/savings_baseline.py b/litellm/router_strategy/savings_baseline.py index 5fa70a75e667..725199f9fc2b 100644 --- a/litellm/router_strategy/savings_baseline.py +++ b/litellm/router_strategy/savings_baseline.py @@ -16,11 +16,22 @@ from typing import TYPE_CHECKING, NamedTuple from litellm._logging import verbose_router_logger +from litellm.types.utils import PromptTokensDetailsWrapper, Usage if TYPE_CHECKING: from litellm.router import Router +# One long cached prompt with a short completion: the shape auto-routed traffic takes, +# and the shape whose ordering a flat-rate comparison gets wrong. +_REFERENCE_REQUEST = Usage( + prompt_tokens=20_000, + completion_tokens=1_000, + total_tokens=21_000, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=19_000, cache_creation_tokens=1_000, text_tokens=0), +) + + class Baseline(NamedTuple): """The counterfactual deployment: what it is called, and which deployment it was. @@ -82,44 +93,46 @@ def candidate(index: int) -> Baseline | None: return tuple(c for index in indices if (c := candidate(index)) is not None) -def _priced(router: "Router", candidate: Baseline) -> tuple[float, float, Baseline] | None: - """``(output_rate, input_rate, candidate)``, or ``None`` when it cannot be priced. +def _priced(router: "Router", candidate: Baseline) -> tuple[float, Baseline] | None: + """``(cost_of_the_reference_request, candidate)``, or ``None`` when unpriceable. - Rates come from `Router.get_deployment_model_info`, which owns what a deployment is - actually charged: it merges the deployment's own configured prices over the built-in - map, folds in `base_model` defaults for deployments whose name is not a model, and - falls back to the model name when the deployment overrides nothing. Every override - shape is its problem, not ours. - - A candidate that costs nothing per token cannot stand in for what the traffic would - otherwise have cost; as a baseline it would report the whole real spend as a loss. + "Most expensive" is a property of a request, not of a rate: a deployment dearer per + output token can be cheaper per cached token, so comparing a chosen pair of rates + orders cache-heavy traffic backwards. Costing one reference request through the same + engine the savings use leaves cache rates, tiered tables and every other billing + dimension to that engine. A candidate that prices to nothing there cannot stand in + for what the traffic would have cost. """ + from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token + + provider, _, model_name = candidate.model.partition("/") try: info = router.get_deployment_model_info(candidate.deployment_id or "", candidate.model) + if info is None: + return None + prompt_cost, completion_cost = generic_cost_per_token( + model=model_name or candidate.model, + usage=_REFERENCE_REQUEST, + custom_llm_provider=provider, + model_info=info, + ) except Exception as e: # noqa: BLE001 # an unpriceable candidate simply cannot be the baseline verbose_router_logger.debug("savings baseline: no pricing for candidate %s (%s)", candidate.model, e) return None - if info is None: - return None - output_rate, input_rate = info.get("output_cost_per_token") or 0.0, info.get("input_cost_per_token") or 0.0 - if output_rate <= 0.0 and input_rate <= 0.0: - verbose_router_logger.debug("savings baseline: candidate %s has no per-token price", candidate.model) + cost = prompt_cost + completion_cost + if cost <= 0.0: + verbose_router_logger.debug("savings baseline: candidate %s prices to nothing", candidate.model) return None - return (output_rate, input_rate, candidate) + return (cost, candidate) def _most_expensive(router: "Router", candidates: Iterable[Baseline]) -> Baseline | None: - """The priciest candidate by output rate, input rate breaking the tie. - - Ranked on what each candidate really costs, so a deployment whose configured price - is the expensive one is chosen as the counterfactual; ranking on the public rate - picks the wrong baseline and then prices it at a rate nobody pays. - """ + """The candidate that would have cost the most on the reference request.""" priced = tuple(r for candidate in candidates if (r := _priced(router, candidate)) is not None) if not priced: verbose_router_logger.debug("savings baseline: no priceable candidates; savings driver disabled") return None - return max(priced)[2] + return max(priced)[1] def resolve_baseline(configured: str | None, router: "Router", group_names: Iterable[str]) -> Baseline | None: From ec72930035a7e31c60de5f59b346f2a308671388 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Mon, 3 Aug 2026 12:33:40 -0700 Subject: [PATCH 09/15] fix(spend): pick the baseline against the request that ran, not a stand-in for one Ranking happened in the pre-routing hook, where the request has not executed yet, so candidates were costed against a hard-coded reference workload: 20k prompt, 19k of it cached, 1k out. Which candidate is dearest depends on that mix, so a pooled hardest tier holding a deployment with non-proportional configured rates could be ranked for a request nothing like the one served. The mix is known on the spend path, so the ranking belongs there. The routing decision now carries the tier's candidates rather than a winner already chosen, and the baseline is resolved against the usage that actually happened. The reference workload is gone; nothing here assumes a traffic shape any more. The router is passed in rather than imported from `proxy_server` inside the computation, so the savings stay a pure function of their arguments and the caller owns where the router comes from. That also makes the spend path testable without a running proxy, which the previous shape was not. --- litellm/proxy/db/db_spend_update_writer.py | 17 ++++++ litellm/proxy/spend_tracking/savings.py | 56 +++++++++++++------ .../complexity_router/complexity_router.py | 31 +++++----- litellm/router_strategy/savings_baseline.py | 55 +++++++----------- litellm/types/utils.py | 6 +- .../proxy/spend_tracking/test_savings.py | 15 +++-- .../router_strategy/test_complexity_router.py | 39 +++++++------ .../router_strategy/test_savings_baseline.py | 33 +++++++---- 8 files changed, 143 insertions(+), 109 deletions(-) diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 8e2d6ff52a53..26ff27ad1944 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -59,6 +59,22 @@ extract_compression_saved_tokens, ) from litellm.proxy.spend_tracking.savings import compute_savings_spend + + +def _get_llm_router(): + """The proxy's router, or None outside a running proxy. + + Injected rather than imported where it is used, so the savings computation stays + a pure function of its arguments and the caller owns where the router comes from. + """ + try: + from litellm.proxy.proxy_server import llm_router + + return llm_router + except Exception: # noqa: BLE001 # no proxy in scope; savings degrade to zero + return None + + from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error if TYPE_CHECKING: @@ -1859,6 +1875,7 @@ async def _common_add_spend_log_transaction_to_daily_transaction( cache_read_input_tokens=cache_read_input_tokens, routing_decision=_metadata.get("routing_decision"), model_id=payload.get("model_id"), + llm_router=_get_llm_router(), usage_object=usage_obj, ) diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index cec2b30583d7..7fac2d30e726 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -9,11 +9,15 @@ """ from collections.abc import Mapping -from typing import NamedTuple +from typing import TYPE_CHECKING, NamedTuple import litellm from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.router_strategy.savings_baseline import Baseline + +if TYPE_CHECKING: + from litellm.router import Router from litellm.types.utils import ModelInfo, PromptTokensDetailsWrapper, Usage @@ -75,7 +79,7 @@ def _resolve_model(model: str | None, custom_llm_provider: str | None) -> _Model return _ModelIdentity(model=resolved_model, provider=provider) -def _effective_model_info(deployment_id: str | None, model: str) -> ModelInfo | None: +def _effective_model_info(router: "Router | None", deployment_id: str | None, model: str) -> ModelInfo | None: """What a deployment is actually charged, or ``None`` to price by name. `Router.get_deployment_model_info` owns this: it merges a deployment's configured @@ -84,17 +88,28 @@ def _effective_model_info(deployment_id: str | None, model: str) -> ModelInfo | Resolving a name here instead reads the public rate, which a deployment with a negotiated price does not pay, and an Azure deployment name prices to nothing at all. """ - if deployment_id is None: + if router is None or deployment_id is None: return None try: - from litellm.proxy.proxy_server import llm_router - - return llm_router.get_deployment_model_info(deployment_id, model) if llm_router else None + return router.get_deployment_model_info(deployment_id, model) except Exception as e: # noqa: BLE001 # a dashboard metric must not fail the spend write verbose_proxy_logger.debug("savings: no deployment pricing for %s (%s)", model, e) return None +def _resolve_baseline_for(router: "Router | None", candidates: object, baseline_usage: Usage) -> "Baseline | None": + """The dearest candidate for this request, or ``None`` when none can be priced.""" + if router is None or not isinstance(candidates, (list, tuple)) or not candidates: + return None + try: + from litellm.router_strategy.savings_baseline import resolve_baseline + + return resolve_baseline(router, [str(c) for c in candidates], baseline_usage) + except Exception as e: # noqa: BLE001 # a dashboard metric must not fail the spend write + verbose_proxy_logger.debug("savings: could not resolve a baseline (%s)", e) + return None + + def _cost_of_usage(model: _ModelIdentity, usage: Usage, model_info: ModelInfo | None = None) -> float | None: """What ``usage`` costs on ``model``, or ``None`` when the model has no pricing.""" try: @@ -181,13 +196,13 @@ def _baseline_usage(usage: Usage, conversation_continuing: bool) -> Usage: def compute_autorouter_savings( - baseline_model: str | None, + baseline: "Baseline | None", selected_model: str | None, selected_provider: str | None, usage: Usage, conversation_continuing: bool = True, - baseline_info: ModelInfo | None = None, selected_info: ModelInfo | None = None, + router: "Router | None" = None, ) -> float: """Net dollars the router saved, or cost, by serving this request on ``selected_model``. @@ -205,17 +220,18 @@ def compute_autorouter_savings( # No provider argument for the baseline on purpose: it arrives from the routing # metadata as a single self-describing string, already qualified by the auto-router, # so there is no second field that could disagree with it. - baseline = _resolve_model(baseline_model, None) + baseline_identity = _resolve_model(baseline.model if baseline else None, None) selected = _resolve_model(selected_model, selected_provider) - if baseline is None or selected is None: + if baseline is None or baseline_identity is None or selected is None: return 0.0 # Same model is only the same cost when it is also the same deployment. Two # deployments of one model can carry different negotiated rates, and routing from # the dear one to the cheap one is a real saving that short-circuiting on the model # name alone reports as zero. - if baseline == selected and baseline_info == selected_info: + baseline_info = _effective_model_info(router, baseline.deployment_id, baseline.model) + if baseline_identity == selected and baseline_info == selected_info: return 0.0 - baseline_cost = _cost_of_usage(baseline, _baseline_usage(usage, conversation_continuing), baseline_info) + baseline_cost = _cost_of_usage(baseline_identity, _baseline_usage(usage, conversation_continuing), baseline_info) selected_cost = _cost_of_usage(selected, usage, selected_info) if baseline_cost is None or selected_cost is None: return 0.0 @@ -244,6 +260,7 @@ def compute_savings_spend( routing_decision: Mapping[str, object] | None = None, usage_object: Mapping[str, object] | None = None, model_id: str | None = None, + llm_router: "Router | None" = None, ) -> SavingsSpend: """ Dollar savings for one request, split by optimization driver. @@ -265,17 +282,22 @@ def compute_savings_spend( return SavingsSpend(compression=compression, prompt_caching=prompt_caching) decision = routing_decision if isinstance(routing_decision, Mapping) else {} - baseline_model = decision.get("savings_baseline_model") - baseline_id = decision.get("savings_baseline_deployment_id") + # Which candidate is dearest depends on this request's token mix, so the baseline is + # picked here, against the usage that actually happened, rather than in the routing + # hook against a stand-in for it. + candidates = decision.get("savings_baseline_candidates") + baseline = _resolve_baseline_for( + llm_router, candidates, _baseline_usage(usage, decision.get("conversation_continuing") is not False) + ) autorouter = compute_autorouter_savings( - baseline_model=baseline_model if isinstance(baseline_model, str) else None, + baseline=baseline, selected_model=model, selected_provider=custom_llm_provider, usage=usage, # Absent means the router never recorded a shape, which is the conservative # reading: charge the cache write rather than claim a first turn's saving. conversation_continuing=decision.get("conversation_continuing") is not False, - baseline_info=_effective_model_info(baseline_id if isinstance(baseline_id, str) else None, str(baseline_model)), - selected_info=_effective_model_info(model_id, model or ""), + selected_info=_effective_model_info(llm_router, model_id, model or ""), + router=llm_router, ) return SavingsSpend(compression=compression, prompt_caching=prompt_caching, autorouter=autorouter) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index c1fef252f599..5a7f06a67f30 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -28,7 +28,6 @@ from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY, RETURN_RAW_MODEL_NAME_METADATA_KEY from litellm.integrations.custom_logger import CustomLogger from litellm.llms.base_llm.base_utils import type_to_response_format_param -from litellm.router_strategy.savings_baseline import Baseline, resolve_baseline from litellm.types.utils import ( AUTOROUTER_CLASSIFIER_CALL_ORIGIN, ModelResponse, @@ -469,18 +468,18 @@ def _hardest_tier_models(self) -> tuple[str, ...]: return () @property - def savings_baseline_model(self) -> Baseline | None: - """The model this router's savings are measured against. - - A complexity router's tier ladder already names the model an operator - would have had to run to serve the hardest request, so the counterfactual - is the priciest model in that tier rather than the priciest model the - router can reach; a cheap tier is a choice the router made, not a ceiling - it was bounded by. + def savings_baseline_candidates(self) -> tuple[str, ...]: + """What this router's savings could be measured against. + + The tier ladder already names what an operator would have had to run to serve + the hardest request, so the counterfactual comes from that tier rather than from + everything the router can reach; a cheap tier is a choice the router made, not a + ceiling it was bounded by. Which of them is dearest is not decided here: that + depends on the request's token mix, which does not exist until it has been + served, so the candidates travel and the spend path picks between them. """ - return resolve_baseline( - self.configured_savings_baseline_model, self.litellm_router_instance, self._hardest_tier_models() - ) + configured = self.configured_savings_baseline_model + return (configured,) if configured else self._hardest_tier_models() def _estimate_tokens(self, text: str) -> int: """ @@ -721,11 +720,9 @@ def _build_routing_decision( cause=cause, conversation_continuing=conversation_continuing, ) - baseline = self.savings_baseline_model - if baseline is not None: - decision["savings_baseline_model"] = baseline.model - if baseline.deployment_id is not None: - decision["savings_baseline_deployment_id"] = baseline.deployment_id + candidates = self.savings_baseline_candidates + if candidates: + decision["savings_baseline_candidates"] = candidates if tier is not None: decision["tier"] = tier.value if score is not None: diff --git a/litellm/router_strategy/savings_baseline.py b/litellm/router_strategy/savings_baseline.py index 725199f9fc2b..0251abc5b939 100644 --- a/litellm/router_strategy/savings_baseline.py +++ b/litellm/router_strategy/savings_baseline.py @@ -16,22 +16,12 @@ from typing import TYPE_CHECKING, NamedTuple from litellm._logging import verbose_router_logger -from litellm.types.utils import PromptTokensDetailsWrapper, Usage +from litellm.types.utils import Usage if TYPE_CHECKING: from litellm.router import Router -# One long cached prompt with a short completion: the shape auto-routed traffic takes, -# and the shape whose ordering a flat-rate comparison gets wrong. -_REFERENCE_REQUEST = Usage( - prompt_tokens=20_000, - completion_tokens=1_000, - total_tokens=21_000, - prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=19_000, cache_creation_tokens=1_000, text_tokens=0), -) - - class Baseline(NamedTuple): """The counterfactual deployment: what it is called, and which deployment it was. @@ -93,15 +83,15 @@ def candidate(index: int) -> Baseline | None: return tuple(c for index in indices if (c := candidate(index)) is not None) -def _priced(router: "Router", candidate: Baseline) -> tuple[float, Baseline] | None: - """``(cost_of_the_reference_request, candidate)``, or ``None`` when unpriceable. +def _priced(router: "Router", candidate: Baseline, usage: Usage) -> tuple[float, Baseline] | None: + """``(cost_of_this_request, candidate)``, or ``None`` when unpriceable. "Most expensive" is a property of a request, not of a rate: a deployment dearer per output token can be cheaper per cached token, so comparing a chosen pair of rates - orders cache-heavy traffic backwards. Costing one reference request through the same + orders cache-heavy traffic backwards. Costing the real request through the same engine the savings use leaves cache rates, tiered tables and every other billing - dimension to that engine. A candidate that prices to nothing there cannot stand in - for what the traffic would have cost. + dimension to that engine. A candidate that prices to nothing cannot stand in for + what the traffic would have cost. """ from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token @@ -112,7 +102,7 @@ def _priced(router: "Router", candidate: Baseline) -> tuple[float, Baseline] | N return None prompt_cost, completion_cost = generic_cost_per_token( model=model_name or candidate.model, - usage=_REFERENCE_REQUEST, + usage=usage, custom_llm_provider=provider, model_info=info, ) @@ -126,35 +116,28 @@ def _priced(router: "Router", candidate: Baseline) -> tuple[float, Baseline] | N return (cost, candidate) -def _most_expensive(router: "Router", candidates: Iterable[Baseline]) -> Baseline | None: - """The candidate that would have cost the most on the reference request.""" - priced = tuple(r for candidate in candidates if (r := _priced(router, candidate)) is not None) +def _most_expensive(router: "Router", candidates: Iterable[Baseline], usage: Usage) -> Baseline | None: + """The candidate that would have cost the most on this request.""" + priced = tuple(r for candidate in candidates if (r := _priced(router, candidate, usage)) is not None) if not priced: verbose_router_logger.debug("savings baseline: no priceable candidates; savings driver disabled") return None return max(priced)[1] -def resolve_baseline(configured: str | None, router: "Router", group_names: Iterable[str]) -> Baseline | None: - """The baseline for a router whose hardest tier offers ``group_names``. - - A configured override wins and is only qualified, never re-derived. +def resolve_baseline(router: "Router", candidates: Iterable[str], usage: Usage) -> Baseline | None: + """The dearest of ``candidates`` for this request's actual usage. - Derived per call rather than cached: the parent router adds and removes deployments - while it runs, so a baseline pinned on first use would keep naming a model the - router no longer has, and a pricier one added later could never become the baseline. - Resolving costs tens of microseconds against a network call. + Resolved against the served request rather than a stand-in for one, because which + candidate is dearest depends on the token mix: a deployment can be dear per output + token and cheap per cached token. That means this runs where the usage is known, on + the spend path, not in the pre-routing hook where the request has not happened yet. - Never raises. This is read on the routing path to decorate a request that is about - to be served, and a dashboard's counterfactual is not worth failing a live request - over; an unresolvable baseline zeroes the savings driver instead. + Never raises. A dashboard's counterfactual is not worth failing a spend write over; + an unresolvable baseline zeroes the savings driver instead. """ try: - if configured: - # No pricing key: a configured override names a model, not a deployment, - # so it prices under its own name like any other unmatched name. - return Baseline(qualified) if (qualified := canonical_model(configured)) else None - return _most_expensive(router, (c for name in group_names for c in _models_in(router, name))) + return _most_expensive(router, (c for name in candidates for c in _models_in(router, name)), usage) except Exception as e: # noqa: BLE001 # see docstring: routing must not fail for a metric verbose_router_logger.warning("savings baseline: could not resolve, savings will read zero (%s)", e) return None diff --git a/litellm/types/utils.py b/litellm/types/utils.py index b5143af1cb5e..09fb8b4d5af1 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2734,8 +2734,7 @@ class StandardLoggingRoutingDecision(TypedDict, total=False): classifier_model: str escalated: bool tier_boundaries: StandardLoggingRoutingDecisionTierBoundaries - savings_baseline_model: str - savings_baseline_deployment_id: str + savings_baseline_candidates: Sequence[str] conversation_continuing: bool @@ -2756,8 +2755,7 @@ class StandardLoggingRoutingDecision(TypedDict, total=False): "classifier_model", "escalated", "tier_boundaries", - "savings_baseline_model", - "savings_baseline_deployment_id", + "savings_baseline_candidates", "conversation_continuing", } ) diff --git a/tests/test_litellm/proxy/spend_tracking/test_savings.py b/tests/test_litellm/proxy/spend_tracking/test_savings.py index 8862b96d76c9..e9138018ec46 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_savings.py +++ b/tests/test_litellm/proxy/spend_tracking/test_savings.py @@ -6,7 +6,9 @@ import pytest import litellm +from litellm.router import Router from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.router_strategy.savings_baseline import Baseline from litellm.proxy.spend_tracking.savings import ( _baseline_usage, compute_autorouter_savings, @@ -137,7 +139,7 @@ def _savings(baseline: str, selected: str, usage: Usage, continuing: bool = True conversation's first turn, where nothing was cached for any model. """ return compute_autorouter_savings( - baseline_model=baseline, + baseline=Baseline(baseline), selected_model=selected, selected_provider="anthropic", usage=usage, @@ -304,8 +306,9 @@ def test_compute_savings_spend_carries_a_losing_switch_through(): custom_llm_provider="anthropic", compression_saved_tokens=0, cache_read_input_tokens=0, - routing_decision={"savings_baseline_model": "claude-sonnet-5"}, + routing_decision={"savings_baseline_candidates": ["dear"]}, usage_object=_cached_usage_object(), + llm_router=Router(model_list=[{"model_name": "dear", "litellm_params": {"model": "claude-sonnet-5"}}]), ) assert result.autorouter < 0 @@ -318,7 +321,7 @@ def test_malformed_usage_object_does_not_fail_the_spend_write(): custom_llm_provider="anthropic", compression_saved_tokens=1000, cache_read_input_tokens=0, - routing_decision={"savings_baseline_model": "claude-opus-5"}, + routing_decision={"savings_baseline_candidates": ["claude-opus-5"]}, usage_object={"prompt_tokens": ["not", "a", "number"]}, ) assert result.autorouter == 0.0 @@ -359,13 +362,13 @@ def test_baseline_is_priced_under_its_own_provider(): decides whether routing reads as a saving or a loss.""" usage = Usage(prompt_tokens=100_000, completion_tokens=10_000, total_tokens=110_000) azure = compute_autorouter_savings( - baseline_model="azure_ai/deepseek-r1", + baseline=Baseline("azure_ai/deepseek-r1"), selected_model="claude-haiku-4-5", selected_provider="anthropic", usage=usage, ) deepseek = compute_autorouter_savings( - baseline_model="deepseek/deepseek-r1", + baseline=Baseline("deepseek/deepseek-r1"), selected_model="claude-haiku-4-5", selected_provider="anthropic", usage=usage, @@ -416,7 +419,7 @@ def test_an_undetermined_conversation_shape_stays_conservative(): """ usage = _usage(fresh=0, cached=0, written=20_000, out=1_000) defaulted = compute_autorouter_savings( - baseline_model="anthropic/claude-opus-5", + baseline=Baseline("anthropic/claude-opus-5"), selected_model="claude-haiku-4-5", selected_provider="anthropic", usage=usage, diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index a0e26594ac75..445c0652e1a6 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -5064,11 +5064,14 @@ def test_a_window_stops_telling_the_model_to_disregard_it(self): assert "rate the work it approves rather than the reply itself" in system_prompt -class TestSavingsBaselineModel: - """The counterfactual model a complexity router's savings are measured against.""" +class TestSavingsBaselineCandidates: + """What the router offers as the counterfactual. Which of them is dearest depends on + the request's token mix, so that is decided on the spend path, not here.""" + @staticmethod def _router_with_tiers(tiers: dict, **kwargs) -> ComplexityRouter: from litellm.router import Router + parent = Router( model_list=[ {"model_name": "cheap", "litellm_params": {"model": "anthropic/claude-haiku-4-5"}}, @@ -5083,33 +5086,35 @@ def _router_with_tiers(tiers: dict, **kwargs) -> ComplexityRouter: default_model="mid", **kwargs, ) - def test_baseline_is_the_reasoning_tier_not_the_priciest_reachable_model(self): + + def test_candidates_are_the_reasoning_tier_not_everything_reachable(self): """The ladder names what an operator would have had to run for the hardest request; a pricier model sitting in a lower tier is a choice, not a ceiling.""" router = self._router_with_tiers({"SIMPLE": "top", "MEDIUM": "cheap", "REASONING": "mid"}) - assert router.savings_baseline_model.model == "anthropic/claude-sonnet-4-5" - def test_baseline_is_the_priciest_model_when_the_reasoning_tier_is_a_pool(self): + assert router.savings_baseline_candidates == ("mid",) + + def test_a_pooled_tier_offers_every_member(self): router = self._router_with_tiers({"SIMPLE": "cheap", "REASONING": ["cheap", "top", "mid"]}) - assert router.savings_baseline_model.model == "anthropic/claude-opus-4-5" + assert sorted(router.savings_baseline_candidates) == ["cheap", "mid", "top"] + def test_falls_back_to_the_hardest_configured_tier_when_reasoning_is_absent(self): router = self._router_with_tiers({"SIMPLE": "cheap", "COMPLEX": "top"}) - assert router.savings_baseline_model.model == "anthropic/claude-opus-4-5" - def test_a_configured_baseline_wins_and_is_provider_qualified(self): + assert router.savings_baseline_candidates == ("top",) + + def test_a_configured_baseline_replaces_the_tier(self): router = self._router_with_tiers( {"SIMPLE": "cheap", "REASONING": "mid"}, savings_baseline_model="claude-opus-4-5" ) - assert router.savings_baseline_model.model == "anthropic/claude-opus-4-5" - def test_an_unpriceable_tier_disables_the_driver_rather_than_inventing_a_baseline(self): - router = self._router_with_tiers({"REASONING": "not-a-real-model-anywhere"}) - assert router.savings_baseline_model is None - def test_the_baseline_travels_on_every_pre_routing_response(self): - """A response without it silently zeroes the savings driver for that path.""" + assert router.savings_baseline_candidates == ("claude-opus-4-5",) + + def test_the_candidates_travel_on_every_routing_decision(self): + """A decision without them silently zeroes the savings driver for that path.""" import inspect + from litellm.router_strategy.complexity_router import complexity_router as module + source = inspect.getsource(module.ComplexityRouter._build_routing_decision) - assert "self.savings_baseline_model" in source, ( - "the baseline rides on the routing decision, so every path that builds one carries it" - ) + assert "self.savings_baseline_candidates" in source class TestConversationShapeDiscriminator: diff --git a/tests/test_litellm/router_strategy/test_savings_baseline.py b/tests/test_litellm/router_strategy/test_savings_baseline.py index cfbbbb9987f8..e5ff0e502ac3 100644 --- a/tests/test_litellm/router_strategy/test_savings_baseline.py +++ b/tests/test_litellm/router_strategy/test_savings_baseline.py @@ -1,5 +1,14 @@ import pytest +from litellm.types.utils import PromptTokensDetailsWrapper, Usage + +REQUEST = Usage( + prompt_tokens=20_000, + completion_tokens=1_000, + total_tokens=21_000, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=0, cache_creation_tokens=0, text_tokens=20_000), +) + from litellm.router import Router from litellm.router_strategy.savings_baseline import ( Baseline, @@ -56,27 +65,27 @@ class TestMostExpensive: router's answer to give: it merges configured prices over the built-in map.""" def test_picks_by_output_rate(self, parent): - picked = _most_expensive(parent, [Baseline("anthropic/claude-haiku-4-5"), Baseline("anthropic/claude-opus-4-5")]) + picked = _most_expensive(parent, [Baseline("anthropic/claude-haiku-4-5"), Baseline("anthropic/claude-opus-4-5")], REQUEST) assert picked.model == "anthropic/claude-opus-4-5" def test_ignores_models_with_no_per_token_price(self, parent): """A free model as baseline would report the whole real spend as a loss.""" - picked = _most_expensive(parent, [Baseline("not-a-real-model-anywhere"), Baseline("anthropic/claude-haiku-4-5")]) + picked = _most_expensive(parent, [Baseline("not-a-real-model-anywhere"), Baseline("anthropic/claude-haiku-4-5")], REQUEST) assert picked.model == "anthropic/claude-haiku-4-5" def test_returns_none_when_nothing_can_be_priced(self, parent): - assert _most_expensive(parent, [Baseline("not-a-real-model-anywhere")]) is None + assert _most_expensive(parent, [Baseline("not-a-real-model-anywhere")], REQUEST) is None def test_returns_none_for_an_empty_candidate_set(self, parent): - assert _most_expensive(parent, []) is None + assert _most_expensive(parent, [], REQUEST) is None class TestResolveBaseline: def test_a_configured_baseline_wins_over_the_candidates(self, parent): - assert resolve_baseline("claude-haiku-4-5", parent, ["top"]).model == "anthropic/claude-haiku-4-5" + assert resolve_baseline(parent, ["claude-haiku-4-5"], REQUEST).model == "anthropic/claude-haiku-4-5" def test_derives_the_priciest_candidate_when_unconfigured(self, parent): - assert resolve_baseline(None, parent, ["cheap", "top"]).model == "anthropic/claude-opus-4-5" + assert resolve_baseline(parent, ["cheap", "top"], REQUEST).model == "anthropic/claude-opus-4-5" def test_never_raises_so_a_metric_cannot_fail_a_live_request(self): """Read on the routing path while decorating a request that is about to be @@ -87,10 +96,10 @@ class Exploding: def model_name_to_deployment_indices(self): raise RuntimeError("router is mid-reload") - assert resolve_baseline(None, Exploding(), ["anything"]) is None + assert resolve_baseline(Exploding(), ["anything"], REQUEST) is None def test_an_empty_candidate_set_zeroes_the_driver_rather_than_inventing_one(self, parent): - assert resolve_baseline(None, parent, []) is None + assert resolve_baseline(parent, [], REQUEST) is None class TestDeploymentsPricedByBaseModel: @@ -142,7 +151,7 @@ def test_an_azure_deployment_can_win_the_priciest_candidate(self): "model_info": {"base_model": "azure/gpt-4.1"}, }, ) - assert resolve_baseline(None, router, ["cheap", "big"]).model == "azure/gpt-4.1" + assert resolve_baseline(router, ["cheap", "big"], REQUEST).model == "azure/gpt-4.1" def test_an_all_azure_pool_still_has_a_baseline(self): """Otherwise nothing prices, the driver is disabled and the card reads $0.00.""" @@ -153,7 +162,7 @@ def test_an_all_azure_pool_still_has_a_baseline(self): "model_info": {"base_model": "azure/gpt-4.1"}, }, ) - assert resolve_baseline(None, router, ["big"]).model == "azure/gpt-4.1" + assert resolve_baseline(router, ["big"], REQUEST).model == "azure/gpt-4.1" class TestDeploymentPricingOverrides: @@ -173,7 +182,7 @@ def test_a_configured_price_decides_the_baseline_not_the_public_rate(self): really have cost. Ranking on the public rate picks the wrong counterfactual and then prices it at a rate nobody pays.""" router = self._router({}) - assert resolve_baseline(None, router, ["cheap", "top"]).model == "anthropic/claude-opus-4-5" + assert resolve_baseline(router, ["cheap", "top"], REQUEST).model == "anthropic/claude-opus-4-5" # haiku configured 1000x above its public rate now outprices opus overridden = Router( @@ -189,4 +198,4 @@ def test_a_configured_price_decides_the_baseline_not_the_public_rate(self): {"model_name": "top", "litellm_params": {"model": "anthropic/claude-opus-4-5"}}, ] ) - assert resolve_baseline(None, overridden, ["cheap", "top"]).model == "anthropic/claude-haiku-4-5" + assert resolve_baseline(overridden, ["cheap", "top"], REQUEST).model == "anthropic/claude-haiku-4-5" From 8d0283c5060a771c41ed2f1e6fca19f2fd7d5412 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Mon, 3 Aug 2026 13:17:01 -0700 Subject: [PATCH 10/15] refactor(spend): measure savings against one configured model, not a derived one The counterfactual was derived per request: enumerate the hardest tier's deployments, resolve each one's effective pricing, price them all, take the dearest. That machinery produced a review finding per input shape it had not anticipated, and every answer it gave was one an operator could have stated in a line of config. So they state it. `litellm_settings.autorouter_savings_baseline_model` names the model the traffic would have run on without a router, for every auto-router on the proxy, and unset means the driver is off rather than a model nobody named being guessed at. `savings_baseline.py` and its tests are deleted outright, along with the tier enumeration, the candidate list on the routing decision, and the per-deployment override that shadowed it. Cache-state handling is untouched: the baseline is still priced on this request's own read and write split, so a switch still pays for re-warming the cache and a first turn still charges the write to both arms. 45 insertions against 482 deletions. --- litellm/__init__.py | 1 + litellm/proxy/spend_tracking/savings.py | 59 ++--- litellm/router.py | 1 - .../complexity_router/complexity_router.py | 33 --- litellm/router_strategy/savings_baseline.py | 143 ------------- litellm/types/router.py | 1 - litellm/types/utils.py | 3 - .../proxy/spend_tracking/test_savings.py | 32 ++- .../router_strategy/test_complexity_router.py | 53 ----- .../router_strategy/test_savings_baseline.py | 201 ------------------ ui/litellm-dashboard/src/lib/http/schema.d.ts | 4 - 11 files changed, 45 insertions(+), 486 deletions(-) delete mode 100644 litellm/router_strategy/savings_baseline.py delete mode 100644 tests/test_litellm/router_strategy/test_savings_baseline.py diff --git a/litellm/__init__.py b/litellm/__init__.py index a9a78846fa14..62c41c2959b2 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -264,6 +264,7 @@ def _dev_env_hot_reload_enabled() -> bool: openai_like_key: Optional[str] = None azure_key: Optional[str] = None anthropic_key: Optional[str] = None +autorouter_savings_baseline_model: Optional[str] = None replicate_key: Optional[str] = None bytez_key: Optional[str] = None gdc_key: Optional[str] = None diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index 7fac2d30e726..ce8c1249a277 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -14,7 +14,6 @@ import litellm from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token -from litellm.router_strategy.savings_baseline import Baseline if TYPE_CHECKING: from litellm.router import Router @@ -97,19 +96,6 @@ def _effective_model_info(router: "Router | None", deployment_id: str | None, mo return None -def _resolve_baseline_for(router: "Router | None", candidates: object, baseline_usage: Usage) -> "Baseline | None": - """The dearest candidate for this request, or ``None`` when none can be priced.""" - if router is None or not isinstance(candidates, (list, tuple)) or not candidates: - return None - try: - from litellm.router_strategy.savings_baseline import resolve_baseline - - return resolve_baseline(router, [str(c) for c in candidates], baseline_usage) - except Exception as e: # noqa: BLE001 # a dashboard metric must not fail the spend write - verbose_proxy_logger.debug("savings: could not resolve a baseline (%s)", e) - return None - - def _cost_of_usage(model: _ModelIdentity, usage: Usage, model_info: ModelInfo | None = None) -> float | None: """What ``usage`` costs on ``model``, or ``None`` when the model has no pricing.""" try: @@ -196,13 +182,12 @@ def _baseline_usage(usage: Usage, conversation_continuing: bool) -> Usage: def compute_autorouter_savings( - baseline: "Baseline | None", + baseline_model: str | None, selected_model: str | None, selected_provider: str | None, usage: Usage, conversation_continuing: bool = True, selected_info: ModelInfo | None = None, - router: "Router | None" = None, ) -> float: """Net dollars the router saved, or cost, by serving this request on ``selected_model``. @@ -220,18 +205,17 @@ def compute_autorouter_savings( # No provider argument for the baseline on purpose: it arrives from the routing # metadata as a single self-describing string, already qualified by the auto-router, # so there is no second field that could disagree with it. - baseline_identity = _resolve_model(baseline.model if baseline else None, None) + baseline = _resolve_model(baseline_model, None) selected = _resolve_model(selected_model, selected_provider) - if baseline is None or baseline_identity is None or selected is None: + if baseline is None or selected is None: return 0.0 # Same model is only the same cost when it is also the same deployment. Two # deployments of one model can carry different negotiated rates, and routing from # the dear one to the cheap one is a real saving that short-circuiting on the model # name alone reports as zero. - baseline_info = _effective_model_info(router, baseline.deployment_id, baseline.model) - if baseline_identity == selected and baseline_info == selected_info: + if baseline == selected: return 0.0 - baseline_cost = _cost_of_usage(baseline_identity, _baseline_usage(usage, conversation_continuing), baseline_info) + baseline_cost = _cost_of_usage(baseline, _baseline_usage(usage, conversation_continuing)) selected_cost = _cost_of_usage(selected, usage, selected_info) if baseline_cost is None or selected_cost is None: return 0.0 @@ -281,23 +265,22 @@ def compute_savings_spend( if usage is None or not model: return SavingsSpend(compression=compression, prompt_caching=prompt_caching) + # The counterfactual is one model an operator would have run instead of the router, + # configured once for the proxy rather than derived per request. Unset means the + # driver is off; a routing decision is what says this request was auto-routed at all. decision = routing_decision if isinstance(routing_decision, Mapping) else {} - # Which candidate is dearest depends on this request's token mix, so the baseline is - # picked here, against the usage that actually happened, rather than in the routing - # hook against a stand-in for it. - candidates = decision.get("savings_baseline_candidates") - baseline = _resolve_baseline_for( - llm_router, candidates, _baseline_usage(usage, decision.get("conversation_continuing") is not False) - ) - autorouter = compute_autorouter_savings( - baseline=baseline, - selected_model=model, - selected_provider=custom_llm_provider, - usage=usage, - # Absent means the router never recorded a shape, which is the conservative - # reading: charge the cache write rather than claim a first turn's saving. - conversation_continuing=decision.get("conversation_continuing") is not False, - selected_info=_effective_model_info(llm_router, model_id, model or ""), - router=llm_router, + autorouter = ( + compute_autorouter_savings( + baseline_model=litellm.autorouter_savings_baseline_model, + selected_model=model, + selected_provider=custom_llm_provider, + usage=usage, + # Absent means the router never recorded a shape, which is the conservative + # reading: charge the cache write rather than claim a first turn's saving. + conversation_continuing=decision.get("conversation_continuing") is not False, + selected_info=_effective_model_info(llm_router, model_id, model or ""), + ) + if decision + else 0.0 ) return SavingsSpend(compression=compression, prompt_caching=prompt_caching, autorouter=autorouter) diff --git a/litellm/router.py b/litellm/router.py index 24fde45f3349..37190bdbf38a 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -7673,7 +7673,6 @@ def init_complexity_router_deployment(self, deployment: Deployment): default_model=default_model, litellm_router_instance=self, complexity_router_config=complexity_router_config, - savings_baseline_model=deployment.litellm_params.auto_router_savings_baseline_model, ) self._register_pre_routing_strategy( registry=self.complexity_routers, diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 5a7f06a67f30..492cf7465c7d 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -392,7 +392,6 @@ def __init__( litellm_router_instance: Router, complexity_router_config: dict[str, Any] | None = None, default_model: str | None = None, - savings_baseline_model: str | None = None, ): """ Initialize ComplexityRouter. @@ -405,7 +404,6 @@ def __init__( """ self.model_name = model_name self.litellm_router_instance = litellm_router_instance - self.configured_savings_baseline_model: str | None = savings_baseline_model # Parse config - always create a new instance to avoid singleton mutation if complexity_router_config: @@ -453,34 +451,6 @@ def __init__( verbose_router_logger.debug(f"ComplexityRouter initialized for {model_name} with tiers: {self.config.tiers}") - def _hardest_tier_models(self) -> tuple[str, ...]: - """The model or pool serving the hardest tier this router configures. - - REASONING when it is configured, since that is the tier a request has to be - hard enough to reach; otherwise the highest-severity tier present, so a - deployment that only defines SIMPLE and MEDIUM is still measured against - the best it could actually have picked. - """ - for tier in reversed(TIER_SEVERITY_ORDER): - models = self.config.tiers.get(tier.value) - if models: - return tuple(models) if isinstance(models, list) else (models,) - return () - - @property - def savings_baseline_candidates(self) -> tuple[str, ...]: - """What this router's savings could be measured against. - - The tier ladder already names what an operator would have had to run to serve - the hardest request, so the counterfactual comes from that tier rather than from - everything the router can reach; a cheap tier is a choice the router made, not a - ceiling it was bounded by. Which of them is dearest is not decided here: that - depends on the request's token mix, which does not exist until it has been - served, so the candidates travel and the spend path picks between them. - """ - configured = self.configured_savings_baseline_model - return (configured,) if configured else self._hardest_tier_models() - def _estimate_tokens(self, text: str) -> int: """ Estimate token count from text. @@ -720,9 +690,6 @@ def _build_routing_decision( cause=cause, conversation_continuing=conversation_continuing, ) - candidates = self.savings_baseline_candidates - if candidates: - decision["savings_baseline_candidates"] = candidates if tier is not None: decision["tier"] = tier.value if score is not None: diff --git a/litellm/router_strategy/savings_baseline.py b/litellm/router_strategy/savings_baseline.py deleted file mode 100644 index 0251abc5b939..000000000000 --- a/litellm/router_strategy/savings_baseline.py +++ /dev/null @@ -1,143 +0,0 @@ -"""The counterfactual model a complexity router's savings are measured against. - -Without the router a deployment has to pick one model, and it has to be one that can -carry the hardest request it will see. That model is the baseline: what the traffic -would have cost had nobody routed it. The router's own tier ladder already names it, -so the candidates are the models in the hardest configured tier; a cheap tier is a -choice the router made, not a ceiling it was bounded by. - -Baselines are always provider-qualified, whether derived or configured, because they -travel to the spend writer as a bare string with no provider beside them; an operator -who writes ``deepseek-r1`` meaning Azure would otherwise be priced against whoever -else owns that name. -""" - -from collections.abc import Iterable -from typing import TYPE_CHECKING, NamedTuple - -from litellm._logging import verbose_router_logger -from litellm.types.utils import Usage - -if TYPE_CHECKING: - from litellm.router import Router - - -class Baseline(NamedTuple): - """The counterfactual deployment: what it is called, and which deployment it was. - - ``model`` is what the operator would recognise, and what decides whether the router - switched away from it. ``deployment_id`` is how its rates are found, because a - deployment can be charged something other than its model's public rate and - `Router.get_deployment_model_info` is what merges the two. - """ - - model: str - deployment_id: str | None = None - - -def canonical_model(model: str, custom_llm_provider: str | None = None) -> str | None: - """``provider/model``, or ``None`` when the pair names no known provider. - - A deployment may name its vendor in the model prefix or in a separate - ``custom_llm_provider``, and the bare name alone is not enough to price: it can - resolve to a different vendor's rates, or to nothing at all. - """ - import litellm - - try: - resolved, provider, _, _ = litellm.get_llm_provider(model=model, custom_llm_provider=custom_llm_provider) - except Exception as e: # noqa: BLE001 # an unroutable candidate cannot be the baseline - verbose_router_logger.debug("savings baseline: cannot resolve candidate %s (%s)", model, e) - return None - return f"{provider}/{resolved}" - - -def _models_in(router: "Router", group_name: str) -> tuple[Baseline, ...]: - """The candidates a tier entry actually calls, each with its own pricing key. - - `litellm_params.model` is not always a model: on Azure it is the deployment name, - absent from the cost map, and `model_info.base_model` names the real one. Wildcard - and aliased deployments behave the same, and router.py resolves pricing through - that same base_model chain. A name matching no deployment is a tier pointing - straight at a provider model rather than at a configured group, and prices under - its own name because there is no deployment to override it. - """ - indices = router.model_name_to_deployment_indices.get(group_name) - if not indices: - return (Baseline(qualified),) if (qualified := canonical_model(group_name)) else () - - def candidate(index: int) -> Baseline | None: - deployment = router.model_list[index] - params = deployment.get("litellm_params") - if not isinstance(params, dict): - return None - info = deployment.get("model_info") - base = info.get("base_model") if isinstance(info, dict) else None - model = base or params.get("base_model") or params.get("model") - qualified = canonical_model(model, params.get("custom_llm_provider")) if model else None - if qualified is None: - return None - deployment_id = info.get("id") if isinstance(info, dict) else None - return Baseline(qualified, str(deployment_id) if deployment_id else None) - - return tuple(c for index in indices if (c := candidate(index)) is not None) - - -def _priced(router: "Router", candidate: Baseline, usage: Usage) -> tuple[float, Baseline] | None: - """``(cost_of_this_request, candidate)``, or ``None`` when unpriceable. - - "Most expensive" is a property of a request, not of a rate: a deployment dearer per - output token can be cheaper per cached token, so comparing a chosen pair of rates - orders cache-heavy traffic backwards. Costing the real request through the same - engine the savings use leaves cache rates, tiered tables and every other billing - dimension to that engine. A candidate that prices to nothing cannot stand in for - what the traffic would have cost. - """ - from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token - - provider, _, model_name = candidate.model.partition("/") - try: - info = router.get_deployment_model_info(candidate.deployment_id or "", candidate.model) - if info is None: - return None - prompt_cost, completion_cost = generic_cost_per_token( - model=model_name or candidate.model, - usage=usage, - custom_llm_provider=provider, - model_info=info, - ) - except Exception as e: # noqa: BLE001 # an unpriceable candidate simply cannot be the baseline - verbose_router_logger.debug("savings baseline: no pricing for candidate %s (%s)", candidate.model, e) - return None - cost = prompt_cost + completion_cost - if cost <= 0.0: - verbose_router_logger.debug("savings baseline: candidate %s prices to nothing", candidate.model) - return None - return (cost, candidate) - - -def _most_expensive(router: "Router", candidates: Iterable[Baseline], usage: Usage) -> Baseline | None: - """The candidate that would have cost the most on this request.""" - priced = tuple(r for candidate in candidates if (r := _priced(router, candidate, usage)) is not None) - if not priced: - verbose_router_logger.debug("savings baseline: no priceable candidates; savings driver disabled") - return None - return max(priced)[1] - - -def resolve_baseline(router: "Router", candidates: Iterable[str], usage: Usage) -> Baseline | None: - """The dearest of ``candidates`` for this request's actual usage. - - Resolved against the served request rather than a stand-in for one, because which - candidate is dearest depends on the token mix: a deployment can be dear per output - token and cheap per cached token. That means this runs where the usage is known, on - the spend path, not in the pre-routing hook where the request has not happened yet. - - Never raises. A dashboard's counterfactual is not worth failing a spend write over; - an unresolvable baseline zeroes the savings driver instead. - """ - try: - return _most_expensive(router, (c for name in candidates for c in _models_in(router, name)), usage) - except Exception as e: # noqa: BLE001 # see docstring: routing must not fail for a metric - verbose_router_logger.warning("savings baseline: could not resolve, savings will read zero (%s)", e) - return None diff --git a/litellm/types/router.py b/litellm/types/router.py index bd2572877de5..837a93367a2c 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -272,7 +272,6 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): auto_router_config: Optional[str] = None auto_router_default_model: Optional[str] = None auto_router_embedding_model: Optional[str] = None - auto_router_savings_baseline_model: Optional[str] = None # complexity-router params complexity_router_config: Optional[Dict] = None diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 09fb8b4d5af1..59e71c908ee9 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2734,7 +2734,6 @@ class StandardLoggingRoutingDecision(TypedDict, total=False): classifier_model: str escalated: bool tier_boundaries: StandardLoggingRoutingDecisionTierBoundaries - savings_baseline_candidates: Sequence[str] conversation_continuing: bool @@ -2755,7 +2754,6 @@ class StandardLoggingRoutingDecision(TypedDict, total=False): "classifier_model", "escalated", "tier_boundaries", - "savings_baseline_candidates", "conversation_continuing", } ) @@ -3405,7 +3403,6 @@ def shared_backend_model_info(model_info: Dict[str, Any]) -> Dict[str, Any]: "auto_router_config", "auto_router_default_model", "auto_router_embedding_model", - "auto_router_savings_baseline_model", "complexity_router_config", "complexity_router_default_model", "adaptive_router_config", diff --git a/tests/test_litellm/proxy/spend_tracking/test_savings.py b/tests/test_litellm/proxy/spend_tracking/test_savings.py index e9138018ec46..c3515dd43bdf 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_savings.py +++ b/tests/test_litellm/proxy/spend_tracking/test_savings.py @@ -8,7 +8,6 @@ import litellm from litellm.router import Router from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token -from litellm.router_strategy.savings_baseline import Baseline from litellm.proxy.spend_tracking.savings import ( _baseline_usage, compute_autorouter_savings, @@ -139,7 +138,7 @@ def _savings(baseline: str, selected: str, usage: Usage, continuing: bool = True conversation's first turn, where nothing was cached for any model. """ return compute_autorouter_savings( - baseline=Baseline(baseline), + baseline_model=baseline, selected_model=selected, selected_provider="anthropic", usage=usage, @@ -298,21 +297,36 @@ def test_autorouter_savings_zero_without_baseline(): assert result.autorouter == 0.0 -def test_compute_savings_spend_carries_a_losing_switch_through(): +def test_compute_savings_spend_carries_a_losing_switch_through(monkeypatch): """The signed value must survive into SavingsSpend; clamping it here would put the dashboard back to only ever showing gains.""" + monkeypatch.setattr(litellm, "autorouter_savings_baseline_model", "claude-sonnet-5") result = compute_savings_spend( model="claude-haiku-4-5", custom_llm_provider="anthropic", compression_saved_tokens=0, cache_read_input_tokens=0, - routing_decision={"savings_baseline_candidates": ["dear"]}, + routing_decision={"conversation_continuing": True}, usage_object=_cached_usage_object(), - llm_router=Router(model_list=[{"model_name": "dear", "litellm_params": {"model": "claude-sonnet-5"}}]), ) assert result.autorouter < 0 +def test_the_driver_is_off_until_a_baseline_is_configured(): + """No configured counterfactual means there is nothing to measure against, so the + driver reports zero rather than inventing a model the operator never named.""" + result = compute_savings_spend( + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + compression_saved_tokens=1000, + cache_read_input_tokens=0, + routing_decision={"conversation_continuing": True}, + usage_object=_cached_usage_object(), + ) + assert result.autorouter == 0.0 + assert result.compression > 0, "the other drivers keep working" + + def test_malformed_usage_object_does_not_fail_the_spend_write(): """The daily spend write must survive an unusable usage_object; losing one row's savings is recoverable, losing the row is not.""" @@ -321,7 +335,7 @@ def test_malformed_usage_object_does_not_fail_the_spend_write(): custom_llm_provider="anthropic", compression_saved_tokens=1000, cache_read_input_tokens=0, - routing_decision={"savings_baseline_candidates": ["claude-opus-5"]}, + routing_decision={"conversation_continuing": True}, usage_object={"prompt_tokens": ["not", "a", "number"]}, ) assert result.autorouter == 0.0 @@ -362,13 +376,13 @@ def test_baseline_is_priced_under_its_own_provider(): decides whether routing reads as a saving or a loss.""" usage = Usage(prompt_tokens=100_000, completion_tokens=10_000, total_tokens=110_000) azure = compute_autorouter_savings( - baseline=Baseline("azure_ai/deepseek-r1"), + baseline_model="azure_ai/deepseek-r1", selected_model="claude-haiku-4-5", selected_provider="anthropic", usage=usage, ) deepseek = compute_autorouter_savings( - baseline=Baseline("deepseek/deepseek-r1"), + baseline_model="deepseek/deepseek-r1", selected_model="claude-haiku-4-5", selected_provider="anthropic", usage=usage, @@ -419,7 +433,7 @@ def test_an_undetermined_conversation_shape_stays_conservative(): """ usage = _usage(fresh=0, cached=0, written=20_000, out=1_000) defaulted = compute_autorouter_savings( - baseline=Baseline("anthropic/claude-opus-5"), + baseline_model="anthropic/claude-opus-5", selected_model="claude-haiku-4-5", selected_provider="anthropic", usage=usage, diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 445c0652e1a6..479c9904e7d4 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -5064,59 +5064,6 @@ def test_a_window_stops_telling_the_model_to_disregard_it(self): assert "rate the work it approves rather than the reply itself" in system_prompt -class TestSavingsBaselineCandidates: - """What the router offers as the counterfactual. Which of them is dearest depends on - the request's token mix, so that is decided on the spend path, not here.""" - - @staticmethod - def _router_with_tiers(tiers: dict, **kwargs) -> ComplexityRouter: - from litellm.router import Router - - parent = Router( - model_list=[ - {"model_name": "cheap", "litellm_params": {"model": "anthropic/claude-haiku-4-5"}}, - {"model_name": "mid", "litellm_params": {"model": "anthropic/claude-sonnet-4-5"}}, - {"model_name": "top", "litellm_params": {"model": "anthropic/claude-opus-4-5"}}, - ] - ) - return ComplexityRouter( - model_name="bench", - litellm_router_instance=parent, - complexity_router_config={"tiers": tiers}, - default_model="mid", - **kwargs, - ) - - def test_candidates_are_the_reasoning_tier_not_everything_reachable(self): - """The ladder names what an operator would have had to run for the hardest - request; a pricier model sitting in a lower tier is a choice, not a ceiling.""" - router = self._router_with_tiers({"SIMPLE": "top", "MEDIUM": "cheap", "REASONING": "mid"}) - assert router.savings_baseline_candidates == ("mid",) - - def test_a_pooled_tier_offers_every_member(self): - router = self._router_with_tiers({"SIMPLE": "cheap", "REASONING": ["cheap", "top", "mid"]}) - assert sorted(router.savings_baseline_candidates) == ["cheap", "mid", "top"] - - def test_falls_back_to_the_hardest_configured_tier_when_reasoning_is_absent(self): - router = self._router_with_tiers({"SIMPLE": "cheap", "COMPLEX": "top"}) - assert router.savings_baseline_candidates == ("top",) - - def test_a_configured_baseline_replaces_the_tier(self): - router = self._router_with_tiers( - {"SIMPLE": "cheap", "REASONING": "mid"}, savings_baseline_model="claude-opus-4-5" - ) - assert router.savings_baseline_candidates == ("claude-opus-4-5",) - - def test_the_candidates_travel_on_every_routing_decision(self): - """A decision without them silently zeroes the savings driver for that path.""" - import inspect - - from litellm.router_strategy.complexity_router import complexity_router as module - - source = inspect.getsource(module.ComplexityRouter._build_routing_decision) - assert "self.savings_baseline_candidates" in source - - class TestConversationShapeDiscriminator: """Whether the counterfactual single model would already have had the prompt cached.""" diff --git a/tests/test_litellm/router_strategy/test_savings_baseline.py b/tests/test_litellm/router_strategy/test_savings_baseline.py deleted file mode 100644 index e5ff0e502ac3..000000000000 --- a/tests/test_litellm/router_strategy/test_savings_baseline.py +++ /dev/null @@ -1,201 +0,0 @@ -import pytest - -from litellm.types.utils import PromptTokensDetailsWrapper, Usage - -REQUEST = Usage( - prompt_tokens=20_000, - completion_tokens=1_000, - total_tokens=21_000, - prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=0, cache_creation_tokens=0, text_tokens=20_000), -) - -from litellm.router import Router -from litellm.router_strategy.savings_baseline import ( - Baseline, - canonical_model, - _models_in, - _most_expensive, - resolve_baseline, -) - - -@pytest.fixture -def parent() -> Router: - return Router( - model_list=[ - {"model_name": "cheap", "litellm_params": {"model": "anthropic/claude-haiku-4-5"}}, - {"model_name": "top", "litellm_params": {"model": "anthropic/claude-opus-4-5"}}, - {"model_name": "pool", "litellm_params": {"model": "anthropic/claude-haiku-4-5"}}, - {"model_name": "pool", "litellm_params": {"model": "anthropic/claude-opus-4-5"}}, - ] - ) - - -class TestCanonicalModel: - def test_qualifies_a_bare_name_with_the_provider_that_owns_it(self): - assert canonical_model("claude-opus-4-5") == "anthropic/claude-opus-4-5" - - def test_keeps_an_already_qualified_name_qualified(self): - assert canonical_model("anthropic/claude-opus-4-5") == "anthropic/claude-opus-4-5" - - def test_honours_a_separately_declared_provider(self): - assert canonical_model("claude-opus-4-5", "openai") == "openai/claude-opus-4-5" - - def test_returns_none_for_a_name_no_provider_claims(self): - assert canonical_model("") is None - - -class TestModelsForGroup: - def test_resolves_a_group_to_the_models_its_deployments_call(self, parent): - assert [c.model for c in _models_in(parent, "cheap")] == ["anthropic/claude-haiku-4-5"] - - def test_returns_every_deployment_in_a_pooled_group(self, parent): - assert sorted(c.model for c in _models_in(parent, "pool")) == [ - "anthropic/claude-haiku-4-5", - "anthropic/claude-opus-4-5", - ] - - def test_treats_an_unknown_group_as_a_model_name(self, parent): - """A tier can point straight at a provider model rather than a configured group.""" - assert [c.model for c in _models_in(parent, "claude-opus-4-5")] == ["anthropic/claude-opus-4-5"] - - -class TestMostExpensive: - """Ranking runs through the router, because what a deployment costs is the - router's answer to give: it merges configured prices over the built-in map.""" - - def test_picks_by_output_rate(self, parent): - picked = _most_expensive(parent, [Baseline("anthropic/claude-haiku-4-5"), Baseline("anthropic/claude-opus-4-5")], REQUEST) - assert picked.model == "anthropic/claude-opus-4-5" - - def test_ignores_models_with_no_per_token_price(self, parent): - """A free model as baseline would report the whole real spend as a loss.""" - picked = _most_expensive(parent, [Baseline("not-a-real-model-anywhere"), Baseline("anthropic/claude-haiku-4-5")], REQUEST) - assert picked.model == "anthropic/claude-haiku-4-5" - - def test_returns_none_when_nothing_can_be_priced(self, parent): - assert _most_expensive(parent, [Baseline("not-a-real-model-anywhere")], REQUEST) is None - - def test_returns_none_for_an_empty_candidate_set(self, parent): - assert _most_expensive(parent, [], REQUEST) is None - - -class TestResolveBaseline: - def test_a_configured_baseline_wins_over_the_candidates(self, parent): - assert resolve_baseline(parent, ["claude-haiku-4-5"], REQUEST).model == "anthropic/claude-haiku-4-5" - - def test_derives_the_priciest_candidate_when_unconfigured(self, parent): - assert resolve_baseline(parent, ["cheap", "top"], REQUEST).model == "anthropic/claude-opus-4-5" - - def test_never_raises_so_a_metric_cannot_fail_a_live_request(self): - """Read on the routing path while decorating a request that is about to be - served; a dashboard counterfactual must not be able to take routing down.""" - - class Exploding: - @property - def model_name_to_deployment_indices(self): - raise RuntimeError("router is mid-reload") - - assert resolve_baseline(Exploding(), ["anything"], REQUEST) is None - - def test_an_empty_candidate_set_zeroes_the_driver_rather_than_inventing_one(self, parent): - assert resolve_baseline(parent, [], REQUEST) is None - - -class TestDeploymentsPricedByBaseModel: - """`litellm_params.model` is not always a model. - - On Azure it is the deployment name, which is absent from the cost map, so pricing it - directly drops the candidate. If that candidate was the priciest, the baseline quietly - becomes the second priciest and every saving is understated; if the whole pool is - Azure, nothing prices and the driver reports zero with nothing at default log level - saying why. `model_info.base_model` is what names the real model, which is the chain - router.py already resolves pricing through. - """ - - @staticmethod - def _router(*deployments: dict) -> Router: - return Router(model_list=list(deployments)) - - def test_model_info_base_model_is_preferred_over_the_deployment_name(self): - router = self._router( - { - "model_name": "big", - "litellm_params": {"model": "azure/my-gpt5-deployment"}, - "model_info": {"base_model": "azure/gpt-4.1"}, - }, - ) - assert [c.model for c in _models_in(router, "big")] == ["azure/gpt-4.1"] - - def test_litellm_params_base_model_is_the_other_accepted_spelling(self): - router = self._router( - { - "model_name": "big", - "litellm_params": {"model": "azure/my-gpt5-deployment", "base_model": "azure/gpt-4.1"}, - }, - ) - assert [c.model for c in _models_in(router, "big")] == ["azure/gpt-4.1"] - - def test_a_deployment_without_a_base_model_still_prices_by_its_model(self): - router = self._router({"model_name": "big", "litellm_params": {"model": "anthropic/claude-opus-4-5"}}) - assert [c.model for c in _models_in(router, "big")] == ["anthropic/claude-opus-4-5"] - - def test_an_azure_deployment_can_win_the_priciest_candidate(self): - """Without the base_model hop the Azure candidate never prices, so the cheaper - model wins by default and the reported saving shrinks.""" - router = self._router( - {"model_name": "cheap", "litellm_params": {"model": "anthropic/claude-haiku-4-5"}}, - { - "model_name": "big", - "litellm_params": {"model": "azure/my-gpt5-deployment"}, - "model_info": {"base_model": "azure/gpt-4.1"}, - }, - ) - assert resolve_baseline(router, ["cheap", "big"], REQUEST).model == "azure/gpt-4.1" - - def test_an_all_azure_pool_still_has_a_baseline(self): - """Otherwise nothing prices, the driver is disabled and the card reads $0.00.""" - router = self._router( - { - "model_name": "big", - "litellm_params": {"model": "azure/my-gpt5-deployment"}, - "model_info": {"base_model": "azure/gpt-4.1"}, - }, - ) - assert resolve_baseline(router, ["big"], REQUEST).model == "azure/gpt-4.1" - - -class TestDeploymentPricingOverrides: - """A deployment may not be charged the public rate for the model it names.""" - - @staticmethod - def _router(top_params: dict) -> Router: - return Router( - model_list=[ - {"model_name": "cheap", "litellm_params": {"model": "anthropic/claude-haiku-4-5"}}, - {"model_name": "top", "litellm_params": {"model": "anthropic/claude-opus-4-5", **top_params}}, - ] - ) - - def test_a_configured_price_decides_the_baseline_not_the_public_rate(self): - """A deployment configured far above its public rate is what the traffic would - really have cost. Ranking on the public rate picks the wrong counterfactual and - then prices it at a rate nobody pays.""" - router = self._router({}) - assert resolve_baseline(router, ["cheap", "top"], REQUEST).model == "anthropic/claude-opus-4-5" - - # haiku configured 1000x above its public rate now outprices opus - overridden = Router( - model_list=[ - { - "model_name": "cheap", - "litellm_params": { - "model": "anthropic/claude-haiku-4-5", - "input_cost_per_token": 0.001, - "output_cost_per_token": 0.002, - }, - }, - {"model_name": "top", "litellm_params": {"model": "anthropic/claude-opus-4-5"}}, - ] - ) - assert resolve_baseline(overridden, ["cheap", "top"], REQUEST).model == "anthropic/claude-haiku-4-5" diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 8402fa6c62ec..f4cc74428b32 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -25941,8 +25941,6 @@ export interface components { auto_router_default_model?: string | null; /** Auto Router Embedding Model */ auto_router_embedding_model?: string | null; - /** Auto Router Savings Baseline Model */ - auto_router_savings_baseline_model?: string | null; /** Aws Access Key Id */ aws_access_key_id?: string | null; /** Aws Bedrock Project Id */ @@ -34080,8 +34078,6 @@ export interface components { auto_router_default_model?: string | null; /** Auto Router Embedding Model */ auto_router_embedding_model?: string | null; - /** Auto Router Savings Baseline Model */ - auto_router_savings_baseline_model?: string | null; /** Aws Access Key Id */ aws_access_key_id?: string | null; /** Aws Bedrock Project Id */ From fe4cb356cb685fae7688631086da5d54fba985a3 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Mon, 3 Aug 2026 15:32:06 -0700 Subject: [PATCH 11/15] refactor(router): compute the conversation shape once and pass it down `_classify_and_route` re-derived it from the messages the hook had already resolved, so an ordinary routed request walked the turn list twice for one boolean. The hook computes it and hands it over, which is also where the affinity-hit path already got it from. Also moves `_get_llm_router` below the imports it sat among. --- litellm/proxy/db/db_spend_update_writer.py | 16 +++++++--------- .../complexity_router/complexity_router.py | 4 +++- .../router_strategy/test_complexity_router.py | 9 ++++----- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 26ff27ad1944..a05534699c9f 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -59,6 +59,13 @@ extract_compression_saved_tokens, ) from litellm.proxy.spend_tracking.savings import compute_savings_spend +from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error + +if TYPE_CHECKING: + from litellm.proxy.utils import PrismaClient, ProxyLogging +else: + PrismaClient = Any + ProxyLogging = Any def _get_llm_router(): @@ -75,15 +82,6 @@ def _get_llm_router(): return None -from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error - -if TYPE_CHECKING: - from litellm.proxy.utils import PrismaClient, ProxyLogging -else: - PrismaClient = Any - ProxyLogging = Any - - def _extract_cache_read_tokens(usage_obj: dict) -> int: """ Anthropic: top-level cache_read_input_tokens field. diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 492cf7465c7d..aada17b0f731 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -1400,6 +1400,7 @@ async def async_pre_routing_hook( messages: list[dict[str, Any]] | None = None, input: str | list | None = None, specific_deployment: bool | None = False, + conversation_continuing: bool = True, ) -> PreRoutingHookResponse | None: """ Pre-routing hook called before the routing decision. @@ -1479,6 +1480,7 @@ async def async_pre_routing_hook( messages=messages, input=input, specific_deployment=specific_deployment, + conversation_continuing=conversation_continuing, ) if cache_key is not None and response is not None: await self.litellm_router_instance.cache.async_set_cache( @@ -1495,6 +1497,7 @@ async def _classify_and_route( messages: list[dict[str, Any]] | None = None, input: str | list | None = None, specific_deployment: bool | None = False, + conversation_continuing: bool = True, ) -> PreRoutingHookResponse | None: """ Classifies the request by complexity and returns the appropriate model. @@ -1514,7 +1517,6 @@ async def _classify_and_route( from litellm.types.router import PreRoutingHookResponse resolved_messages = self._resolve_messages(messages, request_kwargs) - conversation_continuing = _conversation_is_continuing(resolved_messages) if not resolved_messages: verbose_router_logger.debug("ComplexityRouter: No messages could be resolved, skipping routing") diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 479c9904e7d4..d4e662320b9c 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -5197,8 +5197,7 @@ async def test_the_shape_travels_on_every_pre_routing_response(self): source = inspect.getsource(module.ComplexityRouter.async_pre_routing_hook) + inspect.getsource( module.ComplexityRouter._classify_and_route ) - builds = source.count("self._build_routing_decision(") - assert builds > 0 - assert source.count("conversation_continuing=conversation_continuing") == builds, ( - "every routing decision must carry the conversation shape, or that path silently charges the write" - ) + builds = source.split("self._build_routing_decision(")[1:] + assert builds + missing = [i for i, block in enumerate(builds) if "conversation_continuing=conversation_continuing" not in block.split("),")[0]] + assert not missing, f"routing decisions {missing} do not carry the conversation shape" From a08e5dc089578539eb57fa520a1fdbd731781173 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Mon, 3 Aug 2026 16:09:55 -0700 Subject: [PATCH 12/15] fix(router): drop the dead conversation_continuing parameter off the hook It was added to `async_pre_routing_hook` by mistake and immediately overwritten by the value the hook computes, so it never did anything. It also widened a signature every pre-routing strategy shares with the protocol in `types/router.py`, leaving this one router diverged from `AutoRouter` and the interface for no reason. Also records why an unreadable request counts as continuing: no messages is no evidence a turn was served, so it pays the cache write and under-claims rather than being handed a first turn's larger saving on nothing. --- .../complexity_router/complexity_router.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index aada17b0f731..ca2e7fcc0380 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -245,6 +245,13 @@ def _conversation_is_continuing(messages: Sequence[Mapping[str, object]] | None) session id and their failure modes, and it works for callers that send no session header at all. A few-shot prompt's synthetic assistant turns read as prior conversation, which charges the write and under-claims; that is the safe side. + + So is an unreadable request. No messages says nothing about whether a turn was + served, and a surface that carries its turns somewhere this cannot see, or a + genuinely single-turn call arriving with none, is treated as continuing: it pays the + cache write and under-claims rather than being handed a first turn's larger saving + on no evidence. That direction is deliberate in both cases and is the only one that + cannot inflate. """ if not messages: return True @@ -1400,7 +1407,6 @@ async def async_pre_routing_hook( messages: list[dict[str, Any]] | None = None, input: str | list | None = None, specific_deployment: bool | None = False, - conversation_continuing: bool = True, ) -> PreRoutingHookResponse | None: """ Pre-routing hook called before the routing decision. From 22795c2eba05ef1b0f3e7a783dcf8796f8c68a9c Mon Sep 17 00:00:00 2001 From: tin Date: Tue, 4 Aug 2026 01:00:55 +0000 Subject: [PATCH 13/15] fix(spend): charge a baseline its input rate for cache buckets it cannot price A model with no cache_creation_input_token_cost, which is every OpenAI, Azure and Gemini entry, resolved that rate to 0.0 and carried the whole written prompt for free, so a first turn routed onto a cheaper model reported a loss. Same hole on cache reads. Those tokens are plain input on such a model, so they move into the text bucket. --- litellm/proxy/spend_tracking/savings.py | 73 +++++++++++++++---- .../proxy/spend_tracking/test_savings.py | 50 +++++++++++++ 2 files changed, 108 insertions(+), 15 deletions(-) diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index ce8c1249a277..1a1c813d7843 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -96,6 +96,15 @@ def _effective_model_info(router: "Router | None", deployment_id: str | None, mo return None +def _model_info(model: _ModelIdentity) -> ModelInfo | None: + """The public rates for ``model``, or ``None`` when it has none.""" + try: + return litellm.get_model_info(model=model.model, custom_llm_provider=model.provider) + except Exception as e: # noqa: BLE001 # get_model_info raises bare Exception for unmapped models + verbose_proxy_logger.debug("savings: no pricing for provider=%s model=%s (%s)", model.provider, model.model, e) + return None + + def _cost_of_usage(model: _ModelIdentity, usage: Usage, model_info: ModelInfo | None = None) -> float | None: """What ``usage`` costs on ``model``, or ``None`` when the model has no pricing.""" try: @@ -125,7 +134,24 @@ def _cache_token_split(usage: Usage) -> tuple[int, int]: ) -def _baseline_usage(usage: Usage, conversation_continuing: bool) -> Usage: +def _baseline_cache_rate_keys(baseline_info: ModelInfo | None) -> tuple[bool, bool]: + """Whether the baseline model has a ``(cache read, cache write)`` rate of its own. + + A missing rate is not a free bucket. `_get_token_base_cost` resolves an absent + `cache_read_input_token_cost` or `cache_creation_input_token_cost` to 0.0, so a + baseline whose provider prices caching implicitly, which is every OpenAI, Azure and + Gemini entry for cache writes, would carry the whole prompt for nothing and turn a + profitable route into a reported loss. Such a model pays its plain input rate for + those tokens, so the buckets it cannot price become ordinary input below. + """ + if baseline_info is None: + return True, True + return bool(baseline_info.get("cache_read_input_token_cost")), bool( + baseline_info.get("cache_creation_input_token_cost") + ) + + +def _baseline_usage(usage: Usage, conversation_continuing: bool, baseline_info: ModelInfo | None = None) -> Usage: """The same request as a single-model baseline would have met it. The baseline is one model serving every turn, so whether it had this prompt cached @@ -135,8 +161,9 @@ def _baseline_usage(usage: Usage, conversation_continuing: bool) -> Usage: counts against the saving; that write is what switching models costs. On a conversation's first turn nothing was cached anywhere, for any model. The - baseline would have written the same prompt, so the usage passes through untouched - and both arms carry the write at their own rates. Charging the write to this case + baseline would have written the same prompt, so the cache buckets stay where they are + and both arms carry the write at their own rates, unless the baseline has no rate for + a bucket, in which case those tokens are its plain input. Charging the write to this case too, which is all a single rollup row can support, understates a first turn to a few percent of its value and can render a profitable route as a loss. @@ -157,10 +184,26 @@ def _baseline_usage(usage: Usage, conversation_continuing: bool) -> Usage: """ cache_read, cache_creation = _cache_token_split(usage) details = usage.prompt_tokens_details - if details is None or cache_creation <= 0 or not conversation_continuing: + if details is None or (cache_read <= 0 and cache_creation <= 0): return usage - if cache_read > cache_creation: + + # The tokens this request paid to write move into the cached count and the creation + # charge is dropped: on one model that cache was already warm, so the baseline would + # have read them rather than paying to create them. The 5m/1h breakdown goes with + # them; left behind it re-charges the write. + warm = conversation_continuing and cache_creation > 0 and cache_read <= cache_creation + reads = cache_read + cache_creation if warm else cache_read + writes = 0 if warm else cache_creation + + prices_reads, prices_writes = _baseline_cache_rate_keys(baseline_info) + reads = reads if prices_reads else 0 + writes = writes if prices_writes else 0 + if (reads, writes) == (cache_read, cache_creation): return usage + + other_modalities = sum( + (getattr(details, field, 0) or 0) for field in ("audio_tokens", "image_tokens", "video_tokens") + ) return Usage( prompt_tokens=usage.prompt_tokens, completion_tokens=usage.completion_tokens, @@ -168,15 +211,12 @@ def _baseline_usage(usage: Usage, conversation_continuing: bool) -> Usage: completion_tokens_details=usage.completion_tokens_details, prompt_tokens_details=PromptTokensDetailsWrapper( **details.model_dump(exclude=_CACHE_SPLIT_FIELDS), - # The tokens this request paid to write are moved into the cached count and - # the creation charge is dropped: on one model that cache was already warm, - # so the baseline would have read them rather than paying to create them. - # The 5m/1h breakdown goes with it; left behind it re-charges the write. - cached_tokens=cache_read + cache_creation, - cache_creation_tokens=0, - cache_write_tokens=0, - cache_creation_token_details=None, - text_tokens=max(usage.prompt_tokens - cache_read - cache_creation, 0), + cached_tokens=reads, + cache_creation_tokens=writes, + cache_write_tokens=writes, + cache_creation_token_details=details.cache_creation_token_details if writes else None, + # Whatever no longer sits in a cache bucket is plain input on the baseline. + text_tokens=max(usage.prompt_tokens - reads - writes - other_modalities, 0), ), ) @@ -215,7 +255,10 @@ def compute_autorouter_savings( # name alone reports as zero. if baseline == selected: return 0.0 - baseline_cost = _cost_of_usage(baseline, _baseline_usage(usage, conversation_continuing)) + baseline_info = _model_info(baseline) + baseline_cost = _cost_of_usage( + baseline, _baseline_usage(usage, conversation_continuing, baseline_info), baseline_info + ) selected_cost = _cost_of_usage(selected, usage, selected_info) if baseline_cost is None or selected_cost is None: return 0.0 diff --git a/tests/test_litellm/proxy/spend_tracking/test_savings.py b/tests/test_litellm/proxy/spend_tracking/test_savings.py index c3515dd43bdf..6d0195b632bb 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_savings.py +++ b/tests/test_litellm/proxy/spend_tracking/test_savings.py @@ -484,3 +484,53 @@ def test_a_switch_onto_a_partly_cached_model_still_pays_for_the_write(): + 1_000 * haiku["output_cost_per_token"] ) assert reported < if_treated_as_same_model / 10, "a mostly-cold switch must not be priced as a continuation" + +def test_a_baseline_that_prices_caching_implicitly_still_pays_for_its_prompt(): + """OpenAI, Azure and Gemini entries carry no `cache_creation_input_token_cost`, + because those providers cache implicitly and charge nothing to write. Leaving this + request's written tokens in the creation bucket priced them at the 0.0 the cost + resolver falls back to, so the baseline carried a 20k prompt for free and a first + turn that saved money reported a loss. Those tokens are plain input on such a model. + """ + first_turn = _usage(fresh=0, cached=0, written=20_000, out=1_000) + reported = compute_autorouter_savings( + baseline_model="gpt-5", + selected_model="claude-haiku-4-5", + selected_provider="anthropic", + usage=first_turn, + conversation_continuing=False, + ) + + gpt5 = litellm.get_model_info("gpt-5", "openai") + assert gpt5.get("cache_creation_input_token_cost") is None, "pick a baseline with no cache-write rate" + haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") + baseline_pays_input = 20_000 * gpt5["input_cost_per_token"] + 1_000 * gpt5["output_cost_per_token"] + actually_paid = ( + 20_000 * haiku["cache_creation_input_token_cost"] + 1_000 * haiku["output_cost_per_token"] + ) + assert reported == pytest.approx(baseline_pays_input - actually_paid) + assert reported > 0, "routing a cold first turn onto a cheaper model is a saving, not a loss" + + +def test_a_baseline_with_no_cache_read_rate_is_charged_its_input_rate(): + """The same hole on the other bucket. A baseline whose entry has no + `cache_read_input_token_cost` reads for 0.0, so a continuing turn priced the whole + prompt at nothing and every switch away from it reported a loss. + """ + continuing = _usage(fresh=0, cached=0, written=20_000, out=1_000) + reported = compute_autorouter_savings( + baseline_model="xai/grok-4", + selected_model="claude-haiku-4-5", + selected_provider="anthropic", + usage=continuing, + conversation_continuing=True, + ) + + grok = litellm.get_model_info("grok-4", "xai") + assert grok.get("cache_read_input_token_cost") is None, "pick a baseline with no cache-read rate" + haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") + baseline_pays_input = 20_000 * grok["input_cost_per_token"] + 1_000 * grok["output_cost_per_token"] + actually_paid = ( + 20_000 * haiku["cache_creation_input_token_cost"] + 1_000 * haiku["output_cost_per_token"] + ) + assert reported == pytest.approx(baseline_pays_input - actually_paid) From e5e0396b5ca354e9920ee59c337087a6800f9ae9 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Mon, 3 Aug 2026 19:44:40 -0700 Subject: [PATCH 14/15] refactor(spend): build the daily upsert payloads in one shot `common_data` and `update_data` were constructed and then appended to: `request_id` conditionally for tag rows, `endpoint` unconditionally a few lines later. A dict that grows after its literal cannot be reasoned about by reading the literal, which is the whole point of building it at once. The conditional key resolves to a spreadable value before either payload, so both are single expressions and the tag branch appears once instead of twice. Not wrapped in MappingProxyType, though it was suggested: these go straight to prisma, whose query builder branches on `isinstance(value, dict)` to tell a nested node from a scalar. A mappingproxy is a Mapping but not a dict, so it falls through to the serializer and raises `TypeError: Type not serializable` inside the batch upsert, where the surrounding except would log it and leave the rollups silently unwritten. --- litellm/proxy/db/db_spend_update_writer.py | 24 +++++++++++++--------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index a05534699c9f..70187da87d1a 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -1583,6 +1583,16 @@ async def _update_daily_spend( if value is not None } + # Only tag rows carry a request_id. Resolved to a spreadable + # value here so both payloads are built in one shot: a dict + # appended to after construction is one nobody can reason about + # by reading its literal. + tag_request_id = ( + {"request_id": transaction["request_id"]} + if entity_type == "tag" and "request_id" in transaction + else {} + ) + # Common data structure for both create and update common_data = { entity_id_field: entity_id, @@ -1600,12 +1610,9 @@ async def _update_daily_spend( "successful_requests": transaction["successful_requests"], "failed_requests": transaction["failed_requests"], **optional_metrics, + **tag_request_id, } - if entity_type == "tag" and "request_id" in transaction: - common_data["request_id"] = transaction.get("request_id") - - # Create update data structure update_data = { "prompt_tokens": {"increment": transaction["prompt_tokens"]}, "completion_tokens": {"increment": transaction["completion_tokens"]}, @@ -1614,14 +1621,11 @@ async def _update_daily_spend( "successful_requests": {"increment": transaction["successful_requests"]}, "failed_requests": {"increment": transaction["failed_requests"]}, **{field: {"increment": value} for field, value in optional_metrics.items()}, + # An existing row predating the endpoint column gets it filled in here + "endpoint": transaction.get("endpoint") or "", + **tag_request_id, } - if entity_type == "tag" and "request_id" in transaction: - update_data["request_id"] = transaction.get("request_id") - - # Add endpoint to update_data so existing rows get their endpoint field updated - update_data["endpoint"] = transaction.get("endpoint") or "" - table.upsert( where=where_clause, data={ From c4695a432439c21998d82403c94e4d04d836e48a Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Mon, 3 Aug 2026 19:57:27 -0700 Subject: [PATCH 15/15] fix(spend): keep the one-shot upsert payloads under the type-discipline budget Building both payloads as single literals traded a mutation for two dict literals, and LIT002 counts construction rather than mutation, so the change the review asked for is the one the gate charges for. The empty branch is the avoidable half: it is the same value every time, so it moves to a module constant built once instead of a literal per transaction, and it is a read-only mapping so none of the call sites that spread it can fill it in later. --- litellm/proxy/db/db_spend_update_writer.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 70187da87d1a..7c9ee96e809b 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -12,7 +12,9 @@ import random import time import traceback +from collections.abc import Mapping from datetime import datetime, timedelta, timezone +from types import MappingProxyType from typing import ( TYPE_CHECKING, Any, @@ -68,6 +70,12 @@ ProxyLogging = Any +# Only tag rows carry a request_id, so the other entity types spread nothing. Built +# once here rather than as an empty literal per transaction, and read-only so it cannot +# be filled in by accident from one of the call sites that spreads it. +_NO_TAG_REQUEST_ID: Mapping[str, Any] = MappingProxyType({}) + + def _get_llm_router(): """The proxy's router, or None outside a running proxy. @@ -1587,10 +1595,10 @@ async def _update_daily_spend( # value here so both payloads are built in one shot: a dict # appended to after construction is one nobody can reason about # by reading its literal. - tag_request_id = ( - {"request_id": transaction["request_id"]} + tag_request_id: Mapping[str, Any] = ( + MappingProxyType({"request_id": transaction["request_id"]}) if entity_type == "tag" and "request_id" in transaction - else {} + else _NO_TAG_REQUEST_ID ) # Common data structure for both create and update