Skip to content

feat(spend): add net auto-router savings to the cost-optimization dashboard #35521

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tin-berri merged 15 commits into from
Aug 4, 2026
Merged

feat(spend): add net auto-router savings to the cost-optimization dashboard #35521

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
a028f6c
feat(spend): add net auto-router savings to the cost-optimization das…
tin-berri Aug 1, 2026
c6e0f1b
fix(spend): let the baseline pay for a continuing turn's own growth
tin-berri Aug 1, 2026
721790d
fix(spend): price each arm under the key litellm billed it, and see a…
tin-berri Aug 2, 2026
cfc4f13
fix(spend): give the cost-key resolver both inputs the selected arm n…
tin-berri Aug 2, 2026
1b54ea1
fix(spend): same model is only the same cost when it is the same depl…
tin-berri Aug 2, 2026
fa80e64
refactor(spend): price from resolved rates, not from a name we keep r…
tin-berri Aug 2, 2026
6e17303
test(spend): follow _most_expensive onto the router that prices its c…
tin-berri Aug 3, 2026
acc10e8
fix(spend): rank baseline candidates by what a request costs, not by …
tin-berri Aug 3, 2026
ec72930
fix(spend): pick the baseline against the request that ran, not a sta…
tin-berri Aug 3, 2026
8d0283c
refactor(spend): measure savings against one configured model, not a …
tin-berri Aug 3, 2026
fe4cb35
refactor(router): compute the conversation shape once and pass it down
tin-berri Aug 3, 2026
a08e5dc
fix(router): drop the dead conversation_continuing parameter off the …
tin-berri Aug 3, 2026
22795c2
fix(spend): charge a baseline its input rate for cache buckets it can…
tin-berri Aug 4, 2026
e5e0396
refactor(spend): build the daily upsert payloads in one shot
tin-berri Aug 4, 2026
c4695a4
fix(spend): keep the one-shot upsert payloads under the type-discipli…
tin-berri Aug 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions ...litellm_proxy_extras/migrations/20260731000000_add_autorouter_savings_spend/migration.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
-- AlterTable
ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;

-- AlterTable
ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;

-- AlterTable
ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;

-- AlterTable
ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;

-- AlterTable
ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;

-- AlterTable
ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
6 changes: 6 additions & 0 deletions litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -748,6 +748,7 @@ model LiteLLM_DailyUserSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
Expand Down Expand Up @@ -782,6 +783,7 @@ model LiteLLM_DailyOrganizationSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
Expand Down Expand Up @@ -816,6 +818,7 @@ model LiteLLM_DailyEndUserSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
Expand Down Expand Up @@ -849,6 +852,7 @@ model LiteLLM_DailyAgentSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
Expand Down Expand Up @@ -882,6 +886,7 @@ model LiteLLM_DailyTeamSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
Expand Down Expand Up @@ -917,6 +922,7 @@ model LiteLLM_DailyTagSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
Expand Down
1 change: 1 addition & 0 deletions litellm/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -264,6 +264,7 @@ def _dev_env_hot_reload_enabled() -> bool:
openai_like_key: Optional[str] = None
azure_key: Optional[str] = None
anthropic_key: Optional[str] = None
autorouter_savings_baseline_model: Optional[str] = None
replicate_key: Optional[str] = None
bytez_key: Optional[str] = None
gdc_key: Optional[str] = None
Expand Down
8 changes: 7 additions & 1 deletion litellm/litellm_core_utils/llm_cost_calc/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -683,6 +683,7 @@ def generic_cost_per_token(
custom_llm_provider: str,
service_tier: str | None = None,
data_residency: str | None = None,
model_info: ModelInfo | None = None,
) -> tuple[float, float]:
"""
Calculates the cost per token for a given model, prompt tokens, and completion tokens.
Expand All @@ -700,7 +701,12 @@ def generic_cost_per_token(
"""

## GET MODEL INFO
model_info = get_model_info(model=model, custom_llm_provider=custom_llm_provider)
# A caller that already resolved the deployment's effective rates passes them in
# rather than handing back a name for this to re-resolve. A name cannot express a
# per-deployment override: those are registered under the deployment id and kept off
# the shared model-name key, so resolving from the name here reads the public rate.
if model_info is None:
model_info = get_model_info(model=model, custom_llm_provider=custom_llm_provider)

## CALCULATE INPUT COST
### Cost of processing (non-cache hit + cache hit) + Cost of cache-writing (cache writing)
Expand Down
7 changes: 6 additions & 1 deletion litellm/proxy/_types.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
field_validator,
model_validator,
)
from typing_extensions import Required, TypedDict
from typing_extensions import NotRequired, Required, TypedDict

from litellm._uuid import uuid
from litellm.constants import MCP_STDIO_ALLOWED_COMMANDS
Expand Down Expand Up @@ -4564,6 +4564,11 @@ class BaseDailySpendTransaction(TypedDict):
# cost-savings metrics (dollars, priced per request before aggregation)
compression_savings_spend: float
prompt_caching_savings_spend: float
# Not required: rows queued by a pod running the previous release, or replayed from
# the Redis buffer across an upgrade, carry no such key. Every reader coalesces a
# missing value to zero, so requiring it here would describe a shape the aggregation
# is explicitly tested against.
autorouter_savings_spend: NotRequired[float]

# request level metrics
spend: float
Expand Down
121 changes: 67 additions & 54 deletions litellm/proxy/db/db_spend_update_writer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,9 @@
import random
import time
import traceback
from collections.abc import Mapping
from datetime import datetime, timedelta, timezone
from types import MappingProxyType
from typing import (
TYPE_CHECKING,
Any,
Expand Down Expand Up @@ -68,6 +70,26 @@
ProxyLogging = Any


# Only tag rows carry a request_id, so the other entity types spread nothing. Built
# once here rather than as an empty literal per transaction, and read-only so it cannot
# be filled in by accident from one of the call sites that spreads it.
_NO_TAG_REQUEST_ID: Mapping[str, Any] = MappingProxyType({})


def _get_llm_router():
"""The proxy's router, or None outside a running proxy.

Injected rather than imported where it is used, so the savings computation stays
a pure function of its arguments and the caller owns where the router comes from.
"""
try:
from litellm.proxy.proxy_server import llm_router

return llm_router
except Exception: # noqa: BLE001 # no proxy in scope; savings degrade to zero
return None


def _extract_cache_read_tokens(usage_obj: dict) -> int:
"""
Anthropic: top-level cache_read_input_tokens field.
Expand Down Expand Up @@ -1545,6 +1567,40 @@ async def _update_daily_spend(
# Get the table dynamically
table = getattr(batcher, table_name)

# Additive metrics that older queued rows may omit; one
# enumeration feeds both the create and the increment below
optional_metrics = {

@mateo-berri mateo-berri Aug 4, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: I would wrap in a MappingProxyType if you're not mutating it. That way, it's a type error if you try to mutate it unexpectedly

Strongly prefer constructing the dict in one place all at once and marking it as unchangeable, instead of slowly adding to it over time, as the latter can lead to a whole class of "wait, did someone mutate this out from under me?" bugs. AFAICT, you've done pt. 1 here but not pt. 2

Even better if they can be frozen slots=True dataclasses, because of the typing guarantees but that's another discussion

@tin-berri tin-berri Aug 4, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Took the one-shot half, skipped the freeze, because prisma rejects it.

Both payloads go straight to table.upsert, and prisma's builder branches on isinstance(value, dict) to tell a nested node from a scalar (builder.py:851,903). A mappingproxy is a Mapping but not a dict, so it falls to the serializer:

plain dict         -> OK
MappingProxyType   -> TypeError: Type <class 'mappingproxy'> not serializable

That would raise inside the batch upsert, where the surrounding except Exception as batch_error logs it and moves on, so daily rollups would quietly stop updating. Same blocker kills the frozen-dataclass version, plus it would need a to_dict() at the prisma boundary and hand the guarantee straight back.

Your pt. 2 was right though, and worse than the nit suggested: common_data and update_data were both appended to after construction (request_id for tag rows, then endpoint unconditionally). The conditional key is now resolved to a spreadable value first, so both are single literals and the tag branch appears once instead of twice. Verified the payloads are byte-identical across user/tag and both endpoint shapes.

optional_metrics is the one that could be wrapped safely, since it is only ever spread. Happy to if you want the marker somewhere, though it is already built in one shot and never touched.

@mateo-berri mateo-berri Aug 4, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it would need a to_dict() at the prisma boundary and hand the guarantee straight back

What do you mean "hand the guarantee straight back"? I understand we might need to convert to dict at the prisma boundary, but I would argue that doesn't defeat the point of frozen dataclasses. All the dangerous stuff can be put in simple seam functions (e.g., see the repository classes in our codebase) while core heavy-duty business logic is stateless, pure, and strongly typed

field: value
for field, value in (
("cache_read_input_tokens", transaction.get("cache_read_input_tokens")),
(
"cache_creation_input_tokens",
transaction.get("cache_creation_input_tokens"),
),
("compression_saved_tokens", transaction.get("compression_saved_tokens")),
(
"compression_savings_spend",
transaction.get("compression_savings_spend"),
),
(
"prompt_caching_savings_spend",
transaction.get("prompt_caching_savings_spend"),
),
("autorouter_savings_spend", transaction.get("autorouter_savings_spend")),
)
if value is not None
}

# Only tag rows carry a request_id. Resolved to a spreadable
# value here so both payloads are built in one shot: a dict
# appended to after construction is one nobody can reason about
# by reading its literal.
tag_request_id: Mapping[str, Any] = (
MappingProxyType({"request_id": transaction["request_id"]})
if entity_type == "tag" and "request_id" in transaction
else _NO_TAG_REQUEST_ID
)

# Common data structure for both create and update
common_data = {

@mateo-berri mateo-berri Aug 4, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same here. I would approach this with all dicts

@tin-berri tin-berri Aug 4, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same change here: update_data is now a single literal too, with endpoint and the tag request_id folded in rather than appended after. Not frozen, for the prisma reason in the thread above.

entity_id_field: entity_id,
Expand All @@ -1561,71 +1617,23 @@ async def _update_daily_spend(
"api_requests": transaction["api_requests"],
"successful_requests": transaction["successful_requests"],
"failed_requests": transaction["failed_requests"],
**optional_metrics,
**tag_request_id,
}

# Add cache-related fields if they exist
if "cache_read_input_tokens" in transaction:
common_data["cache_read_input_tokens"] = transaction.get(
"cache_read_input_tokens", 0
)
if "cache_creation_input_tokens" in transaction:
common_data["cache_creation_input_tokens"] = transaction.get(
"cache_creation_input_tokens", 0
)
if "compression_saved_tokens" in transaction:
common_data["compression_saved_tokens"] = transaction.get(
"compression_saved_tokens", 0
)
if "compression_savings_spend" in transaction:
common_data["compression_savings_spend"] = transaction.get(
"compression_savings_spend", 0
)
if "prompt_caching_savings_spend" in transaction:
common_data["prompt_caching_savings_spend"] = transaction.get(
"prompt_caching_savings_spend", 0
)

if entity_type == "tag" and "request_id" in transaction:
common_data["request_id"] = transaction.get("request_id")

# Create update data structure
update_data = {
"prompt_tokens": {"increment": transaction["prompt_tokens"]},
"completion_tokens": {"increment": transaction["completion_tokens"]},
"spend": {"increment": transaction["spend"]},
"api_requests": {"increment": transaction["api_requests"]},
"successful_requests": {"increment": transaction["successful_requests"]},
"failed_requests": {"increment": transaction["failed_requests"]},
**{field: {"increment": value} for field, value in optional_metrics.items()},
# An existing row predating the endpoint column gets it filled in here
"endpoint": transaction.get("endpoint") or "",
**tag_request_id,
}

# Add cache-related fields to update if they exist
if "cache_read_input_tokens" in transaction:
update_data["cache_read_input_tokens"] = {
"increment": transaction.get("cache_read_input_tokens", 0)
}
if "cache_creation_input_tokens" in transaction:
update_data["cache_creation_input_tokens"] = {
"increment": transaction.get("cache_creation_input_tokens", 0)
}
if "compression_saved_tokens" in transaction:
update_data["compression_saved_tokens"] = {
"increment": transaction.get("compression_saved_tokens", 0)
}
if "compression_savings_spend" in transaction:
update_data["compression_savings_spend"] = {
"increment": transaction.get("compression_savings_spend", 0)
}
if "prompt_caching_savings_spend" in transaction:
update_data["prompt_caching_savings_spend"] = {
"increment": transaction.get("prompt_caching_savings_spend", 0)
}

if entity_type == "tag" and "request_id" in transaction:
update_data["request_id"] = transaction.get("request_id")

# Add endpoint to update_data so existing rows get their endpoint field updated
update_data["endpoint"] = transaction.get("endpoint") or ""

table.upsert(
where=where_clause,
data={
Expand Down Expand Up @@ -1875,6 +1883,10 @@ async def _common_add_spend_log_transaction_to_daily_transaction(
custom_llm_provider=payload.get("custom_llm_provider", None),
compression_saved_tokens=compression_saved_tokens,
cache_read_input_tokens=cache_read_input_tokens,
routing_decision=_metadata.get("routing_decision"),
model_id=payload.get("model_id"),
llm_router=_get_llm_router(),
usage_object=usage_obj,
)

daily_transaction = BaseDailySpendTransaction(
Expand All @@ -1896,6 +1908,7 @@ async def _common_add_spend_log_transaction_to_daily_transaction(
compression_saved_tokens=compression_saved_tokens,
compression_savings_spend=savings_spend.compression,
prompt_caching_savings_spend=savings_spend.prompt_caching,
autorouter_savings_spend=savings_spend.autorouter,
)
return daily_transaction
except Exception as e:
Expand Down
4 changes: 4 additions & 0 deletions litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,10 @@ def get_aggregated_daily_spend_update_transactions(
payload.get("prompt_caching_savings_spend", 0) or 0
) + daily_transaction.get("prompt_caching_savings_spend", 0)

daily_transaction["autorouter_savings_spend"] = (
payload.get("autorouter_savings_spend", 0) or 0
) + daily_transaction.get("autorouter_savings_spend", 0)

else:
aggregated_daily_spend_update_transactions[_key] = deepcopy(payload)
return aggregated_daily_spend_update_transactions
Expand Down
Loading
Loading