Skip to content

feat(spend): add net auto-router savings to the cost-optimization dashboard #35521

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tin-berri merged 15 commits into from
Aug 4, 2026
Merged

feat(spend): add net auto-router savings to the cost-optimization dashboard #35521

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
a028f6c
feat(spend): add net auto-router savings to the cost-optimization das…
tin-berri Aug 1, 2026
c6e0f1b
fix(spend): let the baseline pay for a continuing turn's own growth
tin-berri Aug 1, 2026
721790d
fix(spend): price each arm under the key litellm billed it, and see a…
tin-berri Aug 2, 2026
cfc4f13
fix(spend): give the cost-key resolver both inputs the selected arm n…
tin-berri Aug 2, 2026
1b54ea1
fix(spend): same model is only the same cost when it is the same depl…
tin-berri Aug 2, 2026
fa80e64
refactor(spend): price from resolved rates, not from a name we keep r…
tin-berri Aug 2, 2026
6e17303
test(spend): follow _most_expensive onto the router that prices its c…
tin-berri Aug 3, 2026
acc10e8
fix(spend): rank baseline candidates by what a request costs, not by …
tin-berri Aug 3, 2026
ec72930
fix(spend): pick the baseline against the request that ran, not a sta…
tin-berri Aug 3, 2026
8d0283c
refactor(spend): measure savings against one configured model, not a …
tin-berri Aug 3, 2026
fe4cb35
refactor(router): compute the conversation shape once and pass it down
tin-berri Aug 3, 2026
a08e5dc
fix(router): drop the dead conversation_continuing parameter off the …
tin-berri Aug 3, 2026
22795c2
fix(spend): charge a baseline its input rate for cache buckets it can…
tin-berri Aug 4, 2026
e5e0396
refactor(spend): build the daily upsert payloads in one shot
tin-berri Aug 4, 2026
c4695a4
fix(spend): keep the one-shot upsert payloads under the type-discipli…
tin-berri Aug 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions ...litellm_proxy_extras/migrations/20260731000000_add_autorouter_savings_spend/migration.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
-- AlterTable
ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;

-- AlterTable
ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;

-- AlterTable
ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;

-- AlterTable
ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;

-- AlterTable
ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;

-- AlterTable
ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
6 changes: 6 additions & 0 deletions litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -748,6 +748,7 @@ model LiteLLM_DailyUserSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
Expand Down Expand Up @@ -782,6 +783,7 @@ model LiteLLM_DailyOrganizationSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
Expand Down Expand Up @@ -816,6 +818,7 @@ model LiteLLM_DailyEndUserSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
Expand Down Expand Up @@ -849,6 +852,7 @@ model LiteLLM_DailyAgentSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
Expand Down Expand Up @@ -882,6 +886,7 @@ model LiteLLM_DailyTeamSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
Expand Down Expand Up @@ -917,6 +922,7 @@ model LiteLLM_DailyTagSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
Expand Down
7 changes: 6 additions & 1 deletion litellm/proxy/_types.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
field_validator,
model_validator,
)
from typing_extensions import Required, TypedDict
from typing_extensions import NotRequired, Required, TypedDict

from litellm._uuid import uuid
from litellm.constants import MCP_STDIO_ALLOWED_COMMANDS
Expand Down Expand Up @@ -4564,6 +4564,11 @@ class BaseDailySpendTransaction(TypedDict):
# cost-savings metrics (dollars, priced per request before aggregation)
compression_savings_spend: float
prompt_caching_savings_spend: float
# Not required: rows queued by a pod running the previous release, or replayed from
# the Redis buffer across an upgrade, carry no such key. Every reader coalesces a
# missing value to zero, so requiring it here would describe a shape the aggregation
# is explicitly tested against.
autorouter_savings_spend: NotRequired[float]

# request level metrics
spend: float
Expand Down
75 changes: 31 additions & 44 deletions litellm/proxy/db/db_spend_update_writer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1545,6 +1545,30 @@ async def _update_daily_spend(
# Get the table dynamically
table = getattr(batcher, table_name)

# Additive metrics that older queued rows may omit; one
# enumeration feeds both the create and the increment below
optional_metrics = {

@mateo-berri mateo-berri Aug 4, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: I would wrap in a MappingProxyType if you're not mutating it. That way, it's a type error if you try to mutate it unexpectedly

Strongly prefer constructing the dict in one place all at once and marking it as unchangeable, instead of slowly adding to it over time, as the latter can lead to a whole class of "wait, did someone mutate this out from under me?" bugs. AFAICT, you've done pt. 1 here but not pt. 2

Even better if they can be frozen slots=True dataclasses, because of the typing guarantees but that's another discussion

@tin-berri tin-berri Aug 4, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Took the one-shot half, skipped the freeze, because prisma rejects it.

Both payloads go straight to table.upsert, and prisma's builder branches on isinstance(value, dict) to tell a nested node from a scalar (builder.py:851,903). A mappingproxy is a Mapping but not a dict, so it falls to the serializer:

plain dict         -> OK
MappingProxyType   -> TypeError: Type <class 'mappingproxy'> not serializable

That would raise inside the batch upsert, where the surrounding except Exception as batch_error logs it and moves on, so daily rollups would quietly stop updating. Same blocker kills the frozen-dataclass version, plus it would need a to_dict() at the prisma boundary and hand the guarantee straight back.

Your pt. 2 was right though, and worse than the nit suggested: common_data and update_data were both appended to after construction (request_id for tag rows, then endpoint unconditionally). The conditional key is now resolved to a spreadable value first, so both are single literals and the tag branch appears once instead of twice. Verified the payloads are byte-identical across user/tag and both endpoint shapes.

optional_metrics is the one that could be wrapped safely, since it is only ever spread. Happy to if you want the marker somewhere, though it is already built in one shot and never touched.

@mateo-berri mateo-berri Aug 4, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it would need a to_dict() at the prisma boundary and hand the guarantee straight back

What do you mean "hand the guarantee straight back"? I understand we might need to convert to dict at the prisma boundary, but I would argue that doesn't defeat the point of frozen dataclasses. All the dangerous stuff can be put in simple seam functions (e.g., see the repository classes in our codebase) while core heavy-duty business logic is stateless, pure, and strongly typed

field: value
for field, value in (
("cache_read_input_tokens", transaction.get("cache_read_input_tokens")),
(
"cache_creation_input_tokens",
transaction.get("cache_creation_input_tokens"),
),
("compression_saved_tokens", transaction.get("compression_saved_tokens")),
(
"compression_savings_spend",
transaction.get("compression_savings_spend"),
),
(
"prompt_caching_savings_spend",
transaction.get("prompt_caching_savings_spend"),
),
("autorouter_savings_spend", transaction.get("autorouter_savings_spend")),
)
if value is not None
}

# Common data structure for both create and update
common_data = {

@mateo-berri mateo-berri Aug 4, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same here. I would approach this with all dicts

@tin-berri tin-berri Aug 4, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same change here: update_data is now a single literal too, with endpoint and the tag request_id folded in rather than appended after. Not frozen, for the prisma reason in the thread above.

entity_id_field: entity_id,
Expand All @@ -1561,30 +1585,9 @@ async def _update_daily_spend(
"api_requests": transaction["api_requests"],
"successful_requests": transaction["successful_requests"],
"failed_requests": transaction["failed_requests"],
**optional_metrics,
}

# Add cache-related fields if they exist
if "cache_read_input_tokens" in transaction:
common_data["cache_read_input_tokens"] = transaction.get(
"cache_read_input_tokens", 0
)
if "cache_creation_input_tokens" in transaction:
common_data["cache_creation_input_tokens"] = transaction.get(
"cache_creation_input_tokens", 0
)
if "compression_saved_tokens" in transaction:
common_data["compression_saved_tokens"] = transaction.get(
"compression_saved_tokens", 0
)
if "compression_savings_spend" in transaction:
common_data["compression_savings_spend"] = transaction.get(
"compression_savings_spend", 0
)
if "prompt_caching_savings_spend" in transaction:
common_data["prompt_caching_savings_spend"] = transaction.get(
"prompt_caching_savings_spend", 0
)

if entity_type == "tag" and "request_id" in transaction:
common_data["request_id"] = transaction.get("request_id")

Expand All @@ -1596,30 +1599,9 @@ async def _update_daily_spend(
"api_requests": {"increment": transaction["api_requests"]},
"successful_requests": {"increment": transaction["successful_requests"]},
"failed_requests": {"increment": transaction["failed_requests"]},
**{field: {"increment": value} for field, value in optional_metrics.items()},
}

# Add cache-related fields to update if they exist
if "cache_read_input_tokens" in transaction:
update_data["cache_read_input_tokens"] = {
"increment": transaction.get("cache_read_input_tokens", 0)
}
if "cache_creation_input_tokens" in transaction:
update_data["cache_creation_input_tokens"] = {
"increment": transaction.get("cache_creation_input_tokens", 0)
}
if "compression_saved_tokens" in transaction:
update_data["compression_saved_tokens"] = {
"increment": transaction.get("compression_saved_tokens", 0)
}
if "compression_savings_spend" in transaction:
update_data["compression_savings_spend"] = {
"increment": transaction.get("compression_savings_spend", 0)
}
if "prompt_caching_savings_spend" in transaction:
update_data["prompt_caching_savings_spend"] = {
"increment": transaction.get("prompt_caching_savings_spend", 0)
}

if entity_type == "tag" and "request_id" in transaction:
update_data["request_id"] = transaction.get("request_id")

Expand Down Expand Up @@ -1875,6 +1857,10 @@ async def _common_add_spend_log_transaction_to_daily_transaction(
custom_llm_provider=payload.get("custom_llm_provider", None),
compression_saved_tokens=compression_saved_tokens,
cache_read_input_tokens=cache_read_input_tokens,
routing_decision=_metadata.get("routing_decision"),
model_map_information=_metadata.get("model_map_information"),
model_id=payload.get("model_id"),
usage_object=usage_obj,
)

daily_transaction = BaseDailySpendTransaction(
Expand All @@ -1896,6 +1882,7 @@ async def _common_add_spend_log_transaction_to_daily_transaction(
compression_saved_tokens=compression_saved_tokens,
compression_savings_spend=savings_spend.compression,
prompt_caching_savings_spend=savings_spend.prompt_caching,
autorouter_savings_spend=savings_spend.autorouter,
)
return daily_transaction
except Exception as e:
Expand Down
4 changes: 4 additions & 0 deletions litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,10 @@ def get_aggregated_daily_spend_update_transactions(
payload.get("prompt_caching_savings_spend", 0) or 0
) + daily_transaction.get("prompt_caching_savings_spend", 0)

daily_transaction["autorouter_savings_spend"] = (
payload.get("autorouter_savings_spend", 0) or 0
) + daily_transaction.get("autorouter_savings_spend", 0)

else:
aggregated_daily_spend_update_transactions[_key] = deepcopy(payload)
return aggregated_daily_spend_update_transactions
Expand Down
9 changes: 9 additions & 0 deletions litellm/proxy/management_endpoints/common_daily_activity.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,9 @@ def compression_savings_spend(self) -> float: ...
@property
def prompt_caching_savings_spend(self) -> float: ...

@property
def autorouter_savings_spend(self) -> float: ...

@property
def api_requests(self) -> int: ...

Expand Down Expand Up @@ -135,6 +138,7 @@ class _GroupingSetsRow(SimpleNamespace):
compression_saved_tokens: int | None
compression_savings_spend: float | None
prompt_caching_savings_spend: float | None
autorouter_savings_spend: float | None
api_requests: int | None
successful_requests: int | None
failed_requests: int | None
Expand All @@ -158,6 +162,7 @@ def update_metrics(existing_metrics: SpendMetrics, record: DailySpendRecord) ->
existing_metrics.compression_saved_tokens += record.compression_saved_tokens or 0
existing_metrics.compression_savings_spend += record.compression_savings_spend or 0
existing_metrics.prompt_caching_savings_spend += record.prompt_caching_savings_spend or 0
existing_metrics.autorouter_savings_spend += record.autorouter_savings_spend or 0
existing_metrics.api_requests += record.api_requests or 0
existing_metrics.successful_requests += record.successful_requests or 0
existing_metrics.failed_requests += record.failed_requests or 0
Expand Down Expand Up @@ -590,6 +595,7 @@ def _build_aggregated_sql_query(
SUM(compression_saved_tokens)::bigint AS compression_saved_tokens,
SUM(compression_savings_spend)::float AS compression_savings_spend,
SUM(prompt_caching_savings_spend)::float AS prompt_caching_savings_spend,
SUM(autorouter_savings_spend)::float AS autorouter_savings_spend,
SUM(api_requests)::bigint AS api_requests,
SUM(successful_requests)::bigint AS successful_requests,
SUM(failed_requests)::bigint AS failed_requests
Expand Down Expand Up @@ -732,6 +738,7 @@ def _record_to_spend_metrics(record: _GroupingSetsRow) -> SpendMetrics:
compression_saved_tokens=record.compression_saved_tokens or 0,
compression_savings_spend=record.compression_savings_spend or 0,
prompt_caching_savings_spend=record.prompt_caching_savings_spend or 0,
autorouter_savings_spend=record.autorouter_savings_spend or 0,
api_requests=record.api_requests or 0,
successful_requests=record.successful_requests or 0,
failed_requests=record.failed_requests or 0,
Expand Down Expand Up @@ -986,6 +993,7 @@ async def get_daily_activity(
total_compression_saved_tokens=metadata_metrics.compression_saved_tokens,
total_compression_savings_spend=metadata_metrics.compression_savings_spend,
total_prompt_caching_savings_spend=metadata_metrics.prompt_caching_savings_spend,
total_autorouter_savings_spend=metadata_metrics.autorouter_savings_spend,

@veria-ai veria-ai Bot Aug 2, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Low: Deployment pricing disclosure

This returns savings calculated from deployment-specific pricing overrides through /user/daily/activity, where non-admin users can read their own traffic. An authenticated user can submit requests with controlled token usage and combine total_spend with this value to infer the baseline deployment's negotiated input and output rates; restrict this metric to administrative responses or avoid using private deployment rates in user-visible analytics.

@tin-berri tin-berri Aug 2, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Declining this one, with the code rather than an argument.

remove_sensitive_info_from_deployment (proxy/common_utils/openai_endpoint_utils.py:13) is the owner of what must not leave the proxy. It pops api_key, client_secret, vertex_credentials, aws_access_key_id and aws_secret_access_key, and deliberately does not touch pricing.

Pricing is published directly, not just inferable. _get_proxy_model_info (proxy_server.py:12912) merges it into /model/info on purpose, comment and all:

# read litellm model_prices_and_context_window.json to get the following:
# input_cost_per_token, output_cost_per_token, max_tokens

so a caller reads the rates rather than solving for them, and the dashboard depends on that to show what models cost. Treating the derived figure as sensitive while the endpoint states the inputs would be a posture change for the proxy, not a fix to this PR.

One residual worth naming rather than hiding: the baseline is the hardest tier's deployment, which a given caller may not be entitled to enumerate through /model/info, so for that deployment this is a slightly narrower channel than the endpoint. It stays low: the router can already route that caller's traffic onto those models. Gating the field would also have to live in the model-access resolver, not in the response builder, and the value is summed across callers at write time in the daily rollup, so a viewer-dependent counterfactual is not something a pre-aggregated column can express.

@veria-ai veria-ai Bot Aug 2, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for the detail. I can't automatically confirm this is safe to dismiss, so I'm leaving the thread open for a maintainer to make the call.

page=page,
total_pages=-(-total_count // page_size), # Ceiling division
has_more=(page * page_size) < total_count,
Expand Down Expand Up @@ -1075,6 +1083,7 @@ async def get_daily_activity_aggregated(
total_compression_saved_tokens=aggregated["totals"].compression_saved_tokens,
total_compression_savings_spend=aggregated["totals"].compression_savings_spend,
total_prompt_caching_savings_spend=aggregated["totals"].prompt_caching_savings_spend,
total_autorouter_savings_spend=aggregated["totals"].autorouter_savings_spend,
page=1,
total_pages=1,
has_more=False,
Expand Down
6 changes: 6 additions & 0 deletions litellm/proxy/schema.prisma
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -748,6 +748,7 @@ model LiteLLM_DailyUserSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
Expand Down Expand Up @@ -782,6 +783,7 @@ model LiteLLM_DailyOrganizationSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
Expand Down Expand Up @@ -816,6 +818,7 @@ model LiteLLM_DailyEndUserSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
Expand Down Expand Up @@ -849,6 +852,7 @@ model LiteLLM_DailyAgentSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
Expand Down Expand Up @@ -882,6 +886,7 @@ model LiteLLM_DailyTeamSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
Expand Down Expand Up @@ -917,6 +922,7 @@ model LiteLLM_DailyTagSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
Expand Down
Loading
Loading