feat(ui): lead the benchmarks tab with savings and the sessions it was earned over

litellm-proxy-extras/litellm_proxy_extras/migrations/20260801000000_add_auto_router_session_rollup/migration.sql

@@ -0,0 +1,37 @@
+-- CreateTable
+CREATE TABLE IF NOT EXISTS "LiteLLM_AutoRouterSession" (
+    "session_id" TEXT NOT NULL,
+    "model_group" TEXT NOT NULL,
+    "router_kind" TEXT NOT NULL,
+    "baseline_model" TEXT,
+    "first_turn_at" TIMESTAMP(3) NOT NULL,
+    "last_turn_at" TIMESTAMP(3) NOT NULL,
+    "turns" INTEGER NOT NULL DEFAULT 0,
+    "turns_with_usage" INTEGER NOT NULL DEFAULT 0,
+    "total_tokens" BIGINT NOT NULL DEFAULT 0,
+    "ephemeral_5m_tokens" BIGINT NOT NULL DEFAULT 0,
+    "ephemeral_1h_tokens" BIGINT NOT NULL DEFAULT 0,
+    "spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0,
+    "baseline_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0,
+    "same_model_turns" INTEGER NOT NULL DEFAULT 0,
+    "same_model_hits" INTEGER NOT NULL DEFAULT 0,
+    "first_visit_turns" INTEGER NOT NULL DEFAULT 0,
+    "first_visit_hits" INTEGER NOT NULL DEFAULT 0,
+    "return_turns" INTEGER NOT NULL DEFAULT 0,
+    "return_hits" INTEGER NOT NULL DEFAULT 0,
+    "stale_return_misses" INTEGER NOT NULL DEFAULT 0,
+    "savable_return_misses" INTEGER NOT NULL DEFAULT 0,
+    "rescued_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0,
+    "replay_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0,
+    "last_model" TEXT,
+    "model_state" JSONB NOT NULL DEFAULT '{}',
+    "updated_at" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "LiteLLM_AutoRouterSession_pkey" PRIMARY KEY ("session_id","model_group")
+);
+
+-- CreateIndex
+CREATE INDEX IF NOT EXISTS "idx_auto_router_session_group_activity" ON "LiteLLM_AutoRouterSession"("model_group", "last_turn_at");
+
+-- CreateIndex
+CREATE INDEX IF NOT EXISTS "idx_auto_router_session_last_turn" ON "LiteLLM_AutoRouterSession"("last_turn_at");

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -1391,6 +1391,46 @@ model LiteLLM_AdaptiveRouterSession {
   @@index([last_activity_at], map: "idx_adaptive_router_session_activity")
 }
 
+// Per-(session, auto-router) rollup behind the auto-router benchmarks dashboard.
+model LiteLLM_AutoRouterSession {
+  session_id            String
+  model_group           String
+  router_kind           String
+  baseline_model        String?
+
+  first_turn_at         DateTime
+  last_turn_at          DateTime
+  turns                 Int      @default(0)
+  turns_with_usage      Int      @default(0)
+
+  total_tokens          BigInt   @default(0)
+  ephemeral_5m_tokens   BigInt   @default(0)
+  ephemeral_1h_tokens   BigInt   @default(0)
+
+  spend                 Float    @default(0.0)
+  baseline_spend        Float    @default(0.0)
+
+  same_model_turns      Int      @default(0)
+  same_model_hits       Int      @default(0)
+  first_visit_turns     Int      @default(0)
+  first_visit_hits      Int      @default(0)
+  return_turns          Int      @default(0)
+  return_hits           Int      @default(0)
+  stale_return_misses   Int      @default(0)
+  savable_return_misses Int      @default(0)
+
+  rescued_spend         Float    @default(0.0)
+  replay_spend          Float    @default(0.0)
+
+  last_model            String?
+  model_state           Json     @default("{}")
+  updated_at            DateTime @default(now()) @updatedAt
+
+  @@id([session_id, model_group])
+  @@index([model_group, last_turn_at], map: "idx_auto_router_session_group_activity")
+  @@index([last_turn_at], map: "idx_auto_router_session_last_turn")
+}
+
 // ---------------------------------------------------------------------------
 // Workflow Run Tracking
 //

litellm/proxy/db/db_spend_update_writer.py

@@ -57,6 +57,11 @@
     ToolDiscoveryQueue,
 )
 from litellm.proxy.route_llm_request import ROUTE_ENDPOINT_MAPPING
+from litellm.proxy.spend_tracking.auto_router_session_queue import AutoRouterSessionQueue
+from litellm.proxy.spend_tracking.auto_router_sessions import (
+    auto_router_group_kinds,
+    turn_from_spend_payload,
+)
 from litellm.proxy.spend_tracking.compression_savings import (
     extract_compression_saved_tokens,
 )
@@ -115,6 +120,16 @@ def _extract_cache_creation_tokens(usage_obj: dict) -> int:
     return int(details.get("cache_write_tokens", 0) or details.get("cache_creation_tokens", 0) or 0)
 
 
+def _parse_start_time(value: object) -> datetime | None:
+    if not isinstance(value, str):
+        return None
+    try:
+        return datetime.fromisoformat(value)
+    except ValueError:
+        verbose_proxy_logger.debug("auto_router_sessions: unparseable startTime %s", value)
+        return None
+
+
 class DBSpendUpdateWriter:
     """
     Module responsible for
@@ -138,6 +153,7 @@ def __init__(
         self.daily_agent_spend_update_queue = DailySpendUpdateQueue()
         self.daily_org_spend_update_queue = DailySpendUpdateQueue()
         self.daily_tag_spend_update_queue = DailySpendUpdateQueue()
+        self.auto_router_session_queue = AutoRouterSessionQueue()
 
     async def update_database(
         # LiteLLM management object fields
@@ -195,6 +211,8 @@ async def update_database(
             if team_id is not None and team_id != "":
                 payload["team_id"] = team_id
 
+            await self._record_auto_router_turn(payload=payload, prisma_client=prisma_client)
+
             if disable_spend_logs is False:
                 await self._insert_spend_log_to_db(
                     payload=payload,
@@ -247,6 +265,86 @@ async def update_database(
                 end_user_id,
             )
 
+    async def _record_auto_router_turn(
+        self,
+        payload: SpendLogsPayload,
+        prisma_client: PrismaClient | None,
+    ) -> None:
+        """Fold one auto-routed turn into its session rollup.
+
+        Hooked here rather than beside the daily transactions because this is the
+        one place a request passes through exactly once; the daily path runs per
+        entity type and would count every turn six times over.
+
+        Independent of ``disable_spend_logs``: the rollup is what the benchmarks
+        dashboard reads, so turning off per-request logging must not also turn off
+        the aggregate that replaced it.
+
+        Never raises. A dashboard rollup is not worth failing spend tracking over,
+        and this runs before the spend log insert, so anything escaping here would
+        cost the deployment money it could not account for.
+        """
+        try:
+            await self._record_auto_router_turn_unsafe(payload=payload, prisma_client=prisma_client)
+        except Exception as e:  # noqa: BLE001  # see docstring: this must never break the spend path
+            verbose_proxy_logger.warning("auto_router_sessions: skipped a turn (%s: %s)", type(e).__name__, e)
+
+    async def _record_auto_router_turn_unsafe(
+        self,
+        payload: SpendLogsPayload,
+        prisma_client: PrismaClient | None,
+    ) -> None:
+        from litellm.proxy.proxy_server import llm_router
+
+        session_id = payload.get("session_id")
+        model_group = payload.get("model_group")
+        model = payload.get("model")
+        start_time = payload.get("startTime")
+        if prisma_client is None or llm_router is None or not session_id or not model_group or not model:
+            return
+        router_kind = auto_router_group_kinds(llm_router).get(model_group)
+        if router_kind is None:
+            return
+        started_at = start_time if isinstance(start_time, datetime) else _parse_start_time(start_time)
+        if started_at is None:
+            return
+
+        _metadata: SpendLogsMetadata = json.loads(payload["metadata"])
+        usage_obj = _metadata.get("usage_object", {}) or {}  # mutable-ok: empty fallback for an absent usage payload
+        cache_read_tokens = _extract_cache_read_tokens(usage_obj)
+        savings_spend = compute_savings_spend(
+            model=model,
+            custom_llm_provider=payload.get("custom_llm_provider", None),
+            compression_saved_tokens=extract_compression_saved_tokens(_metadata),
+            cache_read_input_tokens=cache_read_tokens,
+            routing_decision=_metadata.get("routing_decision"),
+            model_id=payload.get("model_id"),
+            llm_router=_get_llm_router,
+            usage_object=usage_obj,
+            cost_breakdown=_metadata.get("cost_breakdown"),
+        )
+        await self.auto_router_session_queue.record_turn(
+            key=(session_id, model_group),
+            router_kind=router_kind,
+            # The same setting savings.py priced this turn against, stored on the row so
+            # the dashboard names the baseline the numbers were actually computed with
+            # rather than whatever the config says by the time someone opens the tab.
+            baseline_model=litellm.autorouter_savings_baseline_model,
+            turn=turn_from_spend_payload(
+                model=model,
+                started_at=started_at,
+                prompt_tokens=payload.get("prompt_tokens", 0) or 0,
+                completion_tokens=payload.get("completion_tokens", 0) or 0,
+                total_tokens=payload.get("total_tokens", 0) or 0,
+                spend=payload.get("spend", 0.0) or 0.0,
+                autorouter_savings=savings_spend.autorouter,
+                cache_read_tokens=cache_read_tokens,
+                cache_creation_tokens=_extract_cache_creation_tokens(usage_obj),
+                usage_object=usage_obj,
+            ),
+            prisma_client=prisma_client,
+        )
+
     async def _enqueue_tool_usage_transaction(
         self,
         payload: SpendLogsPayload,
@@ -797,6 +895,11 @@ async def db_update_spend_transaction_handler(
         else:
             - Regular flow of this method
         """
+        # Flushed outside the Redis buffer on purpose: session rollup writes are
+        # atomic increments, so two pods committing the same session compose
+        # correctly without being funnelled through a single elected writer.
+        await self.auto_router_session_queue.flush(prisma_client=prisma_client)
+
         if RedisUpdateBuffer._should_commit_spend_updates_to_redis():
             await self._commit_spend_updates_to_db_with_redis(
                 prisma_client=prisma_client,

litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py

@@ -184,6 +184,19 @@ async def _delete_old_tool_index_rows(self, prisma_client: PrismaClient, cutoff_
             time_column="start_time",
         )
 
+    async def _delete_old_auto_router_sessions(self, prisma_client: PrismaClient, cutoff_date: datetime) -> int:
+        # Auto-router session rollups summarize spend logs, so they expire on the
+        # same cutoff. A row is keyed on when the session was last active, not
+        # when it started, so a long-running conversation is not pruned out from
+        # under itself while it is still being written to.
+        return await self._delete_old_rows_batched(
+            prisma_client,
+            cutoff_date,
+            table_name="LiteLLM_AutoRouterSession",
+            key_columns=("session_id", "model_group"),
+            time_column="last_turn_at",
+        )
+
     async def cleanup_old_spend_logs(self, prisma_client: PrismaClient) -> None:
         """
         Main cleanup function. Deletes old spend logs in batches.
@@ -243,6 +256,9 @@ async def cleanup_old_spend_logs(self, prisma_client: PrismaClient) -> None:
             index_deleted = await self._delete_old_tool_index_rows(prisma_client, cutoff_date)
             verbose_proxy_logger.info(f"Deleted {index_deleted} expired tool index rows")
 
+            sessions_deleted = await self._delete_old_auto_router_sessions(prisma_client, cutoff_date)
+            verbose_proxy_logger.info(f"Deleted {sessions_deleted} expired auto-router session rollups")
+
         except Exception as e:
             # .exception() captures the traceback; str(e) alone on a Prisma/DB
             # timeout is often empty and gives operators no signal to diagnose.

litellm/proxy/proxy_server.py

@@ -20,6 +20,7 @@
 from types import UnionType
 from typing import (
     TYPE_CHECKING,
+    Annotated,
     Any,
     Literal,
     Optional,
@@ -16391,6 +16392,63 @@ async def get_adaptive_router_state(
     return {"routers": snapshots}
 
 
+@router.get(
+    "/auto_router/benchmarks",
+    tags=["auto_router"],  # mutable-ok: HTTPException takes a dict detail
+    dependencies=[Depends(user_api_key_auth)],  # mutable-ok: FastAPI takes a list of dependencies
+)
+async def get_auto_router_benchmarks(
+    start_date: str,
+    end_date: str,
+    user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
+):
+    """Session-level benchmarks for every configured auto-router.
+
+    Admin-only. For each auto-router alias, reports turns per session, session
+    length, tokens per session, the dollar savings of the routed mix against the
+    counterfactual baseline, and how the provider prompt cache behaved.
+
+    Reads the per-session rollup, never the per-request spend logs.
+    ``start_date`` / ``end_date`` are ``YYYY-MM-DD``; the window is clamped to
+    the most recent ``BENCHMARKS_MAX_WINDOW_DAYS`` days and the response echoes
+    the window actually served. Sessions are counted whole when they were active
+    in the window. Returns 404 when no auto-router is configured.
+    """
+    from litellm.proxy.spend_tracking.auto_router_benchmarks import compute_benchmarks
+    from litellm.proxy.spend_tracking.auto_router_sessions import auto_router_group_kinds
+
+    if not _user_has_admin_view(user_api_key_dict):
+        raise HTTPException(
+            status_code=403,
+            detail={  # mutable-ok: HTTPException takes a dict detail
+                "error": CommonProxyErrors.not_allowed_access.value
+            },
+        )
+    if llm_router is None:
+        raise HTTPException(
+            status_code=404,
+            detail={  # mutable-ok: HTTPException takes a dict detail
+                "error": "No auto_router is configured on this proxy."
+            },
+        )
+    group_kinds = auto_router_group_kinds(llm_router)
+    if not group_kinds:
+        raise HTTPException(
+            status_code=404,
+            detail={  # mutable-ok: HTTPException takes a dict detail
+                "error": "No auto_router is configured on this proxy."
+            },
+        )
+    if prisma_client is None:
+        raise HTTPException(
+            status_code=500,
+            detail={  # mutable-ok: HTTPException takes a dict detail
+                "error": CommonProxyErrors.db_not_connected_error.value
+            },
+        )
+    return await compute_benchmarks(prisma_client, group_kinds, start_date, end_date)
+
+
 @router.get("/routes", dependencies=[Depends(user_api_key_auth)])
 async def get_routes():
     """

litellm/proxy/schema.prisma

@@ -1391,6 +1391,46 @@ model LiteLLM_AdaptiveRouterSession {
   @@index([last_activity_at], map: "idx_adaptive_router_session_activity")
 }
 
+// Per-(session, auto-router) rollup behind the auto-router benchmarks dashboard.
+model LiteLLM_AutoRouterSession {
+  session_id            String
+  model_group           String
+  router_kind           String
+  baseline_model        String?
+
+  first_turn_at         DateTime
+  last_turn_at          DateTime
+  turns                 Int      @default(0)
+  turns_with_usage      Int      @default(0)
+
+  total_tokens          BigInt   @default(0)
+  ephemeral_5m_tokens   BigInt   @default(0)
+  ephemeral_1h_tokens   BigInt   @default(0)
+
+  spend                 Float    @default(0.0)
+  baseline_spend        Float    @default(0.0)
+
+  same_model_turns      Int      @default(0)
+  same_model_hits       Int      @default(0)
+  first_visit_turns     Int      @default(0)
+  first_visit_hits      Int      @default(0)
+  return_turns          Int      @default(0)
+  return_hits           Int      @default(0)
+  stale_return_misses   Int      @default(0)
+  savable_return_misses Int      @default(0)
+
+  rescued_spend         Float    @default(0.0)
+  replay_spend          Float    @default(0.0)
+
+  last_model            String?
+  model_state           Json     @default("{}")
+  updated_at            DateTime @default(now()) @updatedAt
+
+  @@id([session_id, model_group])
+  @@index([model_group, last_turn_at], map: "idx_auto_router_session_group_activity")
+  @@index([last_turn_at], map: "idx_auto_router_session_last_turn")
+}
+
 // ---------------------------------------------------------------------------
 // Workflow Run Tracking
 //