feat(spend): add a per-session auto-router benchmarks rollup

litellm-proxy-extras/litellm_proxy_extras/migrations/20260804000000_add_auto_router_session_rollup/migration.sql

@@ -0,0 +1,37 @@
+CREATE TABLE IF NOT EXISTS "LiteLLM_AutoRouterSession" (
+    api_key            TEXT NOT NULL,
+    session_id         TEXT NOT NULL,
+    model_group        TEXT NOT NULL,
+    router_kind        TEXT NOT NULL,
+    baseline_model     TEXT,
+    turns              INTEGER NOT NULL DEFAULT 0,
+    turns_with_usage   INTEGER NOT NULL DEFAULT 0,
+    total_tokens       BIGINT NOT NULL DEFAULT 0,
+    spend              DOUBLE PRECISION NOT NULL DEFAULT 0,
+    baseline_spend     DOUBLE PRECISION NOT NULL DEFAULT 0,
+    first_visit_turns  INTEGER NOT NULL DEFAULT 0,
+    first_visit_hits   INTEGER NOT NULL DEFAULT 0,
+    warm_turns         INTEGER NOT NULL DEFAULT 0,
+    warm_hits          INTEGER NOT NULL DEFAULT 0,
+    expired_turns      INTEGER NOT NULL DEFAULT 0,
+    expired_hits       INTEGER NOT NULL DEFAULT 0,
+    unordered_turns    INTEGER NOT NULL DEFAULT 0,
+    unordered_hits     INTEGER NOT NULL DEFAULT 0,
+    unknown_ttl_turns        INTEGER NOT NULL DEFAULT 0,
+    unknown_ttl_hits         INTEGER NOT NULL DEFAULT 0,
+    cache_5m_turns           INTEGER NOT NULL DEFAULT 0,
+    cache_1h_turns           INTEGER NOT NULL DEFAULT 0,
+    cache_ttl_unknown_turns INTEGER NOT NULL DEFAULT 0,
+    tiers              JSONB NOT NULL DEFAULT '{}',
+    first_turn_at      TIMESTAMP(3) NOT NULL,
+    last_turn_at       TIMESTAMP(3) NOT NULL,
+    updated_at         TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "LiteLLM_AutoRouterSession_pkey" PRIMARY KEY (api_key, session_id, model_group)
+);
+
+CREATE INDEX IF NOT EXISTS "idx_auto_router_session_started"
+    ON "LiteLLM_AutoRouterSession" (first_turn_at);
+
+CREATE INDEX IF NOT EXISTS "idx_auto_router_session_activity"
+    ON "LiteLLM_AutoRouterSession" (last_turn_at);

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -1393,6 +1393,45 @@ model LiteLLM_AdaptiveRouterSession {
   @@index([last_activity_at], map: "idx_adaptive_router_session_activity")
 }
 
+model LiteLLM_AutoRouterSession {
+  api_key            String
+  session_id         String
+  model_group        String
+  router_kind        String
+  baseline_model     String?
+
+  turns              Int      @default(0)
+  turns_with_usage   Int      @default(0)
+  total_tokens       BigInt   @default(0)
+  spend              Float    @default(0)
+  baseline_spend     Float    @default(0)
+
+  first_visit_turns  Int      @default(0)
+  first_visit_hits   Int      @default(0)
+  warm_turns         Int      @default(0)
+  warm_hits          Int      @default(0)
+  expired_turns      Int      @default(0)
+  expired_hits       Int      @default(0)
+  unordered_turns    Int      @default(0)
+  unordered_hits     Int      @default(0)
+  unknown_ttl_turns  Int      @default(0)
+  unknown_ttl_hits   Int      @default(0)
+
+  cache_5m_turns          Int @default(0)
+  cache_1h_turns          Int @default(0)
+  cache_ttl_unknown_turns Int @default(0)
+
+  tiers              Json     @default("{}")
+
+  first_turn_at      DateTime
+  last_turn_at       DateTime
+  updated_at         DateTime @default(now()) @updatedAt
+
+  @@id([api_key, session_id, model_group])
+  @@index([first_turn_at], map: "idx_auto_router_session_started")
+  @@index([last_turn_at], map: "idx_auto_router_session_activity")
+}
+
 // ---------------------------------------------------------------------------
 // Workflow Run Tracking
 //

litellm/proxy/db/db_spend_update_writer.py

@@ -51,6 +51,7 @@
     ToolDiscoveryQueue,
 )
 from litellm.proxy.route_llm_request import ROUTE_ENDPOINT_MAPPING
+from litellm.proxy.spend_tracking.auto_router_sessions import AutoRouterSessionQueue, build_turn_facts
 from litellm.proxy.spend_tracking.compression_savings import (
     extract_compression_saved_tokens,
 )
@@ -132,6 +133,46 @@ def __init__(
         self.daily_agent_spend_update_queue = DailySpendUpdateQueue()
         self.daily_org_spend_update_queue = DailySpendUpdateQueue()
         self.daily_tag_spend_update_queue = DailySpendUpdateQueue()
+        self.auto_router_session_queue = AutoRouterSessionQueue()
+
+    def _enqueue_auto_router_turn(self, payload: SpendLogsPayload) -> None:
+        """Stage one auto-routed turn for the benchmarks rollup.
+
+        Runs once per request rather than beside the daily transactions, which are built
+        per entity type and would count every turn six times over. Independent of
+        ``disable_spend_logs``, because the rollup is what the benchmarks dashboard reads
+        now. Never raises, and never blocks: a full queue drops the turn rather than
+        applying backpressure to spend tracking.
+        """
+        try:
+            metadata: Final[SpendLogsMetadata] = json.loads(payload["metadata"])
+            if not metadata.get("routing_decision"):
+                return
+            usage_raw: Final = metadata.get("usage_object")
+            usage_obj: Final = usage_raw if isinstance(usage_raw, dict) else None
+            cache_read_tokens: Final = _extract_cache_read_tokens(usage_obj) if usage_obj is not None else 0
+            savings: Final = compute_savings_spend(
+                model=payload.get("model", None),
+                custom_llm_provider=payload.get("custom_llm_provider", None),
+                compression_saved_tokens=extract_compression_saved_tokens(metadata),
+                cache_read_input_tokens=cache_read_tokens,
+                routing_decision=metadata.get("routing_decision"),
+                model_id=payload.get("model_id"),
+                llm_router=_get_llm_router,
+                usage_object=usage_obj,
+                cost_breakdown=metadata.get("cost_breakdown"),
+            )
+            turn: Final = build_turn_facts(
+                payload=payload,
+                metadata=metadata,
+                autorouter_savings=savings.autorouter,
+                cache_read_tokens=cache_read_tokens,
+                cache_creation_tokens=_extract_cache_creation_tokens(usage_obj) if usage_obj is not None else 0,
+            )
+            if turn is not None:
+                self.auto_router_session_queue.update_queue.put_nowait(turn)
+        except Exception as e:  # noqa: BLE001  # a dashboard rollup must never fail spend tracking
+            verbose_proxy_logger.debug("auto_router_sessions: could not stage turn (%s)", e)
 
     async def update_database(
         # LiteLLM management object fields
@@ -209,6 +250,8 @@ async def update_database(
                     "disable_spend_logs=True. Skipping writing spend logs to db. Other spend updates - Key/User/Team table will still occur."
                 )
 
+            self._enqueue_auto_router_turn(payload=payload)
+
             # Single task replaces 11 create_task() calls
             asyncio.create_task(
                 self._batch_database_updates(
@@ -809,6 +852,9 @@ async def db_update_spend_transaction_handler(
                 proxy_logging_obj=proxy_logging_obj,
             )
 
+        ################## Auto-Router Benchmarks Rollup ##################
+        await self.auto_router_session_queue.flush(prisma_client=prisma_client)
+
     async def _commit_spend_updates_to_db_with_redis(
         self,
         prisma_client: PrismaClient,

litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py

@@ -186,6 +186,20 @@ async def _delete_old_tool_index_rows(self, prisma_client: PrismaClient, cutoff_
             time_column="start_time",
         )
 
+    async def _delete_old_auto_router_sessions(self, prisma_client: PrismaClient, cutoff_date: datetime) -> int:
+        """Expire auto-router rollups on the spend-log cutoff.
+
+        Keyed on last activity rather than session start, so a conversation still running
+        when the cutoff passes is not pruned out from under itself.
+        """
+        return await self._delete_old_rows_batched(
+            prisma_client,
+            cutoff_date,
+            table_name="LiteLLM_AutoRouterSession",
+            key_columns=("api_key", "session_id", "model_group"),
+            time_column="last_turn_at",
+        )
+
     async def cleanup_old_spend_logs(self, prisma_client: PrismaClient) -> None:
         """
         Main cleanup function. Deletes old spend logs in batches.
@@ -245,6 +259,9 @@ async def cleanup_old_spend_logs(self, prisma_client: PrismaClient) -> None:
             index_deleted: Final = await self._delete_old_tool_index_rows(prisma_client, cutoff_date)
             verbose_proxy_logger.info("Deleted %s expired tool index rows", index_deleted)
 
+            sessions_deleted: Final = await self._delete_old_auto_router_sessions(prisma_client, cutoff_date)
+            verbose_proxy_logger.info("Deleted %s expired auto-router session rollups", sessions_deleted)
+
         except Exception as e:
             # .exception() captures the traceback; str(e) alone on a Prisma/DB
             # timeout is often empty and gives operators no signal to diagnose.

litellm/proxy/proxy_server.py

@@ -16,10 +16,11 @@
 import traceback
 import warnings
 from collections.abc import AsyncGenerator, Callable, Mapping
-from datetime import datetime, timedelta, timezone
+from datetime import date, datetime, timedelta, timezone
 from types import UnionType
 from typing import (
     TYPE_CHECKING,
+    Annotated,
     Any,
     Final,
     Literal,
@@ -544,6 +545,10 @@ def generate_feedback_box():
 from litellm.proxy.route_llm_request import route_request
 from litellm.proxy.search_endpoints.endpoints import router as search_router
 from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager
+from litellm.proxy.spend_tracking.auto_router_benchmarks import (
+    AutoRouterBenchmarksResponse,
+    fetch_benchmarks,
+)
 from litellm.proxy.spend_tracking.budget_reservation import get_budget_window_start
 from litellm.proxy.spend_tracking.spend_management_endpoints import (
     router as spend_management_router,
@@ -16375,6 +16380,50 @@ async def get_adaptive_router_state(
     return {"routers": snapshots}
 
 
+def _auto_router_error(message: str) -> dict[str, str]:
+    return {"error": message}  # mutable-ok: HTTPException serializes its detail from a real dict
+
+
+@router.get(
+    "/auto_router/benchmarks",
+    tags=["auto_router"],  # mutable-ok: FastAPI types tags as List[str]
+    response_model=AutoRouterBenchmarksResponse,
+)
+async def get_auto_router_benchmarks(
+    start_date: date,
+    end_date: date,
+    user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
+    model_group: str | None = None,
+):
+    """Savings, session shape and prompt-cache behaviour for every auto-router.
+
+    Admin-only. Reads the per-session rollup only; no per-request table is scanned.
+    `start_date` and `end_date` are inclusive calendar dates, clamped to the most recent
+    30 days. Pass `model_group` to scope every figure to one auto-router.
+    """
+    if not _user_has_admin_view(user_api_key_dict):
+        raise HTTPException(
+            status_code=403,
+            detail=_auto_router_error(CommonProxyErrors.not_allowed_access.value),
+        )
+    if end_date < start_date:
+        raise HTTPException(
+            status_code=400,
+            detail=_auto_router_error("end_date must not be earlier than start_date."),
+        )
+    if prisma_client is None:
+        raise HTTPException(
+            status_code=500,
+            detail=_auto_router_error(CommonProxyErrors.db_not_connected_error.value),
+        )
+    return await fetch_benchmarks(
+        prisma_client=prisma_client,
+        start_date=start_date,
+        end_date=end_date,
+        model_group=model_group,
+    )
+
+
 @router.get("/routes", dependencies=[Depends(user_api_key_auth)])
 async def get_routes():
     """

litellm/proxy/schema.prisma

@@ -1393,6 +1393,45 @@ model LiteLLM_AdaptiveRouterSession {
   @@index([last_activity_at], map: "idx_adaptive_router_session_activity")
 }
 
+model LiteLLM_AutoRouterSession {
+  api_key            String
+  session_id         String
+  model_group        String
+  router_kind        String
+  baseline_model     String?
+
+  turns              Int      @default(0)
+  turns_with_usage   Int      @default(0)
+  total_tokens       BigInt   @default(0)
+  spend              Float    @default(0)
+  baseline_spend     Float    @default(0)
+
+  first_visit_turns  Int      @default(0)
+  first_visit_hits   Int      @default(0)
+  warm_turns         Int      @default(0)
+  warm_hits          Int      @default(0)
+  expired_turns      Int      @default(0)
+  expired_hits       Int      @default(0)
+  unordered_turns    Int      @default(0)
+  unordered_hits     Int      @default(0)
+  unknown_ttl_turns  Int      @default(0)
+  unknown_ttl_hits   Int      @default(0)
+
+  cache_5m_turns          Int @default(0)
+  cache_1h_turns          Int @default(0)
+  cache_ttl_unknown_turns Int @default(0)
+
+  tiers              Json     @default("{}")
+
+  first_turn_at      DateTime
+  last_turn_at       DateTime
+  updated_at         DateTime @default(now()) @updatedAt
+
+  @@id([api_key, session_id, model_group])
+  @@index([first_turn_at], map: "idx_auto_router_session_started")
+  @@index([last_turn_at], map: "idx_auto_router_session_activity")
+}
+
 // ---------------------------------------------------------------------------
 // Workflow Run Tracking
 //