feat(sgr): make the gateway middleware the source of truth for successful requests

backend/routes/allowlist.py

@@ -82,6 +82,9 @@
     "/user_agent",
     "/usage/",
     "/daily/",
+    # Deployment-wide gateway request counts. Scoped to the analytics read rather
+    # than all of /gateway/, which stays free for data-plane routes.
+    "/gateway/daily/",
     # CloudZero cost-export admin (init / settings / export / dry-run / delete)
     "/cloudzero/",
     # Caching admin

litellm-proxy-extras/litellm_proxy_extras/migrations/20260803000000_add_daily_gateway_requests/migration.sql

@@ -0,0 +1,15 @@
+-- CreateTable
+CREATE TABLE IF NOT EXISTS "LiteLLM_DailyGatewayRequests" (
+    "date" TEXT NOT NULL,
+    "category" TEXT NOT NULL,
+    "route" TEXT NOT NULL,
+    "successful_requests" BIGINT NOT NULL DEFAULT 0,
+    "failed_requests" BIGINT NOT NULL DEFAULT 0,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "LiteLLM_DailyGatewayRequests_pkey" PRIMARY KEY ("date","category","route")
+);
+
+-- CreateIndex
+CREATE INDEX IF NOT EXISTS "LiteLLM_DailyGatewayRequests_date_idx" ON "LiteLLM_DailyGatewayRequests"("date");

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -1118,6 +1118,26 @@ model LiteLLM_DailyToolSpend {
   @@id([date, tool_name])
 }
 
+// Gateway request counts recorded at the ASGI edge by
+// BillableRequestMetricsMiddleware. This is the source of truth for SGR
+// (successful gateway requests): it counts what the proxy actually answered,
+// independent of whether the request reached litellm's logging callbacks.
+// The key carries no deployment or caller dimension. Every part of it is
+// chosen by the proxy and drawn from a closed set, so the table is bounded by
+// (days x categories x routes) rather than by anything a caller can vary.
+model LiteLLM_DailyGatewayRequests {
+  date                String
+  category            String
+  route               String
+  successful_requests BigInt   @default(0)
+  failed_requests     BigInt   @default(0)
+  created_at          DateTime @default(now())
+  updated_at          DateTime @updatedAt
+
+  @@id([date, category, route])
+  @@index([date])
+}
+
 // Prompt table for storing prompt configurations
 model LiteLLM_PromptTable {
   id String @id @default(uuid())

litellm/proxy/_types.py

@@ -622,6 +622,8 @@ class LiteLLMRoutes(enum.Enum):
             "/team/permissions_update",
             "/team/permissions_bulk_update",
             "/team/daily/activity",
+            # gateway request counts (SGR); deployment-wide, admin-only
+            "/gateway/daily/activity",
             # model
             "/model/new",
             "/model/update",
@@ -715,6 +717,7 @@ class LiteLLMRoutes(enum.Enum):
         "/global/spend/tags",
         "/global/predict/spend/logs",
         "/global/activity",
+        "/gateway/daily/activity",
         "/health/services",
     ] + info_routes
 

litellm/proxy/db/db_spend_update_writer.py

@@ -1860,6 +1860,12 @@ async def _common_add_spend_log_transaction_to_daily_transaction(
             )
             return None
 
+        # TODO: remove the successful_requests/failed_requests counters below once the
+        # admin UI has fully migrated to LiteLLM_DailyGatewayRequests, which is now the
+        # source of truth for SGR. This path derives the counts from spend-log metadata
+        # rather than from what the gateway answered, so the two intentionally disagree
+        # (see litellm/proxy/middleware/billable_request_metrics_middleware.py). The
+        # spend, token and per-entity columns written here stay either way.
         request_status: Final = prisma_client.get_request_status(payload)
         verbose_proxy_logger.debug("Logged request status: %s", request_status)
         _metadata: Final[SpendLogsMetadata] = json.loads(payload["metadata"])

litellm/proxy/db/gateway_request_tracking.py

@@ -0,0 +1,133 @@
+"""
+Accumulates gateway request counts (SGR) recorded at the ASGI edge and commits
+them to ``LiteLLM_DailyGatewayRequests``.
+
+Unlike the spend queues this keeps no per-request item. A count is a pure
+aggregate, so requests fold into an in-memory map as they finish. Every
+dimension of the key is server-chosen and drawn from a fixed set: the date, the
+category, and a route that the classifier maps to one of a closed list of
+strings rather than passing the raw path through. Nothing a caller sends can
+add a key, so the fold and the table it commits to are bounded by (days x
+routes) however much traffic arrives, and the response path carries no
+unbounded queue that would block once full.
+"""
+
+from dataclasses import asdict
+from datetime import datetime, timezone
+from typing import TYPE_CHECKING, Final
+
+from litellm._logging import verbose_proxy_logger
+from litellm.proxy.middleware.billable_request_metrics_middleware import BillableCategory
+from litellm.types.proxy.gateway_requests import (
+    GatewayRequestCounts,
+    GatewayRequestKey,
+    GatewayRequestSnapshot,
+)
+
+if TYPE_CHECKING:
+    from litellm.proxy.utils import PrismaClient
+
+_EMPTY: Final = GatewayRequestCounts(successful_requests=0, failed_requests=0)
+
+
+def _utc_date() -> str:
+    return datetime.now(timezone.utc).strftime("%Y-%m-%d")
+
+
+class GatewayRequestAccumulator:
+    """Sink for the request-metrics middleware. ``record`` is sync and never awaits."""
+
+    def __init__(self) -> None:
+        self._counts: dict[GatewayRequestKey, GatewayRequestCounts] = {}  # mutable-ok: bounded fold, drained per flush
+
+    def record(self, *, category: BillableCategory, route: str, status_code: int) -> None:
+        key: Final = GatewayRequestKey(date=_utc_date(), category=category.value, route=route)
+        self._counts[key] = self._counts.get(key, _EMPTY).plus(succeeded=200 <= status_code < 300)
+
+    def drain(self) -> GatewayRequestSnapshot:
+        drained: Final = self._counts
+        self._counts = {}  # mutable-ok: the fold restarts empty; the drained map is handed off whole
+        return drained
+
+    def restore(self, snapshot: GatewayRequestSnapshot) -> None:
+        """
+        Merge un-committed counts back so the next flush retries them.
+
+        A dropped flush would silently undercount the metric the dashboard now
+        treats as the source of truth. Merging cannot grow without bound: keys
+        collapse on collision, so the fold stays bounded by (date x category x
+        route) however long the database is unreachable.
+
+        This buys at-least-once, not exactly-once, and the cost is worth stating.
+        The batch commits inside its context manager's ``__aexit__``, so a failure
+        raised after the transaction committed (a connection dropped while reading
+        the acknowledgement) restores counts that are already persisted, and the
+        next flush increments them a second time. Exactly-once would need a dedup
+        key the upserts could ignore on replay. For a traffic-volume metric a rare
+        overcount on a dropped acknowledgement beats losing a whole interval to
+        every database blip, so the trade is deliberate.
+        """
+        for key, counts in snapshot.items():
+            existing = self._counts.get(key, _EMPTY)
+            self._counts[key] = GatewayRequestCounts(
+                successful_requests=existing.successful_requests + counts.successful_requests,
+                failed_requests=existing.failed_requests + counts.failed_requests,
+            )
+
+
+async def commit_gateway_requests_to_db(
+    *,
+    prisma_client: "PrismaClient",
+    snapshot: GatewayRequestSnapshot,
+) -> None:
+    """Upsert one incrementing row per (date, category, route)."""
+    if not snapshot:
+        return
+
+    ordered: Final = sorted(snapshot.items(), key=lambda item: (item[0].date, item[0].category, item[0].route))
+
+    # pyright: ignore[reportAny] on both lines -- prisma's generated client is untyped,
+    # so .db and every table action off it resolve to Any at this boundary. The dict
+    # literals below are the shape prisma's generated inputs require.
+    async with prisma_client.db.batch_() as batcher:  # pyright: ignore[reportAny]  # untyped prisma client
+        for key, counts in ordered:
+            columns = asdict(key)
+            batcher.litellm_dailygatewayrequests.upsert(  # pyright: ignore[reportAny]  # untyped prisma client
+                where={"date_category_route": columns},  # mutable-ok: prisma input is dict-shaped
+                data={  # mutable-ok: prisma input is dict-shaped
+                    "create": {  # mutable-ok: prisma input is dict-shaped
+                        **columns,
+                        "successful_requests": counts.successful_requests,
+                        "failed_requests": counts.failed_requests,
+                    },
+                    "update": {  # mutable-ok: prisma input is dict-shaped
+                        "successful_requests": {"increment": counts.successful_requests},  # mutable-ok: as above
+                        "failed_requests": {"increment": counts.failed_requests},  # mutable-ok: as above
+                    },
+                },
+            )
+
+    verbose_proxy_logger.debug("Gateway request tracking - committed %d aggregated rows", len(ordered))
+
+
+async def flush_gateway_requests(
+    prisma_client: "PrismaClient",
+    accumulator: GatewayRequestAccumulator,
+) -> None:
+    """
+    Scheduler entrypoint. Never raises: a metering failure must not kill the job.
+
+    ``CancelledError`` is deliberately not caught, so a flush cancelled during
+    shutdown drops its snapshot rather than restoring counts onto an accumulator
+    the process is about to discard.
+    """
+    snapshot: Final = accumulator.drain()
+    try:
+        await commit_gateway_requests_to_db(prisma_client=prisma_client, snapshot=snapshot)
+    except Exception:  # noqa: BLE001 -- a failed flush must not stop the scheduler
+        accumulator.restore(snapshot)
+        verbose_proxy_logger.warning(
+            "Gateway request tracking - failed to commit %d rows, retrying on the next flush",
+            len(snapshot),
+            exc_info=True,
+        )

litellm/proxy/management_endpoints/common_daily_activity.py

@@ -571,6 +571,11 @@ def _build_aggregated_sql_query(
     # straight into their buckets without re-summing. The leaf grouping
     # is omitted on purpose: nothing in the response shape needs it once
     # all the rollups are present.
+    #
+    # TODO: drop the successful_requests/failed_requests aggregates (and the
+    # total_successful_requests metadata they feed) once the admin UI reads SGR
+    # only from LiteLLM_DailyGatewayRequests. The remaining spend, token and
+    # api_requests rollups are still served from here.
     sql_query: Final = f"""
         SELECT
             date,

litellm/proxy/management_endpoints/gateway_request_endpoints.py

@@ -0,0 +1,139 @@
+"""
+GATEWAY REQUEST COUNTS (SGR)
+
+GET /gateway/daily/activity - successful/failed gateway requests by date and route
+
+Source of truth is LiteLLM_DailyGatewayRequests, written at the ASGI edge by
+BillableRequestMetricsMiddleware. This counts what the proxy answered, so it is
+independent of whether a request reached litellm's logging callbacks.
+
+The table carries no key/user/team dimension, so these totals are deployment-wide
+and the endpoint is restricted to proxy admin roles.
+"""
+
+from collections.abc import Sequence
+from datetime import datetime, timedelta, timezone
+from typing import Annotated, Final
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+from pydantic import BaseModel, TypeAdapter
+
+from litellm._logging import verbose_proxy_logger
+from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth
+from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+from litellm.types.proxy.gateway_requests import (
+    GatewayRequestActivityResponse,
+    GatewayRequestBreakdownEntry,
+    GatewayRequestDailyEntry,
+)
+
+router: Final = APIRouter()
+
+_DEFAULT_LOOKBACK_DAYS: Final = 30
+
+_AGGREGATE_SQL: Final = """
+    SELECT
+        date,
+        category,
+        route,
+        SUM(successful_requests)::bigint AS successful_requests,
+        SUM(failed_requests)::bigint AS failed_requests
+    FROM "LiteLLM_DailyGatewayRequests"
+    WHERE date >= $1 AND date <= $2
+    GROUP BY date, category, route
+"""
+
+
+class _AggregateRow(BaseModel):
+    """Validates one query_raw row so the handler works with typed values, not Any."""
+
+    date: str
+    category: str
+    route: str
+    successful_requests: int
+    failed_requests: int
+
+
+_ROWS_ADAPTER: Final = TypeAdapter(tuple[_AggregateRow, ...])
+
+
+def _default_range() -> tuple[str, str]:
+    end: Final = datetime.now(timezone.utc)
+    start: Final = end - timedelta(days=_DEFAULT_LOOKBACK_DAYS)
+    return start.strftime("%Y-%m-%d"), end.strftime("%Y-%m-%d")
+
+
+def _fold_by_date(rows: Sequence[_AggregateRow]) -> tuple[GatewayRequestDailyEntry, ...]:
+    dates: Final = sorted(frozenset(row.date for row in rows))
+    return tuple(
+        GatewayRequestDailyEntry(
+            date=date,
+            successful_requests=sum(row.successful_requests for row in rows if row.date == date),
+            failed_requests=sum(row.failed_requests for row in rows if row.date == date),
+        )
+        for date in dates
+    )
+
+
+def _fold_by_route(rows: Sequence[_AggregateRow]) -> tuple[GatewayRequestBreakdownEntry, ...]:
+    pairs: Final = sorted(frozenset((row.category, row.route) for row in rows))
+    entries: Final = tuple(
+        GatewayRequestBreakdownEntry(
+            category=category,
+            route=route,
+            successful_requests=sum(
+                row.successful_requests for row in rows if row.category == category and row.route == route
+            ),
+            failed_requests=sum(row.failed_requests for row in rows if row.category == category and row.route == route),
+        )
+        for category, route in pairs
+    )
+    return tuple(sorted(entries, key=lambda entry: entry.successful_requests, reverse=True))
+
+
+@router.get(
+    "/gateway/daily/activity",
+    tags=["Budget & Spend Tracking"],  # mutable-ok: fastapi's decorator signature types tags as a list
+    response_model=GatewayRequestActivityResponse,
+)
+async def get_gateway_daily_activity(
+    user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
+    start_date: str | None = Query(default=None, description="Start date in YYYY-MM-DD format"),
+    end_date: str | None = Query(default=None, description="End date in YYYY-MM-DD format"),
+) -> GatewayRequestActivityResponse:
+    """
+    Successful and failed gateway requests, counted at the ASGI edge.
+
+    Deployment-wide: the underlying table has no per-key or per-user dimension,
+    so this is admin-only.
+    """
+    from litellm.proxy.proxy_server import prisma_client
+
+    if user_api_key_dict.user_role not in (
+        LitellmUserRoles.PROXY_ADMIN,
+        LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY,
+    ):
+        raise HTTPException(
+            status_code=403,
+            detail="Only proxy admin roles can view gateway request counts across the deployment",
+        )
+
+    if prisma_client is None:
+        raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value)
+
+    default_start, default_end = _default_range()
+    raw_rows: Final = await prisma_client.db.query_raw(  # pyright: ignore[reportAny]  # untyped prisma client
+        _AGGREGATE_SQL,
+        start_date or default_start,
+        end_date or default_end,
+    )
+    # Every downstream use is typed: the adapter returns _AggregateRow or raises.
+    rows: Final = _ROWS_ADAPTER.validate_python(raw_rows or ())
+    verbose_proxy_logger.debug("/gateway/daily/activity - aggregated %d rows", len(rows))
+
+    return GatewayRequestActivityResponse(
+        total_successful_requests=sum(row.successful_requests for row in rows),
+        total_failed_requests=sum(row.failed_requests for row in rows),
+        by_date=_fold_by_date(rows),
+        by_route=_fold_by_route(rows),
+    )