feat: add model leaderboard page

litellm-proxy-extras/litellm_proxy_extras/migrations/20260928000000_add_daily_model_usage/migration.sql

@@ -0,0 +1,19 @@
+CREATE TABLE "LiteLLM_DailyModelUsage" (
+    "date" TEXT NOT NULL,
+    "model_group" TEXT NOT NULL,
+    "model" TEXT NOT NULL,
+    "custom_llm_provider" TEXT NOT NULL,
+    "task_type" TEXT NOT NULL,
+    "spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0,
+    "prompt_tokens" BIGINT NOT NULL DEFAULT 0,
+    "completion_tokens" BIGINT NOT NULL DEFAULT 0,
+    "request_count" BIGINT NOT NULL DEFAULT 0,
+    "successful_requests" BIGINT NOT NULL DEFAULT 0,
+    "failed_requests" BIGINT NOT NULL DEFAULT 0,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL,
+    CONSTRAINT "LiteLLM_DailyModelUsage_pkey" PRIMARY KEY ("date", "model_group", "model", "custom_llm_provider", "task_type")
+);
+
+CREATE INDEX "LiteLLM_DailyModelUsage_date_idx" ON "LiteLLM_DailyModelUsage"("date");
+CREATE INDEX "LiteLLM_DailyModelUsage_model_group_idx" ON "LiteLLM_DailyModelUsage"("model_group");

litellm/constants.py

@@ -1777,6 +1777,34 @@
     os.getenv("SCHEDULED_JOB_SHUTDOWN_CANCEL_TIMEOUT_SECONDS", "5")
 )
 TOOL_SPEND_TOP_TOOLS: Final = 100
+MODEL_INSIGHTS_TOP_MODELS: Final = 10
+MODEL_INSIGHTS_MAX_RANGE_DAYS: Final = 365
+MODEL_INSIGHTS_TASK_TYPES: Final = frozenset(
+    {
+        "classification",
+        "content_writing",
+        "roleplay_fiction",
+        "conversation",
+        "research_reports",
+        "qa_knowledge",
+        "customer_support",
+        "summarization",
+        "translation",
+        "workflow_execution",
+        "multi_step_planning",
+        "tool_dispatch",
+        "code_generation",
+        "debugging",
+        "code_review",
+        "frontend_ui",
+        "file_io",
+        "shell_execution",
+        "data_extraction",
+        "data_transformation",
+    }
+)
+MODEL_INSIGHTS_DEFAULT_TASK: Final = "uncategorized"
+MODEL_INSIGHTS_TASK_TAG_PREFIX: Final = "task:"
 SPEND_LOG_PARTITION_INTERVAL: Final = os.getenv("SPEND_LOG_PARTITION_INTERVAL", "day")
 SPEND_LOG_PARTITION_PRECREATE_AHEAD: Final = int(os.getenv("SPEND_LOG_PARTITION_PRECREATE_AHEAD", 7))
 SPEND_LOG_WRITE_BATCH_MAX_BYTES: Final = max(1, int(os.getenv("SPEND_LOG_WRITE_BATCH_MAX_BYTES", 2_000_000)))

litellm/proxy/_lazy_features.py

@@ -128,6 +128,11 @@ def matches(self, path: str) -> bool:
         module_path="litellm.proxy.management_endpoints.tool_management_endpoints",
         path_prefixes=("/v1/tool", "/tool"),
     ),
+    LazyFeature(
+        name="model_insights",
+        module_path="litellm.proxy.management_endpoints.model_insights_endpoints",
+        path_prefixes=("/model-insights",),
+    ),
     LazyFeature(
         name="search_tools",
         module_path="litellm.proxy.search_endpoints.search_tool_management",

litellm/proxy/db/db_spend_update_writer.py

@@ -1148,6 +1148,16 @@ async def _batch_database_updates(
                 traceback.format_exc(),
             )
 
+        try:
+            from litellm.proxy.db.model_usage_rollup import increment_daily_model_usage
+
+            await increment_daily_model_usage(prisma_client=prisma_client, payload=payload_copy)
+        except Exception:
+            verbose_proxy_logger.debug(
+                "_batch_database_updates: increment_daily_model_usage failed: %s",
+                traceback.format_exc(),
+            )
+
     async def _update_key_db(
         self,
         response_cost: float | None,

litellm/proxy/db/model_usage_rollup.py

@@ -0,0 +1,85 @@
+from datetime import datetime
+from typing import Final
+
+from pydantic import TypeAdapter, ValidationError
+
+from litellm.constants import (
+    INTERNAL_CALL_ORIGIN_METADATA_KEY,
+    MODEL_INSIGHTS_DEFAULT_TASK,
+    MODEL_INSIGHTS_TASK_TAG_PREFIX,
+    MODEL_INSIGHTS_TASK_TYPES,
+)
+from litellm.proxy._types import SpendLogsPayload
+from litellm.proxy.utils import PrismaClient
+from litellm.repositories.table_repositories import DailyModelUsageRepository
+
+_METADATA: Final = TypeAdapter(dict[str, object])
+_TAGS: Final = TypeAdapter(list[object])
+
+
+def model_usage_task_type(request_tags: str) -> str:
+    try:
+        tags: Final = _TAGS.validate_json(request_tags)
+    except ValidationError:
+        return MODEL_INSIGHTS_DEFAULT_TASK
+    for tag in tags:
+        task = tag.removeprefix(MODEL_INSIGHTS_TASK_TAG_PREFIX) if isinstance(tag, str) else None
+        if task in MODEL_INSIGHTS_TASK_TYPES:
+            return task
+    return MODEL_INSIGHTS_DEFAULT_TASK
+
+
+def _is_internal_call(metadata: str) -> bool:
+    try:
+        decoded: Final = _METADATA.validate_json(metadata)
+    except ValidationError:
+        return False
+    return bool(decoded.get(INTERNAL_CALL_ORIGIN_METADATA_KEY))
+
+
+def _date_from_start_time(start_time: datetime | str) -> str | None:
+    if isinstance(start_time, datetime):
+        return start_time.date().isoformat()
+    return start_time[:10] if len(start_time) >= 10 else None
+
+
+async def increment_daily_model_usage(prisma_client: PrismaClient, payload: SpendLogsPayload) -> None:
+    date: Final = _date_from_start_time(payload["startTime"])
+    if date is None or _is_internal_call(payload["metadata"]):
+        return
+
+    model: Final = payload["model"] or "unknown"
+    model_group: Final = payload["model_group"] or model
+    provider: Final = payload["custom_llm_provider"] or "unknown"
+    task_type: Final = model_usage_task_type(payload["request_tags"])
+    successful: Final = 1 if payload["status"] == "success" else 0
+    failed: Final = 1 - successful
+    key: Final = {
+        "date": date,
+        "model_group": model_group,
+        "model": model,
+        "custom_llm_provider": provider,
+        "task_type": task_type,
+    }
+    await DailyModelUsageRepository(prisma_client).table.upsert(
+        where={"date_model_group_model_custom_llm_provider_task_type": key},
+        data={
+            "create": {
+                **key,
+                "spend": payload["spend"],
+                "prompt_tokens": payload["prompt_tokens"],
+                "completion_tokens": payload["completion_tokens"],
+                "request_count": 1,
+                "successful_requests": successful,
+                "failed_requests": failed,
+            },
+            "update": {
+                "spend": {"increment": payload["spend"]},
+                "prompt_tokens": {"increment": payload["prompt_tokens"]},
+                "completion_tokens": {"increment": payload["completion_tokens"]},
+                "request_count": {"increment": 1},
+                "successful_requests": {"increment": successful},
+                "failed_requests": {"increment": failed},
+            },
+        },
+    )

litellm/proxy/management_endpoints/model_insights_endpoints.py

@@ -0,0 +1,161 @@
+from collections.abc import Mapping
+from datetime import date, datetime, timedelta, timezone
+from typing import Annotated, Final
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+from pydantic import BaseModel, Field, TypeAdapter
+
+from litellm.constants import MODEL_INSIGHTS_MAX_RANGE_DAYS, MODEL_INSIGHTS_TOP_MODELS
+from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth
+from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+from litellm.repositories.table_repositories import DailyModelUsageRepository
+from litellm.types.model_insights import (
+    ModelInsightDailyMetric,
+    ModelInsightMetric,
+    ModelInsightsResponse,
+    ModelInsightTaskMetric,
+)
+
+router: Final = APIRouter()
+
+
+class _Sums(BaseModel):
+    spend: float = 0.0
+    prompt_tokens: int = 0
+    completion_tokens: int = 0
+    request_count: int = 0
+    successful_requests: int = 0
+    failed_requests: int = 0
+
+
+class _GroupedModel(BaseModel):
+    model_group: str
+    model: str
+    custom_llm_provider: str
+    sums: _Sums = Field(alias="_sum")
+
+
+class _GroupedDaily(_GroupedModel):
+    date: str
+
+
+class _GroupedTask(_GroupedModel):
+    task_type: str
+
+
+_MODEL_ROWS: Final = TypeAdapter(list[_GroupedModel])
+_DAILY_ROWS: Final = TypeAdapter(list[_GroupedDaily])
+_TASK_ROWS: Final = TypeAdapter(list[_GroupedTask])
+_SUM_FIELDS: Final = {
+    "spend": True,
+    "prompt_tokens": True,
+    "completion_tokens": True,
+    "request_count": True,
+    "successful_requests": True,
+    "failed_requests": True,
+}
+
+
+def _parse_date(value: str | None, fallback: date) -> date:
+    if value is None:
+        return fallback
+    try:
+        return date.fromisoformat(value)
+    except ValueError as exc:
+        raise HTTPException(status_code=400, detail="Dates must use YYYY-MM-DD") from exc
+
+
+def _metric(row: _GroupedModel) -> ModelInsightMetric:
+    return ModelInsightMetric(
+        model_group=row.model_group,
+        model=row.model,
+        provider=row.custom_llm_provider,
+        spend=row.sums.spend,
+        prompt_tokens=row.sums.prompt_tokens,
+        completion_tokens=row.sums.completion_tokens,
+        requests=row.sums.request_count,
+        successful_requests=row.sums.successful_requests,
+        failed_requests=row.sums.failed_requests,
+    )
+
+
+def _total_tokens(row: _GroupedModel) -> int:
+    return row.sums.prompt_tokens + row.sums.completion_tokens
+
+
+def _top_model_rows(rows: list[_GroupedModel]) -> list[_GroupedModel]:
+    return sorted(rows, key=_total_tokens, reverse=True)[:MODEL_INSIGHTS_TOP_MODELS]
+
+
+def _daily_metric(row: _GroupedDaily) -> ModelInsightDailyMetric:
+    return ModelInsightDailyMetric(date=row.date, **_metric(row).model_dump())
+
+
+def _task_metric(row: _GroupedTask) -> ModelInsightTaskMetric:
+    return ModelInsightTaskMetric(task_type=row.task_type, **_metric(row).model_dump())
+
+
+@router.get(
+    "/model-insights",
+    tags=["model insights"],
+    dependencies=[Depends(user_api_key_auth)],
+    response_model=ModelInsightsResponse,
+)
+async def get_model_insights(
+    user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
+    start_date: Annotated[str | None, Query(description="YYYY-MM-DD, defaults to 365 days ago")] = None,
+    end_date: Annotated[str | None, Query(description="YYYY-MM-DD, defaults to today")] = None,
+) -> ModelInsightsResponse:
+    from litellm.proxy.proxy_server import prisma_client
+
+    if user_api_key_dict.user_role not in (LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY):
+        raise HTTPException(status_code=403, detail="Only proxy admins can view deployment-wide model insights")
+    if prisma_client is None:
+        raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value)
+
+    end_day: Final = _parse_date(end_date, datetime.now(timezone.utc).date())
+    start_day: Final = _parse_date(start_date, end_day - timedelta(days=MODEL_INSIGHTS_MAX_RANGE_DAYS - 1))
+    if start_day > end_day or (end_day - start_day).days >= MODEL_INSIGHTS_MAX_RANGE_DAYS:
+        raise HTTPException(
+            status_code=400, detail=f"Date range must be between 1 and {MODEL_INSIGHTS_MAX_RANGE_DAYS} days"
+        )
+
+    date_window: Final[Mapping[str, object]] = {"date": {"gte": start_day.isoformat(), "lte": end_day.isoformat()}}
+    table: Final = DailyModelUsageRepository(prisma_client).table
+    grouped_model_rows: Final = _MODEL_ROWS.validate_python(
+        await table.group_by(
+            by=["model_group", "model", "custom_llm_provider"],
+            sum=_SUM_FIELDS,
+            where=date_window,
+        )
+    )
+    model_rows: Final = _top_model_rows(grouped_model_rows)
+    selected_models: Final = [row.model_group for row in model_rows]
+    selected_window: Final = {**date_window, "model_group": {"in": selected_models}}
+    daily_rows: Final = _DAILY_ROWS.validate_python(
+        await table.group_by(
+            by=["date", "model_group", "model", "custom_llm_provider"],
+            sum=_SUM_FIELDS,
+            where=selected_window,
+            order={"date": "asc"},
+        )
+        if selected_models
+        else []
+    )
+    task_rows: Final = _TASK_ROWS.validate_python(
+        await table.group_by(
+            by=["task_type", "model_group", "model", "custom_llm_provider"],
+            sum=_SUM_FIELDS,
+            where=selected_window,
+            order={"_sum": {"completion_tokens": "desc"}},
+        )
+        if selected_models
+        else []
+    )
+    return ModelInsightsResponse(
+        start_date=start_day.isoformat(),
+        end_date=end_day.isoformat(),
+        top_models=[_metric(row) for row in model_rows],
+        daily=[_daily_metric(row) for row in daily_rows],
+        by_task=[_task_metric(row) for row in task_rows],
+    )

litellm/proxy/schema.prisma

@@ -1259,6 +1259,26 @@ model LiteLLM_DailyToolSpend {
   @@id([date, tool_name])
 }
 
+model LiteLLM_DailyModelUsage {
+  date                String
+  model_group         String
+  model               String
+  custom_llm_provider String
+  task_type           String
+  spend               Float    @default(0.0)
+  prompt_tokens       BigInt   @default(0)
+  completion_tokens   BigInt   @default(0)
+  request_count       BigInt   @default(0)
+  successful_requests BigInt   @default(0)
+  failed_requests     BigInt   @default(0)
+  created_at          DateTime @default(now())
+  updated_at          DateTime @updatedAt
+
+  @@id([date, model_group, model, custom_llm_provider, task_type])
+  @@index([date])
+  @@index([model_group])
+}
+
 // Gateway request counts recorded at the ASGI edge by
 // BillableRequestMetricsMiddleware. This is the source of truth for SGR
 // (successful gateway requests): it counts what the proxy actually answered,

litellm/repositories/__init__.py

@@ -30,6 +30,7 @@
     ConfigOverridesRepository,
     DailyGuardrailMetricsRepository,
     DailyGuardrailUsageUnitsRepository,
+    DailyModelUsageRepository,
     DailyPolicyMetricsRepository,
     DailyTagSpendRepository,
     DailyToolSpendRepository,
@@ -105,6 +106,7 @@
     "CredentialsRepository",
     "DailyGuardrailMetricsRepository",
     "DailyGuardrailUsageUnitsRepository",
+    "DailyModelUsageRepository",
     "DailyPolicyMetricsRepository",
     "DailyTagSpendRepository",
     "DailyToolSpendRepository",

litellm/repositories/table_repositories.py

@@ -212,6 +212,10 @@ class DailyToolSpendRepository(PrismaTableRepository["prisma_models.LiteLLM_Dail
     table_name = "litellm_dailytoolspend"
 
 
+class DailyModelUsageRepository(PrismaTableRepository["prisma_models.LiteLLM_DailyModelUsage"]):
+    table_name = "litellm_dailymodelusage"
+
+
 class SpendLogGuardrailIndexRepository(PrismaTableRepository["prisma_models.LiteLLM_SpendLogGuardrailIndex"]):
     table_name = "litellm_spendlogguardrailindex"