diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx new file mode 100644 index 000000000000..51e9e125cb9f --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx @@ -0,0 +1,243 @@ +import { fireEvent, render, screen } from "@testing-library/react"; +import React from "react"; +import { describe, expect, it, vi } from "vitest"; + +import { ApiError } from "@/lib/http/client"; + +vi.mock("./useAutoRouterBenchmarks", () => ({ useAutoRouterBenchmarks: vi.fn() })); + +import AutoRouterBenchmarksTab from "./AutoRouterBenchmarksTab"; +import type { + AutoRouterBenchmarkGroup, + AutoRouterBenchmarksResponse, + AutoRouterCacheStats, +} from "./autoRouterBenchmarks"; +import { useAutoRouterBenchmarks } from "./useAutoRouterBenchmarks"; + +type HookResult = ReturnType; + +const mockHook = (result: { data?: AutoRouterBenchmarksResponse; isPending?: boolean; error?: Error }) => { + vi.mocked(useAutoRouterBenchmarks).mockReturnValue({ + data: result.data, + isPending: result.isPending ?? false, + error: result.error ?? null, + } as unknown as HookResult); +}; + +const cache = (overrides: Partial = {}): AutoRouterCacheStats => ({ + coverage_pct: 99.6, + hit_rate_pct: 93.3, + same_model: { turns: 400, hits: 391, hit_rate_pct: 97.7 }, + first_visit: { turns: 37, hits: 9, hit_rate_pct: 24.3 }, + return_to_tier: { turns: 381, hits: 311, hit_rate_pct: 81.6 }, + unordered_turns: 0, + return_misses_expired: 19, + return_misses_within_ttl: 51, + return_misses_unknown: 0, + ttl_5m_turns: 0, + ttl_1h_turns: 818, + ...overrides, +}); + +type Totals = AutoRouterBenchmarksResponse["totals"]; + +const totals = (overrides: Partial = {}): Totals => ({ + sessions: 94, + turns: 3073, + avg_turns_per_session: 32.7, + avg_session_seconds: 7560, + avg_tokens_per_session: 5_300_000, + spend: 359.86, + saved_spend: 2174.59, + baseline_spend: 2534.45, + saved_pct: 85.8, + saved_per_session: 23.13, + cache: cache(), + ...overrides, +}); + +const group = (overrides: Partial = {}): AutoRouterBenchmarkGroup => ({ + router_name: "claude-auto", + router_type: "complexity", + ...totals(), + ...overrides, +}); + +const response = (groups: AutoRouterBenchmarkGroup[], shared: Totals = totals()): AutoRouterBenchmarksResponse => ({ + start_date: "2026-07-06", + end_date: "2026-08-05", + routers_in_scope: groups.length, + totals: shared, + groups, +}); + +const renderTab = () => render(); + +describe("AutoRouterBenchmarksTab", () => { + it("leads with total estimated savings, before the three session-shape metrics", () => { + mockHook({ data: response([group(), group({ router_name: "gpt-auto" })]) }); + renderTab(); + + const labels = screen + .getAllByText(/Total estimated savings|Avg turns per session|Avg session length|Avg tokens per session/) + .map((node) => node.textContent); + expect(labels).toEqual([ + "Total estimated savings", + "Avg turns per session", + "Avg session length", + "Avg tokens per session", + ]); + }); + + it("renders the headline numbers the tiles exist for", () => { + mockHook({ data: response([group(), group({ router_name: "gpt-auto" })]) }); + renderTab(); + + expect(screen.getByText("$2,174.59")).toBeInTheDocument(); + expect(screen.getByText("-86%")).toBeInTheDocument(); + expect(screen.getByText("Actual auto-router spend")).toBeInTheDocument(); + expect(screen.getByText("$359.86")).toBeInTheDocument(); + expect(screen.getByText("Estimated spend at highest-cost model")).toBeInTheDocument(); + expect(screen.getByText("$2,534.45")).toBeInTheDocument(); + expect(screen.getByText("32.7")).toBeInTheDocument(); + expect(screen.getByText("2.1h")).toBeInTheDocument(); + expect(screen.getByText("5.3M")).toBeInTheDocument(); + }); + + it("pairs the savings with the session count it was earned over", () => { + mockHook({ data: response([group(), group({ router_name: "gpt-auto" })]) }); + renderTab(); + + expect(screen.getByText("Total sessions")).toBeInTheDocument(); + expect(screen.getByText("94")).toBeInTheDocument(); + expect(screen.getByText("Total turns")).toBeInTheDocument(); + expect(screen.getByText("3,073")).toBeInTheDocument(); + expect(screen.getByText("Avg saved per session")).toBeInTheDocument(); + expect(screen.getByText("$23.13")).toBeInTheDocument(); + }); + + it("shows a cost increase as a positive delta rather than a saving", () => { + const overBaseline = { spend: 120, baseline_spend: 100, saved_spend: -20, saved_pct: -20 }; + const dearer = totals(overBaseline); + mockHook({ data: response([group(dearer)], dearer) }); + renderTab(); + + expect(screen.getByText("+20%")).toBeInTheDocument(); + }); + + it("renders all three cache buckets with their turn counts and hit rates", () => { + mockHook({ data: response([group()]) }); + renderTab(); + + expect(screen.getByText("Same model")).toBeInTheDocument(); + expect(screen.getByText("previous turn → same tier")).toBeInTheDocument(); + expect(screen.getByText("First visit")).toBeInTheDocument(); + expect(screen.getByText("previous turn → a tier not used yet")).toBeInTheDocument(); + expect(screen.getByText("Return to tier")).toBeInTheDocument(); + expect(screen.getByText("previous turn → a tier used earlier")).toBeInTheDocument(); + expect(screen.getByText("400")).toBeInTheDocument(); + expect(screen.getByText("37")).toBeInTheDocument(); + expect(screen.getByText("381")).toBeInTheDocument(); + expect(screen.getByText("49%")).toBeInTheDocument(); + expect(screen.getByText("5%")).toBeInTheDocument(); + expect(screen.getByText("47%")).toBeInTheDocument(); + expect(screen.getByText("97.7%")).toBeInTheDocument(); + expect(screen.getByText("24.3%")).toBeInTheDocument(); + expect(screen.getByText("81.6%")).toBeInTheDocument(); + }); + + it("summarizes the cache column from the bucketed turns, not the session turns", () => { + mockHook({ data: response([group()]) }); + renderTab(); + + expect(screen.getByText("93.3%")).toBeInTheDocument(); + expect(screen.getByText("818")).toBeInTheDocument(); + expect(screen.getByText(/turns measured/)).toBeInTheDocument(); + }); + + it("recomputes the expired-miss share from the miss counts", () => { + mockHook({ data: response([group()]) }); + renderTab(); + + expect(screen.getByText("Expired-miss")).toBeInTheDocument(); + expect(screen.getByText("27.1%")).toBeInTheDocument(); + }); + + it("hides the expired-miss row when every return turn hit", () => { + const allHits = totals({ + cache: cache({ return_to_tier: { turns: 381, hits: 381, hit_rate_pct: 100 }, return_misses_expired: 0 }), + }); + mockHook({ data: response([group(allHits)], allHits) }); + renderTab(); + + expect(screen.queryByText("Expired-miss")).not.toBeInTheDocument(); + }); + + it("mentions out-of-order turns only when there are any", () => { + const unordered = totals({ cache: cache({ unordered_turns: 12 }) }); + mockHook({ data: response([group(unordered)], unordered) }); + renderTab(); + + expect(screen.getByText(/12 turns arrived out of order across pods and are not bucketed/)).toBeInTheDocument(); + }); + + it("labels the default selection instead of leaking the __all__ sentinel", () => { + mockHook({ data: response([group()]) }); + renderTab(); + + expect(screen.getByText("All auto-routers")).toBeInTheDocument(); + expect(screen.queryByText("__all__")).not.toBeInTheDocument(); + }); + + it("says so while the benchmarks are loading", () => { + mockHook({ isPending: true }); + renderTab(); + + expect(screen.getByText("Loading auto-router usage...")).toBeInTheDocument(); + }); + + it("names the admin requirement when the proxy answers 403", () => { + mockHook({ error: new ApiError("forbidden", 403, {}) }); + renderTab(); + + expect(screen.getByText("Auto-router usage is visible to proxy admin roles only")).toBeInTheDocument(); + }); + + it("degrades to a message when the endpoint is unavailable", () => { + mockHook({ error: new ApiError("boom", 500, {}) }); + renderTab(); + + expect(screen.getByText("Auto-router usage is unavailable right now")).toBeInTheDocument(); + }); + + it("says so when there are no auto-router sessions at all", () => { + mockHook({ data: response([]) }); + renderTab(); + + expect(screen.getByText("No auto-router sessions in this window yet")).toBeInTheDocument(); + }); + + it("requests the default thirty day window and widens or narrows it from the picker", () => { + mockHook({ data: response([group()]) }); + renderTab(); + + expect(vi.mocked(useAutoRouterBenchmarks)).toHaveBeenCalledWith("sk-test", "30d"); + expect(screen.getByText("Last 30 days")).toBeInTheDocument(); + + fireEvent.click(screen.getByRole("tab", { name: "7d" })); + expect(vi.mocked(useAutoRouterBenchmarks)).toHaveBeenCalledWith("sk-test", "7d"); + expect(screen.getByText("Last 7 days")).toBeInTheDocument(); + + fireEvent.click(screen.getByRole("tab", { name: "24h" })); + expect(vi.mocked(useAutoRouterBenchmarks)).toHaveBeenCalledWith("sk-test", "24h"); + expect(screen.getByText("Last 24 hours")).toBeInTheDocument(); + }); + + it("keeps the window picker reachable while a window has no sessions", () => { + mockHook({ data: response([]) }); + renderTab(); + + expect(screen.getByRole("tab", { name: "30d" })).toBeInTheDocument(); + expect(screen.getByText("All auto-routers")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx new file mode 100644 index 000000000000..2b71c36c5970 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx @@ -0,0 +1,323 @@ +"use client"; + +import React, { useState } from "react"; + +import { Badge } from "@/components/ui/badge"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; +import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; +import { ApiError } from "@/lib/http/client"; +import { formatNumberWithCommas } from "@/utils/dataUtils"; + +import { + ALL_ROUTERS, + WINDOW_LABELS, + bucketRows, + bucketTurnsTotal, + durationLabel, + groupKey, + expiredMissShare, + groupLabel, + pctLabel, + viewFor, + type AutoRouterBenchmarksResponse, + type AutoRouterCacheStats, + type BenchmarkView, + type BenchmarkWindow, + type BucketRow, +} from "./autoRouterBenchmarks"; +import { usd } from "./costOptimizationUtils"; +import { useAutoRouterBenchmarks } from "./useAutoRouterBenchmarks"; + +const Message: React.FC<{ children: React.ReactNode }> = ({ children }) => ( +

{children}

+); + +const Metric: React.FC<{ label: string; value: string }> = ({ label, value }) => ( + + + {label} + + +

{value}

+ + +); + +const HeroCard: React.FC<{ view: BenchmarkView }> = ({ view }) => { + const stats = view.stats; + const cheaper = stats.saved_spend >= 0; + return ( + +
+
+

Total estimated savings

+
+

{usd(stats.saved_spend)}

+ + {cheaper ? "-" : "+"} + {Math.abs(stats.saved_pct).toFixed(0)}% + +
+
+ +
+
+
+
Actual auto-router spend
+
{usd(stats.spend)}
+
+
+
Estimated spend at highest-cost model
+
{usd(stats.baseline_spend)}
+
+
+
+ +
+
+
+

Total sessions

+

{stats.sessions.toLocaleString()}

+
+
+

Total turns

+

{stats.turns.toLocaleString()}

+
+
+
+
+
Avg saved per session
+
{usd(stats.saved_per_session)}
+
+
+
+
+ + ); +}; + +const StackedTurnBar: React.FC<{ buckets: BucketRow[] }> = ({ buckets }) => { + const segments = buckets.filter((b) => b.turns > 0); + return ( +
+
+ {segments.map((b) => ( +
+ ))} +
+
+ {segments.map((b) => ( + + {b.sharePct}% + + ))} +
+
+ ); +}; + +const BucketTable: React.FC<{ buckets: BucketRow[] }> = ({ buckets }) => ( + + + + Bucket + Turns + + Hit rate + + + + {buckets.map((b) => ( + + + + + + {b.label} + {b.sublabel} + + + + + {b.turns.toLocaleString()} + + +
+
+
+ + + {pctLabel(b.hitRatePct)} + + + ))} + +
+); + +const CachingCard: React.FC<{ cache: AutoRouterCacheStats }> = ({ cache }) => { + const buckets = bucketRows(cache); + const total = bucketTurnsTotal(cache); + const expiredMissPct = expiredMissShare(cache); + return ( + +
+
+
+

Cache hit rate

+

{pctLabel(cache.hit_rate_pct)}

+
+ {expiredMissPct === null ? null : ( +
+ + + + Expired-miss +

+ } + /> + + percentage of return-to-tier cache misses caused by cache expiring + + + +

{pctLabel(expiredMissPct)}

+
+ )} +
+ +
+
+

Share of turns

+

+ {total.toLocaleString()} turns + measured +

+
+ + + {cache.unordered_turns > 0 && ( +

+ {cache.unordered_turns.toLocaleString()} turns arrived out of order across pods and are not bucketed +

+ )} +
+
+ + ); +}; + +interface BenchmarksBodyProps { + isPending: boolean; + error: unknown; + data: AutoRouterBenchmarksResponse | undefined; + selectedKey: string; +} + +const BenchmarksBody: React.FC = ({ isPending, error, data, selectedKey }) => { + if (isPending) return Loading auto-router usage...; + if (error instanceof ApiError && error.status === 403) { + return Auto-router usage is visible to proxy admin roles only; + } + if (error || !data) return Auto-router usage is unavailable right now; + if (data.groups.length === 0) return No auto-router sessions in this window yet; + + const view = viewFor(data, selectedKey); + const stats = view.stats; + return ( + <> + + +
+ + + +
+ +

+ Compares your actual routed spend with the estimated cost of using only the most expensive model configured in + the auto-router. It accounts for both the cache savings from staying on one model and the added cache costs from + switching models. +

+ +
+
+

Auto-router prompt caching

+

+ every turn falls in exactly one bucket, by what the router did +

+
+ +
+ + ); +}; + +interface AutoRouterBenchmarksTabProps { + accessToken: string | null; +} + +const AutoRouterBenchmarksTab: React.FC = ({ accessToken }) => { + const [range, setRange] = useState("30d"); + const { data, isPending, error } = useAutoRouterBenchmarks(accessToken, range); + const [selectedKey, setSelectedKey] = useState(ALL_ROUTERS); + + const groups = data?.groups ?? []; + const selectedLabel = data ? viewFor(data, selectedKey).label : "All auto-routers"; + + return ( +
+
+
+

Auto-router usage

+

{WINDOW_LABELS[range]}

+
+
+ setRange(value === "7d" || value === "24h" ? value : "30d")}> + + 30d + 7d + 24h + + +
+ +
+
+
+ + +
+ ); +}; + +export default AutoRouterBenchmarksTab; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.test.tsx index 42dc77191445..96ef75e8dd18 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.test.tsx @@ -4,19 +4,23 @@ import { describe, expect, it, vi } from "vitest"; vi.mock("./UsageTab", () => ({ __esModule: true, default: () =>
})); vi.mock("./PromptCompressionTab", () => ({ __esModule: true, default: () =>
})); vi.mock("./PromptCachingTab", () => ({ __esModule: true, default: () =>
})); +vi.mock("./AutoRouterBenchmarksTab", () => ({ + __esModule: true, + default: () =>
, +})); import CostOptimizationView from "./CostOptimizationView"; const renderView = () => render(); describe("CostOptimizationView", () => { - it("renders the three cost-optimization tabs and no autorouter tab", () => { - const { getByText, queryByText } = renderView(); + it("renders the four cost-optimization tabs", () => { + const { getByText } = renderView(); expect(getByText("Usage")).toBeInTheDocument(); expect(getByText("Prompt Compression")).toBeInTheDocument(); expect(getByText("Prompt Caching")).toBeInTheDocument(); - expect(queryByText("Autorouter")).not.toBeInTheDocument(); + expect(getByText("Auto-Router Usage")).toBeInTheDocument(); }); it("defaults to the Usage tab and switches the active tab on click", () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.tsx index f6593e809993..0d986fdcaa89 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.tsx @@ -7,6 +7,7 @@ import { Alert, Tabs } from "antd"; import UsageTab from "./UsageTab"; import PromptCompressionTab from "./PromptCompressionTab"; import PromptCachingTab from "./PromptCachingTab"; +import AutoRouterBenchmarksTab from "./AutoRouterBenchmarksTab"; import { useDailyActivityRange } from "./useDailyActivityRange"; interface CostOptimizationViewProps { @@ -34,6 +35,11 @@ const CostOptimizationView: React.FC = ({ accessToken label: "Prompt Caching", children: , }, + { + key: "autorouter-usage", + label: "Auto-Router Usage", + children: , + }, ]; return ( @@ -57,7 +63,7 @@ const CostOptimizationView: React.FC = ({ accessToken Have feedback? Join the discussion{" "} = {}): AutoRouterCacheStats => ({ + coverage_pct: 99.6, + hit_rate_pct: 93.3, + same_model: { turns: 400, hits: 391, hit_rate_pct: 97.7 }, + first_visit: { turns: 37, hits: 9, hit_rate_pct: 24.3 }, + return_to_tier: { turns: 381, hits: 311, hit_rate_pct: 81.6 }, + unordered_turns: 0, + return_misses_expired: 19, + return_misses_within_ttl: 51, + return_misses_unknown: 0, + ttl_5m_turns: 0, + ttl_1h_turns: 818, + ...overrides, +}); + +const totals = (overrides: Partial = {}) => ({ + sessions: 94, + turns: 3073, + avg_turns_per_session: 32.7, + avg_session_seconds: 7560, + avg_tokens_per_session: 5_300_000, + spend: 359.86, + saved_spend: 2174.59, + baseline_spend: 2534.45, + saved_pct: 85.8, + saved_per_session: 23.13, + cache: cache(), + ...overrides, +}); + +const group = (overrides: Partial = {}): AutoRouterBenchmarkGroup => ({ + router_name: "claude-auto", + router_type: "complexity", + ...totals(), + ...overrides, +}); + +const response = (groups: AutoRouterBenchmarkGroup[]): AutoRouterBenchmarksResponse => ({ + start_date: "2026-07-06", + end_date: "2026-08-05", + routers_in_scope: groups.length, + totals: totals(), + groups, +}); + +describe("viewFor", () => { + it("maps the all-routers selection to the server totals, never a client sum", () => { + const data = response([group(), group({ router_name: "gpt-auto", sessions: 7 })]); + const view = viewFor(data, ALL_ROUTERS); + expect(view.stats).toBe(data.totals); + expect(view.label).toBe("All auto-routers"); + }); + + it("maps a selected router to that group's slice with a scope of one", () => { + const other = group({ router_name: "gpt-auto", sessions: 7, saved_spend: 12.5 }); + const data = response([group(), other]); + const view = viewFor(data, groupKey(other)); + expect(view.stats).toBe(other); + expect(view.label).toBe("gpt-auto"); + }); + + it("falls back to the all-routers view when the selected key no longer exists", () => { + const data = response([group()]); + const view = viewFor(data, "vanished complexity"); + expect(view.stats).toBe(data.totals); + expect(view.label).toBe("All auto-routers"); + }); + + it("distinguishes two groups sharing an alias by their router type", () => { + const a = group({ router_type: "complexity" }); + const b = group({ router_type: "adaptive" }); + const data = response([a, b]); + expect(groupKey(a)).not.toBe(groupKey(b)); + expect(viewFor(data, groupKey(b)).stats).toBe(b); + expect(viewFor(data, groupKey(b)).label).toBe("claude-auto (adaptive)"); + }); +}); + +describe("groupLabel", () => { + it("uses the bare alias when it is unique", () => { + const groups = [group(), group({ router_name: "gpt-auto" })]; + expect(groupLabel(groups[0], groups)).toBe("claude-auto"); + }); + + it("appends the router type only when the alias is duplicated", () => { + const groups = [group({ router_type: "complexity" }), group({ router_type: "adaptive" })]; + expect(groupLabel(groups[0], groups)).toBe("claude-auto (complexity)"); + expect(groupLabel(groups[1], groups)).toBe("claude-auto (adaptive)"); + }); +}); + +describe("bucketRows", () => { + it("keeps the three buckets summing to the bucketed turn total", () => { + const stats = cache(); + const rows = bucketRows(stats); + expect(rows.map((r) => r.turns)).toEqual([400, 37, 381]); + expect(bucketTurnsTotal(stats)).toBe(818); + }); + + it("renders the server's per-bucket rates as-is", () => { + expect(bucketRows(cache()).map((r) => r.hitRatePct)).toEqual([97.7, 24.3, 81.6]); + }); + + it("derives each bucket's share of the measured turns", () => { + expect(bucketRows(cache()).map((r) => r.sharePct)).toEqual([49, 5, 47]); + }); + + it("reports zero shares instead of dividing by zero when nothing was bucketed", () => { + const empty = { turns: 0, hits: 0, hit_rate_pct: 0 }; + const rows = bucketRows(cache({ same_model: empty, first_visit: empty, return_to_tier: empty })); + expect(rows.map((r) => r.sharePct)).toEqual([0, 0, 0]); + }); +}); + +describe("expiredMissShare", () => { + it("recomputes the expired share from the miss counts", () => { + expect(expiredMissShare(cache())).toBeCloseTo((100 * 19) / 70); + }); + + it("is absent when every return turn hit", () => { + expect(expiredMissShare(cache({ return_to_tier: { turns: 10, hits: 10, hit_rate_pct: 100 } }))).toBeNull(); + }); +}); + +describe("windowFor", () => { + const noon = new Date("2026-08-05T12:00:00Z"); + + it("derives each picker range as UTC calendar days ending today", () => { + expect(windowFor("30d", noon)).toEqual({ start_date: "2026-07-06", end_date: "2026-08-05" }); + expect(windowFor("7d", noon)).toEqual({ start_date: "2026-07-29", end_date: "2026-08-05" }); + expect(windowFor("24h", noon)).toEqual({ start_date: "2026-08-04", end_date: "2026-08-05" }); + }); + + it("uses UTC days, not the local calendar", () => { + const lateEvening = new Date("2026-08-05T23:30:00-05:00"); + expect(windowFor("24h", lateEvening)).toEqual({ start_date: "2026-08-05", end_date: "2026-08-06" }); + }); +}); + +describe("formatting", () => { + it("renders session length in the largest sensible unit", () => { + expect(durationLabel(42)).toBe("42s"); + expect(durationLabel(150)).toBe("2.5m"); + expect(durationLabel(7560)).toBe("2.1h"); + }); + + it("renders percentages at the requested precision", () => { + expect(pctLabel(93.3)).toBe("93.3%"); + expect(pctLabel(85.8, 0)).toBe("86%"); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/autoRouterBenchmarks.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/autoRouterBenchmarks.ts new file mode 100644 index 000000000000..8b6a4fa41051 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/autoRouterBenchmarks.ts @@ -0,0 +1,105 @@ +import type { components } from "@/lib/http/schema"; + +export type AutoRouterBenchmarksResponse = components["schemas"]["AutoRouterBenchmarksResponse"]; +export type AutoRouterBenchmarkTotals = components["schemas"]["AutoRouterBenchmarkTotals"]; +export type AutoRouterBenchmarkGroup = components["schemas"]["AutoRouterBenchmarkGroup"]; +export type AutoRouterCacheStats = components["schemas"]["AutoRouterCacheStats"]; + +export const ALL_ROUTERS = "__all__"; + +export type BenchmarkWindow = "30d" | "7d" | "24h"; + +const WINDOW_DAYS: Record = { "30d": 30, "7d": 7, "24h": 1 }; + +export const WINDOW_LABELS: Record = { + "30d": "Last 30 days", + "7d": "Last 7 days", + "24h": "Last 24 hours", +}; + +export const windowFor = (range: BenchmarkWindow, now: Date): { start_date: string; end_date: string } => ({ + start_date: new Date(now.getTime() - WINDOW_DAYS[range] * 24 * 60 * 60 * 1000).toISOString().slice(0, 10), + end_date: now.toISOString().slice(0, 10), +}); + +export interface BenchmarkView { + label: string; + stats: AutoRouterBenchmarkTotals; +} + +export const groupKey = (group: AutoRouterBenchmarkGroup): string => `${group.router_name} ${group.router_type}`; + +export const groupLabel = (group: AutoRouterBenchmarkGroup, groups: readonly AutoRouterBenchmarkGroup[]): string => { + const duplicated = groups.some((g) => g !== group && g.router_name === group.router_name); + return duplicated ? `${group.router_name} (${group.router_type})` : group.router_name; +}; + +export const viewFor = (data: AutoRouterBenchmarksResponse, selectedKey: string): BenchmarkView => { + const group = data.groups.find((g) => groupKey(g) === selectedKey); + if (selectedKey === ALL_ROUTERS || !group) { + return { label: "All auto-routers", stats: data.totals }; + } + return { label: groupLabel(group, data.groups), stats: group }; +}; + +export interface BucketRow { + key: "same_model" | "first_visit" | "return_to_tier"; + label: string; + sublabel: string; + turns: number; + sharePct: number; + hitRatePct: number; + fill: string; +} + +export const bucketTurnsTotal = (cache: AutoRouterCacheStats): number => + cache.same_model.turns + cache.first_visit.turns + cache.return_to_tier.turns; + +const sharePctOf = (turns: number, total: number): number => (total > 0 ? Math.round((100 * turns) / total) : 0); + +export const bucketRows = (cache: AutoRouterCacheStats): BucketRow[] => { + const total = bucketTurnsTotal(cache); + return [ + { + key: "same_model", + label: "Same model", + sublabel: "previous turn → same tier", + turns: cache.same_model.turns, + sharePct: sharePctOf(cache.same_model.turns, total), + hitRatePct: cache.same_model.hit_rate_pct, + fill: "bg-foreground", + }, + { + key: "first_visit", + label: "First visit", + sublabel: "previous turn → a tier not used yet", + turns: cache.first_visit.turns, + sharePct: sharePctOf(cache.first_visit.turns, total), + hitRatePct: cache.first_visit.hit_rate_pct, + fill: "bg-foreground/30", + }, + { + key: "return_to_tier", + label: "Return to tier", + sublabel: "previous turn → a tier used earlier", + turns: cache.return_to_tier.turns, + sharePct: sharePctOf(cache.return_to_tier.turns, total), + hitRatePct: cache.return_to_tier.hit_rate_pct, + fill: "bg-foreground/60", + }, + ]; +}; + +export const expiredMissShare = (cache: AutoRouterCacheStats): number | null => { + const misses = cache.return_to_tier.turns - cache.return_to_tier.hits; + if (misses <= 0) return null; + return (100 * cache.return_misses_expired) / misses; +}; + +export const pctLabel = (value: number, digits: number = 1): string => `${value.toFixed(digits)}%`; + +export const durationLabel = (seconds: number): string => { + if (seconds < 60) return `${Math.round(seconds)}s`; + if (seconds < 3600) return `${(seconds / 60).toFixed(1)}m`; + return `${(seconds / 3600).toFixed(1)}h`; +}; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useAutoRouterBenchmarks.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useAutoRouterBenchmarks.ts new file mode 100644 index 000000000000..ab87c5ef4307 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useAutoRouterBenchmarks.ts @@ -0,0 +1,11 @@ +import { $api } from "@/lib/http/api"; + +import { windowFor, type BenchmarkWindow } from "./autoRouterBenchmarks"; + +export const useAutoRouterBenchmarks = (accessToken: string | null, range: BenchmarkWindow) => + $api.useQuery( + "get", + "/auto_router/benchmarks", + { params: { query: windowFor(range, new Date()) } }, + { enabled: Boolean(accessToken), retry: false }, + );