From 957e0f87af553a3d109b23028e95360af3676ce7 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Sat, 1 Aug 2026 16:05:54 -0700 Subject: [PATCH] feat(ui): show auto-router savings on the cost-optimization dashboard Adds the auto-router as a third optimization driver beside compression and prompt caching: a summary card, a donut segment, and a series in the savings graph across both the cumulative and per-day views. The number is signed, because a switch that thrashes the prompt cache can cost more than the cheaper rates save and an operator needs to see that. The donut plots only drivers that saved, since a negative slice has no meaning, while the card and the range total keep the sign. `usd()` sizes and signs off the magnitude so a small loss renders as -$0.01 rather than "$-0.00". The card's popover states the counterfactual and its two consequences: that a switch pays to re-warm the cache, and that a first turn the router could not identify is charged that write and therefore under-reported. --- .../_components/UsageTab.test.tsx | 112 +++++++++++++++++- .../_components/UsageTab.tsx | 74 +++++++----- .../_components/costOptimizationUtils.test.ts | 59 ++++++++- .../_components/costOptimizationUtils.ts | 30 ++++- .../src/components/UsagePage/types.ts | 1 + 5 files changed, 238 insertions(+), 38 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx index 5c26ac304777..fe3e792eeeee 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx @@ -29,12 +29,14 @@ vi.mock("@/components/shared/charts", () => ({ colors, showLegend, maxBarSize, + stack, }: { data: unknown; categories: string[]; colors?: readonly string[]; showLegend?: boolean; maxBarSize?: number; + stack?: boolean; }) => (
({ data-colors={(colors ?? []).join(",")} data-show-legend={String(showLegend ?? true)} data-max-bar-size={maxBarSize === undefined ? "" : String(maxBarSize)} + data-stack={String(stack ?? false)} data-series={JSON.stringify(data)} /> ), @@ -216,8 +219,8 @@ describe("UsageTab", () => { const slices = JSON.parse(getByTestId("donut-chart").getAttribute("data-slices") ?? "[]"); expect(slices).toEqual([ - { driver: "Compression", usd: expect.closeTo(0.14, 5) }, - { driver: "Prompt caching", usd: expect.closeTo(0.016, 5) }, + { driver: "Compression", color: "emerald", usd: expect.closeTo(0.14, 5) }, + { driver: "Prompt caching", color: "blue", usd: expect.closeTo(0.016, 5) }, ]); }); @@ -225,7 +228,110 @@ describe("UsageTab", () => { const { getByTestId } = renderWith([day("2026-07-12", { compression_savings_spend: 0.04 })]); const slices = JSON.parse(getByTestId("donut-chart").getAttribute("data-slices") ?? "[]"); - expect(slices).toEqual([{ driver: "Compression", usd: expect.closeTo(0.04, 5) }]); + expect(slices).toEqual([{ driver: "Compression", color: "emerald", usd: expect.closeTo(0.04, 5) }]); + }); + + it("does not stack the per-day drivers, because one of them can be negative", async () => { + // Stacking sums the series into one bar. Auto-router savings go negative when a + // model switch pays for a cold cache, and that segment would be drawn below the + // axis while the rest of the bar still read as the day's total. + const { getByRole, getByTestId } = renderWith([ + day("2026-07-12", { + compression_savings_spend: 0.1, + prompt_caching_savings_spend: 0.02, + autorouter_savings_spend: -0.05, + }), + ]); + + await userEvent.click(getByRole("tab", { name: "Per day" })); + const bars = getByTestId("bar-chart"); + expect(bars.getAttribute("data-stack")).toBe("false"); + expect(readSeries(bars)[0]).toMatchObject({ "Auto-router": -0.05 }); + }); + + it("lays the savings header out with the card's own slots so nothing shifts between tabs", async () => { + // The subtitle differs in length between the tabs ("Running total saved" vs "Saved + // per day"). Hand-rolled rows made it compete with the legend and the toggle for + // width, so the header grew a line on one tab and the chart moved with it. CardHeader + // sizes the action column to its content and gives the rest to the title column. + const { getByRole, getByTestId, container } = renderWith(twoDays()); + + const header = () => { + const legend = getByTestId("chart-legend"); + const action = legend.closest('[data-slot="card-action"]') as HTMLElement; + const cardHeader = action.parentElement as HTMLElement; + const description = cardHeader.querySelector('[data-slot="card-description"]') as HTMLElement; + return { action, cardHeader, description }; + }; + + const before = header(); + expect(before.action).toBeTruthy(); + expect(before.description).toBeTruthy(); + // the toggle rides in the same action slot as the legend, so neither moves alone + expect(before.action.contains(getByRole("tablist"))).toBe(true); + // the subtitle lives outside that slot, so its length cannot reposition the controls + expect(before.action.contains(before.description)).toBe(false); + expect(before.description.textContent).toContain("Running total saved"); + + await userEvent.click(getByRole("tab", { name: "Per day" })); + + const after = header(); + expect(after.action).toBe(before.action); + expect(after.cardHeader).toBe(before.cardHeader); + expect(after.action.contains(after.description)).toBe(false); + expect(after.description.textContent).toContain("Saved per day"); + expect(container.textContent).toContain("Savings"); + }); + + it("subtracts a losing auto-router route from the total and keeps it out of the donut", () => { + // Switching models leaves the new one with a cold cache, so a route can cost more + // than the baseline would have. A negative slice is meaningless in a donut, but the + // total has to keep the loss or the page can only ever report good news. + const { getByText, getByTestId } = renderWith([ + day("2026-07-12", { + compression_savings_spend: 0.1, + prompt_caching_savings_spend: 0.02, + autorouter_savings_spend: -0.05, + }), + ]); + + expect(getByText("$0.0700")).toBeInTheDocument(); + expect(getByText("-$0.0500")).toBeInTheDocument(); + + const slices = JSON.parse(getByTestId("donut-chart").getAttribute("data-slices") ?? "[]"); + expect(slices.map((d: { driver: string }) => d.driver)).toEqual(["Compression", "Prompt caching"]); + expect(getByTestId("donut-chart").getAttribute("data-label")).toBe("$0.1200"); + }); + + it("carries auto-router savings into the summary card, donut slice, and cumulative series", () => { + const { getByText, getByTestId } = renderWith([ + day("2026-07-12", { + compression_savings_spend: 0.04, + prompt_caching_savings_spend: 0.006, + autorouter_savings_spend: 0.02, + }), + day("2026-07-13", { + compression_savings_spend: 0.1, + prompt_caching_savings_spend: 0.01, + autorouter_savings_spend: 0.05, + }), + ]); + + // Total saved now sums three drivers, and the auto-router card carries its own total. + expect(getByText("$0.2260")).toBeInTheDocument(); + expect(getByText("$0.0700")).toBeInTheDocument(); + + // The driver donut gains a third slice priced from the range totals. + const slices = JSON.parse(getByTestId("donut-chart").getAttribute("data-slices") ?? "[]"); + expect(slices).toEqual([ + { driver: "Compression", color: "emerald", usd: expect.closeTo(0.14, 5) }, + { driver: "Prompt caching", color: "blue", usd: expect.closeTo(0.016, 5) }, + { driver: "Auto-router", color: "amber", usd: expect.closeTo(0.07, 5) }, + ]); + + // And the cumulative line accumulates the auto-router series alongside the others. + const series = readSeries(getByTestId("area-chart")); + expect(series[2]["Auto-router"]).toBeCloseTo(0.07, 5); }); it("renders spend-by-tool bars from the tool spend endpoint", async () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.tsx index ec37418e0b55..b62876022100 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.tsx @@ -5,7 +5,7 @@ import { Info } from "lucide-react"; import { AreaChart, BarChart, CustomLegend, DonutChart, SEQUENTIAL_COLOR_RAMP } from "@/components/shared/charts"; import AdvancedDatePicker from "@/components/shared/advanced_date_picker"; -import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Card, CardAction, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { getToolSpend, ToolSpendResponse } from "@/components/networking"; @@ -16,6 +16,8 @@ import { formatRangeLabel, localIsoDay, MAX_POINTS_WITH_DOTS, + SAVINGS_COLORS, + SAVINGS_DRIVERS, SAVINGS_SERIES, SavingsAccumulation, SavingsPoint, @@ -38,8 +40,6 @@ const EMPTY_TOOL_SPEND: ToolSpendResponse = { end_date: null, }; -const SAVINGS_COLORS = ["emerald", "blue"] as const; - const shortDate = (iso: string): string => new Date(`${iso}T00:00:00`).toLocaleDateString("en-US", { month: "short", day: "numeric" }); @@ -47,6 +47,7 @@ const isoDay = (d: Date): string => d.toISOString().slice(0, 10); const compressionOf = (m: SpendMetrics): number => m.compression_savings_spend ?? 0; const cachingOf = (m: SpendMetrics): number => m.prompt_caching_savings_spend ?? 0; +const autorouterOf = (m: SpendMetrics): number => m.autorouter_savings_spend ?? 0; const savedTokensOf = (m: SpendMetrics): number => m.compression_saved_tokens ?? 0; const SummaryCard = ({ label, value, hint, info }: { label: string; value: string; hint?: string; info?: string }) => ( @@ -105,8 +106,9 @@ const UsageTab: React.FC = ({ accessToken, activity }) => { const compressionTotal = useMemo(() => results.reduce((sum, d) => sum + compressionOf(d.metrics), 0), [results]); const cachingTotal = useMemo(() => results.reduce((sum, d) => sum + cachingOf(d.metrics), 0), [results]); + const autorouterTotal = useMemo(() => results.reduce((sum, d) => sum + autorouterOf(d.metrics), 0), [results]); const savedTokensTotal = useMemo(() => results.reduce((sum, d) => sum + savedTokensOf(d.metrics), 0), [results]); - const totalSaved = compressionTotal + cachingTotal; + const totalSaved = compressionTotal + cachingTotal + autorouterTotal; const [accumulation, setAccumulation] = useState("cumulative"); @@ -122,6 +124,7 @@ const UsageTab: React.FC = ({ accessToken, activity }) => { date: shortDate(d.date), Compression: compressionOf(d.metrics), "Prompt caching": cachingOf(d.metrics), + "Auto-router": autorouterOf(d.metrics), })), [results], ); @@ -143,14 +146,19 @@ const UsageTab: React.FC = ({ accessToken, activity }) => { .filter(Boolean) .join(" \u00b7 "); + // A driver can come out negative (auto-router pays a cold-cache write on every + // model switch), and a negative slice has no meaning in a donut, so only drivers + // that actually saved are plotted; the range total keeps the signed truth. const byDriver = useMemo( () => - [ - { driver: "Compression", usd: compressionTotal }, - { driver: "Prompt caching", usd: cachingTotal }, - ].filter((d) => d.usd > 0), - [compressionTotal, cachingTotal], + SAVINGS_DRIVERS.map(({ name, color }) => ({ + driver: name, + color, + usd: { Compression: compressionTotal, "Prompt caching": cachingTotal, "Auto-router": autorouterTotal }[name], + })).filter((d) => d.usd > 0), + [compressionTotal, cachingTotal, autorouterTotal], ); + const plottedDriverTotal = useMemo(() => byDriver.reduce((sum, d) => sum + d.usd, 0), [byDriver]); const topTools = useMemo(() => topToolsBySpend(toolSpend?.by_tool ?? []), [toolSpend]); const topToolNames = useMemo(() => topTools.map((t) => t.tool_name), [topTools]); @@ -174,11 +182,11 @@ const UsageTab: React.FC = ({ accessToken, activity }) => {
-
+
= ({ accessToken, activity }) => { hint="Cache read discount" info="Tokens the provider served from cache, priced at the discount between the input and cache-read rates." /> +
+ {/* CardHeader's own slots rather than hand-rolled rows: the action column is + sized to its content and the title column takes the rest, so the subtitle + never competes with the controls for width and neither moves when it grows. + The controls wrap within their column instead of pushing past the card */} -
-
- Savings -

{savingsSubtitle}

-
-
- - setAccumulation(value as SavingsAccumulation)}> - - Cumulative - {intervalLabel} - - -
-
+ Savings + {savingsSubtitle} + + + setAccumulation(value as SavingsAccumulation)}> + + Cumulative + {intervalLabel} + + + {accumulation === "cumulative" ? ( @@ -225,12 +239,14 @@ const UsageTab: React.FC = ({ accessToken, activity }) => { showDots={overTime.length <= MAX_POINTS_WITH_DOTS} /> ) : ( + // Not stacked: a driver can be negative once a model switch is charged + // for its cold cache, and stacking would draw that segment below the axis + // while the remaining bar still read as the day's total @@ -247,10 +263,10 @@ const UsageTab: React.FC = ({ accessToken, activity }) => { data={byDriver} index="driver" category="usd" - colors={["emerald", "blue"]} + colors={byDriver.map((d) => d.color)} valueFormatter={usd} showLabel - label={usd(totalSaved)} + label={usd(plottedDriverTotal)} /> diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts index dc08799a3c4d..14fb26c53efb 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts @@ -3,6 +3,9 @@ import { describe, expect, it } from "vitest"; import type { DailyData, SpendMetrics } from "@/components/UsagePage/types"; import type { ToolSpendDailyEntry, ToolSpendEntry } from "@/components/networking"; import { + SAVINGS_COLORS, + SAVINGS_DRIVERS, + SAVINGS_SERIES, buildDailyToolSeries, computeCacheLeakage, formatRangeLabel, @@ -10,6 +13,7 @@ import { localIsoDay, toCumulative, topToolsBySpend, + usd, withStartAnchor, } from "./costOptimizationUtils"; @@ -239,22 +243,25 @@ describe("localIsoDay", () => { }); describe("toCumulative", () => { - const point = (date: string, compression: number, caching: number) => ({ + const point = (date: string, compression: number, caching: number, autorouter: number = 0) => ({ date, Compression: compression, "Prompt caching": caching, + "Auto-router": autorouter, }); it("turns each reading into everything saved up to that point", () => { const running = toCumulative([point("Jul 1", 1, 10), point("Jul 2", 2, 20), point("Jul 3", 3, 30)]); expect(running.map((p) => p.Compression)).toEqual([1, 3, 6]); expect(running.map((p) => p["Prompt caching"])).toEqual([10, 30, 60]); + expect(running.map((p) => p["Auto-router"])).toEqual([0, 0, 0]); }); it("accumulates each driver on its own, so one flat series cannot lift the other", () => { const running = toCumulative([point("Jul 1", 0, 5), point("Jul 2", 0, 5)]); expect(running.map((p) => p.Compression)).toEqual([0, 0]); expect(running.map((p) => p["Prompt caching"])).toEqual([5, 10]); + expect(running.map((p) => p["Auto-router"])).toEqual([0, 0]); }); it("never falls, even across a quiet interval", () => { @@ -267,13 +274,19 @@ describe("toCumulative", () => { expect(running.map((p) => p.date)).toEqual(["9am", "10am"]); expect(toCumulative([])).toEqual([]); }); + + it("accumulates auto-router savings like other drivers", () => { + const running = toCumulative([point("Jul 1", 1, 1, 5), point("Jul 2", 1, 1, 10)]); + expect(running.map((p) => p["Auto-router"])).toEqual([5, 15]); + }); }); describe("withStartAnchor", () => { - const point = (date: string, compression: number, caching: number) => ({ + const point = (date: string, compression: number, caching: number, autorouter: number = 0) => ({ date, Compression: compression, "Prompt caching": caching, + "Auto-router": autorouter, }); it("lifts a single-day cumulative off a floating dot by prepending a $0 origin", () => { @@ -285,6 +298,7 @@ describe("withStartAnchor", () => { const anchored = withStartAnchor([point("Jul 16", 5, 1), point("Jul 17", 9, 4)], "Jul 16"); expect(anchored.map((p) => p.Compression)).toEqual([0, 5, 9]); expect(anchored.map((p) => p["Prompt caching"])).toEqual([0, 1, 4]); + expect(anchored.map((p) => p["Auto-router"])).toEqual([0, 0, 0]); }); it("leaves an empty series alone so the chart's own no-data state can show", () => { @@ -306,3 +320,44 @@ describe("formatRangeLabel", () => { expect(formatRangeLabel(new Date(2026, 6, 23), undefined)).toBe(""); }); }); + +describe("usd", () => { + it("keeps four decimals for sub-dollar amounts so small savings stay visible", () => { + expect(usd(0.05)).toBe("$0.0500"); + expect(usd(1.5)).toBe("$1.50"); + expect(usd(0)).toBe("$0.00"); + }); + + it("signs a loss ahead of the symbol and keeps its precision", () => { + // A driver can be negative once a model switch is charged for its cold cache. + // Sizing decimals off the raw value would render this as "$-0.00". + expect(usd(-0.05)).toBe("-$0.0500"); + expect(usd(-0.0004)).toBe("-$0.0004"); + expect(usd(-12.4)).toBe("-$12.40"); + }); +}); + +describe("savings driver colours", () => { + it("keeps a driver's colour when a driver above it is filtered out", () => { + // Charts colour by position in the data they are given, and the donut is given + // only drivers that saved something. Compression is zero on any deployment not + // running the compression guardrail, so the survivors must not slide onto the + // colours of the drivers dropped above them. + const totals = { Compression: 0, "Prompt caching": 4, "Auto-router": 7 } as const; + const plotted = SAVINGS_DRIVERS.map(({ name, color }) => ({ name, color, usd: totals[name] })).filter( + (d) => d.usd > 0, + ); + + expect(plotted.map((d) => [d.name, d.color])).toEqual([ + ["Prompt caching", "blue"], + ["Auto-router", "amber"], + ]); + }); + + it("agrees with the legend, which is built from the unfiltered list", () => { + const legend = new Map(SAVINGS_SERIES.map((name, i) => [SAVINGS_COLORS[i], name])); + for (const { name, color } of SAVINGS_DRIVERS) { + expect(legend.get(color)).toBe(name); + } + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts index 32eb6ae198db..d63266c5ee71 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts @@ -3,8 +3,11 @@ import { ToolSpendDailyEntry, ToolSpendEntry } from "@/components/networking"; import { formatNumberWithCommas } from "@/utils/dataUtils"; export const usd = (value: number): string => { - const decimals = value > 0 && value < 1 ? 4 : 2; - return `$${formatNumberWithCommas(value, decimals)}`; + // Sized and signed off the magnitude: a driver can come out negative, and a small + // loss rendered at two decimals would read as "$-0.00" + const magnitude = Math.abs(value); + const decimals = magnitude > 0 && magnitude < 1 ? 4 : 2; + return `${value < 0 ? "-" : ""}$${formatNumberWithCommas(magnitude, decimals)}`; }; export const pct = (ratio: number): string => `${formatNumberWithCommas(ratio * 100, 1)}%`; @@ -161,9 +164,27 @@ export type SavingsPoint = { date: string; Compression: number; "Prompt caching": number; + "Auto-router": number; }; -export const SAVINGS_SERIES = ["Compression", "Prompt caching"] as const; +/** + * The savings drivers, each owning its own colour. + * + * One list rather than a names list beside a colours list, because the donut is + * given only the drivers that saved anything and charts assign colours by position + * in the data they receive. Two lists that line up by index therefore stop lining + * up the moment a driver is filtered out: the survivors slide down and inherit the + * colours of the drivers above them, while the legend still reports the original + * mapping. Colour travels with the driver so filtering cannot separate them. + */ +export const SAVINGS_DRIVERS = [ + { name: "Compression", color: "emerald" }, + { name: "Prompt caching", color: "blue" }, + { name: "Auto-router", color: "amber" }, +] as const; + +export const SAVINGS_SERIES = SAVINGS_DRIVERS.map((d) => d.name); +export const SAVINGS_COLORS = SAVINGS_DRIVERS.map((d) => d.color); /** * Running total of each series across the selected window. The total restarts @@ -179,6 +200,7 @@ export const toCumulative = (points: readonly SavingsPoint[]): SavingsPoint[] => date: point.date, Compression: (previous?.Compression ?? 0) + point.Compression, "Prompt caching": (previous?.["Prompt caching"] ?? 0) + point["Prompt caching"], + "Auto-router": (previous?.["Auto-router"] ?? 0) + point["Auto-router"], }, ]; }, []); @@ -193,7 +215,7 @@ export const toCumulative = (points: readonly SavingsPoint[]): SavingsPoint[] => export const withStartAnchor = (cumulative: readonly SavingsPoint[], startLabel: string): SavingsPoint[] => cumulative.length === 0 ? [...cumulative] - : [{ date: startLabel, Compression: 0, "Prompt caching": 0 }, ...cumulative]; + : [{ date: startLabel, Compression: 0, "Prompt caching": 0, "Auto-router": 0 }, ...cumulative]; /** "Jul 16 – Jul 23", collapsing to a single date when the range is one day. */ export const formatRangeLabel = (from: Date | undefined, to: Date | undefined): string => { diff --git a/ui/litellm-dashboard/src/components/UsagePage/types.ts b/ui/litellm-dashboard/src/components/UsagePage/types.ts index bf33fa371117..b10fc79be159 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/types.ts +++ b/ui/litellm-dashboard/src/components/UsagePage/types.ts @@ -11,6 +11,7 @@ export interface SpendMetrics { compression_saved_tokens?: number; compression_savings_spend?: number; prompt_caching_savings_spend?: number; + autorouter_savings_spend?: number; } export type DailyData = {