Skip to content

feat(ui): show auto-router savings on the cost-optimization dashboard #35522

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tin-berri merged 1 commit into from
Aug 4, 2026
Merged

feat(ui): show auto-router savings on the cost-optimization dashboard #35522

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
112 changes: 109 additions & 3 deletions ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,19 +29,22 @@ vi.mock("@/components/shared/charts", () => ({
colors,
showLegend,
maxBarSize,
stack,
}: {
data: unknown;
categories: string[];
colors?: readonly string[];
showLegend?: boolean;
maxBarSize?: number;
stack?: boolean;
}) => (
<div
data-testid="bar-chart"
data-categories={categories.join(",")}
data-colors={(colors ?? []).join(",")}
data-show-legend={String(showLegend ?? true)}
data-max-bar-size={maxBarSize === undefined ? "" : String(maxBarSize)}
data-stack={String(stack ?? false)}
data-series={JSON.stringify(data)}
/>
),
Expand Down Expand Up @@ -216,16 +219,119 @@ describe("UsageTab", () => {

const slices = JSON.parse(getByTestId("donut-chart").getAttribute("data-slices") ?? "[]");
expect(slices).toEqual([
{ driver: "Compression", usd: expect.closeTo(0.14, 5) },
{ driver: "Prompt caching", usd: expect.closeTo(0.016, 5) },
{ driver: "Compression", color: "emerald", usd: expect.closeTo(0.14, 5) },
{ driver: "Prompt caching", color: "blue", usd: expect.closeTo(0.016, 5) },
]);
});

it("omits a driver slice when that driver has no savings", () => {
const { getByTestId } = renderWith([day("2026-07-12", { compression_savings_spend: 0.04 })]);

const slices = JSON.parse(getByTestId("donut-chart").getAttribute("data-slices") ?? "[]");
expect(slices).toEqual([{ driver: "Compression", usd: expect.closeTo(0.04, 5) }]);
expect(slices).toEqual([{ driver: "Compression", color: "emerald", usd: expect.closeTo(0.04, 5) }]);
});

it("does not stack the per-day drivers, because one of them can be negative", async () => {
// Stacking sums the series into one bar. Auto-router savings go negative when a
// model switch pays for a cold cache, and that segment would be drawn below the
// axis while the rest of the bar still read as the day's total.
const { getByRole, getByTestId } = renderWith([
day("2026-07-12", {
compression_savings_spend: 0.1,
prompt_caching_savings_spend: 0.02,
autorouter_savings_spend: -0.05,
}),
]);

await userEvent.click(getByRole("tab", { name: "Per day" }));
const bars = getByTestId("bar-chart");
expect(bars.getAttribute("data-stack")).toBe("false");
expect(readSeries(bars)[0]).toMatchObject({ "Auto-router": -0.05 });
});

it("lays the savings header out with the card's own slots so nothing shifts between tabs", async () => {
// The subtitle differs in length between the tabs ("Running total saved" vs "Saved
// per day"). Hand-rolled rows made it compete with the legend and the toggle for
// width, so the header grew a line on one tab and the chart moved with it. CardHeader
// sizes the action column to its content and gives the rest to the title column.
const { getByRole, getByTestId, container } = renderWith(twoDays());

const header = () => {
const legend = getByTestId("chart-legend");
const action = legend.closest('[data-slot="card-action"]') as HTMLElement;
const cardHeader = action.parentElement as HTMLElement;
const description = cardHeader.querySelector('[data-slot="card-description"]') as HTMLElement;
return { action, cardHeader, description };
};

const before = header();
expect(before.action).toBeTruthy();
expect(before.description).toBeTruthy();
// the toggle rides in the same action slot as the legend, so neither moves alone
expect(before.action.contains(getByRole("tablist"))).toBe(true);
// the subtitle lives outside that slot, so its length cannot reposition the controls
expect(before.action.contains(before.description)).toBe(false);
expect(before.description.textContent).toContain("Running total saved");

await userEvent.click(getByRole("tab", { name: "Per day" }));

const after = header();
expect(after.action).toBe(before.action);
expect(after.cardHeader).toBe(before.cardHeader);
expect(after.action.contains(after.description)).toBe(false);
expect(after.description.textContent).toContain("Saved per day");
expect(container.textContent).toContain("Savings");
});

it("subtracts a losing auto-router route from the total and keeps it out of the donut", () => {
// Switching models leaves the new one with a cold cache, so a route can cost more
// than the baseline would have. A negative slice is meaningless in a donut, but the
// total has to keep the loss or the page can only ever report good news.
const { getByText, getByTestId } = renderWith([
day("2026-07-12", {
compression_savings_spend: 0.1,
prompt_caching_savings_spend: 0.02,
autorouter_savings_spend: -0.05,
}),
]);

expect(getByText("$0.0700")).toBeInTheDocument();
expect(getByText("-$0.0500")).toBeInTheDocument();

const slices = JSON.parse(getByTestId("donut-chart").getAttribute("data-slices") ?? "[]");
expect(slices.map((d: { driver: string }) => d.driver)).toEqual(["Compression", "Prompt caching"]);
expect(getByTestId("donut-chart").getAttribute("data-label")).toBe("$0.1200");
});

it("carries auto-router savings into the summary card, donut slice, and cumulative series", () => {
const { getByText, getByTestId } = renderWith([
day("2026-07-12", {
compression_savings_spend: 0.04,
prompt_caching_savings_spend: 0.006,
autorouter_savings_spend: 0.02,
}),
day("2026-07-13", {
compression_savings_spend: 0.1,
prompt_caching_savings_spend: 0.01,
autorouter_savings_spend: 0.05,
}),
]);

// Total saved now sums three drivers, and the auto-router card carries its own total.
expect(getByText("$0.2260")).toBeInTheDocument();
expect(getByText("$0.0700")).toBeInTheDocument();

// The driver donut gains a third slice priced from the range totals.
const slices = JSON.parse(getByTestId("donut-chart").getAttribute("data-slices") ?? "[]");
expect(slices).toEqual([
{ driver: "Compression", color: "emerald", usd: expect.closeTo(0.14, 5) },
{ driver: "Prompt caching", color: "blue", usd: expect.closeTo(0.016, 5) },
{ driver: "Auto-router", color: "amber", usd: expect.closeTo(0.07, 5) },
]);

// And the cumulative line accumulates the auto-router series alongside the others.
const series = readSeries(getByTestId("area-chart"));
expect(series[2]["Auto-router"]).toBeCloseTo(0.07, 5);
});

it("renders spend-by-tool bars from the tool spend endpoint", async () => {
Expand Down
74 changes: 45 additions & 29 deletions ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import { Info } from "lucide-react";

import { AreaChart, BarChart, CustomLegend, DonutChart, SEQUENTIAL_COLOR_RAMP } from "@/components/shared/charts";
import AdvancedDatePicker from "@/components/shared/advanced_date_picker";
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
import { Card, CardAction, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";
import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover";
import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs";
import { getToolSpend, ToolSpendResponse } from "@/components/networking";
Expand All @@ -16,6 +16,8 @@ import {
formatRangeLabel,
localIsoDay,
MAX_POINTS_WITH_DOTS,
SAVINGS_COLORS,
SAVINGS_DRIVERS,
SAVINGS_SERIES,
SavingsAccumulation,
SavingsPoint,
Expand All @@ -38,15 +40,14 @@ const EMPTY_TOOL_SPEND: ToolSpendResponse = {
end_date: null,
};

const SAVINGS_COLORS = ["emerald", "blue"] as const;

const shortDate = (iso: string): string =>
new Date(`${iso}T00:00:00`).toLocaleDateString("en-US", { month: "short", day: "numeric" });

const isoDay = (d: Date): string => d.toISOString().slice(0, 10);

const compressionOf = (m: SpendMetrics): number => m.compression_savings_spend ?? 0;
const cachingOf = (m: SpendMetrics): number => m.prompt_caching_savings_spend ?? 0;
const autorouterOf = (m: SpendMetrics): number => m.autorouter_savings_spend ?? 0;
const savedTokensOf = (m: SpendMetrics): number => m.compression_saved_tokens ?? 0;

const SummaryCard = ({ label, value, hint, info }: { label: string; value: string; hint?: string; info?: string }) => (
Expand Down Expand Up @@ -105,8 +106,9 @@ const UsageTab: React.FC<UsageTabProps> = ({ accessToken, activity }) => {

const compressionTotal = useMemo(() => results.reduce((sum, d) => sum + compressionOf(d.metrics), 0), [results]);
const cachingTotal = useMemo(() => results.reduce((sum, d) => sum + cachingOf(d.metrics), 0), [results]);
const autorouterTotal = useMemo(() => results.reduce((sum, d) => sum + autorouterOf(d.metrics), 0), [results]);
const savedTokensTotal = useMemo(() => results.reduce((sum, d) => sum + savedTokensOf(d.metrics), 0), [results]);
const totalSaved = compressionTotal + cachingTotal;
const totalSaved = compressionTotal + cachingTotal + autorouterTotal;

const [accumulation, setAccumulation] = useState<SavingsAccumulation>("cumulative");

Expand All @@ -122,6 +124,7 @@ const UsageTab: React.FC<UsageTabProps> = ({ accessToken, activity }) => {
date: shortDate(d.date),
Compression: compressionOf(d.metrics),
"Prompt caching": cachingOf(d.metrics),
"Auto-router": autorouterOf(d.metrics),
})),
[results],
);
Expand All @@ -143,14 +146,19 @@ const UsageTab: React.FC<UsageTabProps> = ({ accessToken, activity }) => {
.filter(Boolean)
.join(" \u00b7 ");

// A driver can come out negative (auto-router pays a cold-cache write on every
// model switch), and a negative slice has no meaning in a donut, so only drivers
// that actually saved are plotted; the range total keeps the signed truth.
const byDriver = useMemo(
() =>
[
{ driver: "Compression", usd: compressionTotal },
{ driver: "Prompt caching", usd: cachingTotal },
].filter((d) => d.usd > 0),
[compressionTotal, cachingTotal],
SAVINGS_DRIVERS.map(({ name, color }) => ({
driver: name,
color,
usd: { Compression: compressionTotal, "Prompt caching": cachingTotal, "Auto-router": autorouterTotal }[name],
})).filter((d) => d.usd > 0),
[compressionTotal, cachingTotal, autorouterTotal],
);
const plottedDriverTotal = useMemo(() => byDriver.reduce((sum, d) => sum + d.usd, 0), [byDriver]);

const topTools = useMemo(() => topToolsBySpend(toolSpend?.by_tool ?? []), [toolSpend]);
const topToolNames = useMemo(() => topTools.map((t) => t.tool_name), [topTools]);
Expand All @@ -174,11 +182,11 @@ const UsageTab: React.FC<UsageTabProps> = ({ accessToken, activity }) => {
<AdvancedDatePicker value={dateValue} onValueChange={onDateChange} />
</div>

<div className="grid grid-cols-1 gap-6 sm:grid-cols-2 lg:grid-cols-3">
<div className="grid grid-cols-1 gap-6 sm:grid-cols-2 lg:grid-cols-4">
<SummaryCard
label="Total saved"
value={usd(totalSaved)}
hint={loading || isFetchingMore ? "Loading..." : "Compression + prompt caching"}
hint={loading || isFetchingMore ? "Loading..." : "Compression + prompt caching + auto-router"}
/>
<SummaryCard
label="Compression savings"
Expand All @@ -192,26 +200,32 @@ const UsageTab: React.FC<UsageTabProps> = ({ accessToken, activity }) => {
hint="Cache read discount"
info="Tokens the provider served from cache, priced at the discount between the input and cache-read rates."
/>
<SummaryCard
label="Auto-router savings"
value={usd(autorouterTotal)}
hint="vs. the priciest model it could pick"
info="What this traffic would have cost had every request gone to the most expensive model the auto-router can route to, minus what it actually cost. Switching leaves the new model with a cold cache, so it pays to write the prompt again while the baseline is priced as already warm; a route that thrashes the cache can total below zero, and a genuine first turn, where neither side had anything cached, is undercounted."
/>
</div>

<div className="grid grid-cols-1 gap-6 lg:grid-cols-3">
<Card className="lg:col-span-2">
{/* CardHeader's own slots rather than hand-rolled rows: the action column is
sized to its content and the title column takes the rest, so the subtitle
never competes with the controls for width and neither moves when it grows.
The controls wrap within their column instead of pushing past the card */}
<CardHeader>
<div className="flex flex-wrap items-start justify-between gap-3">
<div>
<CardTitle>Savings</CardTitle>
<p className="text-sm text-muted-foreground">{savingsSubtitle}</p>
</div>
<div className="flex items-center gap-4">
<CustomLegend categories={SAVINGS_SERIES} colors={SAVINGS_COLORS} />
<Tabs value={accumulation} onValueChange={(value) => setAccumulation(value as SavingsAccumulation)}>
<TabsList>
<TabsTrigger value="cumulative">Cumulative</TabsTrigger>
<TabsTrigger value="per-interval">{intervalLabel}</TabsTrigger>
</TabsList>
</Tabs>
</div>
</div>
<CardTitle>Savings</CardTitle>
<CardDescription>{savingsSubtitle}</CardDescription>
<CardAction className="flex flex-wrap items-center justify-end gap-x-4 gap-y-2">
<CustomLegend categories={SAVINGS_SERIES} colors={SAVINGS_COLORS} />
<Tabs value={accumulation} onValueChange={(value) => setAccumulation(value as SavingsAccumulation)}>
<TabsList>
<TabsTrigger value="cumulative">Cumulative</TabsTrigger>
<TabsTrigger value="per-interval">{intervalLabel}</TabsTrigger>
</TabsList>
</Tabs>
</CardAction>
</CardHeader>
<CardContent>
{accumulation === "cumulative" ? (
Expand All @@ -225,12 +239,14 @@ const UsageTab: React.FC<UsageTabProps> = ({ accessToken, activity }) => {
showDots={overTime.length <= MAX_POINTS_WITH_DOTS}
/>
) : (
// Not stacked: a driver can be negative once a model switch is charged
// for its cold cache, and stacking would draw that segment below the axis
// while the remaining bar still read as the day's total
<BarChart
data={overTime}
index="date"
categories={SAVINGS_SERIES}
colors={SAVINGS_COLORS}
stack
valueFormatter={usd}
showLegend={false}
/>
Expand All @@ -247,10 +263,10 @@ const UsageTab: React.FC<UsageTabProps> = ({ accessToken, activity }) => {
data={byDriver}
index="driver"
category="usd"
colors={["emerald", "blue"]}
colors={byDriver.map((d) => d.color)}
valueFormatter={usd}
showLabel
label={usd(totalSaved)}
label={usd(plottedDriverTotal)}
Comment thread
tin-berri marked this conversation as resolved.
/>
</CardContent>
</Card>
Expand Down
Loading
Loading