Skip to content
Closed
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions litellm/types/management_endpoints/auto_router_endpoints.py
Original file line number Diff line number Diff line change
Expand Up @@ -126,8 +126,8 @@ class AutoRouterBenchmarkGroup(AutoRouterBenchmarkTotals):
description="Turns per tier, keyed by the tier name the routing decision recorded at "
"request time (never re-derived at read time, since the tier-to-model mapping is "
"mutable config). Tier names are scoped to this group's router_type and are not "
"comparable across types: a complexity router reports 'simple'/'medium'/'complex'/"
"'reasoning', a quality router reports its numeric quality tier, and an adaptive router "
"comparable across types: a complexity router reports 'SIMPLE'/'MEDIUM'/'COMPLEX'/"
"'REASONING', a quality router reports its numeric quality tier, and an adaptive router "
"records no tier at all. Turns no tier served (the classifier fell back to default_model) "
"are absent rather than pooled under a sentinel key, so the values may sum to less than turns",
)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,15 @@
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
import { fireEvent, render, screen } from "@testing-library/react";
import React from "react";
import { describe, expect, it, vi } from "vitest";
import { beforeEach, describe, expect, it, vi } from "vitest";

import type { AutoRouterDeployment } from "@/app/(dashboard)/hooks/models/useModels";
import { ApiError } from "@/lib/http/client";

vi.mock("./useAutoRouterBenchmarks", () => ({ useAutoRouterBenchmarks: vi.fn() }));
vi.mock("@/app/(dashboard)/hooks/models/useModels", () => ({ useAutoRouters: vi.fn() }));

import { useAutoRouters } from "@/app/(dashboard)/hooks/models/useModels";

import AutoRouterBenchmarksTab from "./AutoRouterBenchmarksTab";
import type {
Expand All @@ -16,6 +21,10 @@ import { useAutoRouterBenchmarks } from "./useAutoRouterBenchmarks";

type HookResult = ReturnType<typeof useAutoRouterBenchmarks>;

const mockAutoRouters = (deployments: AutoRouterDeployment[] = []) => {
vi.mocked(useAutoRouters).mockReturnValue({ data: deployments } as unknown as ReturnType<typeof useAutoRouters>);
};

const mockHook = (result: { data?: AutoRouterBenchmarksResponse; isPending?: boolean; error?: Error }) => {
vi.mocked(useAutoRouterBenchmarks).mockReturnValue({
data: result.data,
Expand Down Expand Up @@ -71,9 +80,20 @@ const response = (groups: AutoRouterBenchmarkGroup[], shared: Totals = totals())
groups,
});

const renderTab = () => render(<AutoRouterBenchmarksTab accessToken="sk-test" />);
const renderTab = () => {
const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } });
return render(
<QueryClientProvider client={queryClient}>
<AutoRouterBenchmarksTab accessToken="sk-test" />
</QueryClientProvider>,
);
};

describe("AutoRouterBenchmarksTab", () => {
beforeEach(() => {
mockAutoRouters();
});

it("leads with total estimated savings, before the three session-shape metrics", () => {
mockHook({ data: response([group(), group({ router_name: "gpt-auto" })]) });
renderTab();
Expand All @@ -97,7 +117,7 @@ describe("AutoRouterBenchmarksTab", () => {
expect(screen.getByText("-86%")).toBeInTheDocument();
expect(screen.getByText("Actual auto-router spend")).toBeInTheDocument();
expect(screen.getByText("$359.86")).toBeInTheDocument();
expect(screen.getByText("Estimated spend at highest-cost model")).toBeInTheDocument();
expect(screen.getByText("Estimated spend at highest-tier model")).toBeInTheDocument();
expect(screen.getByText("$2,534.45")).toBeInTheDocument();
expect(screen.getByText("32.7")).toBeInTheDocument();
expect(screen.getByText("2.1h")).toBeInTheDocument();
Expand All @@ -108,12 +128,9 @@ describe("AutoRouterBenchmarksTab", () => {
mockHook({ data: response([group(), group({ router_name: "gpt-auto" })]) });
renderTab();

expect(screen.getByText("Total sessions")).toBeInTheDocument();
expect(screen.getByText("94")).toBeInTheDocument();
expect(screen.getByText("Total turns")).toBeInTheDocument();
expect(screen.getByText("3,073")).toBeInTheDocument();
expect(screen.getByText("Avg saved per session")).toBeInTheDocument();
expect(screen.getByText("$23.13")).toBeInTheDocument();
expect(screen.getByText("across 94 sessions")).toBeInTheDocument();
});

it("shows a cost increase as a positive delta rather than a saving", () => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

import React, { useState } from "react";

import type { AutoRouterDeployment } from "@/app/(dashboard)/hooks/models/useModels";
import { useAutoRouters } from "@/app/(dashboard)/hooks/models/useModels";
import { Badge } from "@/components/ui/badge";
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
Expand Down Expand Up @@ -29,6 +31,7 @@ import {
type BucketRow,
} from "./autoRouterBenchmarks";
import { usd } from "./costOptimizationUtils";
import TierTurnsChart from "./TierTurnsChart";
import { useAutoRouterBenchmarks } from "./useAutoRouterBenchmarks";

const Message: React.FC<{ children: React.ReactNode }> = ({ children }) => (
Expand All @@ -51,7 +54,7 @@ const HeroCard: React.FC<{ view: BenchmarkView }> = ({ view }) => {
const cheaper = stats.saved_spend >= 0;
return (
<Card className="overflow-hidden py-0">
<div className="grid md:grid-cols-[4fr_3fr_5fr]">
<div className="grid md:grid-cols-[1fr_1fr]">
<div className="flex flex-col justify-center gap-3 p-6">
<p className="text-sm text-muted-foreground">Total estimated savings</p>
<div className="flex flex-wrap items-center gap-3">
Expand All @@ -64,38 +67,22 @@ const HeroCard: React.FC<{ view: BenchmarkView }> = ({ view }) => {
{Math.abs(stats.saved_pct).toFixed(0)}%
</Badge>
</div>
</div>

<div className="flex flex-col justify-center px-6 pb-6 md:py-6">
<dl className="divide-y text-sm">
<div className="flex items-baseline justify-between gap-6 py-3">
<dt className="text-muted-foreground">Actual auto-router spend</dt>
<dd className="font-medium tabular-nums text-foreground">{usd(stats.spend)}</dd>
</div>
<div className="flex items-baseline justify-between gap-6 py-3">
<dt className="text-muted-foreground">Estimated spend at highest-cost model</dt>
<dt className="text-muted-foreground">Estimated spend at highest-tier model</dt>
<dd className="font-medium tabular-nums text-foreground">{usd(stats.baseline_spend)}</dd>
</div>
</dl>
</div>

<div className="flex flex-col border-t md:border-t-0 md:border-l">
<div className="grid flex-1 grid-cols-2 divide-x">
<div className="flex flex-col justify-center gap-1 px-6 py-4">
<p className="text-[11px] uppercase tracking-wide text-muted-foreground">Total sessions</p>
<p className="text-3xl font-semibold text-foreground">{stats.sessions.toLocaleString()}</p>
</div>
<div className="flex flex-col justify-center gap-1 px-6 py-4">
<p className="text-[11px] uppercase tracking-wide text-muted-foreground">Total turns</p>
<p className="text-3xl font-semibold text-foreground">{stats.turns.toLocaleString()}</p>
</div>
</div>
<dl className="flex flex-col divide-y border-t text-sm">
<div className="flex items-center justify-between gap-2 px-6 py-3">
<dt className="text-[11px] uppercase tracking-wide text-muted-foreground">Avg saved per session</dt>
<dd className="text-lg font-semibold tabular-nums text-foreground">{usd(stats.saved_per_session)}</dd>
</div>
</dl>
<div className="flex flex-col items-center justify-center gap-2 border-t p-6 md:border-t-0 md:border-l">
<p className="text-[11px] uppercase tracking-wide text-muted-foreground">Avg saved per session</p>
<p className="text-5xl font-semibold tracking-tight text-foreground">{usd(stats.saved_per_session)}</p>
<p className="text-sm text-muted-foreground">across {stats.sessions.toLocaleString()} sessions</p>
</div>
</div>
</Card>
Expand Down Expand Up @@ -233,9 +220,10 @@ interface BenchmarksBodyProps {
error: unknown;
data: AutoRouterBenchmarksResponse | undefined;
selectedKey: string;
autoRouters: readonly AutoRouterDeployment[];
}

const BenchmarksBody: React.FC<BenchmarksBodyProps> = ({ isPending, error, data, selectedKey }) => {
const BenchmarksBody: React.FC<BenchmarksBodyProps> = ({ isPending, error, data, selectedKey, autoRouters }) => {
if (isPending) return <Message>Loading auto-router usage...</Message>;
if (error instanceof ApiError && error.status === 403) {
return <Message>Auto-router usage is visible to proxy admin roles only</Message>;
Expand All @@ -249,6 +237,8 @@ const BenchmarksBody: React.FC<BenchmarksBodyProps> = ({ isPending, error, data,
<>
<HeroCard view={view} />

<TierTurnsChart view={view} autoRouters={autoRouters} />

<div className="grid grid-cols-1 gap-4 sm:grid-cols-3">
<Metric label="Avg turns per session" value={stats.avg_turns_per_session.toFixed(1)} />
<Metric label="Avg session length" value={durationLabel(stats.avg_session_seconds)} />
Expand Down Expand Up @@ -282,6 +272,7 @@ const AutoRouterBenchmarksTab: React.FC<AutoRouterBenchmarksTabProps> = ({ acces
const [range, setRange] = useState<BenchmarkWindow>("30d");
const { data, isPending, error } = useAutoRouterBenchmarks(accessToken, range);
const [selectedKey, setSelectedKey] = useState<string>(ALL_ROUTERS);
const { data: autoRouters } = useAutoRouters();

const groups = data?.groups ?? [];
const selectedLabel = data ? viewFor(data, selectedKey).label : "All auto-routers";
Expand Down Expand Up @@ -319,7 +310,13 @@ const AutoRouterBenchmarksTab: React.FC<AutoRouterBenchmarksTabProps> = ({ acces
</div>
</div>

<BenchmarksBody isPending={isPending} error={error} data={data} selectedKey={selectedKey} />
<BenchmarksBody
isPending={isPending}
error={error}
data={data}
selectedKey={selectedKey}
autoRouters={autoRouters ?? []}
/>
</div>
);
};
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,165 @@
import { render, screen } from "@testing-library/react";
import React from "react";
import { describe, expect, it, vi } from "vitest";

import type { AutoRouterDeployment } from "@/app/(dashboard)/hooks/models/useModels";

vi.mock("@/components/shared/charts", () => ({
DonutChart: ({ label }: { label: string }) => <div data-testid="donut">{label}</div>,
SEQUENTIAL_COLOR_RAMP: ["indigo", "blue"],
chartColorValue: (color: string) => color,
}));

import TierTurnsChart, { tierDisplayLabel } from "./TierTurnsChart";
import type { AutoRouterBenchmarkGroup, BenchmarkView } from "./autoRouterBenchmarks";

const totalsOnly = {
sessions: 3,
turns: 9,
avg_turns_per_session: 3,
avg_session_seconds: 60,
avg_tokens_per_session: 100,
spend: 1,
saved_spend: 1,
baseline_spend: 2,
saved_pct: 50,
saved_per_session: 0.33,
cache: {
coverage_pct: 0,
hit_rate_pct: 0,
same_model: { turns: 0, hits: 0, hit_rate_pct: 0 },
first_visit: { turns: 0, hits: 0, hit_rate_pct: 0 },
return_to_tier: { turns: 0, hits: 0, hit_rate_pct: 0 },
unordered_turns: 0,
return_misses_expired: 0,
return_misses_within_ttl: 0,
return_misses_unknown: 0,
ttl_5m_turns: 0,
ttl_1h_turns: 0,
},
};

const groupView = (overrides: Partial<AutoRouterBenchmarkGroup> = {}): BenchmarkView => ({
label: "claude-auto",
stats: {
...totalsOnly,
router_name: "claude-auto",
router_type: "complexity",
tier_turns: { SIMPLE: 3, COMPLEX: 1 },
...overrides,
} as AutoRouterBenchmarkGroup,
});

const deployment = (config: unknown): AutoRouterDeployment => ({
model_name: "claude-auto",
litellm_params: { model: "auto_router/claude-auto", complexity_router_config: config },
});

describe("tierDisplayLabel", () => {
it("prefers the admin's custom label for a canonical complexity tier", () => {
expect(tierDisplayLabel("SIMPLE", { SIMPLE: "Cheap" })).toBe("Cheap");
});

it("falls back to the canonical name when that tier has no custom label", () => {
expect(tierDisplayLabel("COMPLEX", { SIMPLE: "Cheap" })).toBe("Complex");
expect(tierDisplayLabel("REASONING", undefined)).toBe("Reasoning");
});

it("shows a non-complexity tier verbatim, since no label map covers a quality router's tier", () => {
expect(tierDisplayLabel("3", { SIMPLE: "Cheap" })).toBe("3");
});
});

describe("TierTurnsChart", () => {
it("labels each slice with its tier and share of the tiered turns", () => {
render(<TierTurnsChart view={groupView()} autoRouters={[deployment({ tier_labels: { SIMPLE: "Cheap" } })]} />);

expect(screen.getByText("Cheap 75%")).toBeInTheDocument();
expect(screen.getByText("Complex 25%")).toBeInTheDocument();
expect(screen.getByTestId("donut")).toHaveTextContent("4 total turns");
});

it("reads tier_labels out of a config stored as a JSON string", () => {
const stored = JSON.stringify({ tier_labels: { SIMPLE: "Cheap" } });
render(<TierTurnsChart view={groupView()} autoRouters={[deployment(stored)]} />);

expect(screen.getByText("Cheap 75%")).toBeInTheDocument();
});

it("uses canonical names when the router is not in the deployment list", () => {
render(<TierTurnsChart view={groupView()} autoRouters={[]} />);

expect(screen.getByText("Simple 75%")).toBeInTheDocument();
expect(screen.getByText("Complex 25%")).toBeInTheDocument();
});

it("lists each tier's assigned models below its name and share", () => {
render(
<TierTurnsChart
view={groupView()}
autoRouters={[deployment({ tiers: { SIMPLE: ["gpt-4o-mini"], COMPLEX: ["gpt-4o", "claude-3-opus"] } })]}
/>,
);

expect(screen.getByText("gpt-4o-mini")).toBeInTheDocument();
expect(screen.getByText("gpt-4o, claude-3-opus")).toBeInTheDocument();
});

it("widens a bare string tier (pinned single model) into its one-model list", () => {
render(<TierTurnsChart view={groupView()} autoRouters={[deployment({ tiers: { SIMPLE: "gpt-4o-mini" } })]} />);

expect(screen.getByText("gpt-4o-mini")).toBeInTheDocument();
});

it("omits the model line for a tier with no configured models", () => {
render(<TierTurnsChart view={groupView()} autoRouters={[deployment({ tiers: { SIMPLE: [] } })]} />);

expect(screen.getByText("Simple 75%")).toBeInTheDocument();
});

it("shows no models for a quality router's numeric tier, which has no per-tier model list", () => {
render(
<TierTurnsChart
view={groupView({ router_type: "quality", tier_turns: { "3": 3, "1": 1 } })}
autoRouters={[deployment({ quality_router_config: { available_models: ["gpt-4o"] } })]}
/>,
);

expect(screen.getByText("3 75%")).toBeInTheDocument();
expect(screen.getByText("1 25%")).toBeInTheDocument();
expect(screen.queryByText("gpt-4o")).not.toBeInTheDocument();
});

it("ignores a same-named deployment of a different router type", () => {
const qualityDeployment = {
model_name: "claude-auto",
litellm_params: { model: "auto_router/claude-auto", quality_router_config: { available_models: ["gpt-4o"] } },
};

render(
<TierTurnsChart
view={groupView()} // complexity router
autoRouters={[qualityDeployment] as AutoRouterDeployment[]}
/>,
);

// No complexity_router_config on the quality deployment, so no tier labels are read,
// falling back to canonical names. No models are shown (quality config doesn't have tiers).
expect(screen.getByText("Simple 75%")).toBeInTheDocument();
Comment on lines +141 to +146

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Remove prohibited test comments

These newly added comments violate the repository convention against adding source-code comments; the test name and assertions already communicate the scenario without them.

Suggested change
view={groupView()} // complexity router
autoRouters={[qualityDeployment] as AutoRouterDeployment[]}
/>,
);
// No complexity_router_config on the quality deployment, so no tier labels are read,
// falling back to canonical names. No models are shown (quality config doesn't have tiers).
expect(screen.getByText("Simple 75%")).toBeInTheDocument();
view={groupView()}
autoRouters={[qualityDeployment] as AutoRouterDeployment[]}
/>,
);
expect(screen.getByText("Simple 75%")).toBeInTheDocument();

Context Used: CLAUDE.md (source)

Note: If this suggestion doesn't match your team's coding style, reply to this and let me know. I'll remember it for next time!

expect(screen.queryByText("gpt-4o")).not.toBeInTheDocument();
});

it("renders nothing for the all-routers view, which carries no router identity", () => {
const { container } = render(
<TierTurnsChart view={{ label: "All auto-routers", stats: totalsOnly }} autoRouters={[]} />,
);

expect(container).toBeEmptyDOMElement();
});

it("renders nothing when the router recorded no tiers", () => {
const { container } = render(<TierTurnsChart view={groupView({ tier_turns: {} })} autoRouters={[]} />);

expect(container).toBeEmptyDOMElement();
});
});
Loading