Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,16 @@ import { ApiError } from "@/lib/http/client";
vi.mock("./useAutoRouterBenchmarks", () => ({ useAutoRouterBenchmarks: vi.fn() }));
vi.mock("@/app/(dashboard)/hooks/models/useModels", () => ({ useAutoRouters: vi.fn() }));
vi.mock("./ShadowEvalSection", () => ({ default: () => <div data-testid="shadow-eval-section" /> }));
vi.mock("@/components/shared/advanced_date_picker", () => ({
__esModule: true,
default: ({ onValueChange }: { onValueChange: (value: { from?: Date; to?: Date }) => void }) => (
<button
type="button"
data-testid="date-picker"
onClick={() => onValueChange({ from: new Date(2026, 7, 1), to: new Date(2026, 7, 5) })}
/>
),
}));

import { useAutoRouters } from "@/app/(dashboard)/hooks/models/useModels";

Expand Down Expand Up @@ -112,12 +122,28 @@ const response = (groups: AutoRouterBenchmarkGroup[], shared: Totals = totals())
});

const renderTab = () => {
const dateValue = { from: new Date(2026, 6, 6), to: new Date(2026, 7, 5) };
const onDateChange = vi.fn();
const activity = {
dateValue,
onDateChange,
results: [],
loading: false,
isFetchingMore: false,
progress: { currentPage: 1, totalPages: 1 },
cancelled: false,
cancel: vi.fn(),
};
const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } });
return render(
<QueryClientProvider client={queryClient}>
<AutoRouterBenchmarksTab accessToken="sk-test" />
</QueryClientProvider>,
);
return {
dateValue,
onDateChange,
...render(
<QueryClientProvider client={queryClient}>
<AutoRouterBenchmarksTab accessToken="sk-test" activity={activity} />
</QueryClientProvider>,
),
};
};

describe("AutoRouterBenchmarksTab", () => {
Expand Down Expand Up @@ -304,20 +330,15 @@ describe("AutoRouterBenchmarksTab", () => {
expect(screen.queryByText("-0%")).not.toBeInTheDocument();
});

it("requests the default thirty day window and widens or narrows it from the picker", () => {
it("queries the shared picker's range and pushes picker changes back to the shared state", () => {
mockHook({ data: response([group()]) });
renderTab();

expect(vi.mocked(useAutoRouterBenchmarks)).toHaveBeenCalledWith("sk-test", "30d");
expect(screen.getByText("Last 30 days")).toBeInTheDocument();
const { dateValue, onDateChange } = renderTab();

fireEvent.click(screen.getByRole("tab", { name: "7d" }));
expect(vi.mocked(useAutoRouterBenchmarks)).toHaveBeenCalledWith("sk-test", "7d");
expect(screen.getByText("Last 7 days")).toBeInTheDocument();
expect(vi.mocked(useAutoRouterBenchmarks)).toHaveBeenCalledWith("sk-test", dateValue);
expect(screen.getByText("Jul 6 – Aug 5 (UTC)")).toBeInTheDocument();

fireEvent.click(screen.getByRole("tab", { name: "24h" }));
expect(vi.mocked(useAutoRouterBenchmarks)).toHaveBeenCalledWith("sk-test", "24h");
expect(screen.getByText("Last 24 hours")).toBeInTheDocument();
fireEvent.click(screen.getByTestId("date-picker"));
expect(onDateChange).toHaveBeenCalledWith({ from: new Date(2026, 7, 1), to: new Date(2026, 7, 5) });
});

it("shows usage by default and mounts shadow evals only when its sub-tab is selected", () => {
Expand Down Expand Up @@ -347,11 +368,11 @@ describe("AutoRouterBenchmarksTab", () => {
expect(screen.getByTestId("shadow-eval-section")).toBeInTheDocument();
});

it("keeps the window picker reachable while a window has no sessions", () => {
it("keeps the range picker reachable while a window has no sessions", () => {
mockHook({ data: response([], zeroTotals) });
renderTab();

expect(screen.getByRole("tab", { name: "30d" })).toBeInTheDocument();
expect(screen.getByTestId("date-picker")).toBeInTheDocument();
expect(screen.getByText("All auto-routers")).toBeInTheDocument();
});
});
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import React, { useState } from "react";

import type { AutoRouterDeployment } from "@/app/(dashboard)/hooks/models/useModels";
import { useAutoRouters } from "@/app/(dashboard)/hooks/models/useModels";
import AdvancedDatePicker from "@/components/shared/advanced_date_picker";
import { Badge } from "@/components/ui/badge";
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
Expand All @@ -15,7 +16,6 @@ import { formatNumberWithCommas } from "@/utils/dataUtils";

import {
ALL_ROUTERS,
WINDOW_LABELS,
bucketRows,
bucketTurnsTotal,
durationLabel,
Expand All @@ -27,13 +27,13 @@ import {
type AutoRouterBenchmarksResponse,
type AutoRouterCacheStats,
type BenchmarkView,
type BenchmarkWindow,
type BucketRow,
} from "./autoRouterBenchmarks";
import { usd } from "./costOptimizationUtils";
import { formatRangeLabel, usd } from "./costOptimizationUtils";
import ShadowEvalSection from "./ShadowEvalSection";
import TierTurnsChart from "./TierTurnsChart";
import { useAutoRouterBenchmarks } from "./useAutoRouterBenchmarks";
import { DailyActivityRange } from "./useDailyActivityRange";

const Message: React.FC<{ children: React.ReactNode }> = ({ children }) => (
<p className="py-8 text-center text-sm text-muted-foreground">{children}</p>
Expand Down Expand Up @@ -248,7 +248,8 @@ const BenchmarksBody: React.FC<BenchmarksBodyProps> = ({ isPending, error, data,
<p className="text-xs text-muted-foreground">
Compares your actual routed spend with the estimated cost of using only the most expensive model configured in
the auto-router. It accounts for both the cache savings from staying on one model and the added cache costs from
switching models.
switching models. The range counts whole sessions that overlap it, so totals can differ slightly from the
Overall tab, which buckets savings by UTC day.
</p>

<div className="space-y-4">
Expand All @@ -266,32 +267,28 @@ const BenchmarksBody: React.FC<BenchmarksBodyProps> = ({ isPending, error, data,

interface AutoRouterBenchmarksTabProps {
accessToken: string | null;
activity: DailyActivityRange;
}

const UsageView: React.FC<AutoRouterBenchmarksTabProps> = ({ accessToken }) => {
const [range, setRange] = useState<BenchmarkWindow>("30d");
const { data, isPending, error } = useAutoRouterBenchmarks(accessToken, range);
const UsageView: React.FC<AutoRouterBenchmarksTabProps> = ({ accessToken, activity }) => {
const { dateValue, onDateChange } = activity;
const { data, isPending, error } = useAutoRouterBenchmarks(accessToken, dateValue);
const [selectedKey, setSelectedKey] = useState<string>(ALL_ROUTERS);
const { data: autoRouters } = useAutoRouters();

const groups = data?.groups ?? [];
const selectedLabel = data ? viewFor(data, selectedKey).label : "All auto-routers";
const rangeLabel = formatRangeLabel(dateValue.from, dateValue.to);

return (
<div className="w-full space-y-6">
<div className="flex flex-col gap-3 sm:flex-row sm:items-start sm:justify-between">
<div>
<h2 className="text-xl font-semibold text-foreground">Auto-router usage</h2>
<p className="mt-1 text-sm text-muted-foreground">{WINDOW_LABELS[range]}</p>
{rangeLabel && <p className="mt-1 text-sm text-muted-foreground">{rangeLabel} (UTC)</p>}
</div>
<div className="flex w-full flex-col gap-3 sm:w-auto sm:flex-row sm:items-center">
<Tabs value={range} onValueChange={(value) => setRange(value === "7d" || value === "24h" ? value : "30d")}>
<TabsList>
<TabsTrigger value="30d">30d</TabsTrigger>
<TabsTrigger value="7d">7d</TabsTrigger>
<TabsTrigger value="24h">24h</TabsTrigger>
</TabsList>
</Tabs>
<AdvancedDatePicker value={dateValue} onValueChange={onDateChange} />
<div className="w-full sm:w-64">
<Select value={selectedKey} onValueChange={(value: string | null) => setSelectedKey(value ?? ALL_ROUTERS)}>
<SelectTrigger className="w-full">
Expand Down Expand Up @@ -321,7 +318,7 @@ const UsageView: React.FC<AutoRouterBenchmarksTabProps> = ({ accessToken }) => {
);
};

const AutoRouterBenchmarksTab: React.FC<AutoRouterBenchmarksTabProps> = ({ accessToken }) => {
const AutoRouterBenchmarksTab: React.FC<AutoRouterBenchmarksTabProps> = ({ accessToken, activity }) => {
const [visitedTabs, setVisitedTabs] = useState<readonly string[]>(["usage"]);

const handleTabChange = (value: unknown) => {
Expand All @@ -344,7 +341,7 @@ const AutoRouterBenchmarksTab: React.FC<AutoRouterBenchmarksTabProps> = ({ acces
</TabsList>

<TabsContent value="usage" keepMounted={visitedTabs.includes("usage")}>
<UsageView accessToken={accessToken} />
<UsageView accessToken={accessToken} activity={activity} />
</TabsContent>
<TabsContent value="shadow-evals" keepMounted={visitedTabs.includes("shadow-evals")}>
<ShadowEvalSection />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ const CostOptimizationView: React.FC<CostOptimizationViewProps> = ({ accessToken
<PromptCachingTab accessToken={accessToken} activity={activity} />
</TabsContent>
<TabsContent value="autorouter-usage" keepMounted={visitedTabs.includes("autorouter-usage")}>
<AutoRouterBenchmarksTab accessToken={accessToken} />
<AutoRouterBenchmarksTab accessToken={accessToken} activity={activity} />
</TabsContent>
</>
)}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import {
groupLabel,
pctLabel,
viewFor,
windowFor,
type AutoRouterBenchmarkGroup,
type AutoRouterBenchmarksResponse,
type AutoRouterCacheStats,
Expand Down Expand Up @@ -153,21 +152,6 @@ describe("expiredMissShare", () => {
});
});

describe("windowFor", () => {
const noon = new Date("2026-08-05T12:00:00Z");

it("derives each picker range as UTC calendar days ending today", () => {
expect(windowFor("30d", noon)).toEqual({ start_date: "2026-07-06", end_date: "2026-08-05" });
expect(windowFor("7d", noon)).toEqual({ start_date: "2026-07-29", end_date: "2026-08-05" });
expect(windowFor("24h", noon)).toEqual({ start_date: "2026-08-04", end_date: "2026-08-05" });
});

it("uses UTC days, not the local calendar", () => {
const lateEvening = new Date("2026-08-05T23:30:00-05:00");
expect(windowFor("24h", lateEvening)).toEqual({ start_date: "2026-08-05", end_date: "2026-08-06" });
});
});

describe("formatting", () => {
it("renders session length in the largest sensible unit", () => {
expect(durationLabel(42)).toBe("42s");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,21 +7,6 @@ export type AutoRouterCacheStats = components["schemas"]["AutoRouterCacheStats"]

export const ALL_ROUTERS = "__all__";

export type BenchmarkWindow = "30d" | "7d" | "24h";

const WINDOW_DAYS: Record<BenchmarkWindow, number> = { "30d": 30, "7d": 7, "24h": 1 };

export const WINDOW_LABELS: Record<BenchmarkWindow, string> = {
"30d": "Last 30 days",
"7d": "Last 7 days",
"24h": "Last 24 hours",
};

export const windowFor = (range: BenchmarkWindow, now: Date): { start_date: string; end_date: string } => ({
start_date: new Date(now.getTime() - WINDOW_DAYS[range] * 24 * 60 * 60 * 1000).toISOString().slice(0, 10),
end_date: now.toISOString().slice(0, 10),
});

export interface BenchmarkView {
label: string;
stats: AutoRouterBenchmarkTotals | AutoRouterBenchmarkGroup;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
import { describe, expect, it, vi } from "vitest";

vi.mock("@/components/networking", () => ({ formatDate: vi.fn() }));
vi.mock("@/lib/http/api", () => ({ $api: { useQuery: vi.fn() } }));

import { benchmarksWindow } from "./useAutoRouterBenchmarks";

const localDay =
(offsetHours: number) =>
(d: Date): string =>
new Date(d.getTime() + offsetHours * 3_600_000).toISOString().slice(0, 10);

const pacific = localDay(-7);
const tokyo = localDay(+9);

describe("benchmarksWindow", () => {
it("passes a historical range through as the picked local calendar days", () => {
const now = new Date("2026-08-21T19:00:00Z");
const range = { from: new Date("2026-07-06T19:00:00Z"), to: new Date("2026-08-05T19:00:00Z") };
expect(benchmarksWindow(range, now, pacific)).toEqual({ start_date: "2026-07-06", end_date: "2026-08-05" });
});

it("keeps a range ending today unchanged while the local and UTC days still agree", () => {
const now = new Date("2026-08-21T19:00:00Z");
const range = { from: new Date("2026-07-22T19:00:00Z"), to: now };
expect(benchmarksWindow(range, now, pacific)).toEqual({ start_date: "2026-07-22", end_date: "2026-08-21" });
});

it("extends a range ending today to the current UTC day once UTC rolls past local midnight", () => {
const now = new Date("2026-08-22T03:00:00Z");
const range = { from: new Date("2026-07-22T19:00:00Z"), to: now };
expect(benchmarksWindow(range, now, pacific)).toEqual({ start_date: "2026-07-22", end_date: "2026-08-22" });
});

it("never shrinks a range for a caller east of UTC whose local day is already tomorrow", () => {
const now = new Date("2026-08-21T19:00:00Z");
const range = { from: new Date("2026-07-22T19:00:00Z"), to: now };
expect(benchmarksWindow(range, now, tokyo)).toEqual({ start_date: "2026-07-23", end_date: "2026-08-22" });
});

it("sends no dates while the picker is missing either end", () => {
const now = new Date("2026-08-21T19:00:00Z");
expect(benchmarksWindow({ from: now }, now, pacific)).toEqual({});
expect(benchmarksWindow({ to: now }, now, pacific)).toEqual({});
});
});
Original file line number Diff line number Diff line change
@@ -1,11 +1,32 @@
import { formatDate } from "@/components/networking";
import { $api } from "@/lib/http/api";

import { windowFor, type BenchmarkWindow } from "./autoRouterBenchmarks";
import type { DateRange } from "./useDailyActivityRange";

export const useAutoRouterBenchmarks = (accessToken: string | null, range: BenchmarkWindow) =>
/**
* The endpoint cuts on UTC days but the picker hands back local dates, so west of UTC a
* range ending today would hide sessions started after UTC midnight. Extend it to the
* current UTC day (only the empty future is added), mirroring include_current_utc_day.
*/
export const benchmarksWindow = (
range: DateRange,
now: Date,
toLocalDay: (d: Date) => string = formatDate,
): { start_date: string; end_date: string } | Record<string, never> => {
if (!range.from || !range.to) return {};
const end_date = toLocalDay(range.to);
const utcToday = now.toISOString().slice(0, 10);
const endsToday = end_date >= toLocalDay(now);
return {
start_date: toLocalDay(range.from),
end_date: endsToday && utcToday > end_date ? utcToday : end_date,
};
};

export const useAutoRouterBenchmarks = (accessToken: string | null, range: DateRange) =>
$api.useQuery(
"get",
"/auto_router/benchmarks",
{ params: { query: windowFor(range, new Date()) } },
{ enabled: Boolean(accessToken), retry: false },
{ params: { query: benchmarksWindow(range, new Date()) } },
{ enabled: Boolean(accessToken && range.from && range.to), retry: false },
);
Loading