Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ vi.mock("@/app/(dashboard)/hooks/models/useModels", () => ({
{ model_name: "gpt-auto", litellm_params: { model: "auto_router/gpt-auto" } },
],
})),
usePlainModelGroups: vi.fn(() => new Set(["prod-claude"])),
}));

vi.mock("@/app/(dashboard)/hooks/models/useModelCostMap", () => ({
Expand All @@ -72,6 +73,8 @@ const job = (overrides: Partial<ShadowEvalJob> = {}): ShadowEvalJob => ({
job_id: "job-1",
status: "running",
router_name: "claude-auto",
direction: "forward",
baseline_model: null,
judge_model: "anthropic/claude-sonnet-5",
shadow_percentage: 10,
max_turns: 200,
Expand Down Expand Up @@ -367,6 +370,58 @@ describe("ShadowEvalSection", () => {
expect(start.mutate).toHaveBeenCalledWith(expectedBody);
});

it("requires a baseline model in reverse mode and submits it, while forward mode never shows the picker", async () => {
const user = userEvent.setup();
const { start } = mockHooks({});
render(<ShadowEvalSection />);

expect(screen.queryByPlaceholderText("Select a baseline model")).not.toBeInTheDocument();

await user.click(screen.getByText("Adoption check: key's traffic vs the router"));
await user.click(await screen.findByText("Regression check: router's picks vs a baseline"));
await user.click(screen.getByPlaceholderText("Search keys by alias"));
await user.click(await screen.findByText("prod-alpha"));
await user.click(screen.getByPlaceholderText("Select an auto-router"));
await user.click(await screen.findByText("gpt-auto"));
await user.click(screen.getByPlaceholderText("Select a judge model"));
await user.click(await screen.findByRole("option", { name: /anthropic\/claude-sonnet-5/ }));

expect(screen.getByText("Start shadow eval")).toBeDisabled();

await user.click(screen.getByPlaceholderText("Select a baseline model"));
expect(await screen.findByRole("option", { name: /openai\/gpt-4o/ })).toBeInTheDocument();
await user.click(screen.getByRole("option", { name: /prod-claude/ }));
await user.click(screen.getByText("Start shadow eval"));

const expectedBody = {
api_key_id: "hash-alpha",
router_name: "gpt-auto",
direction: "reverse",
baseline_model: "prod-claude",
shadow_percentage: 10,
duration_days: 7,
max_turns: 200,
judge_model: "anthropic/claude-sonnet-5",
};
expect(start.mutate).toHaveBeenCalledWith(expectedBody);
});

it("flips the arm labels and headline for a reverse job's results", () => {
const j = job({ direction: "reverse", baseline_model: "openai/gpt-4o" });
mockHooks({ jobs: [j], detailsById: { "job-1": j } });
render(<ShadowEvalSection />);

expect(screen.getByText(/on 10% of its traffic/)).toBeInTheDocument();
expect(screen.getByText("Router matched or beat the baseline")).toBeInTheDocument();
expect(screen.getByText("52.0%")).toBeInTheDocument();
expect(screen.getByText(/Router won 30.0%/)).toBeInTheDocument();
expect(screen.getByText(/Baseline won 48.0%/)).toBeInTheDocument();
expect(screen.getAllByText("Baseline wins")).toHaveLength(2);
expect(screen.getByText("Router pick")).toBeInTheDocument();
expect(screen.queryByText(/Current model/)).not.toBeInTheDocument();
expect(screen.queryByText("Compared against")).not.toBeInTheDocument();
});

it("keeps an older job's verdicts reachable through the previous evaluations list", async () => {
const user = userEvent.setup();
const emptyOverrides: Partial<ShadowEvalJob> = {
Expand Down
Loading
Loading