feat(shadow-eval): name the shadowed key in job responses and the UI headline

litellm/proxy/management_endpoints/auto_router_endpoints.py

@@ -574,6 +574,33 @@ def _slices(rows: Sequence[_AttemptAggRow]) -> tuple[ShadowEvalSlice, ...]:
     )
 
 
+_NO_KEY_LABELS: Final[tuple[str | None, str | None]] = (None, None)
+
+
+async def _with_key_labels(
+    prisma_client: "PrismaClient", responses: Sequence[ShadowEvalJobResponse]
+) -> tuple[ShadowEvalJobResponse, ...]:
+    """Resolve each job's key hash to the key's alias and masked name in one batched read,
+    so the UI can say whose traffic a job shadows. Deleted keys resolve to None."""
+    if not responses:
+        return ()
+    key_rows: Final = await prisma_client.db.litellm_verificationtoken.find_many(
+        where={"token": {"in": sorted({response.api_key_id for response in responses})}}  # mutable-ok: Prisma filter
+    )
+    labels: Final[Mapping[str, tuple[str | None, str | None]]] = {
+        row.token: (row.key_alias, row.key_name) for row in key_rows or ()
+    }
+    return tuple(
+        response.model_copy(
+            update={  # mutable-ok: pydantic update payload
+                "key_alias": labels.get(response.api_key_id, _NO_KEY_LABELS)[0],
+                "key_name": labels.get(response.api_key_id, _NO_KEY_LABELS)[1],
+            }
+        )
+        for response in responses
+    )
+
+
 async def _shadow_eval_results(prisma_client: "PrismaClient", job_id: str) -> ShadowEvalResult | None:
     """Both stratifications of one job's verdicts. Tier answers "where does the router do
     well"; the model stratification groups by whichever model served the real arm, so it
@@ -686,7 +713,9 @@ async def start_shadow_eval(
                 f"Key already has an active {data.direction} shadow eval job (started concurrently). Stop it first."
             ),
         ) from e
-    return ShadowEvalJobResponse.model_validate(job, from_attributes=True)
+    return ShadowEvalJobResponse.model_validate(job, from_attributes=True).model_copy(
+        update={"key_alias": key_row.key_alias, "key_name": key_row.key_name}  # mutable-ok: pydantic update payload
+    )
 
 
 @router.get(
@@ -711,7 +740,10 @@ async def list_shadow_eval_jobs(
         order={"created_at": "desc"},  # mutable-ok: Prisma order
         take=limit,
     )
-    return tuple(ShadowEvalJobResponse.model_validate(record, from_attributes=True) for record in records or ())
+    return await _with_key_labels(
+        prisma_client,
+        tuple(ShadowEvalJobResponse.model_validate(record, from_attributes=True) for record in records or ()),
+    )
 
 
 @router.get(
@@ -742,7 +774,10 @@ async def get_shadow_eval_job(
         where={"job_id": job_id, "outcome": "error"},  # mutable-ok: Prisma filter
         order={"created_at": "desc"},  # mutable-ok: Prisma order
     )
-    return ShadowEvalJobResponse.model_validate(record, from_attributes=True).model_copy(
+    labeled: Final = await _with_key_labels(
+        prisma_client, (ShadowEvalJobResponse.model_validate(record, from_attributes=True),)
+    )
+    return labeled[0].model_copy(
         update={  # mutable-ok: pydantic update payload
             "judged_count": totals[0].judged_count if totals else 0,
             "error_count": totals[0].error_count if totals else 0,
@@ -781,4 +816,7 @@ async def stop_shadow_eval_job(
         where={"id": job_id},  # mutable-ok: Prisma filter
         data={"stopped_at": datetime.now(timezone.utc)},  # mutable-ok: Prisma payload
     )
-    return ShadowEvalJobResponse.model_validate(updated, from_attributes=True)
+    labeled: Final = await _with_key_labels(
+        prisma_client, (ShadowEvalJobResponse.model_validate(updated, from_attributes=True),)
+    )
+    return labeled[0]

litellm/types/management_endpoints/auto_router_endpoints.py

@@ -266,6 +266,14 @@ class ShadowEvalJobResponse(BaseModel):
 
     job_id: str = Field(validation_alias=AliasChoices("id", "job_id"))
     api_key_id: str = Field(description="The hashed virtual key whose traffic this job evaluates, and only that key's")
+    key_alias: str | None = Field(
+        default=None,
+        description="Alias of the shadowed key, resolved from the key row at read time; None when unset or deleted",
+    )
+    key_name: str | None = Field(
+        default=None,
+        description="Masked display name (sk-...) of the shadowed key, resolved at read time like key_alias",
+    )
     router_name: str
     direction: ShadowEvalDirection = "forward"
     baseline_model: str | None = None

tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py

@@ -521,9 +521,20 @@ def _job_record(**overrides: object) -> MagicMock:
     return record
 
 
+def _key_record(
+    token: str = "key-hash", key_alias: str | None = "prod-alpha", key_name: str | None = "sk-...lpha"
+) -> MagicMock:
+    record = MagicMock(spec=["token", "key_alias", "key_name"])
+    record.token = token
+    record.key_alias = key_alias
+    record.key_name = key_name
+    return record
+
+
 def _shadow_prisma(active_job=None, agg_rows=None) -> MagicMock:
     prisma = MagicMock()
-    prisma.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=MagicMock())
+    prisma.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=_key_record())
+    prisma.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[_key_record()])
     prisma.db.execute_raw = AsyncMock(return_value=0)
     prisma.db.litellm_shadowevaljob.find_first = AsyncMock(return_value=active_job)
     prisma.db.litellm_shadowevaljob.find_unique = AsyncMock(return_value=None)
@@ -791,6 +802,30 @@ async def test_list_shadow_eval_jobs_returns_derived_status_without_aggregates(m
     assert prisma.db.query_raw.await_count == 0
 
 
+@pytest.mark.asyncio
+async def test_shadow_eval_responses_name_the_shadowed_key(monkeypatch: pytest.MonkeyPatch):
+    import litellm.proxy.proxy_server as proxy_server
+
+    prisma = _shadow_prisma()
+    prisma.db.litellm_shadowevaljob.find_many = AsyncMock(
+        return_value=[_job_record(), _job_record(id="job-2", api_key_id="deleted-key-hash")]
+    )
+    monkeypatch.setattr(proxy_server, "llm_router", _shadow_router())
+    monkeypatch.setattr(proxy_server, "prisma_client", prisma)
+
+    started = await start_shadow_eval(_start_request(), ADMIN)
+    assert (started.key_alias, started.key_name) == ("prod-alpha", "sk-...lpha")
+
+    jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50)
+    assert [(job.key_alias, job.key_name) for job in jobs] == [("prod-alpha", "sk-...lpha"), (None, None)]
+    batched_where = prisma.db.litellm_verificationtoken.find_many.call_args.kwargs["where"]
+    assert batched_where == {"token": {"in": ["deleted-key-hash", "key-hash"]}}
+
+    prisma.db.litellm_shadowevaljob.find_unique = AsyncMock(return_value=_job_record())
+    detail = await get_shadow_eval_job("job-1", VIEWER)
+    assert detail.key_alias == "prod-alpha"
+
+
 @pytest.mark.asyncio
 async def test_stop_shadow_eval_sets_stopped_at_and_rejects_non_running(monkeypatch: pytest.MonkeyPatch):
     import litellm.proxy.proxy_server as proxy_server

ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx

@@ -60,7 +60,7 @@ vi.mock("@/app/(dashboard)/hooks/models/useModelCostMap", () => ({
   })),
 }));
 
-import ShadowEvalSection from "./ShadowEvalSection";
+import ShadowEvalSection, { shadowedKeyLabel } from "./ShadowEvalSection";
 import {
   useShadowEvalJob,
   useShadowEvalJobs,
@@ -117,6 +117,8 @@ const job = (overrides: Partial<ShadowEvalJob> = {}): ShadowEvalJob => ({
   ends_at: "2026-09-07T00:00:00Z",
   stopped_at: null,
   api_key_id: "hashed-key-abc",
+  key_alias: "prod-alpha",
+  key_name: "sk-...alpha",
   last_error: null,
   ...overrides,
 });
@@ -411,7 +413,11 @@ describe("ShadowEvalSection", () => {
     mockHooks({ jobs: [j], detailsById: { "job-1": j } });
     render(<ShadowEvalSection />);
 
-    expect(screen.getByText(/on 10% of its traffic/)).toBeInTheDocument();
+    expect(
+      screen.getByText(
+        (_, element) => element?.textContent === "Comparing claude-auto to openai/gpt-4o on 10% of prod-alpha traffic",
+      ),
+    ).toBeInTheDocument();
     expect(screen.getByText("Router matched or beat the baseline")).toBeInTheDocument();
     expect(screen.getByText("52.0%")).toBeInTheDocument();
     expect(screen.getByText(/Router won 30.0%/)).toBeInTheDocument();
@@ -422,6 +428,12 @@ describe("ShadowEvalSection", () => {
     expect(screen.queryByText("Compared against")).not.toBeInTheDocument();
   });
 
+  it("labels the shadowed key by alias, then masked name, then truncated hash", () => {
+    expect(shadowedKeyLabel(job())).toBe("prod-alpha");
+    expect(shadowedKeyLabel(job({ key_alias: null }))).toBe("sk-...alpha");
+    expect(shadowedKeyLabel(job({ key_alias: null, key_name: null }))).toBe("hashed-key…");
+  });
+
   it("keeps an older job's verdicts reachable through the previous evaluations list", async () => {
     const user = userEvent.setup();
     const emptyOverrides: Partial<ShadowEvalJob> = {
@@ -440,7 +452,7 @@ describe("ShadowEvalSection", () => {
 
     await user.click(screen.getByRole("button", { name: /Previous evaluations \(1\)/ }));
     expect(screen.getByText("view results")).toBeInTheDocument();
-    await user.click(screen.getByRole("button", { name: /10% via claude-auto/ }));
+    await user.click(screen.getByRole("button", { name: /10% of prod-alpha traffic via claude-auto/ }));
 
     expect(await screen.findByText("SIMPLE")).toBeInTheDocument();
     expect(screen.getByText("REASONING")).toBeInTheDocument();

ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx

@@ -50,15 +50,20 @@ const routerMatchedOrBeatPct = (
     ? 100 - results.overall_shadow_win_rate_pct
     : results.overall_shadow_win_rate_pct + results.overall_tie_rate_pct;
 
+export const shadowedKeyLabel = (job: ShadowEvalJob): string =>
+  job.key_alias || job.key_name || `${job.api_key_id.slice(0, 10)}…`;
+
 const jobHeadline = (job: ShadowEvalJob): React.ReactNode =>
   job.direction === "reverse" ? (
     <>
       Comparing <span className="font-mono text-xs">{job.router_name}</span> to{" "}
-      <span className="font-mono text-xs">{job.baseline_model}</span> on {job.shadow_percentage}% of its traffic
+      <span className="font-mono text-xs">{job.baseline_model}</span> on {job.shadow_percentage}% of{" "}
+      <span className="font-mono text-xs">{shadowedKeyLabel(job)}</span> traffic
     </>
   ) : (
     <>
-      Shadowing {job.shadow_percentage}% via <span className="font-mono text-xs">{job.router_name}</span>
+      Shadowing {job.shadow_percentage}% of <span className="font-mono text-xs">{shadowedKeyLabel(job)}</span> traffic
+      via <span className="font-mono text-xs">{job.router_name}</span>
     </>
   );