feat(auto-router)!: scope shadow eval jobs to multiple keys

litellm-proxy-extras/litellm_proxy_extras/migrations/20260817000000_shadow_eval_multi_key/migration.sql

@@ -0,0 +1,7 @@
+ALTER TABLE "LiteLLM_ShadowEvalJob" ADD COLUMN IF NOT EXISTS "group_id" TEXT;
+
+UPDATE "LiteLLM_ShadowEvalJob" SET "group_id" = "id" WHERE "group_id" IS NULL;
+
+ALTER TABLE "LiteLLM_ShadowEvalJob" ALTER COLUMN "group_id" SET NOT NULL;
+
+CREATE INDEX IF NOT EXISTS "LiteLLM_ShadowEvalJob_group_id_idx" ON "LiteLLM_ShadowEvalJob"("group_id");

litellm-proxy-extras/litellm_proxy_extras/migrations/20260818224500_add_shadow_eval_stopped_by/migration.sql

@@ -0,0 +1,5 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_ShadowEvalJob" ADD COLUMN     "stopped_by" TEXT;
+
+UPDATE "LiteLLM_ShadowEvalJob" SET stopped_by = 'unknown'
+WHERE stopped_at IS NOT NULL AND ends_at > (NOW() AT TIME ZONE 'utc');

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -1465,28 +1465,38 @@ model LiteLLM_AutoRouterSession {
   @@index([last_turn_at], map: "idx_autorouter_session_last_turn")
 }
 
-// Shadow eval: evaluation of an auto-router against a key's live traffic, in either
-// direction. forward duplicates the requests the key did not route through the router
-// through it, answering whether the key should adopt it; reverse duplicates the requests
-// the router did serve against a fixed baseline model, answering whether a key already on
-// it still benefits. Either way a sampled slice runs in a detached task and an LLM judge
-// compares real vs shadow responses blind. The job row is immutable config plus
-// stopped_at; every count, status, and spend figure is derived from the append-only
-// attempt rows, so nothing can disagree across pods or stop races.
+// Shadow eval: evaluation of an auto-router against one or more keys' live traffic, in
+// either direction. forward duplicates the requests the keys did not route through the
+// router through it, answering whether they should adopt it; reverse duplicates the
+// requests the router did serve against a fixed baseline model, answering whether a key
+// already on it still benefits. Either way a sampled slice runs in a detached task and an
+// LLM judge compares real vs shadow responses blind. Each row is ONE key's leg of a job:
+// immutable config plus that key's own turn budget and stop state, so one key exhausting
+// its budget never ends a sibling's sampling. A job is the set of legs sharing group_id
+// (the id the API reports), written together by one atomic create_many with identical
+// config; single-key jobs predating group_id were backfilled group_id = id. "One active
+// job per (key, direction)" is a partial unique index on (api_key_id, direction) WHERE
+// stopped_at IS NULL, expressed only in the migration because schema.prisma cannot state
+// partial indexes; it is what makes a concurrent start on another pod race-safe rather
+// than read-then-create. Every count, status, and spend figure is derived from the
+// append-only attempt rows, so nothing can disagree across pods or stop races.
 model LiteLLM_ShadowEvalJob {
   id                String   @id @default(cuid())
-  api_key_id        String   // hashed virtual key whose traffic is shadowed
+  group_id          String   // legs of one job share this; the API's job id
+  api_key_id        String   // hashed virtual key whose traffic this leg shadows
   router_name       String   // the auto-router under evaluation, in either direction
   direction         String   @default("forward") // forward | reverse
   baseline_model    String?  // reverse only: the fixed model the router is judged against
   judge_model       String
   shadow_percentage Float
-  max_turns         Int      // sample budget: judge at most this many turns
+  max_turns         Int      // this key's sample budget: judge at most this many turns
   created_at        DateTime @default(now())
   created_by        String?
   ends_at           DateTime
   stopped_at        DateTime?
+  stopped_by        String?  // operator who stopped it early; null when it ended on its own
 
+  @@index([group_id])
   @@index([api_key_id])
   @@index([created_at])
 }