feat: pre-adoption shadow eval for the auto-router (backend)

litellm-proxy-extras/litellm_proxy_extras/migrations/20260811125249_add_shadow_eval/migration.sql

@@ -0,0 +1,55 @@
+-- CreateTable
+CREATE TABLE "LiteLLM_ShadowEvalJob" (
+    "id" TEXT NOT NULL,
+    "api_key_id" TEXT NOT NULL,
+    "router_name" TEXT NOT NULL,
+    "shadow_percentage" DOUBLE PRECISION NOT NULL,
+    "judge_model" TEXT NOT NULL,
+    "status" TEXT NOT NULL DEFAULT 'pending',
+    "request_count" INTEGER NOT NULL DEFAULT 0,
+    "completed_count" INTEGER NOT NULL DEFAULT 0,
+    "failed_count" INTEGER NOT NULL DEFAULT 0,
+    "last_error" TEXT,
+    "cost_estimate" DOUBLE PRECISION,
+    "cost_actual" DOUBLE PRECISION NOT NULL DEFAULT 0,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "created_by" TEXT,
+    "ends_at" TIMESTAMP(3),
+    "completed_at" TIMESTAMP(3),
+
+    CONSTRAINT "LiteLLM_ShadowEvalJob_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "LiteLLM_ShadowEvalVerdict" (
+    "id" TEXT NOT NULL,
+    "job_id" TEXT NOT NULL,
+    "request_id" TEXT NOT NULL,
+    "tier_classification" TEXT,
+    "real_model" TEXT NOT NULL,
+    "shadow_model" TEXT NOT NULL,
+    "judge_preference" TEXT NOT NULL,
+    "judge_confidence" DOUBLE PRECISION,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "LiteLLM_ShadowEvalVerdict_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "LiteLLM_ShadowEvalJob_api_key_id_status_idx" ON "LiteLLM_ShadowEvalJob"("api_key_id", "status");
+
+-- CreateIndex
+CREATE INDEX "LiteLLM_ShadowEvalJob_status_idx" ON "LiteLLM_ShadowEvalJob"("status");
+
+-- CreateIndex
+CREATE INDEX "LiteLLM_ShadowEvalJob_created_at_idx" ON "LiteLLM_ShadowEvalJob"("created_at");
+
+-- CreateIndex
+CREATE INDEX "LiteLLM_ShadowEvalVerdict_job_id_idx" ON "LiteLLM_ShadowEvalVerdict"("job_id");
+
+-- One active job per key, enforced by the database rather than a read-then-create in
+-- the start endpoint, which races against a concurrent start on another pod. Partial
+-- indexes are not expressible in schema.prisma, so this lives here only.
+CREATE UNIQUE INDEX "LiteLLM_ShadowEvalJob_one_active_per_key"
+    ON "LiteLLM_ShadowEvalJob"("api_key_id") WHERE status IN ('pending', 'running');
+

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -1450,6 +1450,50 @@ model LiteLLM_AutoRouterSession {
   @@index([last_turn_at], map: "idx_autorouter_session_last_turn")
 }
 
+// Shadow Eval: pre-adoption evaluation of an auto-router against a key's live traffic.
+// A sampled slice of requests is duplicated through the router in a detached task and
+// an LLM judge compares real vs shadow responses blind; verdicts stratify by tier.
+model LiteLLM_ShadowEvalJob {
+  id                  String   @id @default(cuid())
+  api_key_id          String   // the hashed virtual key whose traffic is shadowed
+  router_name         String   // the auto-router config to shadow through
+  shadow_percentage   Float
+  judge_model         String
+
+  status              String   @default("pending") // pending | running | completed
+
+  request_count       Int      @default(0) // requests seen on the key while active
+  completed_count     Int      @default(0) // verdicts written
+  failed_count        Int      @default(0) // shadow or judge calls that errored
+  last_error          String?
+
+  cost_estimate       Float?   // upfront judge-spend estimate shown at start
+  cost_actual         Float    @default(0) // running judge-call spend
+
+  created_at          DateTime @default(now())
+  created_by          String?
+  ends_at             DateTime?
+  completed_at        DateTime?
+
+  @@index([api_key_id, status])
+  @@index([status])
+  @@index([created_at])
+}
+
+model LiteLLM_ShadowEvalVerdict {
+  id                   String   @id @default(cuid())
+  job_id               String
+  request_id           String   // the judged real request
+  tier_classification  String?  // the router's tier for the prompt, when classified
+  real_model           String   // model that actually served the request
+  shadow_model         String   // model the router picked
+  judge_preference     String   // real | shadow | tie
+  judge_confidence     Float?
+  created_at           DateTime @default(now())
+
+  @@index([job_id])
+}
+
 // ---------------------------------------------------------------------------
 // Workflow Run Tracking
 //