Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
-- AlterTable
ALTER TABLE "LiteLLM_ShadowEvalJob" ADD COLUMN "baseline_model" TEXT,
ADD COLUMN "direction" TEXT NOT NULL DEFAULT 'forward';

DROP INDEX IF EXISTS "LiteLLM_ShadowEvalJob_one_active_per_key";

CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_ShadowEvalJob_one_active_per_key_direction"
ON "LiteLLM_ShadowEvalJob"("api_key_id", "direction") WHERE "stopped_at" IS NULL;
13 changes: 9 additions & 4 deletions litellm-proxy-extras/litellm_proxy_extras/schema.prisma
Original file line number Diff line number Diff line change
Expand Up @@ -1450,15 +1450,20 @@ model LiteLLM_AutoRouterSession {
@@index([last_turn_at], map: "idx_autorouter_session_last_turn")
}

// Shadow eval: pre-adoption evaluation of an auto-router against a key's live traffic.
// A sampled slice of requests is duplicated through the router in a detached task and an
// LLM judge compares real vs shadow responses blind. The job row is immutable config plus
// Shadow eval: evaluation of an auto-router against a key's live traffic, in either
// direction. forward duplicates the requests the key did not route through the router
// through it, answering whether the key should adopt it; reverse duplicates the requests
// the router did serve against a fixed baseline model, answering whether a key already on
// it still benefits. Either way a sampled slice runs in a detached task and an LLM judge
// compares real vs shadow responses blind. The job row is immutable config plus
// stopped_at; every count, status, and spend figure is derived from the append-only
// attempt rows, so nothing can disagree across pods or stop races.
model LiteLLM_ShadowEvalJob {
id String @id @default(cuid())
api_key_id String // hashed virtual key whose traffic is shadowed
router_name String
router_name String // the auto-router under evaluation, in either direction
direction String @default("forward") // forward | reverse
baseline_model String? // reverse only: the fixed model the router is judged against
judge_model String
shadow_percentage Float
max_turns Int // sample budget: judge at most this many turns
Expand Down
209 changes: 134 additions & 75 deletions litellm/integrations/shadow_eval_logger.py

Large diffs are not rendered by default.

67 changes: 44 additions & 23 deletions litellm/proxy/management_endpoints/auto_router_endpoints.py
Original file line number Diff line number Diff line change
Expand Up @@ -464,35 +464,38 @@ def _is_configured_pre_routing_strategy(llm_router: "Router", router_name: str)
)


def _validate_judge_model(llm_router: "Router | None", judge_model: str) -> None:
"""Reject a judge model the dispatch path cannot resolve, at start rather than as a
silently growing error count once the job is already sampling and billing."""
if llm_router is not None and _is_configured_pre_routing_strategy(llm_router, judge_model):
def _validate_plain_model(llm_router: "Router | None", model: str, field_name: str) -> None:
"""Reject a model the dispatch path cannot resolve, at start rather than as a silently
growing error count once the job is already sampling and billing. Both the judge and a
reverse job's baseline must be plain models: an auto-router in either slot would
re-route per turn, so the comparison would have no fixed arm to attribute results to."""
if llm_router is not None and _is_configured_pre_routing_strategy(llm_router, model):
raise HTTPException(
status_code=400,
detail=f"judge_model '{judge_model}' is an auto-router; the judge must be a plain model",
detail=f"{field_name} '{model}' is an auto-router; it must be a plain model",
)
if router_resolves_model(llm_router, judge_model):
if router_resolves_model(llm_router, model):
return
import litellm

try:
litellm.get_llm_provider(model=judge_model)
litellm.get_llm_provider(model=model)
except Exception as e:
raise HTTPException(
status_code=400,
detail=(
f"judge_model '{judge_model}' is neither a model configured on this proxy nor a "
f"{field_name} '{model}' is neither a model configured on this proxy nor a "
"provider-qualified public model name (e.g. 'anthropic/claude-sonnet-5')"
),
) from e


def _is_unique_violation(error: Exception) -> bool:
"""Whether a Prisma create failed on a unique index. One active job per key lives in
a partial unique index (raw SQL in the migration; schema.prisma cannot express partial
indexes), so the read-then-create check above it is advisory: two concurrent starts
pass the read, and the loser must surface as the same 409 rather than a 500."""
"""Whether a Prisma create failed on a unique index. One active job per key and
direction lives in a partial unique index (raw SQL in the migration; schema.prisma
cannot express partial indexes), so the read-then-create check above it is advisory:
two concurrent starts pass the read, and the loser must surface as the same 409
rather than a 500."""
try:
from prisma.errors import UniqueViolationError
except ImportError:
Expand Down Expand Up @@ -573,8 +576,10 @@ def _slices(rows: Sequence[_AttemptAggRow]) -> tuple[ShadowEvalSlice, ...]:

async def _shadow_eval_results(prisma_client: "PrismaClient", job_id: str) -> ShadowEvalResult | None:
"""Both stratifications of one job's verdicts. Tier answers "where does the router do
well"; current-model answers "which of the models this key uses today would the router
beat". Reads are bounded by the job's own attempts (<= max_turns) via the job_id index."""
well"; the model stratification groups by whichever model served the real arm, so it
answers "which of the models this key uses today would the router beat" forward, and
"for the turns the router sent to X, did X beat the baseline" in reverse. Reads are
bounded by the job's own attempts (<= max_turns) via the job_id index."""
by_tier: Final = _ATTEMPT_AGG_ROWS.validate_python(
await prisma_client.db.query_raw(_ATTEMPT_AGG_BY_TIER_SQL, job_id) or ()
)
Expand Down Expand Up @@ -604,9 +609,15 @@ async def start_shadow_eval(
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
) -> ShadowEvalJobResponse:
"""
Start a pre-adoption shadow eval: duplicate a sampled slice of a key's live traffic
through an auto-router, judge real vs. shadow responses blind, and stratify win rates
by the router's tier classification and by the incumbent model.
Start a shadow eval: duplicate a sampled slice of a key's live traffic against a second
arm, judge the two responses blind, and stratify win rates by tier and by the model that
served the real arm.

A forward job answers whether the key should adopt router_name: it samples the requests
the router did not serve and duplicates them through it. A reverse job answers whether a
key already on the router still gains from it: it samples the requests the router did
serve and duplicates them against baseline_model. A key can hold one active job per
direction, so both questions can run at once.

Shadow responses are never served to users. The job samples until it has judged
max_turns turns, reaches the end of its window, or is stopped; sampling changes
Expand All @@ -620,7 +631,9 @@ async def start_shadow_eval(
raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value)
if llm_router is None or not _is_configured_pre_routing_strategy(llm_router, data.router_name):
raise HTTPException(status_code=400, detail=f"'{data.router_name}' is not a configured auto-router")
_validate_judge_model(llm_router, data.judge_model)
_validate_plain_model(llm_router, data.judge_model, "judge_model")
if data.baseline_model is not None:
_validate_plain_model(llm_router, data.baseline_model, "baseline_model")
key_row: Final = await prisma_client.db.litellm_verificationtoken.find_unique(
where={"token": data.api_key_id} # mutable-ok: Prisma filter
)
Expand All @@ -634,23 +647,29 @@ async def start_shadow_eval(
)

# A job that expired or exhausted its turn budget stopped sampling on its own, but
# still holds the one-active-per-key partial unique index until stamped; free it so
# a new eval can start.
# still holds its slot in the per-key, per-direction partial unique index until
# stamped; free it so a new eval can start. Sweeping both directions is deliberate.
await prisma_client.db.execute_raw(_SWEEP_FINISHED_JOBS_SQL, data.api_key_id)
active: Final = await prisma_client.db.litellm_shadowevaljob.find_first(
where={"api_key_id": data.api_key_id, "stopped_at": None}, # mutable-ok: Prisma filter
where={ # mutable-ok: Prisma filter
"api_key_id": data.api_key_id,
"direction": data.direction,
"stopped_at": None,
},
)
if active is not None:
raise HTTPException(
status_code=409,
detail=f"Key already has an active shadow eval job ({active.id}). Stop it first.",
detail=f"Key already has an active {data.direction} shadow eval job ({active.id}). Stop it first.",
)
now: Final = datetime.now(timezone.utc)
try:
job: Final = await prisma_client.db.litellm_shadowevaljob.create(
data={ # mutable-ok: Prisma payload
"api_key_id": data.api_key_id,
"router_name": data.router_name,
"direction": data.direction,
"baseline_model": data.baseline_model,
"judge_model": data.judge_model,
"shadow_percentage": data.shadow_percentage,
"max_turns": data.max_turns,
Expand All @@ -663,7 +682,9 @@ async def start_shadow_eval(
raise
raise HTTPException(
status_code=409,
detail="Key already has an active shadow eval job (started concurrently). Stop it first.",
detail=(
f"Key already has an active {data.direction} shadow eval job (started concurrently). Stop it first."
),
) from e
return ShadowEvalJobResponse.model_validate(job, from_attributes=True)

Expand Down
13 changes: 9 additions & 4 deletions litellm/proxy/schema.prisma
Original file line number Diff line number Diff line change
Expand Up @@ -1450,15 +1450,20 @@ model LiteLLM_AutoRouterSession {
@@index([last_turn_at], map: "idx_autorouter_session_last_turn")
}

// Shadow eval: pre-adoption evaluation of an auto-router against a key's live traffic.
// A sampled slice of requests is duplicated through the router in a detached task and an
// LLM judge compares real vs shadow responses blind. The job row is immutable config plus
// Shadow eval: evaluation of an auto-router against a key's live traffic, in either
// direction. forward duplicates the requests the key did not route through the router
// through it, answering whether the key should adopt it; reverse duplicates the requests
// the router did serve against a fixed baseline model, answering whether a key already on
// it still benefits. Either way a sampled slice runs in a detached task and an LLM judge
// compares real vs shadow responses blind. The job row is immutable config plus
// stopped_at; every count, status, and spend figure is derived from the append-only
// attempt rows, so nothing can disagree across pods or stop races.
model LiteLLM_ShadowEvalJob {
id String @id @default(cuid())
api_key_id String // hashed virtual key whose traffic is shadowed
router_name String
router_name String // the auto-router under evaluation, in either direction
direction String @default("forward") // forward | reverse
baseline_model String? // reverse only: the fixed model the router is judged against
judge_model String
shadow_percentage Float
max_turns Int // sample budget: judge at most this many turns
Expand Down
57 changes: 50 additions & 7 deletions litellm/types/management_endpoints/auto_router_endpoints.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from datetime import datetime, timezone
from typing import Final, Literal, TypeAlias

from pydantic import AliasChoices, BaseModel, ConfigDict, Field, computed_field, field_validator
from pydantic import AliasChoices, BaseModel, ConfigDict, Field, computed_field, field_validator, model_validator

from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig
from litellm.types.utils import StandardLoggingRoutingDecision
Expand Down Expand Up @@ -146,19 +146,37 @@ class AutoRouterBenchmarksResponse(BaseModel):

ShadowEvalStatus: TypeAlias = Literal["running", "completed", "stopped"]

ShadowEvalDirection: TypeAlias = Literal["forward", "reverse"]

DEFAULT_SHADOW_EVAL_JUDGE_MODEL: Final[str] = "anthropic/claude-sonnet-5"


class StartShadowEvalRequest(BaseModel):
"""Start shadowing a key's traffic through an auto-router for blind comparison."""
"""Start duplicating a key's traffic for blind comparison against an auto-router."""

api_key_id: str = Field(
description=(
"The hashed virtual key whose traffic will be shadowed. Shadow evaluation runs ONLY on this "
"key's traffic; requests made with any other key are not sampled."
)
)
router_name: str = Field(description="The auto-router config to shadow requests through")
router_name: str = Field(description="The auto-router under evaluation, in either direction")
direction: ShadowEvalDirection = Field(
default="forward",
description=(
"forward answers 'should this key adopt router_name': it samples the requests the key did NOT "
"route through the router and duplicates them through it. reverse answers 'is the router still "
"worth it for a key already on it': it samples the requests the router did serve and duplicates "
"them against baseline_model. The response the caller received is always the real arm"
),
)
baseline_model: str | None = Field(
default=None,
description=(
"Required when direction is reverse and rejected otherwise: the fixed model the router's own "
"responses are judged against. Must be a plain model rather than another auto-router"
),
)
shadow_percentage: float = Field(
ge=0.1,
le=100.0,
Expand Down Expand Up @@ -193,15 +211,33 @@ class StartShadowEvalRequest(BaseModel):
def _round_percentage(cls, value: float) -> float:
return round(value, 2)

@model_validator(mode="after")
def _baseline_model_matches_direction(self) -> "StartShadowEvalRequest":
if self.direction == "reverse" and self.baseline_model is None:
raise ValueError("baseline_model is required when direction is 'reverse'")
if self.direction == "forward" and self.baseline_model is not None:
raise ValueError("baseline_model is only meaningful when direction is 'reverse'")
return self


class ShadowEvalSlice(BaseModel):
"""Judge outcomes for one slice of a job's verdicts (a router tier, or one of the
models the shadowed key currently uses)."""
models that served the real arm)."""

group: str
turn_count: int
real_win_rate_pct: float = Field(description="Share of judged turns where the real (control) model won")
shadow_win_rate_pct: float = Field(description="Share of judged turns where the shadowed router's pick won")
real_win_rate_pct: float = Field(
description=(
"Share of judged turns the real arm won, meaning the response the caller actually received: "
"the key's own model in forward mode, the router's pick in reverse"
)
)
shadow_win_rate_pct: float = Field(
description=(
"Share of judged turns the shadow arm won, meaning the duplicated response nobody was served: "
"the router's pick in forward mode, baseline_model in reverse"
)
)
tie_rate_pct: float
avg_judge_confidence: float

Expand All @@ -210,7 +246,12 @@ class ShadowEvalResult(BaseModel):
"""Stratified results of a shadow-eval job's verdicts so far."""

by_tier: tuple[ShadowEvalSlice, ...]
by_current_model: tuple[ShadowEvalSlice, ...]
by_current_model: tuple[ShadowEvalSlice, ...] = Field(
description=(
"Sliced by the model that served the real arm: the key's incumbent models in forward mode, "
"and in reverse the models the router itself picked"
)
)
overall_shadow_win_rate_pct: float
overall_tie_rate_pct: float

Expand All @@ -226,6 +267,8 @@ class ShadowEvalJobResponse(BaseModel):
job_id: str = Field(validation_alias=AliasChoices("id", "job_id"))
api_key_id: str = Field(description="The hashed virtual key whose traffic this job evaluates, and only that key's")
router_name: str
direction: ShadowEvalDirection = "forward"
baseline_model: str | None = None
judge_model: str
shadow_percentage: float
max_turns: int
Expand Down
13 changes: 9 additions & 4 deletions schema.prisma
Original file line number Diff line number Diff line change
Expand Up @@ -1450,15 +1450,20 @@ model LiteLLM_AutoRouterSession {
@@index([last_turn_at], map: "idx_autorouter_session_last_turn")
}

// Shadow eval: pre-adoption evaluation of an auto-router against a key's live traffic.
// A sampled slice of requests is duplicated through the router in a detached task and an
// LLM judge compares real vs shadow responses blind. The job row is immutable config plus
// Shadow eval: evaluation of an auto-router against a key's live traffic, in either
// direction. forward duplicates the requests the key did not route through the router
// through it, answering whether the key should adopt it; reverse duplicates the requests
// the router did serve against a fixed baseline model, answering whether a key already on
// it still benefits. Either way a sampled slice runs in a detached task and an LLM judge
// compares real vs shadow responses blind. The job row is immutable config plus
// stopped_at; every count, status, and spend figure is derived from the append-only
// attempt rows, so nothing can disagree across pods or stop races.
model LiteLLM_ShadowEvalJob {
id String @id @default(cuid())
api_key_id String // hashed virtual key whose traffic is shadowed
router_name String
router_name String // the auto-router under evaluation, in either direction
direction String @default("forward") // forward | reverse
baseline_model String? // reverse only: the fixed model the router is judged against
judge_model String
shadow_percentage Float
max_turns Int // sample budget: judge at most this many turns
Expand Down
Loading
Loading