diff --git a/litellm/integrations/shadow_eval_logger.py b/litellm/integrations/shadow_eval_logger.py index d8292574ac5d..cb895ccecd0d 100644 --- a/litellm/integrations/shadow_eval_logger.py +++ b/litellm/integrations/shadow_eval_logger.py @@ -238,6 +238,19 @@ async def _key_or_team_is_over_budget(metadata: Mapping[str, object]) -> bool: return False +def _request_was_routed_by(request_metadata: Mapping[str, object], router_name: str) -> bool: + """Whether this request was already served by the router being shadowed. + + Duplicating such a request compares the router to itself: guaranteed ties, judge + spend for zero information. Requests routed by a *different* auto-router still + sample, which is a meaningful router-vs-router comparison. + """ + decision: Final = request_metadata.get("routing_decision") + if not isinstance(decision, Mapping): + return False + return decision.get("router_model_name") == router_name + + def _job_is_over_spend_cap(job: ActiveShadowEvalJob) -> bool: """Whether the job has spent past what its start-time estimate justifies. @@ -328,6 +341,8 @@ async def async_log_success_event( return if payload.get("call_type") not in (None, "completion", "acompletion", "chat_completion"): return # only chat-shaped traffic is comparable + if _request_was_routed_by(request_metadata, job.router_name): + return if self._inflight_shadow_tasks >= _MAX_CONCURRENT_SHADOW_TASKS: return if await _key_or_team_is_over_budget(metadata): diff --git a/tests/test_litellm/integrations/test_shadow_eval_logger.py b/tests/test_litellm/integrations/test_shadow_eval_logger.py index 78ac8ccb0f45..20b08f66b699 100644 --- a/tests/test_litellm/integrations/test_shadow_eval_logger.py +++ b/tests/test_litellm/integrations/test_shadow_eval_logger.py @@ -179,6 +179,47 @@ async def test_counts_seen_for_active_job(self): # 0% sampling: request seen but never shadowed. assert logger._pending_seen == {"j1": 1} + @staticmethod + def _routed_kwargs(router_model_name: str): + return { + "standard_logging_object": { + "id": "req-1", + "model": "gpt-4o-mini", + "call_type": "acompletion", + "metadata": {"user_api_key_hash": "key-hash"}, + }, + "litellm_params": {"metadata": {"routing_decision": {"router_model_name": router_model_name}}}, + "messages": [{"role": "user", "content": "hi"}], + } + + async def test_skips_requests_already_served_by_the_shadowed_router(self): + """Duplicating the router's own traffic compares it to itself: paid ties, no signal.""" + job = ActiveShadowEvalJob( + id="j1", router_name="claude-auto", shadow_percentage=100.0, judge_model="m", status="running" + ) + logger, _, router = _logger_with_mocks(job) + logger._run_shadow_eval = AsyncMock() + + await logger.async_log_success_event(self._routed_kwargs("claude-auto"), MagicMock(), None, None) + seen_before_flush = dict(logger._pending_seen) + await asyncio.sleep(0) + + logger._run_shadow_eval.assert_not_awaited() + router.acompletion.assert_not_called() + assert seen_before_flush == {"j1": 1}, "skipped for judging, but still counted toward requests seen" + + async def test_traffic_from_a_different_router_still_samples(self): + job = ActiveShadowEvalJob( + id="j1", router_name="claude-auto", shadow_percentage=100.0, judge_model="m", status="running" + ) + logger, _, _ = _logger_with_mocks(job) + logger._run_shadow_eval = AsyncMock() + + await logger.async_log_success_event(self._routed_kwargs("other-router"), MagicMock(), None, None) + await asyncio.sleep(0.01) + + logger._run_shadow_eval.assert_awaited_once() + @pytest.mark.asyncio class TestCallRouterShadowForwardsParameters: