feat(shadow-eval): skip self-shadowed requests

litellm/integrations/shadow_eval_logger.py

@@ -238,6 +238,19 @@ async def _key_or_team_is_over_budget(metadata: Mapping[str, object]) -> bool:
     return False
 
 
+def _request_was_routed_by(request_metadata: Mapping[str, object], router_name: str) -> bool:
+    """Whether this request was already served by the router being shadowed.
+
+    Duplicating such a request compares the router to itself: guaranteed ties, judge
+    spend for zero information. Requests routed by a *different* auto-router still
+    sample, which is a meaningful router-vs-router comparison.
+    """
+    decision: Final = request_metadata.get("routing_decision")
+    if not isinstance(decision, Mapping):
+        return False
+    return decision.get("router_model_name") == router_name
+
+
 def _job_is_over_spend_cap(job: ActiveShadowEvalJob) -> bool:
     """Whether the job has spent past what its start-time estimate justifies.
 
@@ -328,6 +341,8 @@ async def async_log_success_event(
                 return
             if payload.get("call_type") not in (None, "completion", "acompletion", "chat_completion"):
                 return  # only chat-shaped traffic is comparable
+            if _request_was_routed_by(request_metadata, job.router_name):
+                return
             if self._inflight_shadow_tasks >= _MAX_CONCURRENT_SHADOW_TASKS:
                 return
             if await _key_or_team_is_over_budget(metadata):

tests/test_litellm/integrations/test_shadow_eval_logger.py

@@ -179,6 +179,47 @@ async def test_counts_seen_for_active_job(self):
         # 0% sampling: request seen but never shadowed.
         assert logger._pending_seen == {"j1": 1}
 
+    @staticmethod
+    def _routed_kwargs(router_model_name: str):
+        return {
+            "standard_logging_object": {
+                "id": "req-1",
+                "model": "gpt-4o-mini",
+                "call_type": "acompletion",
+                "metadata": {"user_api_key_hash": "key-hash"},
+            },
+            "litellm_params": {"metadata": {"routing_decision": {"router_model_name": router_model_name}}},
+            "messages": [{"role": "user", "content": "hi"}],
+        }
+
+    async def test_skips_requests_already_served_by_the_shadowed_router(self):
+        """Duplicating the router's own traffic compares it to itself: paid ties, no signal."""
+        job = ActiveShadowEvalJob(
+            id="j1", router_name="claude-auto", shadow_percentage=100.0, judge_model="m", status="running"
+        )
+        logger, _, router = _logger_with_mocks(job)
+        logger._run_shadow_eval = AsyncMock()
+
+        await logger.async_log_success_event(self._routed_kwargs("claude-auto"), MagicMock(), None, None)
+        seen_before_flush = dict(logger._pending_seen)
+        await asyncio.sleep(0)
+
+        logger._run_shadow_eval.assert_not_awaited()
+        router.acompletion.assert_not_called()
+        assert seen_before_flush == {"j1": 1}, "skipped for judging, but still counted toward requests seen"
+
+    async def test_traffic_from_a_different_router_still_samples(self):
+        job = ActiveShadowEvalJob(
+            id="j1", router_name="claude-auto", shadow_percentage=100.0, judge_model="m", status="running"
+        )
+        logger, _, _ = _logger_with_mocks(job)
+        logger._run_shadow_eval = AsyncMock()
+
+        await logger.async_log_success_event(self._routed_kwargs("other-router"), MagicMock(), None, None)
+        await asyncio.sleep(0.01)
+
+        logger._run_shadow_eval.assert_awaited_once()
+
 
 @pytest.mark.asyncio
 class TestCallRouterShadowForwardsParameters: