From 8d5e0339298297eb148164309d0ba9743c128fd3 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 14 Jul 2026 11:22:46 -0700 Subject: [PATCH 1/9] feat(router): resolve auto-router routing plugins from proxy YAML config Router(plugins=[...]) was Python-SDK constructor only, so proxy/YAML users had no way to configure it, and the merged pipeline narrowed candidates from the outer model alias rather than the auto-router's actual tier pool, making it a no-op for auto_router deployments. Add complexity_router_config.plugins (dotted-path strings resolved via get_instance_fn, the same convention litellm_settings.callbacks uses) and run the resolved plugins against ComplexityRouter's tier pool at every model-pick site, so a policy plugin narrows what get_model_for_tier actually returns instead of the outer alias list. adaptive=True with plugins set now raises at config validation instead of silently ignoring the plugins, since the bandit selector doesn't consume narrowed pools yet. Also fixes a latent bug in Router._generate_model_id: it json.dumps every litellm_params dict value to build a deployment hash id, which crashed once a live plugin object could land inside complexity_router_config. --- litellm/proxy/proxy_server.py | 10 ++ litellm/router.py | 4 +- .../complexity_router/complexity_router.py | 41 ++++- .../complexity_router/config.py | 18 ++- litellm/types/router.py | 3 +- .../router_strategy/test_complexity_router.py | 149 ++++++++++++++++++ 6 files changed, 217 insertions(+), 8 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 37f3d6e49e09..6ce41a48b97e 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -4720,6 +4720,16 @@ async def load_config(self, router: Optional[litellm.Router], config_file_path: for k, v in model["litellm_params"].items(): if isinstance(v, str) and v.startswith("os.environ/"): model["litellm_params"][k] = get_secret(v) + complexity_router_config = model["litellm_params"].get("complexity_router_config") + if isinstance(complexity_router_config, dict): + plugin_paths = complexity_router_config.get("plugins") + if isinstance(plugin_paths, list): + complexity_router_config["plugins"] = [ + get_instance_fn(value=plugin_path, config_file_path=config_file_path) + if isinstance(plugin_path, str) + else plugin_path + for plugin_path in plugin_paths + ] print(f"\033[32m {model.get('model_name', '')}\033[0m") # noqa: T201 litellm_model_name = model["litellm_params"]["model"] litellm_model_api_base = model["litellm_params"].get("api_base", None) diff --git a/litellm/router.py b/litellm/router.py index 6e8127110cc7..d73f739cb8e8 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -7299,14 +7299,14 @@ def _generate_model_id(self, model_group: str, litellm_params: dict): if isinstance(k, str): parts.append(k) elif isinstance(k, dict): - parts.append(json.dumps(k)) + parts.append(json.dumps(k, default=str)) else: parts.append(str(k)) if isinstance(v, str): parts.append(v) elif isinstance(v, dict): - parts.append(json.dumps(v)) + parts.append(json.dumps(v, default=str)) else: parts.append(str(v)) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index bd3b300b5585..3b3444ba6844 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -468,6 +468,38 @@ def _pick_from_tier_value(model: str | list[str], tier_key: str) -> str: def _tier_pools(self) -> dict[str, list[str]]: return {tier: (models if isinstance(models, list) else [models]) for tier, models in self.config.tiers.items()} + async def _pick_model_for_tier( + self, + tier: ComplexityTier, + raw_messages: list[dict[str, Any]] | None, + resolved_messages: list[dict[str, Any]] | None, + request_kwargs: dict, + ) -> str: + if not self.config.plugins: + return self.get_model_for_tier(tier) + + from litellm.types.router import RoutingContext + + tier_key = tier.value + metadata_key = "litellm_metadata" if "litellm_metadata" in request_kwargs else "metadata" + context = RoutingContext( + raw_messages=raw_messages or [], + structured_messages=resolved_messages or [], + candidate_models=list(self._tier_pools().get(tier_key, [])), + metadata=request_kwargs.get(metadata_key) or {}, + ) + for plugin in self.config.plugins: + context = await plugin.run(context) + + if not context.candidate_models: + if self.config.default_model: + return self.config.default_model + raise ValueError( + f"No candidate models left for tier {tier_key} after routing-plugin filtering, " + "and no default_model configured" + ) + return self._pick_from_tier_value(context.candidate_models, tier_key) + def _ensure_adaptive_router(self) -> Any | None: if not self.config.adaptive: return None @@ -947,14 +979,17 @@ async def _classify_and_route( if user_message is None: verbose_router_logger.debug("ComplexityRouter: No user message found, routing to default model") + routed_model = self.config.default_model or await self._pick_model_for_tier( + ComplexityTier.MEDIUM, messages, resolved_messages, request_kwargs + ) return PreRoutingHookResponse( - model=self.config.default_model or self.get_model_for_tier(ComplexityTier.MEDIUM), + model=routed_model, messages=messages if has_original_messages else None, ) override_tier = await self._resolve_keyword_tier_override(user_message, request_kwargs) if override_tier is not None: - routed_model = self.get_model_for_tier(override_tier) + routed_model = await self._pick_model_for_tier(override_tier, messages, resolved_messages, request_kwargs) cause = "semantic_keyword_match" if self.config.semantic_keyword_matching else "literal_keyword_match" verbose_router_logger.info( f"ComplexityRouter: routing decision cause={cause}, " @@ -980,7 +1015,7 @@ async def _classify_and_route( f"signals={signals}, routed_model={routed_model}" ) else: - routed_model = self.get_model_for_tier(tier) + routed_model = await self._pick_model_for_tier(tier, messages, resolved_messages, request_kwargs) verbose_router_logger.info( f"ComplexityRouter: routing decision cause=complexity_scorer, tier={tier.value}, " f"score={score:.3f}, signals={signals}, routed_model={routed_model}" diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index e4bd36505e67..b7ffa2866f22 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -10,7 +10,7 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator -from litellm.types.router import AdaptiveRouterWeights +from litellm.types.router import AdaptiveRouterWeights, RoutingPlugin class ComplexityTier(str, Enum): @@ -375,7 +375,12 @@ class ComplexityRouterConfig(BaseModel): description="TTL for the session affinity pin; refreshed on every cache hit", ) - model_config = ConfigDict(extra="allow") # Allow additional fields + plugins: list[RoutingPlugin] | None = Field( + default=None, + description="RoutingPlugin instances that narrow the classified tier's candidate models before selection", + ) + + model_config = ConfigDict(extra="allow", arbitrary_types_allowed=True) # Allow additional fields @field_validator("tiers", mode="before") @classmethod @@ -421,6 +426,15 @@ def _validate_semantic_matching(self) -> "ComplexityRouterConfig": raise ValueError("keyword_tier_rules must be non-empty when semantic_keyword_matching is enabled") return self + @model_validator(mode="after") + def _validate_plugins_adaptive_combo(self) -> "ComplexityRouterConfig": + if self.plugins and self.adaptive: + raise ValueError( + "plugins and adaptive=True cannot both be set: adaptive's bandit selection doesn't yet " + "consume plugin-narrowed candidate pools. Disable adaptive or remove plugins." + ) + return self + # Combined default config DEFAULT_COMPLEXITY_CONFIG = ComplexityRouterConfig() diff --git a/litellm/types/router.py b/litellm/types/router.py index 3bedd97c20ce..d62c613bf577 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -9,7 +9,7 @@ import httpx from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator -from typing_extensions import Protocol, Required, TypedDict +from typing_extensions import Protocol, Required, TypedDict, runtime_checkable from litellm._uuid import uuid @@ -852,6 +852,7 @@ class RoutingContext(BaseModel): signals: dict[str, Any] = Field(default_factory=dict) +@runtime_checkable class RoutingPlugin(Protocol): """Interface a custom routing plugin must implement to run in `Router(plugins=[...])`.""" diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index f7c9f343f80c..88920e35e059 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -2700,3 +2700,152 @@ async def test_adaptive_pinned_turn_still_stamps_chosen_model_metadata(self, moc spy_aclassify.assert_not_called() assert second.model == "cheap" assert request_kwargs_2["metadata"]["adaptive_router_chosen_model"] == "cheap" + + +class _DummyPlugin: + async def run(self, context): + return context + + +class TestRoutingPlugins: + """Test the `complexity_router_config.plugins` field: narrows the classified + tier's candidate pool before a model is picked. Discussion: + https://github.com/BerriAI/litellm/discussions/32168""" + + @pytest.mark.asyncio + async def test_plugin_narrows_tier_candidates(self, mock_router_instance): + class ExcludeGpt4oMini: + async def run(self, context): + context.candidate_models = [m for m in context.candidate_models if m != "gpt-4o-mini"] + return context + + router = ComplexityRouter( + model_name="test-complexity-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={ + "tiers": {"SIMPLE": ["gpt-4o-mini", "gpt-4o-nano"]}, + "plugins": [ExcludeGpt4oMini()], + }, + ) + result = await router.async_pre_routing_hook( + model="test-model", + request_kwargs={}, + messages=[{"role": "user", "content": "hi"}], + ) + assert result is not None + assert result.model == "gpt-4o-nano" + + @pytest.mark.asyncio + async def test_plugin_narrowing_to_zero_falls_back_to_default_model(self, mock_router_instance): + class BlockEverything: + async def run(self, context): + context.candidate_models = [] + return context + + router = ComplexityRouter( + model_name="test-complexity-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={ + "tiers": {"SIMPLE": "gpt-4o-mini"}, + "default_model": "gpt-4o-fallback", + "plugins": [BlockEverything()], + }, + ) + result = await router.async_pre_routing_hook( + model="test-model", + request_kwargs={}, + messages=[{"role": "user", "content": "hi"}], + ) + assert result is not None + assert result.model == "gpt-4o-fallback" + + @pytest.mark.asyncio + async def test_plugin_narrowing_to_zero_without_default_model_raises(self, mock_router_instance): + class BlockEverything: + async def run(self, context): + context.candidate_models = [] + return context + + router = ComplexityRouter( + model_name="test-complexity-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={ + "tiers": {"SIMPLE": "gpt-4o-mini"}, + "plugins": [BlockEverything()], + }, + ) + with pytest.raises(ValueError, match="No candidate models left for tier"): + await router.async_pre_routing_hook( + model="test-model", + request_kwargs={}, + messages=[{"role": "user", "content": "hi"}], + ) + + @pytest.mark.asyncio + async def test_plugin_receives_metadata_from_request_kwargs(self, mock_router_instance): + captured = {} + + class CaptureMetadata: + async def run(self, context): + captured.update(context.metadata) + return context + + router = ComplexityRouter( + model_name="test-complexity-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={ + "tiers": {"SIMPLE": "gpt-4o-mini"}, + "plugins": [CaptureMetadata()], + }, + ) + await router.async_pre_routing_hook( + model="test-model", + request_kwargs={"metadata": {"tenant": "acme-corp"}}, + messages=[{"role": "user", "content": "hi"}], + ) + assert captured.get("tenant") == "acme-corp" + + @pytest.mark.asyncio + async def test_plugin_applies_to_keyword_tier_override(self, mock_router_instance): + """A policy plugin must not be bypassable via the keyword_tier_rules override path.""" + + class ExcludeGpt4oMini: + async def run(self, context): + context.candidate_models = [m for m in context.candidate_models if m != "gpt-4o-mini"] + return context + + router = ComplexityRouter( + model_name="test-complexity-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={ + "tiers": {"SIMPLE": ["gpt-4o-mini", "gpt-4o-nano"]}, + "keyword_tier_rules": [{"keywords": ["hello"], "tier": "SIMPLE"}], + "plugins": [ExcludeGpt4oMini()], + }, + ) + result = await router.async_pre_routing_hook( + model="test-model", + request_kwargs={}, + messages=[{"role": "user", "content": "hello there"}], + ) + assert result is not None + assert result.model == "gpt-4o-nano" + + def test_plugins_and_adaptive_together_raises(self): + with pytest.raises(ValidationError, match="plugins and adaptive=True cannot both be set"): + ComplexityRouterConfig( + tiers={"SIMPLE": ["gpt-4o-mini"]}, + adaptive=True, + plugins=[_DummyPlugin()], + ) + + @pytest.mark.asyncio + async def test_no_plugins_configured_is_unaffected(self, complexity_router): + """Regression guard: a ComplexityRouter with no `plugins` configured behaves exactly as before.""" + result = await complexity_router.async_pre_routing_hook( + model="test-model", + request_kwargs={}, + messages=[{"role": "user", "content": "Hello!"}], + ) + assert result is not None + assert result.model == "gpt-4o-mini" From 64bf705d1b3bc5684bf8fb2882defd643f579fd7 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 14 Jul 2026 12:27:13 -0700 Subject: [PATCH 2/9] fix(router): use stable class name, not object repr, in model-id json fallback json.dumps(v, default=str) on a litellm_params dict containing a live RoutingPlugin instance fell back to object.__repr__'s default , embedding the instance's memory address. _generate_model_id's hash (and therefore the deployment id) changed on every process restart/hot-reload for any deployment with complexity_router_config.plugins configured, defeating the function's own "consistently generate the same id" contract and orphaning anything keyed on that id across restarts (e.g. Redis-backed per-deployment state). Use the plugin's fully-qualified class name instead, which is stable across restarts. --- litellm/router.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index d73f739cb8e8..3d49d7048e42 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -7284,6 +7284,14 @@ async def async_callback_filter_deployments( raise e return returned_healthy_deployments + @staticmethod + def _json_default_stable_id(value: object) -> str: + """json.dumps default= for _generate_model_id: plain str() on an arbitrary + object (e.g. a RoutingPlugin instance) falls back to object.__repr__'s + ``, so the hash -- and deployment id -- would + change every restart. Use the class name instead, stable across restarts.""" + return f"{type(value).__module__}.{type(value).__qualname__}" + def _generate_model_id(self, model_group: str, litellm_params: dict): """ Helper function to consistently generate the same id for a deployment @@ -7299,14 +7307,14 @@ def _generate_model_id(self, model_group: str, litellm_params: dict): if isinstance(k, str): parts.append(k) elif isinstance(k, dict): - parts.append(json.dumps(k, default=str)) + parts.append(json.dumps(k, default=self._json_default_stable_id)) else: parts.append(str(k)) if isinstance(v, str): parts.append(v) elif isinstance(v, dict): - parts.append(json.dumps(v, default=str)) + parts.append(json.dumps(v, default=self._json_default_stable_id)) else: parts.append(str(v)) From 038f9e1e1b7461ba64298f934972d780ccaa5a49 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 14 Jul 2026 12:48:43 -0700 Subject: [PATCH 3/9] test(router): cover _json_default_stable_id for router_code_coverage gate router_code_coverage.py's AST scanner requires every router.py function be called by name somewhere in tests/, and flagged the new _json_default_stable_id helper from the previous commit. --- .../test_router_routing_plugins.py | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/test_litellm/router_strategy/test_router_routing_plugins.py b/tests/test_litellm/router_strategy/test_router_routing_plugins.py index e9c12d009e27..78ed71f5ffd2 100644 --- a/tests/test_litellm/router_strategy/test_router_routing_plugins.py +++ b/tests/test_litellm/router_strategy/test_router_routing_plugins.py @@ -218,3 +218,36 @@ def test_filter_by_routing_plugin_candidates_narrows_and_raises_when_empty(): healthy_deployments=healthy_deployments, request_kwargs={"metadata": {"_routing_plugin_candidate_models": ["nonexistent/model"]}}, ) + + +def test_json_default_stable_id_is_stable_across_instances(): + """_generate_model_id's json.dumps `default=` fallback must not embed an object's + memory address (e.g. plain str() on an object with no custom __repr__ falls back + to object.__repr__'s ``) -- that would make the + deployment id churn on every process restart for any deployment whose + litellm_params contain a live plugin instance.""" + router = Router(model_list=_smart_router_model_list()) + + assert router._json_default_stable_id(LanguageDetector()) == router._json_default_stable_id(LanguageDetector()) + assert router._json_default_stable_id(LanguageDetector()) != router._json_default_stable_id(TenantPolicy()) + + +def test_generate_model_id_is_stable_when_litellm_params_contain_a_plugin_instance(): + """End-to-end: a deployment id built from litellm_params containing a routing + plugin instance (e.g. complexity_router_config.plugins) must be identical across + separate calls, not just non-crashing.""" + router = Router(model_list=_smart_router_model_list()) + litellm_params = { + "model": "auto_router/complexity_router", + "complexity_router_config": {"plugins": [LanguageDetector()]}, + } + + id1 = router._generate_model_id("smart-router", litellm_params) + id2 = router._generate_model_id( + "smart-router", + { + "model": "auto_router/complexity_router", + "complexity_router_config": {"plugins": [LanguageDetector()]}, + }, + ) + assert id1 == id2 From 585121b128634be0d1439b2f27c35e3db95ef8d9 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 14 Jul 2026 13:14:30 -0700 Subject: [PATCH 4/9] fix(router): close two routing-plugin policy-bypass gaps flagged by Veria AI Session-affinity pin shortcut: async_pre_routing_hook returned a session's first-turn pinned model on every later turn without ever re-running it through the plugin pipeline, so a policy plugin (e.g. a budget cap crossed mid-session) was only enforced on turn one. Now the pin shortcut is disabled whenever plugins are configured, so every turn re-runs _classify_and_route (and therefore the plugins). Plugin resolution validation: get_instance_fn accepts any dotted path and returns whatever object it finds there, so a misconfigured complexity_router_config.plugins entry passed proxy startup silently and only surfaced as a confusing AttributeError on the first request that reached the plugin pipeline. Extracted the resolution logic into resolve_complexity_router_plugins() and added an isinstance(..., RoutingPlugin) check that fails proxy startup immediately with a clear error instead. --- litellm/proxy/proxy_server.py | 47 +++++++++++++++--- .../complexity_router/complexity_router.py | 8 ++- .../proxy/proxy_server/test_proxy_config.py | 49 +++++++++++++++++++ .../router_strategy/test_complexity_router.py | 35 +++++++++++++ 4 files changed, 130 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 6ce41a48b97e..18333922eff5 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -556,6 +556,7 @@ def generate_feedback_box(): from litellm.types.router import ModelInfo as RouterModelInfo from litellm.types.router import ( RouterGeneralSettings, + RoutingPlugin, SearchToolTypedDict, updateDeployment, ) @@ -3660,6 +3661,39 @@ def _attach_redis_usage_cache(redis_cache: RedisCache, enable_redis_auth_cache: litellm_config_cache.redis_cache = redis_cache +def resolve_complexity_router_plugins( + model_name: str, + complexity_router_config: dict, + config_file_path: Optional[str], +) -> None: + """ + Resolves `complexity_router_config["plugins"]` dotted-path strings to live + instances via `get_instance_fn` (the same convention `litellm_settings.callbacks` + uses), in place. Raises at config-load time if a path resolves to something that + doesn't implement `RoutingPlugin`, rather than deferring to a confusing + `AttributeError` on the first request that reaches the plugin pipeline. + """ + plugin_paths = complexity_router_config.get("plugins") + if not isinstance(plugin_paths, list): + return + + resolved_plugins = [ + get_instance_fn(value=plugin_path, config_file_path=config_file_path) + if isinstance(plugin_path, str) + else plugin_path + for plugin_path in plugin_paths + ] + for plugin_path, resolved_plugin in zip(plugin_paths, resolved_plugins): + if not isinstance(resolved_plugin, RoutingPlugin): + raise ValueError( + f"complexity_router_config.plugins entry {plugin_path!r} on model {model_name!r} " + f"resolved to {resolved_plugin!r}, which does not implement the RoutingPlugin " + "interface (an async `run(context)` method). Fix the referenced module before " + "starting the proxy." + ) + complexity_router_config["plugins"] = resolved_plugins + + class ProxyConfig: """ Abstraction class on top of config loading/updating logic. Gives us one place to control all config updating logic. @@ -4722,14 +4756,11 @@ async def load_config(self, router: Optional[litellm.Router], config_file_path: model["litellm_params"][k] = get_secret(v) complexity_router_config = model["litellm_params"].get("complexity_router_config") if isinstance(complexity_router_config, dict): - plugin_paths = complexity_router_config.get("plugins") - if isinstance(plugin_paths, list): - complexity_router_config["plugins"] = [ - get_instance_fn(value=plugin_path, config_file_path=config_file_path) - if isinstance(plugin_path, str) - else plugin_path - for plugin_path in plugin_paths - ] + resolve_complexity_router_plugins( + model_name=model.get("model_name", ""), + complexity_router_config=complexity_router_config, + config_file_path=config_file_path, + ) print(f"\033[32m {model.get('model_name', '')}\033[0m") # noqa: T201 litellm_model_name = model["litellm_params"]["model"] litellm_model_api_base = model["litellm_params"].get("api_base", None) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 3b3444ba6844..c954975dcefa 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -893,10 +893,16 @@ async def async_pre_routing_hook( When `session_affinity` is enabled and a session_id is resolvable on the request, pins the model chosen on the session's first turn and reuses it for every later turn, skipping classification entirely. Otherwise delegates to `_classify_and_route`. + + Skipped entirely when `plugins` are configured: reusing a stale pin would bypass + the plugin pipeline on every turn after the first, since a pinned model was never + re-checked against a policy plugin whose decision can change between turns (e.g. a + budget plugin, once the session's spend crosses its cap). """ from litellm.types.router import PreRoutingHookResponse - session_id = self._get_session_id_from_request_kwargs(request_kwargs) if self.config.session_affinity else None + use_session_affinity = self.config.session_affinity and not self.config.plugins + session_id = self._get_session_id_from_request_kwargs(request_kwargs) if use_session_affinity else None cache_key = self._get_session_affinity_cache_key(session_id, request_kwargs) if session_id is not None else None if cache_key is not None: diff --git a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py index 98b270788dc6..7b43a5baf86b 100644 --- a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py @@ -21,6 +21,7 @@ _is_remote_module_url, _scrub_db_overlay_remote_module_loads, _scrub_guardrail_inner, + resolve_complexity_router_plugins, ) from .conftest import normalize @@ -112,6 +113,54 @@ def test__scrub_db_overlay_remote_module_loads_invalid_non_dict_returns_input(): assert _scrub_db_overlay_remote_module_loads("litellm_settings", "raw") == "raw" +# --------------------------------------------------------------------------- +# resolve_complexity_router_plugins +# --------------------------------------------------------------------------- + + +def test_resolve_complexity_router_plugins_no_plugins_key_is_a_noop(): + config: Dict[str, Any] = {"tiers": {"SIMPLE": "gpt-4o-mini"}} + resolve_complexity_router_plugins( + model_name="smart-router", complexity_router_config=config, config_file_path=None + ) + assert config == {"tiers": {"SIMPLE": "gpt-4o-mini"}} + + +def test_resolve_complexity_router_plugins_resolves_dotted_path_to_live_instance(tmp_path): + plugin_file = tmp_path / "my_plugin.py" + plugin_file.write_text( + "class _Plugin:\n" + " async def run(self, context):\n" + " return context\n" + "\n" + "my_plugin_instance = _Plugin()\n" + ) + config: Dict[str, Any] = {"plugins": ["my_plugin.my_plugin_instance"]} + + resolve_complexity_router_plugins( + model_name="smart-router", + complexity_router_config=config, + config_file_path=str(tmp_path / "config.yaml"), + ) + + assert len(config["plugins"]) == 1 + assert hasattr(config["plugins"][0], "run") + assert type(config["plugins"][0]).__name__ == "_Plugin" + + +def test_resolve_complexity_router_plugins_rejects_non_routing_plugin_object(tmp_path): + plugin_file = tmp_path / "bad_plugin.py" + plugin_file.write_text("not_a_plugin = object()\n") + config: Dict[str, Any] = {"plugins": ["bad_plugin.not_a_plugin"]} + + with pytest.raises(ValueError, match="does not implement the RoutingPlugin interface"): + resolve_complexity_router_plugins( + model_name="smart-router", + complexity_router_config=config, + config_file_path=str(tmp_path / "config.yaml"), + ) + + # --------------------------------------------------------------------------- # ProxyConfig.__init__ # --------------------------------------------------------------------------- diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 88920e35e059..4d0c40e80474 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -2848,4 +2848,39 @@ async def test_no_plugins_configured_is_unaffected(self, complexity_router): messages=[{"role": "user", "content": "Hello!"}], ) assert result is not None + + @pytest.mark.asyncio + async def test_session_affinity_pin_shortcut_disabled_when_plugins_configured(self, mock_router_instance): + """Regression: the session_affinity cache-pin shortcut returned a stale pinned + model without ever re-running it through plugins, so a policy plugin's decision + (e.g. a budget cap crossed mid-session) was only ever enforced on a session's + first turn. With plugins configured, every turn must go through + _classify_and_route (and therefore the plugin pipeline) again.""" + mock_router_instance.cache = DualCache() + + class AllowAll: + async def run(self, context): + return context + + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={ + "tiers": {"SIMPLE": ["gpt-4o-mini"]}, + "session_affinity": True, + "plugins": [AllowAll()], + }, + ) + request_kwargs = {"metadata": {"session_id": "session-1"}} + + with patch.object(router, "_classify_and_route", wraps=router._classify_and_route) as spy: + first = await router.async_pre_routing_hook( + model="test-model", request_kwargs=request_kwargs, messages=[{"role": "user", "content": "hi"}] + ) + second = await router.async_pre_routing_hook( + model="test-model", request_kwargs=request_kwargs, messages=[{"role": "user", "content": "hi again"}] + ) + assert first.model == "gpt-4o-mini" + assert second.model == "gpt-4o-mini" + assert spy.call_count == 2 assert result.model == "gpt-4o-mini" From 5e676d62ad096f2a53321c10546b58bb06f8157c Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 14 Jul 2026 15:04:06 -0700 Subject: [PATCH 5/9] fix(router): raise instead of falling back to default_model on empty plugin-narrowed tier default_model was never checked against the configured plugins, so it functioned as an unconditional escape hatch around whatever policy a plugin enforces -- a tenant/budget plugin narrowing a tier to zero candidates could still be bypassed by the fallback. Drop the fallback entirely for this path; a plugin narrowing to zero is a policy decision, not something to route around, matching the fail-closed behavior the Router-level plugin pipeline already uses for the same situation. Flagged by Veria AI on PR #33251. --- .../complexity_router/complexity_router.py | 10 +++++---- .../router_strategy/test_complexity_router.py | 21 +++++++++++-------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index c954975dcefa..beeda0f9a689 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -492,11 +492,13 @@ async def _pick_model_for_tier( context = await plugin.run(context) if not context.candidate_models: - if self.config.default_model: - return self.config.default_model + # A plugin narrowing a tier to zero candidates is a policy decision (e.g. no + # model this tenant's budget allows) -- falling back to default_model here + # (which was never checked against the plugins) would let that policy be + # silently bypassed. Raise instead, matching the Router-level plugin + # pipeline's own fail-closed behavior for the same situation. raise ValueError( - f"No candidate models left for tier {tier_key} after routing-plugin filtering, " - "and no default_model configured" + f"No candidate models left for tier {tier_key} after routing-plugin filtering" ) return self._pick_from_tier_value(context.candidate_models, tier_key) diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 4d0c40e80474..25cb74964c7f 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -2736,7 +2736,12 @@ async def run(self, context): assert result.model == "gpt-4o-nano" @pytest.mark.asyncio - async def test_plugin_narrowing_to_zero_falls_back_to_default_model(self, mock_router_instance): + async def test_plugin_narrowing_to_zero_raises_even_with_default_model_configured(self, mock_router_instance): + """Regression: default_model must never be used as an escape hatch around a + plugin's narrowing decision -- it was never checked against the plugins, so + falling back to it would let a tenant/budget policy be silently bypassed. + Reported by Veria AI on PR #33251.""" + class BlockEverything: async def run(self, context): context.candidate_models = [] @@ -2751,13 +2756,12 @@ async def run(self, context): "plugins": [BlockEverything()], }, ) - result = await router.async_pre_routing_hook( - model="test-model", - request_kwargs={}, - messages=[{"role": "user", "content": "hi"}], - ) - assert result is not None - assert result.model == "gpt-4o-fallback" + with pytest.raises(ValueError, match="No candidate models left for tier"): + await router.async_pre_routing_hook( + model="test-model", + request_kwargs={}, + messages=[{"role": "user", "content": "hi"}], + ) @pytest.mark.asyncio async def test_plugin_narrowing_to_zero_without_default_model_raises(self, mock_router_instance): @@ -2883,4 +2887,3 @@ async def run(self, context): assert first.model == "gpt-4o-mini" assert second.model == "gpt-4o-mini" assert spy.call_count == 2 - assert result.model == "gpt-4o-mini" From 76e53d91fc9f5c3a464bfd59babd220df2e527e8 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 14 Jul 2026 15:24:49 -0700 Subject: [PATCH 6/9] style: ruff format complexity_router.py --- .../router_strategy/complexity_router/complexity_router.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index beeda0f9a689..ea6fa21b6f60 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -497,9 +497,7 @@ async def _pick_model_for_tier( # (which was never checked against the plugins) would let that policy be # silently bypassed. Raise instead, matching the Router-level plugin # pipeline's own fail-closed behavior for the same situation. - raise ValueError( - f"No candidate models left for tier {tier_key} after routing-plugin filtering" - ) + raise ValueError(f"No candidate models left for tier {tier_key} after routing-plugin filtering") return self._pick_from_tier_value(context.candidate_models, tier_key) def _ensure_adaptive_router(self) -> Any | None: From a41586e34bbdcf2df1f291220971596a82ab6383 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 14 Jul 2026 16:15:21 -0700 Subject: [PATCH 7/9] style(proxy): use modern str | None instead of Optional[str] in resolve_complexity_router_plugins --- litellm/proxy/proxy_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 85f17d83c7da..73976b119405 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -3664,7 +3664,7 @@ def _attach_redis_usage_cache(redis_cache: RedisCache, enable_redis_auth_cache: def resolve_complexity_router_plugins( model_name: str, complexity_router_config: dict, - config_file_path: Optional[str], + config_file_path: str | None, ) -> None: """ Resolves `complexity_router_config["plugins"]` dotted-path strings to live From 8ea0f196cabe3e08b023978c724a196b2d3f8d63 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 14 Jul 2026 19:19:45 -0700 Subject: [PATCH 8/9] fix(router): stop default_model short-circuit from skipping plugins on no-user-message path self.config.default_model or await self._pick_model_for_tier(...) -- Python's `or` short-circuits on a truthy default_model, so _pick_model_for_tier (and therefore the plugin pipeline) never ran at all for the no-user-message path whenever default_model was configured. A tenant/budget plugin's decision was silently bypassable this way even after the other two policy-bypass fixes, since this call site had a different shape from the other three pick sites. Removed the short-circuit; falls through to _pick_model_for_tier -> get_model_for_tier, which already checks the MEDIUM tier before default_model -- the same priority every other call site uses. Flagged by Veria AI on PR #33251. --- .../complexity_router/complexity_router.py | 7 +++- .../router_strategy/test_complexity_router.py | 32 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index ea6fa21b6f60..99df2a7e3a8e 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -985,7 +985,12 @@ async def _classify_and_route( if user_message is None: verbose_router_logger.debug("ComplexityRouter: No user message found, routing to default model") - routed_model = self.config.default_model or await self._pick_model_for_tier( + # No `self.config.default_model or ...` short-circuit here: default_model was + # never checked against the plugins, so it would function as an unconditional + # escape hatch around whatever policy a plugin enforces. Falls through to + # _pick_model_for_tier -> get_model_for_tier, which checks the MEDIUM tier + # before default_model -- the same priority every other call site already uses. + routed_model = await self._pick_model_for_tier( ComplexityTier.MEDIUM, messages, resolved_messages, request_kwargs ) return PreRoutingHookResponse( diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 25cb74964c7f..591b7f928863 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -2835,6 +2835,38 @@ async def run(self, context): assert result is not None assert result.model == "gpt-4o-nano" + @pytest.mark.asyncio + async def test_plugin_applies_to_no_user_message_default_tier_path(self, mock_router_instance): + """Regression: `self.config.default_model or await self._pick_model_for_tier(...)` + short-circuited on a truthy default_model, so the no-user-message path never ran + the plugin pipeline at all when default_model was configured. A policy plugin + must not be bypassable via this path either. Reported by Veria AI on PR #33251.""" + + class ExcludeDefaultModel: + async def run(self, context): + context.candidate_models = [m for m in context.candidate_models if m != "gpt-4o-default"] + return context + + router = ComplexityRouter( + model_name="test-complexity-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={ + "tiers": {"MEDIUM": ["gpt-4o-default", "gpt-4o-nano"]}, + "default_model": "gpt-4o-default", + "plugins": [ExcludeDefaultModel()], + }, + ) + result = await router.async_pre_routing_hook( + model="test-model", + request_kwargs={}, + messages=[ + {"role": "system", "content": "You are helpful."}, + {"role": "assistant", "content": "Hello!"}, + ], + ) + assert result is not None + assert result.model == "gpt-4o-nano" + def test_plugins_and_adaptive_together_raises(self): with pytest.raises(ValidationError, match="plugins and adaptive=True cannot both be set"): ComplexityRouterConfig( From bae865cdf8238babdd50e683ea12d30fa0d30056 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 14 Jul 2026 19:52:19 -0700 Subject: [PATCH 9/9] fix(router): address Greptile findings on the plugin-bypass fixes Preserve default_model-first priority in the no-user-message path when no plugins are configured, instead of unconditionally flipping to the MEDIUM tier -- the plugin-bypass fix must not silently change model selection for the (much larger) population of users who don't use plugins at all. Gated on self.config.plugins, matching the pattern already used elsewhere in this PR, per CLAUDE.md's guidance against backwards-compat flags when a plain conditional does the job. Also close a gap in the plugin validation added earlier: @runtime_checkable only checks that `run` exists as an attribute, not that it's a coroutine function, so a synchronous `def run(self, context)` passed isinstance(resolved_plugin, RoutingPlugin) at startup and only failed at request time with a confusing TypeError. Added an inspect.iscoroutinefunction check. Both flagged by Greptile on PR #33251. --- litellm/proxy/proxy_server.py | 8 +++++- .../complexity_router/complexity_router.py | 20 +++++++------ .../proxy/proxy_server/test_proxy_config.py | 24 ++++++++++++++++ .../router_strategy/test_complexity_router.py | 28 +++++++++++++++++++ 4 files changed, 71 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 73976b119405..b04f12a0e901 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -3684,7 +3684,13 @@ def resolve_complexity_router_plugins( for plugin_path in plugin_paths ] for plugin_path, resolved_plugin in zip(plugin_paths, resolved_plugins): - if not isinstance(resolved_plugin, RoutingPlugin): + # `@runtime_checkable` only checks that `run` exists as an attribute, not that + # it's a coroutine function -- a synchronous `def run(self, context)` would pass + # isinstance() here and only fail at request time with a confusing `TypeError: + # object RoutingContext can't be used in 'await' expression`. + if not isinstance(resolved_plugin, RoutingPlugin) or not inspect.iscoroutinefunction( + getattr(resolved_plugin, "run", None) + ): raise ValueError( f"complexity_router_config.plugins entry {plugin_path!r} on model {model_name!r} " f"resolved to {resolved_plugin!r}, which does not implement the RoutingPlugin " diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 99df2a7e3a8e..eb0f74a58e7a 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -985,14 +985,18 @@ async def _classify_and_route( if user_message is None: verbose_router_logger.debug("ComplexityRouter: No user message found, routing to default model") - # No `self.config.default_model or ...` short-circuit here: default_model was - # never checked against the plugins, so it would function as an unconditional - # escape hatch around whatever policy a plugin enforces. Falls through to - # _pick_model_for_tier -> get_model_for_tier, which checks the MEDIUM tier - # before default_model -- the same priority every other call site already uses. - routed_model = await self._pick_model_for_tier( - ComplexityTier.MEDIUM, messages, resolved_messages, request_kwargs - ) + if not self.config.plugins and self.config.default_model: + # No plugins configured: preserve the pre-existing default_model-first + # priority exactly (changing it would be a silent behavior change for + # every non-plugin user, not just a security fix). + routed_model = self.config.default_model + else: + # Plugins configured: default_model must never bypass them, so it's not + # checked here at all -- _pick_model_for_tier -> get_model_for_tier still + # falls back to it (after the MEDIUM tier) once the plugin pipeline runs. + routed_model = await self._pick_model_for_tier( + ComplexityTier.MEDIUM, messages, resolved_messages, request_kwargs + ) return PreRoutingHookResponse( model=routed_model, messages=messages if has_original_messages else None, diff --git a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py index 7b43a5baf86b..45d354196803 100644 --- a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py @@ -161,6 +161,30 @@ def test_resolve_complexity_router_plugins_rejects_non_routing_plugin_object(tmp ) +def test_resolve_complexity_router_plugins_rejects_synchronous_run_method(tmp_path): + """Regression: @runtime_checkable only checks that `run` exists as an attribute, + not that it's a coroutine function. A plugin with a synchronous `run` passes a bare + isinstance() check and would only fail at request time with a confusing + `TypeError: object RoutingContext can't be used in 'await' expression`. Reported + by Greptile on PR #33251.""" + plugin_file = tmp_path / "sync_plugin.py" + plugin_file.write_text( + "class _SyncPlugin:\n" + " def run(self, context):\n" + " return context\n" + "\n" + "sync_plugin_instance = _SyncPlugin()\n" + ) + config: Dict[str, Any] = {"plugins": ["sync_plugin.sync_plugin_instance"]} + + with pytest.raises(ValueError, match="does not implement the RoutingPlugin interface"): + resolve_complexity_router_plugins( + model_name="smart-router", + complexity_router_config=config, + config_file_path=str(tmp_path / "config.yaml"), + ) + + # --------------------------------------------------------------------------- # ProxyConfig.__init__ # --------------------------------------------------------------------------- diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 591b7f928863..41dc7269372a 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -2867,6 +2867,34 @@ async def run(self, context): assert result is not None assert result.model == "gpt-4o-nano" + @pytest.mark.asyncio + async def test_no_user_message_prefers_default_model_over_medium_tier_without_plugins( + self, mock_router_instance + ): + """Regression: without plugins configured, the no-user-message path must keep its + pre-existing default_model-first priority over the MEDIUM tier exactly as before -- + closing the plugin-bypass gap must not silently flip model selection for the (much + larger) population of users who don't use plugins at all. Flagged by Greptile on + PR #33251 after the plugin-bypass fix changed this priority unconditionally.""" + router = ComplexityRouter( + model_name="test-complexity-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={ + "tiers": {"MEDIUM": ["gpt-4o-medium-tier"]}, + "default_model": "gpt-4o-configured-default", + }, + ) + result = await router.async_pre_routing_hook( + model="test-model", + request_kwargs={}, + messages=[ + {"role": "system", "content": "You are helpful."}, + {"role": "assistant", "content": "Hello!"}, + ], + ) + assert result is not None + assert result.model == "gpt-4o-configured-default" + def test_plugins_and_adaptive_together_raises(self): with pytest.raises(ValidationError, match="plugins and adaptive=True cannot both be set"): ComplexityRouterConfig(