feat(auto_router): keyword tier overrides and semantic keyword matching for the complexity router

litellm/proxy/dev_config.yaml

@@ -33,6 +33,12 @@ model_list:
       model: anthropic/claude-sonnet-5
       api_key: os.environ/ANTHROPIC_API_KEY
 
+  # ---------- Embeddings (for complexity router semantic keyword matching) ----------
+  - model_name: voyage-4-large
+    litellm_params:
+      model: voyage/voyage-4-large
+      api_key: os.environ/VOYAGE_API_KEY
+
   # ---------- Bedrock Invoke ----------
   - model_name: bedrock-invoke-haiku-4-5
     litellm_params:

litellm/proxy/management_endpoints/model_management_endpoints.py

@@ -1040,6 +1040,22 @@ async def can_user_make_model_call(
         return True
 
 
+def _deployment_name_and_model(deployment: Optional[Union[Deployment, Dict[str, object]]]) -> Tuple[Optional[str], str]:
+    """Return (model_name, litellm_params.model) for a deployment.
+
+    delete_deployment is annotated to return a Deployment but hands back the raw
+    model_list dict at runtime, so both shapes are handled; the model defaults to "".
+    """
+    if deployment is None:
+        return None, ""
+    if isinstance(deployment, dict):
+        name = deployment.get("model_name")
+        params = deployment.get("litellm_params")
+        model = params.get("model") if isinstance(params, dict) else None
+        return (name if isinstance(name, str) else None), (model if isinstance(model, str) else "")
+    return deployment.model_name, str(getattr(deployment.litellm_params, "model", "") or "")
+
+
 #### [BETA] - This is a beta endpoint, format might change based on user feedback. - https://github.com/BerriAI/litellm/issues/964
 @router.post(
     "/model/delete",
@@ -1111,7 +1127,19 @@ async def delete_model(
 
             ## DELETE FROM ROUTER ##
             if llm_router is not None:
-                llm_router.delete_deployment(id=model_info.id)
+                deleted_deployment = llm_router.delete_deployment(id=model_info.id)
+                # delete_deployment only drops the deployment from model_list; the auto/
+                # complexity router registries are keyed by model_name and would otherwise
+                # retain a stale (now unbacked) entry, so evict it here too. Guard on the
+                # auto_router/ prefix (as clear_cache does): a regular DB model that merely
+                # shares a model_name with a config-defined router must not evict that router,
+                # since add_deployment never restores config-defined routers.
+                deleted_name, deleted_model = _deployment_name_and_model(deleted_deployment)
+                if deleted_name is not None and deleted_model.startswith("auto_router/"):
+                    llm_router.auto_routers.pop(deleted_name, None)
+                    llm_router.complexity_routers.pop(deleted_name, None)
+                    llm_router.adaptive_routers.pop(deleted_name, None)
+                    llm_router.quality_routers.pop(deleted_name, None)
 
             # Runs after the row delete so the sibling check sees post-delete state.
             if model_params.model_info.team_id is not None:
@@ -1715,8 +1743,27 @@ async def clear_cache():
         for model_id in db_model_ids:
             llm_router.delete_deployment(id=model_id)
 
-        # Clear auto routers
-        llm_router.auto_routers.clear()
+        # Clear only DB-backed auto-router-family entries, keyed by model_name, so the
+        # reload below rebuilds them fresh. A blanket .clear() would also drop config-defined
+        # routers, which are never re-added below (add_deployment only reloads DB models),
+        # leaving them permanently unroutable until a full proxy restart for every tenant.
+        # Restrict to deployments whose model is actually an auto_router/* so a config
+        # router that merely shares a model_name with a regular DB model isn't evicted. The
+        # auto_router/ prefix also covers quality_router/ and adaptive_router/, so pop the
+        # name from every router registry (no-op where absent); missing quality/adaptive
+        # entries would otherwise make init raise "already exists" on reload and abort it.
+        db_router_names = {
+            model.get("model_name")
+            for model in current_models
+            if model.get("model_name") is not None
+            and model.get("model_info", {}).get("db_model", False)
+            and str(model.get("litellm_params", {}).get("model", "")).startswith("auto_router/")
+        }
+        for model_name in db_router_names:
+            llm_router.auto_routers.pop(model_name, None)
+            llm_router.complexity_routers.pop(model_name, None)
+            llm_router.adaptive_routers.pop(model_name, None)
+            llm_router.quality_routers.pop(model_name, None)
 
         # Reload only DB models
         await proxy_config.add_deployment(prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj)

litellm/router_strategy/complexity_router/complexity_router.py

@@ -7,12 +7,15 @@
 By default, scoring is local (regex/keyword-based) with no external API calls and <1ms
 latency. Optionally, classifier_type="llm" routes classification through a configured
 model instead, trading that latency/cost guarantee for potentially better accuracy.
+keyword_tier_rules (lexical or, with semantic_keyword_matching, embedding-based) are
+evaluated before either classification strategy and force a tier outright when matched.
 
 Inspired by ClawRouter: https://github.com/BlockRunAI/ClawRouter
 """
 
+import asyncio
 import re
-from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Tuple, Union
+from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Tuple, Union, cast
 
 from pydantic import BaseModel
 
@@ -25,16 +28,20 @@
     DEFAULT_REASONING_KEYWORDS,
     DEFAULT_SIMPLE_KEYWORDS,
     DEFAULT_TECHNICAL_KEYWORDS,
+    TIER_SEVERITY_ORDER,
     ComplexityRouterConfig,
     ComplexityTier,
 )
 
 if TYPE_CHECKING:
+    from semantic_router.routers import SemanticRouter
+
     from litellm.router import Router
     from litellm.types.router import PreRoutingHookResponse
 else:
     Router = Any
     PreRoutingHookResponse = Any
+    SemanticRouter = Any
 
 
 class TierClassification(BaseModel):
@@ -63,18 +70,38 @@ def _append_custom_keywords(base_keywords: list[str], custom_keywords: Optional[
     return [*base_keywords, *deduped_custom.values()]
 
 
-# Metadata keys that carry the parent request's budget reservation. These must not
-# reach the classifier's internal acompletion call: the reservation belongs to the
-# routed completion that the classifier is deciding on, not to the classifier call
-# itself, and forwarding it would let the classifier's cost-tracking reconcile
-# against a reservation it isn't responsible for.
-_BUDGET_RESERVATION_METADATA_KEYS = frozenset({"user_api_key_budget_reservation", "user_api_key_auth"})
+# Metadata keys that carry only the parent request's budget reservation state. These
+# must not reach internal sub-calls (classifier, embedding): the reservation belongs to
+# the routed completion being decided on, not to the sub-call itself, and forwarding it
+# would let the sub-call's cost callback finalize the reservation, causing the routed
+# completion's callback to skip incrementing key/team budget counters.
+#
+# Note: user_api_key_auth itself is intentionally kept; it is required by
+# _filter_deployments_by_model_access_groups to scope embedding/classifier model
+# selection to the caller's authorized access groups. It is forwarded as a sanitized
+# copy with its budget_reservation sub-field removed, because the proxy cost callback
+# (_get_budget_reservation_from_metadata) falls back to reading the reservation from
+# inside the auth object when the top-level key is absent; forwarding it unsanitized
+# would re-create the exact double-finalization this stripping exists to prevent.
+_BUDGET_RESERVATION_METADATA_KEYS = frozenset({"user_api_key_budget_reservation"})
+
+
+def _sanitize_user_api_key_auth(auth: Any) -> Any:
+    if isinstance(auth, dict):
+        return {k: v for k, v in auth.items() if k != "budget_reservation"}
+    if getattr(auth, "budget_reservation", None) is not None and hasattr(auth, "model_copy"):
+        return auth.model_copy(update={"budget_reservation": None})
+    return auth
 
 
 def _classifier_call_metadata(metadata: Optional[dict[str, Any]]) -> Optional[dict[str, Any]]:
     if not metadata:
         return metadata
-    return {k: v for k, v in metadata.items() if k not in _BUDGET_RESERVATION_METADATA_KEYS}
+    return {
+        k: _sanitize_user_api_key_auth(v) if k == "user_api_key_auth" else v
+        for k, v in metadata.items()
+        if k not in _BUDGET_RESERVATION_METADATA_KEYS
+    }
 
 
 class DimensionScore:
@@ -141,6 +168,13 @@ def __init__(
         )
         self.simple_keywords = self.config.simple_keywords or DEFAULT_SIMPLE_KEYWORDS
 
+        # Lazily built on first semantic request and cached for reuse (route
+        # embeddings are static, only the prompt is embedded per request). The lock
+        # serializes the one-time build so concurrent cold-start requests don't each
+        # construct the index and fire duplicate embedding calls.
+        self._semantic_routelayer: Optional[SemanticRouter] = None
+        self._semantic_routelayer_lock = asyncio.Lock()
+
         # Pre-compile regex patterns for efficiency
         # Use non-greedy .*? to prevent ReDoS on pathological inputs
         self._multi_step_patterns = [
@@ -419,6 +453,138 @@ def get_model_for_tier(self, tier: ComplexityTier) -> str:
 
         raise ValueError(f"No model configured for tier {tier_key} and no default_model set")
 
+    def _lexical_tier_override(self, user_message: str) -> Optional[ComplexityTier]:
+        """When keyword_tier_rules match literally, the most-severe matched tier wins.
+
+        Escalating to the highest tier (rather than the first rule in the list) keeps
+        routing independent of the order rules were authored in: a prompt hitting both a
+        SIMPLE and a REASONING keyword routes to REASONING.
+        """
+        rules = self.config.keyword_tier_rules
+        if not rules:
+            return None
+        text = user_message.lower()
+        matched_tiers = [
+            rule.tier for rule in rules if any(self._keyword_matches(text, keyword) for keyword in rule.keywords)
+        ]
+        if not matched_tiers:
+            return None
+        return max(matched_tiers, key=TIER_SEVERITY_ORDER.index)
+
+    def _get_or_create_semantic_routelayer(self) -> "SemanticRouter":
+        """Build (once) a SemanticRouter with one route per tier, utterances = that tier's keywords."""
+        if self._semantic_routelayer is not None:
+            return self._semantic_routelayer
+
+        from semantic_router.routers import SemanticRouter
+        from semantic_router.routers.base import Route
+
+        from litellm.router_strategy.auto_router.litellm_encoder import (
+            LiteLLMRouterEncoder,
+        )
+
+        embedding_model = self.config.embedding_model
+        if embedding_model is None:
+            raise ValueError("embedding_model is required for semantic keyword matching")
+
+        rules = self.config.keyword_tier_rules or []
+        ordered_tiers = tuple(dict.fromkeys(rule.tier.value for rule in rules))
+        routes = [
+            Route(
+                name=tier,
+                utterances=[keyword for rule in rules if rule.tier.value == tier for keyword in rule.keywords],
+                score_threshold=self.config.match_threshold,
+            )
+            for tier in ordered_tiers
+        ]
+        routelayer = SemanticRouter(
+            routes=routes,
+            encoder=LiteLLMRouterEncoder(
+                litellm_router_instance=self.litellm_router_instance,
+                model_name=embedding_model,
+                score_threshold=self.config.match_threshold,
+            ),
+            auto_sync="local",
+            aggregation="max",
+        )
+        self._semantic_routelayer = routelayer
+        return routelayer
+
+    async def _ensure_semantic_routelayer(self) -> "SemanticRouter":
+        """Return the cached route layer, building it once under a lock if needed.
+
+        The build embeds the static route utterances via the encoder's synchronous path,
+        so it runs in a worker thread to avoid blocking the event loop. A double-checked
+        asyncio lock ensures concurrent cold-start requests build it exactly once rather
+        than each firing duplicate embedding calls.
+        """
+        if self._semantic_routelayer is not None:
+            return self._semantic_routelayer
+        async with self._semantic_routelayer_lock:
+            routelayer = self._semantic_routelayer
+            if routelayer is None:
+                routelayer = await asyncio.to_thread(self._get_or_create_semantic_routelayer)
+            return routelayer
+
+    async def _semantic_tier_override(self, user_message: str, request_kwargs: Dict) -> Optional[ComplexityTier]:
+        """Match the prompt against keyword_tier_rules by embedding similarity.
+
+        Embeds the query ourselves (instead of letting SemanticRouter.acall embed it
+        internally) so the caller's metadata/litellm_metadata flows into aembedding()
+        and this spend is attributed and budget-checked against the originating key/team,
+        the same as any other litellm call. SemanticRouter.acall() has no parameter to
+        pass such kwargs through to the encoder, so it's bypassed for the query embedding;
+        the route index itself (static utterances, embedded once at build time with no
+        caller context) is unaffected and still reused via the precomputed `vector=` path.
+        """
+        from semantic_router.schema import RouteChoice
+
+        from litellm.router_strategy.auto_router.litellm_encoder import (
+            LiteLLMRouterEncoder,
+        )
+
+        routelayer = await self._ensure_semantic_routelayer()
+        encoder = cast(LiteLLMRouterEncoder, routelayer.encoder)  # cast-ok: always the encoder we built above
+        # Strip the parent request's budget reservation before forwarding: the reservation
+        # belongs to the routed completion this embedding is helping select, not to the
+        # embedding call. Forwarding it would let the embedding's cost callback finalize the
+        # reservation, so the routed completion's own callback then skips incrementing the
+        # key/team budget. Key/team attribution fields are preserved for spend logging.
+        metadata = _classifier_call_metadata(request_kwargs.get("metadata")) or {}
+        litellm_metadata = _classifier_call_metadata(request_kwargs.get("litellm_metadata")) or {}
+        query_vector = (
+            await encoder.aencode_queries([user_message], metadata=metadata, litellm_metadata=litellm_metadata)
+        )[0]
+        route_choice = await routelayer.acall(vector=query_vector)
+
+        if isinstance(route_choice, list):
+            route_choice = route_choice[0] if route_choice else None
+        if not isinstance(route_choice, RouteChoice) or not route_choice.name:
+            return None
+        try:
+            return ComplexityTier(route_choice.name)
+        except ValueError:
+            return None
+
+    async def _resolve_keyword_tier_override(self, user_message: str, request_kwargs: Dict) -> Optional[ComplexityTier]:
+        """Resolve a keyword_tier_rule override, semantically or lexically per config.
+
+        Returns None (no override -> fall through to the scorer) not only when no rule
+        matches, but also when the semantic path fails: the embedding call can error or
+        time out, and a routing helper must never turn that into a failed user request.
+        """
+        if not self.config.keyword_tier_rules:
+            return None
+        if not self.config.semantic_keyword_matching:
+            return self._lexical_tier_override(user_message)
+        try:
+            return await self._semantic_tier_override(user_message, request_kwargs)
+        except Exception as e:  # noqa: BLE001 -- embedding call can fail many ways (timeout, provider/network/parse error); any failure must fall back to scoring, never fail the request
+            verbose_router_logger.warning(
+                f"ComplexityRouter: semantic keyword matching failed ({e}), falling back to complexity scoring"
+            )
+            return None
+
     def _resolve_messages(
         self,
         messages: Optional[List[Dict[str, Any]]],
@@ -539,6 +705,17 @@ async def async_pre_routing_hook(
                 messages=messages if has_original_messages else None,
             )
 
+        override_tier = await self._resolve_keyword_tier_override(user_message, request_kwargs)
+        if override_tier is not None:
+            routed_model = self.get_model_for_tier(override_tier)
+            verbose_router_logger.info(
+                f"ComplexityRouter: keyword rule fired, tier={override_tier.value}, routed_model={routed_model}"
+            )
+            return PreRoutingHookResponse(
+                model=routed_model,
+                messages=messages if has_original_messages else None,
+            )
+
         tier, score, signals = await self.aclassify(user_message, system_prompt, request_kwargs)
         routed_model = self.get_model_for_tier(tier)