From 0c57c7a8acfe8c9eb3373e5e37e1b297fe76e806 Mon Sep 17 00:00:00 2001 From: AlexsanderHamir Date: Wed, 8 Oct 2025 10:16:51 -0700 Subject: [PATCH 1/2] fix(router): update model_name_to_deployment_indices on deployment removal When a deployment is deleted, the model_name_to_deployment_indices map was not being updated, causing stale index references. This could lead to incorrect routing behavior when deployments with the same model_name were dynamically removed. Changes: - Update _update_deployment_indices_after_removal to maintain model_name_to_deployment_indices mapping - Remove deleted indices and decrement indices greater than removed index - Clean up empty entries when no deployments remain for a model name - Update test to verify proper index shifting and cleanup behavior --- litellm/router.py | 20 +++++++++ .../test_router_index_management.py | 43 ++++++++++++------- 2 files changed, 48 insertions(+), 15 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index ef7bcba635f3..6bedeb2f9a5f 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -5367,6 +5367,26 @@ def _update_deployment_indices_after_removal( # Remove the deleted model from index if model_id in self.model_id_to_deployment_index_map: del self.model_id_to_deployment_index_map[model_id] + + # Update model_name_to_deployment_indices + for model_name, indices in list(self.model_name_to_deployment_indices.items()): + # Remove the deleted index + if removal_idx in indices: + indices.remove(removal_idx) + + # Decrement all indices greater than removal_idx + updated_indices = [] + for idx in indices: + if idx > removal_idx: + updated_indices.append(idx - 1) + else: + updated_indices.append(idx) + + # Update or remove the entry + if len(updated_indices) > 0: + self.model_name_to_deployment_indices[model_name] = updated_indices + else: + del self.model_name_to_deployment_indices[model_name] def _add_model_to_list_and_index_map( self, model: dict, model_id: Optional[str] = None diff --git a/tests/router_unit_tests/test_router_index_management.py b/tests/router_unit_tests/test_router_index_management.py index 04ea92149917..584d3995f44e 100644 --- a/tests/router_unit_tests/test_router_index_management.py +++ b/tests/router_unit_tests/test_router_index_management.py @@ -16,25 +16,38 @@ def router(self): """Create a router instance for testing""" return Router(model_list=[]) - def test_update_deployment_indices_after_removal(self, router): - """Test _update_deployment_indices_after_removal function""" - # Setup: Add models to router with proper structure + def test_deletion_updates_model_name_indices(self, router): + """Test that deleting a deployment updates model_name_to_deployment_indices correctly""" router.model_list = [ - {"model": "test1", "model_info": {"id": "model-1"}}, - {"model": "test2", "model_info": {"id": "model-2"}}, - {"model": "test3", "model_info": {"id": "model-3"}} + {"model_name": "gpt-3.5", "model_info": {"id": "model-1"}}, + {"model_name": "gpt-4", "model_info": {"id": "model-2"}}, + {"model_name": "gpt-4", "model_info": {"id": "model-3"}}, + {"model_name": "claude", "model_info": {"id": "model-4"}} ] - router.model_id_to_deployment_index_map = {"model-1": 0, "model-2": 1, "model-3": 2} - - # Test: Remove model-2 (index 1) + router.model_id_to_deployment_index_map = { + "model-1": 0, "model-2": 1, "model-3": 2, "model-4": 3 + } + router.model_name_to_deployment_indices = { + "gpt-3.5": [0], + "gpt-4": [1, 2], + "claude": [3] + } + + # Remove one of the duplicate gpt-4 deployments router._update_deployment_indices_after_removal(model_id="model-2", removal_idx=1) - # Verify: model-2 is removed from index - assert "model-2" not in router.model_id_to_deployment_index_map - # Verify: model-3 index is updated (2 -> 1) - assert router.model_id_to_deployment_index_map["model-3"] == 1 - # Verify: model-1 index remains unchanged - assert router.model_id_to_deployment_index_map["model-1"] == 0 + # Verify indices are shifted correctly + assert router.model_name_to_deployment_indices["gpt-3.5"] == [0] + assert router.model_name_to_deployment_indices["gpt-4"] == [1] # was [1,2], removed 1, shifted 2->1 + assert router.model_name_to_deployment_indices["claude"] == [2] # was [3], shifted to [2] + + # Remove the last gpt-4 deployment + router._update_deployment_indices_after_removal(model_id="model-3", removal_idx=1) + + # Verify gpt-4 is removed from dict when no deployments remain + assert "gpt-4" not in router.model_name_to_deployment_indices + assert router.model_name_to_deployment_indices["gpt-3.5"] == [0] + assert router.model_name_to_deployment_indices["claude"] == [1] def test_build_model_id_to_deployment_index_map(self, router): """Test _build_model_id_to_deployment_index_map function""" From 891a3463322f1fdd370f622878f288539858ed11 Mon Sep 17 00:00:00 2001 From: AlexsanderHamir Date: Sat, 11 Oct 2025 10:11:47 -0700 Subject: [PATCH 2/2] fix(router): remove redundant index building during initialization Remove duplicate index building operations that were causing unnecessary work during router initialization: 1. Removed redundant `_build_model_id_to_deployment_index_map` call in __init__ - `set_model_list` already builds all indices from scratch 2. Removed redundant `_build_model_name_index` call at end of `set_model_list` - the index is already built incrementally via `_create_deployment` -> `_add_model_to_list_and_index_map` Both indices (model_id_to_deployment_index_map and model_name_to_deployment_indices) are properly maintained as lookup indexes through existing helper methods. This change eliminates O(N) duplicate work during initialization without any behavioral changes. The indices continue to be correctly synchronized with model_list on all operations (add/remove/upsert). --- litellm/router.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 6bedeb2f9a5f..a07d7950e4aa 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -421,8 +421,7 @@ def __init__( # noqa: PLR0915 self.model_name_to_deployment_indices: Dict[str, List[int]] = {} if model_list is not None: - # Build model index immediately to enable O(1) lookups from the start - self._build_model_id_to_deployment_index_map(model_list) + # set_model_list will build indices automatically self.set_model_list(model_list) self.healthy_deployments: List = self.model_list # type: ignore for m in model_list: @@ -5143,8 +5142,8 @@ def set_model_list(self, model_list: list): ) self.model_names = [m["model_name"] for m in model_list] - # Build model_name index for O(1) lookups - self._build_model_name_index(self.model_list) + # Note: model_name_to_deployment_indices is already built incrementally + # by _create_deployment -> _add_model_to_list_and_index_map def _add_deployment(self, deployment: Deployment) -> Deployment: import os