From bea11ddedd414db960cbc57670e4370c08ef624b Mon Sep 17 00:00:00 2001 From: Shivam Rawat Date: Thu, 16 Jul 2026 21:14:45 -0700 Subject: [PATCH 1/4] fix(proxy): resolve team wildcard credentials for vector store files Team-scoped wildcard deployments like openai/* are indexed separately from global router models, so vector store file requests failed with api_key=None when a team also had other yaml/db models. Pass team_id into credential lookup and consult team model indexes and pattern routers. Co-authored-by: Cursor --- .../vector_store_files_endpoints/endpoints.py | 8 ++++++-- litellm/router.py | 17 ++++++++++++++++- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/vector_store_files_endpoints/endpoints.py b/litellm/proxy/vector_store_files_endpoints/endpoints.py index 890db2f73a41..44935fc57c9b 100644 --- a/litellm/proxy/vector_store_files_endpoints/endpoints.py +++ b/litellm/proxy/vector_store_files_endpoints/endpoints.py @@ -227,6 +227,8 @@ async def _update_request_data_with_model_routing_hint( model_hint = data.get("model") or user_controlled_model_hint should_authorize_model_hint = isinstance(model_hint, str) and model_hint == user_controlled_model_hint + caller_team_id = getattr(user_api_key_dict, "team_id", None) if user_api_key_dict else None + should_route = False credentials = None if isinstance(model_hint, str) and "*" in model_hint: @@ -237,7 +239,9 @@ async def _update_request_data_with_model_routing_hint( llm_router=llm_router, user_api_key_dict=user_api_key_dict, ) - credentials = llm_router.get_deployment_credentials_with_provider(model_id=model_hint) + credentials = llm_router.get_deployment_credentials_with_provider( + model_id=model_hint, team_id=caller_team_id + ) should_route = credentials is not None else: if isinstance(model_hint, str) and should_authorize_model_hint: @@ -285,7 +289,7 @@ async def _update_request_data_with_model_routing_hint( openai_credentials = None for model_name in model_names_to_check: - credentials = llm_router.get_deployment_credentials_with_provider(model_id=model_name) + credentials = llm_router.get_deployment_credentials_with_provider(model_id=model_name, team_id=caller_team_id) if credentials is None: continue diff --git a/litellm/router.py b/litellm/router.py index 78e156801f85..dbc6da106e7d 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -8459,7 +8459,9 @@ def get_deployment_by_model_group_name(self, model_group_name: str) -> Optional[ raise Exception("Model Name invalid - {}".format(type(model))) return None - def get_deployment_credentials_with_provider(self, model_id: str) -> Optional[Dict[str, Any]]: + def get_deployment_credentials_with_provider( + self, model_id: str, team_id: Optional[str] = None + ) -> Optional[Dict[str, Any]]: """ Get API credentials and provider info from a model name in model_list. Useful for passthrough endpoints (files, batches, etc.) that need credentials. @@ -8469,6 +8471,9 @@ def get_deployment_credentials_with_provider(self, model_id: str) -> Optional[Di Args: model_id: Model ID or model name from model_list (e.g., "gpt-4o-litellm") + team_id: Optional team id of the caller. When set, team-scoped + deployments (indexed by team public model name, including team + wildcard models like "openai/*") are also considered. Returns: Dictionary containing api_key, api_base, custom_llm_provider, etc. @@ -8487,9 +8492,19 @@ def get_deployment_credentials_with_provider(self, model_id: str) -> Optional[Di if deployment is None: deployment = self.get_deployment_by_model_group_name(model_group_name=model_id) + # If not found, check team-scoped deployments (team public model names, + # e.g. team wildcard models like "openai/*", live in a separate index). + if deployment is None and team_id is not None: + team_indices = self.team_model_to_deployment_indices.get((team_id, model_id), []) + if team_indices: + team_model = self.model_list[team_indices[0]] + deployment = Deployment(**team_model) if isinstance(team_model, dict) else team_model + # If still not found, check for wildcard pattern matches if deployment is None: potential_wildcard_models = self.pattern_router.route(model_id) or [] + if not potential_wildcard_models and team_id is not None and team_id in self.team_pattern_routers: + potential_wildcard_models = self.team_pattern_routers[team_id].route(model_id) or [] if potential_wildcard_models: # Use the first matching wildcard deployment deployment_dict = potential_wildcard_models[0] From 836bf0807b62fe346697e3a1b987cc5b05afbbf9 Mon Sep 17 00:00:00 2001 From: Shivam Rawat Date: Fri, 17 Jul 2026 18:19:02 -0700 Subject: [PATCH 2/4] fix(router): keep team wildcard routers fresh and prioritize them over global patterns team_pattern_routers retained deleted/replaced deployments, so team users could keep resolving stale credentials; now set_model_list resets the registry and deployment removal prunes it. Also consult the team wildcard router before the global pattern_router in get_deployment_credentials_with_provider so a global pattern like "openai/*" no longer shadows the team's own entry Co-authored-by: Cursor --- litellm/router.py | 26 ++++-- .../router_utils/pattern_match_deployments.py | 11 +++ tests/test_litellm/test_router.py | 92 +++++++++++++++++++ 3 files changed, 121 insertions(+), 8 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index dbc6da106e7d..6a055f54b0e5 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -7876,6 +7876,7 @@ def set_model_list(self, model_list: list): self.model_id_to_deployment_index_map = {} # Reset the index self.model_name_to_deployment_indices = {} # Reset the model_name index self.team_model_to_deployment_indices = {} # Reset the team_model index + self.team_pattern_routers = {} self.team_public_model_names = frozenset() # Reset per-strategy router registries so hot-reload doesn't leave # stale routers pointing at the old model_list. @@ -8232,6 +8233,12 @@ def _update_deployment_indices_after_removal(self, model_id: str, removal_idx: i public_model_name for _, public_model_name in self.team_model_to_deployment_indices ) + for team_id in list(self.team_pattern_routers.keys()): + team_pattern_router = self.team_pattern_routers[team_id] + team_pattern_router.remove_deployment(model_id) + if not team_pattern_router.patterns: + del self.team_pattern_routers[team_id] + def _update_team_model_index(self, model: dict, idx: int) -> None: """ Helper to update team_model_to_deployment_indices for a single deployment. @@ -8460,8 +8467,8 @@ def get_deployment_by_model_group_name(self, model_group_name: str) -> Optional[ return None def get_deployment_credentials_with_provider( - self, model_id: str, team_id: Optional[str] = None - ) -> Optional[Dict[str, Any]]: + self, model_id: str, team_id: str | None = None + ) -> dict[str, Any] | None: """ Get API credentials and provider info from a model name in model_list. Useful for passthrough endpoints (files, batches, etc.) that need credentials. @@ -8492,19 +8499,22 @@ def get_deployment_credentials_with_provider( if deployment is None: deployment = self.get_deployment_by_model_group_name(model_group_name=model_id) - # If not found, check team-scoped deployments (team public model names, - # e.g. team wildcard models like "openai/*", live in a separate index). + # If not found, check team-scoped deployments whose team public model + # name exactly matches model_id (wildcard team names are matched via + # team_pattern_routers below). if deployment is None and team_id is not None: team_indices = self.team_model_to_deployment_indices.get((team_id, model_id), []) if team_indices: team_model = self.model_list[team_indices[0]] deployment = Deployment(**team_model) if isinstance(team_model, dict) else team_model - # If still not found, check for wildcard pattern matches + # If still not found, check for wildcard pattern matches. Team wildcard + # matches take priority so a global pattern (e.g. "openai/*") doesn't + # shadow the team's own entry. if deployment is None: - potential_wildcard_models = self.pattern_router.route(model_id) or [] - if not potential_wildcard_models and team_id is not None and team_id in self.team_pattern_routers: - potential_wildcard_models = self.team_pattern_routers[team_id].route(model_id) or [] + team_pattern_router = self.team_pattern_routers.get(team_id) if team_id is not None else None + team_wildcard_models = (team_pattern_router.route(model_id) or []) if team_pattern_router else [] + potential_wildcard_models = team_wildcard_models or self.pattern_router.route(model_id) or [] if potential_wildcard_models: # Use the first matching wildcard deployment deployment_dict = potential_wildcard_models[0] diff --git a/litellm/router_utils/pattern_match_deployments.py b/litellm/router_utils/pattern_match_deployments.py index c08f8e95cf49..7e1ed739ef8b 100644 --- a/litellm/router_utils/pattern_match_deployments.py +++ b/litellm/router_utils/pattern_match_deployments.py @@ -73,6 +73,17 @@ def add_pattern(self, pattern: str, llm_deployment: Dict): self.patterns[regex] = [] self.patterns[regex].append(llm_deployment) + def remove_deployment(self, model_id: str) -> None: + """ + Remove every deployment with the given model id from the pattern registry, + dropping any pattern whose deployment list becomes empty. + """ + self.patterns = { + regex: remaining + for regex, deployments in self.patterns.items() + if (remaining := [d for d in deployments if (d.get("model_info") or {}).get("id") != model_id]) + } + def _pattern_to_regex(self, pattern: str) -> str: """ Convert a wildcard pattern to a regex pattern diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index c2c98c8869c6..0fe855151b07 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -3535,6 +3535,98 @@ def test_get_deployment_credentials_with_provider_resolves_credential_name(): litellm.credential_list = [] +def _team_wildcard_model(api_key: str, model_id: str = "team-wildcard-id") -> dict: + return { + "model_name": f"model_name_team-1_{model_id}", + "litellm_params": {"model": "openai/*", "api_key": api_key}, + "model_info": { + "id": model_id, + "team_id": "team-1", + "team_public_model_name": "openai/*", + }, + } + + +def test_get_deployment_credentials_with_provider_team_wildcard_priority(): + """ + Regression: a global wildcard pattern (e.g. "openai/*") must not shadow a + team's own wildcard entry. When team_id is provided, the team wildcard + deployment's credentials win; without team_id the global one is used. + """ + router = litellm.Router( + model_list=[ + { + "model_name": "openai/*", + "litellm_params": {"model": "openai/*", "api_key": "global-key"}, + }, + _team_wildcard_model(api_key="team-key"), + ], + ) + + team_credentials = router.get_deployment_credentials_with_provider( + model_id="openai/gpt-5.2", team_id="team-1" + ) + assert team_credentials is not None + assert team_credentials["api_key"] == "team-key" + + global_credentials = router.get_deployment_credentials_with_provider( + model_id="openai/gpt-5.2" + ) + assert global_credentials is not None + assert global_credentials["api_key"] == "global-key" + + +def test_team_wildcard_credentials_not_usable_after_delete_deployment(): + """ + Regression: team_pattern_routers retained deleted deployments, so a team + user could keep resolving credentials of a deleted wildcard deployment. + """ + router = litellm.Router(model_list=[_team_wildcard_model(api_key="old-key")]) + + assert ( + router.get_deployment_credentials_with_provider( + model_id="openai/gpt-5.2", team_id="team-1" + ) + is not None + ) + + router.delete_deployment(id="team-wildcard-id") + + assert ( + router.get_deployment_credentials_with_provider( + model_id="openai/gpt-5.2", team_id="team-1" + ) + is None + ) + + +def test_team_wildcard_credentials_refreshed_on_upsert_and_set_model_list(): + """ + Regression: replacing a team wildcard deployment (upsert or model list + reload) must serve the new credentials, not the stale cached ones. + """ + from litellm.types.router import Deployment + + router = litellm.Router(model_list=[_team_wildcard_model(api_key="old-key")]) + + router.upsert_deployment( + deployment=Deployment(**_team_wildcard_model(api_key="new-key")) + ) + credentials = router.get_deployment_credentials_with_provider( + model_id="openai/gpt-5.2", team_id="team-1" + ) + assert credentials is not None + assert credentials["api_key"] == "new-key" + + router.set_model_list(model_list=[]) + assert ( + router.get_deployment_credentials_with_provider( + model_id="openai/gpt-5.2", team_id="team-1" + ) + is None + ) + + def test_get_available_guardrail_single_deployment(): """ Test get_available_guardrail returns the single guardrail when only one exists. From b792fd7c5fb1e448fba5ae910d4c6ba230fd04a9 Mon Sep 17 00:00:00 2001 From: Shivam Rawat Date: Fri, 17 Jul 2026 18:24:58 -0700 Subject: [PATCH 3/4] test(router): cover PatternMatchRouter.remove_deployment for router code coverage gate Co-authored-by: Cursor --- tests/test_litellm/test_router.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 0fe855151b07..f9360abea517 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -3600,6 +3600,33 @@ def test_team_wildcard_credentials_not_usable_after_delete_deployment(): ) +def test_pattern_match_router_remove_deployment(): + """ + remove_deployment must drop only the deployment with the given model id and + delete patterns whose deployment list becomes empty. + """ + from litellm.router_utils.pattern_match_deployments import PatternMatchRouter + + pattern_router = PatternMatchRouter() + pattern_router.add_pattern( + "openai/*", + {"litellm_params": {"model": "openai/*", "api_key": "key-a"}, "model_info": {"id": "dep-a"}}, + ) + pattern_router.add_pattern( + "openai/*", + {"litellm_params": {"model": "openai/*", "api_key": "key-b"}, "model_info": {"id": "dep-b"}}, + ) + + pattern_router.remove_deployment(model_id="dep-a") + matches = pattern_router.route("openai/gpt-5.2") + assert matches is not None + assert [m["model_info"]["id"] for m in matches] == ["dep-b"] + + pattern_router.remove_deployment(model_id="dep-b") + assert pattern_router.patterns == {} + assert pattern_router.route("openai/gpt-5.2") is None + + def test_team_wildcard_credentials_refreshed_on_upsert_and_set_model_list(): """ Regression: replacing a team wildcard deployment (upsert or model list From 72ac741e33843978cea502d5c0dcae3bd2a0397a Mon Sep 17 00:00:00 2001 From: yucheng Date: Sat, 18 Jul 2026 18:41:12 +0000 Subject: [PATCH 4/4] test(vector_store): update credential resolution assertion for team_id kwarg Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../proxy/vector_store_endpoints/test_vector_store_endpoints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py b/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py index 1434dd6b1b24..e7de8b54e4ec 100644 --- a/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py +++ b/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py @@ -210,7 +210,7 @@ async def test_vector_store_file_list_resolves_single_openai_team_deployment(): assert result["model"] == "openai/gpt-4o-mini" assert "custom_llm_provider" not in result llm_router.get_deployment_credentials_with_provider.assert_called_once_with( - model_id="team-openai" + model_id="team-openai", team_id=None )