diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 9c5c545fe76b..e84ef4326999 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -92,6 +92,7 @@ # Global registry to track registered pass-through routes and prevent memory leaks _registered_pass_through_routes: Dict[str, Dict[str, Union[str, bool, List[str], Dict[str, Any]]]] = {} +_STATIC_OPENAI_ROUTES = frozenset(LiteLLMRoutes.openai_routes.value) def get_response_body(response: httpx.Response) -> Optional[dict]: @@ -2623,19 +2624,15 @@ def remove_endpoint_routes(endpoint_id: str): keys_to_remove = [ key for key, value in _registered_pass_through_routes.items() if value["endpoint_id"] == endpoint_id ] + removed_openai_routes = frozenset( + route + for key in keys_to_remove + if (route := _get_registered_pass_through_openai_route(_registered_pass_through_routes[key])) is not None + ) for key in keys_to_remove: - route_info = _registered_pass_through_routes[key] - path = route_info.get("path") - if isinstance(path, str): - openai_routes = LiteLLMRoutes.openai_routes.value - if path in openai_routes: - openai_routes.remove(path) - if route_info.get("type") == "subpath": - wildcard_path = path.rstrip("/") + "/*" - if wildcard_path in openai_routes: - openai_routes.remove(wildcard_path) del _registered_pass_through_routes[key] verbose_proxy_logger.debug("Removed pass-through route from registry: %s", key) + _remove_unused_pass_through_openai_routes(removed_openai_routes) @staticmethod def clear_all_pass_through_routes(): @@ -2741,6 +2738,48 @@ def _get_combined_pass_through_endpoints( return pass_through_endpoints + config_pass_through_endpoints +def _get_registered_pass_through_openai_route( + route_info: Mapping[str, object], +) -> Optional[str]: + if route_info.get("auth") is not True: + return None + path = route_info.get("path") + if not isinstance(path, str): + return None + match route_info.get("type"): + case "exact": + return path + case "subpath": + return path.rstrip("/") + "/*" + case _: + return None + + +def _remove_unused_pass_through_openai_routes( + previously_registered_routes: frozenset[str], +) -> None: + active_routes = frozenset( + route + for route_info in _registered_pass_through_routes.values() + if (route := _get_registered_pass_through_openai_route(route_info)) is not None + ) + routes_to_remove = previously_registered_routes.difference(active_routes, _STATIC_OPENAI_ROUTES) + LiteLLMRoutes.openai_routes.value[:] = [ + route for route in LiteLLMRoutes.openai_routes.value if route not in routes_to_remove + ] + + +def _remove_stale_pass_through_routes( + registered_keys: tuple[str, ...], + visited_keys: set[str], + previously_registered_openai_routes: frozenset[str], +) -> None: + for endpoint_key in registered_keys: + if endpoint_key not in visited_keys: + _registered_pass_through_routes.pop(endpoint_key, None) + _remove_unused_pass_through_openai_routes(previously_registered_openai_routes) + + async def _register_pass_through_endpoint( endpoint: Union[Dict[str, Any], PassThroughGenericEndpoint], app: FastAPI, @@ -2884,7 +2923,12 @@ async def initialize_pass_through_endpoints( # get a list of all registered pass-through endpoints # mark the ones that are visited in the list # remove the ones that are not visited from the list - registered_pass_through_endpoints = InitPassThroughEndpointHelpers.get_all_registered_pass_through_routes() + registered_pass_through_endpoints = tuple(InitPassThroughEndpointHelpers.get_all_registered_pass_through_routes()) + previously_registered_openai_routes = frozenset( + route + for route_info in _registered_pass_through_routes.values() + if (route := _get_registered_pass_through_openai_route(route_info)) is not None + ) visited_endpoints: set[str] = set() @@ -2897,14 +2941,11 @@ async def initialize_pass_through_endpoints( config_file_path=config_file_path, ) - # Drop stale registry entries by their exact route key. registered_pass_through_endpoints - # holds route keys ("{id}:{type}:{path}:{methods}"), not endpoint ids, so remove_endpoint_routes - # (which matches on endpoint_id) never matched and left the registry growing every reload cycle. - # We pop the key directly and leave openai_routes alone: its append is path-deduped, and the path - # is still owned by the live endpoint that was just re-registered under a new id this same cycle. - for endpoint_key in registered_pass_through_endpoints: - if endpoint_key not in visited_endpoints: - _registered_pass_through_routes.pop(endpoint_key, None) + _remove_stale_pass_through_routes( + registered_keys=registered_pass_through_endpoints, + visited_keys=visited_endpoints, + previously_registered_openai_routes=previously_registered_openai_routes, + ) def _get_pass_through_endpoints_from_config() -> List[PassThroughGenericEndpoint]: diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index 9bddeda07239..b1f63552ce3a 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -32,7 +32,7 @@ resolve_llm_passthrough_timeout, ) from litellm.integrations.custom_logger import CustomLogger -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy._types import LiteLLMRoutes, UserAPIKeyAuth from litellm.types.passthrough_endpoints.pass_through_endpoints import ( LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, ) @@ -3672,9 +3672,11 @@ class TestStaleRouteCleanupOnReload: def setup_method(self): _registered_pass_through_routes.clear() + self.original_openai_routes = list(LiteLLMRoutes.openai_routes.value) def teardown_method(self): _registered_pass_through_routes.clear() + LiteLLMRoutes.openai_routes.value[:] = self.original_openai_routes @staticmethod def _patches(): @@ -3770,6 +3772,103 @@ async def test_live_route_survives_reload_and_stays_resolvable(self): "/live-passthrough/some/subpath" ) + @pytest.mark.asyncio + async def test_departed_authenticated_endpoint_is_removed_from_openai_routes(self): + path = "/departed-authenticated-passthrough" + wildcard_path = path + "/*" + with self._patches(): + await initialize_pass_through_endpoints( + [ + { + "id": "departed-endpoint", + "path": path, + "target": "http://example.com", + "include_subpath": True, + "auth": True, + } + ] + ) + assert path in LiteLLMRoutes.openai_routes.value + assert wildcard_path in LiteLLMRoutes.openai_routes.value + + await initialize_pass_through_endpoints([]) + + assert path not in LiteLLMRoutes.openai_routes.value + assert wildcard_path not in LiteLLMRoutes.openai_routes.value + + @pytest.mark.asyncio + async def test_live_authenticated_path_survives_idless_reload(self): + path = "/live-authenticated-passthrough" + wildcard_path = path + "/*" + with self._patches(): + for _ in range(3): + await initialize_pass_through_endpoints( + [ + { + "path": path, + "target": "http://example.com", + "include_subpath": True, + "auth": True, + } + ] + ) + + assert LiteLLMRoutes.openai_routes.value.count(path) == 1 + assert LiteLLMRoutes.openai_routes.value.count(wildcard_path) == 1 + + @pytest.mark.asyncio + async def test_shared_authenticated_path_survives_partial_removal(self): + path = "/shared-authenticated-passthrough" + endpoints = [ + { + "id": endpoint_id, + "path": path, + "target": "http://example.com", + "auth": True, + } + for endpoint_id in ("departed", "live") + ] + with self._patches(): + await initialize_pass_through_endpoints(endpoints) + await initialize_pass_through_endpoints([endpoints[1]]) + + assert path in LiteLLMRoutes.openai_routes.value + + @pytest.mark.asyncio + async def test_disabling_auth_removes_dynamic_openai_route(self): + path = "/auth-toggle-passthrough" + endpoint = { + "id": "auth-toggle", + "path": path, + "target": "http://example.com", + "auth": True, + } + with self._patches(): + await initialize_pass_through_endpoints([endpoint]) + assert path in LiteLLMRoutes.openai_routes.value + + await initialize_pass_through_endpoints([{**endpoint, "auth": False}]) + + assert path not in LiteLLMRoutes.openai_routes.value + + @pytest.mark.asyncio + async def test_builtin_openai_route_is_never_removed(self): + path = "/v1/chat/completions" + with self._patches(): + await initialize_pass_through_endpoints( + [ + { + "id": "builtin-collision", + "path": path, + "target": "http://example.com", + "auth": True, + } + ] + ) + await initialize_pass_through_endpoints([]) + + assert path in LiteLLMRoutes.openai_routes.value + # Regression (LIT-3538): a pre-call guardrail block on a passthrough endpoint # must be logged at WARNING without a traceback, not as an ERROR with a full