revert: "fix(caching): close evicted LLM clients so their connections are reclaimed (#35492)"

litellm/caching/llm_caching_handler.py

@@ -5,44 +5,21 @@
 import asyncio
 from typing import Final
 
-from .evicted_client_closer import EvictedClientCloser, default_evicted_client_closer
 from .in_memory_cache import InMemoryCache
 
 
 class LLMClientCache(InMemoryCache):
     """Cache for LLM HTTP clients (OpenAI, Azure, httpx, etc.).
 
-    An evicted client is never closed on the spot: a request handed the client
-    just before eviction is still using it, and closing it there raises
-    ``RuntimeError: Cannot send a request, as the client has been closed.``
+    IMPORTANT: This cache intentionally does NOT close clients on eviction.
+    Evicted clients may still be in use by in-flight requests. Closing them
+    eagerly causes ``RuntimeError: Cannot send a request, as the client has
+    been closed.`` errors in production after the TTL (1 hour) expires.
 
-    Nor can eviction be left to rely on garbage collection. The SDK clients are
-    reference cycles, so an evicted client and its open TCP connections survive
-    until a generational collection runs. Instead a client litellm created is
-    handed to ``EvictedClientCloser``, which closes it once a grace window has
-    passed. Clients the caller supplied are left untouched.
+    Clients that are no longer referenced will be garbage-collected normally.
+    For explicit shutdown cleanup, use ``close_litellm_async_clients()``.
     """
 
-    def __init__(
-        self,
-        max_size_in_memory: int | None = 200,
-        default_ttl: int | None = 600,
-        max_size_per_item: int | None = 1024,
-        evicted_client_closer: EvictedClientCloser | None = None,
-    ):
-        super().__init__(
-            max_size_in_memory=max_size_in_memory,
-            default_ttl=default_ttl,
-            max_size_per_item=max_size_per_item,
-        )
-        self.evicted_client_closer = evicted_client_closer or default_evicted_client_closer
-
-    def _remove_key(self, key: str) -> None:
-        evicted: Final[object] = self.cache_dict.get(key)
-        super()._remove_key(key)
-        self.evicted_client_closer.schedule(evicted)
-        self.evicted_client_closer.reap()
-
     def update_cache_key_with_event_loop(self, key):
         """
         Add the event loop to the cache key, to prevent event loop closed errors.
@@ -55,22 +32,16 @@ def update_cache_key_with_event_loop(self, key):
         except RuntimeError:  # handle no current running event loop
             return key
 
-    def set_cache(self, key: str, value: object, litellm_owned_client: bool = False, **kwargs):
-        """``litellm_owned_client`` marks a client litellm built, so it may be closed once evicted."""
-        if litellm_owned_client:
-            self.evicted_client_closer.mark_owned(value)
+    def set_cache(self, key, value, **kwargs):
         key = self.update_cache_key_with_event_loop(key)
         return super().set_cache(key, value, **kwargs)
 
-    async def async_set_cache(self, key: str, value: object, litellm_owned_client: bool = False, **kwargs):
-        if litellm_owned_client:
-            self.evicted_client_closer.mark_owned(value)
+    async def async_set_cache(self, key, value, **kwargs):
         key = self.update_cache_key_with_event_loop(key)
         return await super().async_set_cache(key, value, **kwargs)
 
     def get_cache(self, key, **kwargs):
         key = self.update_cache_key_with_event_loop(key)
-        self.evicted_client_closer.reap()
 
         return super().get_cache(key, **kwargs)
 

litellm/constants.py

@@ -197,16 +197,6 @@
 ########## Networking constants ##############################################################
 _DEFAULT_TTL_FOR_HTTPX_CLIENTS: Final = 3600  # 1 hour, re-use the same httpx client for 1 hour
 
-# The earliest an evicted, litellm-created client may be closed. A request handed the
-# client just before eviction is still using it, so nothing is closed inside this window;
-# past it, the client is closed once it reports no connection in flight.
-EVICTED_LLM_CLIENT_CLOSE_GRACE_SECONDS: Final = 900
-
-# How many evicted clients may be queued for closing at once. Past this, an evicted client
-# is left to the collector rather than letting a cache-churning workload grow the queue
-# without bound. Each queued entry is ~100 bytes and comes due within one grace window.
-EVICTED_LLM_CLIENT_CLOSE_MAX_PENDING: Final = 10_000
-
 # Aiohttp connection pooling - prevents memory leaks from unbounded connection growth
 # Set to 0 for unlimited (not recommended for production)
 AIOHTTP_CONNECTOR_LIMIT: Final = int(os.getenv("AIOHTTP_CONNECTOR_LIMIT", 1000))

litellm/llms/azure/common_utils.py

@@ -509,8 +509,6 @@ async def _async_v1_api_key() -> str:
             openai_client=openai_client,
             client_initialization_params=client_initialization_params,
             client_type="azure",
-            litellm_owned_client=client is None
-            and self.owns_wrapped_http_client(azure_client_params.get("http_client")),
         )
         return openai_client
 

litellm/llms/custom_httpx/http_handler.py

@@ -1408,7 +1408,6 @@ def get_async_httpx_client(
         key=_cache_key_name,
         value=_new_client,
         ttl=_DEFAULT_TTL_FOR_HTTPX_CLIENTS,
-        litellm_owned_client=True,
     )
     return _new_client
 
@@ -1454,6 +1453,5 @@ def _get_httpx_client(params: dict | None = None) -> HTTPHandler:
         key=_cache_key_name,
         value=_new_client,
         ttl=_DEFAULT_TTL_FOR_HTTPX_CLIENTS,
-        litellm_owned_client=True,
     )
     return _new_client