Support max_retries for non-OpenAI/Azure providers

litellm/litellm_core_utils/get_supported_openai_params.py

@@ -53,6 +53,16 @@ def get_supported_openai_params(
 
     if provider_config and request_type == "chat_completion":
         supported_params = provider_config.get_supported_openai_params(model=model)
+        allowed_openai_params = getattr(provider_config, "allowed_openai_params", None)
+        supported_params = supported_params or []
+        allowed_openai_params = allowed_openai_params or []
+        supported_params.extend(allowed_openai_params)
+        # ``max_retries`` is supported for providers with actual retry wiring
+        # (OpenAI/Azure retry at the SDK layer; anthropic retries at LiteLLM's
+        # httpx transport layer when ``max_retries`` is set).
+        providers_with_retry_support = {"openai", "azure", "anthropic"}
+        if custom_llm_provider in providers_with_retry_support and "max_retries" not in supported_params:
+            supported_params.append("max_retries")
         if base_model and base_model != model:
             base_model_params = provider_config.get_supported_openai_params(model=base_model)
             supported_params = list(dict.fromkeys([*supported_params, *base_model_params]))

litellm/llms/anthropic/chat/handler.py

@@ -11,6 +11,7 @@
     Dict,
     List,
     Literal,
+    Optional,
     Tuple,
     Union,
     cast,
@@ -229,8 +230,16 @@ async def acompletion_stream_function(
 
         data["stream"] = True
 
+        _stream_client = client
+        _stream_max_retries = optional_params.get("max_retries") if isinstance(optional_params, dict) else None
+        if _stream_client is None and isinstance(_stream_max_retries, int):
+            _stream_client = get_async_httpx_client(
+                llm_provider=litellm.LlmProviders.ANTHROPIC,
+                params={"max_retries": _stream_max_retries},
+            )
+
         completion_stream, headers = await make_call(
-            client=client,
+            client=_stream_client,
             api_base=api_base,
             headers=headers,
             data=json.dumps(data),
@@ -276,7 +285,13 @@ async def acompletion_function(
         headers={},
         client: AsyncHTTPHandler | None = None,
     ) -> Union[ModelResponse, "CustomStreamWrapper"]:
-        async_handler = client or get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC)
+        _async_client_params: Optional[dict] = None
+        _async_max_retries = optional_params.get("max_retries") if isinstance(optional_params, dict) else None
+        if isinstance(_async_max_retries, int):
+            _async_client_params = {"max_retries": _async_max_retries}
+        async_handler = client or get_async_httpx_client(
+            llm_provider=litellm.LlmProviders.ANTHROPIC, params=_async_client_params
+        )
 
         try:
             response = await async_handler.post(
@@ -450,8 +465,16 @@ def completion(
                 stream is True
             ):  # if function call - fake the streaming (need complete blocks for output parsing in openai format)
                 data["stream"] = stream
+                _sync_stream_client = client
+                _sync_stream_max_retries = (
+                    optional_params.get("max_retries") if isinstance(optional_params, dict) else None
+                )
+                if _sync_stream_client is None and isinstance(_sync_stream_max_retries, int):
+                    _sync_stream_client = _get_httpx_client(
+                        params={"timeout": timeout, "max_retries": _sync_stream_max_retries}
+                    )
                 completion_stream, headers = make_sync_call(
-                    client=client,
+                    client=_sync_stream_client,
                     api_base=api_base,
                     headers=headers,  # type: ignore
                     data=json.dumps(data),
@@ -477,7 +500,11 @@ def completion(
 
             else:
                 if client is None or not isinstance(client, HTTPHandler):
-                    client = _get_httpx_client(params={"timeout": timeout})
+                    _sync_client_params: dict = {"timeout": timeout}
+                    _sync_max_retries = optional_params.get("max_retries")
+                    if isinstance(_sync_max_retries, int):
+                        _sync_client_params["max_retries"] = _sync_max_retries
+                    client = _get_httpx_client(params=_sync_client_params)
                 else:
                     client = client
 

litellm/llms/anthropic/chat/transformation.py

@@ -447,6 +447,7 @@ def get_supported_openai_params(self, model: str):
             "speed",
             "context_management",
             "cache_control",
+            "max_retries",
         ]
 
         if (
@@ -1509,6 +1510,11 @@ def map_openai_params(
             elif param == "cache_control" and isinstance(value, dict):
                 # Pass through top-level cache_control for automatic prompt caching
                 optional_params["cache_control"] = value
+            elif param == "max_retries" and isinstance(value, int):
+                # Consumed by LiteLLM's httpx transport layer (see
+                # AsyncHTTPHandler/HTTPHandler). Kept off the request body below
+                # in transform_request so it is never sent to the Anthropic API.
+                optional_params["max_retries"] = value
 
         ## handle thinking tokens
         self.update_optional_params_with_thinking_tokens(
@@ -1870,6 +1876,8 @@ def transform_request(
         # Remove internal LiteLLM parameters that should not be sent to Anthropic API
         optional_params.pop("is_vertex_request", None)
         optional_params.pop("client_metadata", None)
+        # max_retries is consumed by LiteLLM's httpx transport, not the Anthropic API.
+        optional_params.pop("max_retries", None)
 
         # ``top_k`` is a provider-specific kwarg that bypasses
         # ``map_openai_params``; gate it here, the single boundary shared by

litellm/llms/custom_httpx/http_handler.py

@@ -515,14 +515,17 @@ def __init__(
         client_alias: Optional[str] = None,  # name for client in logs
         ssl_verify: Optional[VerifyTypes] = None,
         shared_session: Optional["ClientSession"] = None,
+        max_retries: Optional[int] = None,
     ):
         self.timeout = timeout
         self.event_hooks = event_hooks
+        self.max_retries = max_retries
         self.client = self.create_client(
             timeout=timeout,
             event_hooks=event_hooks,
             ssl_verify=ssl_verify,
             shared_session=shared_session,
+            max_retries=max_retries,
         )
         self.client_alias = client_alias
 
@@ -532,6 +535,7 @@ def create_client(
         event_hooks: Optional[Mapping[str, List[Callable[..., Any]]]],
         ssl_verify: Optional[VerifyTypes] = None,
         shared_session: Optional["ClientSession"] = None,
+        max_retries: Optional[int] = None,
     ) -> httpx.AsyncClient:
         # Get unified SSL configuration
         ssl_config = get_ssl_configuration(ssl_verify)
@@ -548,6 +552,7 @@ def create_client(
             ssl_context=ssl_config if isinstance(ssl_config, ssl.SSLContext) else None,
             ssl_verify=ssl_config if isinstance(ssl_config, bool) else None,
             shared_session=shared_session,
+            max_retries=max_retries,
         )
 
         # Get default headers (User-Agent, overridable via LITELLM_USER_AGENT)
@@ -890,13 +895,21 @@ def _create_async_transport(
         ssl_context: Optional[ssl.SSLContext] = None,
         ssl_verify: Optional[bool] = None,
         shared_session: Optional["ClientSession"] = None,
+        max_retries: Optional[int] = None,
     ) -> Optional[Union[LiteLLMAiohttpTransport, AsyncHTTPTransport]]:
         """
         - Creates a transport for httpx.AsyncClient
             - if litellm.force_ipv4 is True, it will return AsyncHTTPTransport with local_address="0.0.0.0"
             - [Default] It will return AiohttpTransport
             - Users can opt out of using AiohttpTransport by setting litellm.use_aiohttp_transport to False
 
+        - When ``max_retries`` is requested we use the httpx transport with a
+          built-in ``httpx.Retry`` policy (aiohttp has no native retry support in
+          LiteLLM's transport). This is an opt-in that only affects clients created
+          with ``max_retries`` set, so the default high-throughput aiohttp path is
+          unchanged.
+
+
 
         Notes on this handler:
         - Why AiohttpTransport?
@@ -905,6 +918,12 @@ def _create_async_transport(
         - Why force ipv4?
             - Some users have seen httpx ConnectionError when using ipv6 - forcing ipv4 resolves the issue for them
         """
+        #########################################################
+        # Retry transport (opt-in via max_retries)
+        #########################################################
+        if max_retries is not None and max_retries > 0:
+            return AsyncHTTPHandler._create_httpx_transport(max_retries=max_retries)
+
         #########################################################
         # AIOHTTP TRANSPORT is off by default
         #########################################################
@@ -1050,13 +1069,23 @@ def _create_aiohttp_transport(
         )
 
     @staticmethod
-    def _create_httpx_transport() -> Optional[AsyncHTTPTransport]:
+    def _create_httpx_transport(
+        max_retries: Optional[int] = None,
+    ) -> Optional[AsyncHTTPTransport]:
         """
         Creates an AsyncHTTPTransport
 
         - If force_ipv4 is True, it will create an AsyncHTTPTransport with local_address set to "0.0.0.0"
-        - [Default] If force_ipv4 is False, it will return None
+        - If ``max_retries`` is provided, the transport retries transient
+          connection errors using httpx's built-in ``retries`` policy before
+          surfacing the failure to the caller.
+        - [Default] If force_ipv4 is False and max_retries is None, it will return None
         """
+        if max_retries is not None and max_retries > 0:
+            if litellm.force_ipv4:
+                return AsyncHTTPTransport(retries=max_retries, local_address="0.0.0.0")
+            return AsyncHTTPTransport(retries=max_retries)
+
         if litellm.force_ipv4:
             return AsyncHTTPTransport(local_address="0.0.0.0")
         else:
@@ -1073,6 +1102,7 @@ def __init__(
         disable_default_headers: Optional[
             bool
         ] = False,  # arize phoenix returns different API responses when user agent header in request
+        max_retries: Optional[int] = None,
     ):
         if timeout is None:
             timeout = _DEFAULT_TIMEOUT
@@ -1088,7 +1118,7 @@ def __init__(
         default_headers = get_default_headers() if not disable_default_headers else None
 
         if client is None:
-            transport = self._create_sync_transport()
+            transport = self._create_sync_transport(max_retries=max_retries)
 
             # Create a client with a connection pool
             self.client = httpx.Client(
@@ -1351,13 +1381,21 @@ def __del__(self) -> None:
         except Exception:
             pass
 
-    def _create_sync_transport(self) -> Optional[HTTPTransport]:
+    def _create_sync_transport(self, max_retries: Optional[int] = None) -> Optional[HTTPTransport]:
         """
         Create an HTTP transport with IPv4 only if litellm.force_ipv4 is True.
         Otherwise, return None.
 
         Some users have seen httpx ConnectionError when using ipv6 - forcing ipv4 resolves the issue for them
+
+        When ``max_retries`` is provided, the transport retries transient
+        connection errors via httpx's built-in ``retries`` policy.
         """
+        if max_retries is not None and max_retries > 0:
+            if litellm.force_ipv4:
+                return HTTPTransport(retries=max_retries, local_address="0.0.0.0")
+            return HTTPTransport(retries=max_retries)
+
         if litellm.force_ipv4:
             return HTTPTransport(local_address="0.0.0.0")
         else:

litellm/utils.py

@@ -3817,10 +3817,6 @@ def _check_valid_arg(supported_params: List[str]):
                     continue
                 if k == "n" and n == 1:  # langchain sends n=1 as a default value
                     continue  # skip this param
-                if (
-                    k == "max_retries"
-                ):  # TODO: This is a patch. We support max retries for OpenAI, Azure. For non OpenAI LLMs we need to add support for max retries
-                    continue  # skip this param
                 # Always keeps this in elif code blocks
                 else:
                     unsupported_params[k] = non_default_params[k]
@@ -3845,6 +3841,9 @@ def _check_valid_arg(supported_params: List[str]):
     supported_params = supported_params or []
     allowed_openai_params = allowed_openai_params or []
     supported_params.extend(allowed_openai_params)
+    providers_with_retry_support = {"openai", "azure", "anthropic"}
+    if custom_llm_provider in providers_with_retry_support and "max_retries" not in supported_params:
+        supported_params.append("max_retries")
 
     _check_valid_arg(
         supported_params=supported_params or [],