feat(guardrails): bound every guardrail check with litellm_params.timeout

litellm/integrations/custom_guardrail.py

@@ -176,6 +176,9 @@ class CustomGuardrail(CustomLogger):
 
     records_own_guardrail_information: ClassVar[bool] = False
 
+    hook_timeout: float | None = None
+    hook_timeout_fallback: Literal["fail_closed", "fail_open"] = "fail_closed"
+
     def __init_subclass__(cls, **kwargs: object) -> None:  # kwargs-ok: forwarded to cooperative __init_subclass__ hooks
         super().__init_subclass__(**kwargs)
         own_apply_guardrail: Final[object] = cls.__dict__.get("apply_guardrail")

litellm/proxy/_lazy_openapi_snapshot.json

@@ -10377,12 +10377,12 @@
                   "type": "null"
                 }
               ],
-              "description": "Per-request timeout for the guardrail provider API call (seconds). Accepts int, float, or numeric string; coerced to float on load. Each guardrail handler chooses its own default when unset.",
+              "description": "Maximum seconds one guardrail check may take, covering every call it makes. On expiry the check fails with a 408 timeout, or is skipped when unreachable_fallback='fail_open'. Some handlers also apply it to their provider HTTP call. Accepts int, float, or numeric string; coerced to float on load. Unset means no limit.",
               "title": "Timeout"
             },
             "unreachable_fallback": {
               "default": "fail_closed",
-              "description": "Behavior when a guardrail endpoint is unreachable due to network errors. Implemented by guardrail='generic_guardrail_api', 'agent_365', 'akto', 'vigil_guard', 'repelloai', 'headroom', 'compresr', and 'typesafe'. 'fail_closed' raises an error (default). 'fail_open' logs a critical error and allows the request to proceed.",
+              "description": "Behavior when a guardrail endpoint is unreachable due to network errors. Implemented by guardrail='generic_guardrail_api', 'agent_365', 'akto', 'vigil_guard', 'repelloai', 'headroom', 'compresr', and 'typesafe'. 'fail_closed' raises an error (default). 'fail_open' logs a critical error and allows the request to proceed. Every guardrail also honors it when its check exceeds timeout.",
               "enum": [
                 "fail_closed",
                 "fail_open"
@@ -13173,7 +13173,7 @@
                   "type": "null"
                 }
               ],
-              "description": "Per-request timeout for the guardrail provider API call (seconds). Accepts int, float, or numeric string; coerced to float on load. Each guardrail handler chooses its own default when unset.",
+              "description": "Maximum seconds one guardrail check may take, covering every call it makes. On expiry the check fails with a 408 timeout, or is skipped when unreachable_fallback='fail_open'. Some handlers also apply it to their provider HTTP call. Accepts int, float, or numeric string; coerced to float on load. Unset means no limit.",
               "title": "Timeout"
             },
             "tool_selection_quality_check": {

litellm/proxy/guardrails/guardrail_registry.py

@@ -459,6 +459,8 @@ def _configure_callback_scoping(
             "scanning, so no request content would ever be scanned. Remove one of the two."
         )
     _apply_configured_bool_overrides(custom_guardrail_callback, litellm_params)
+    custom_guardrail_callback.hook_timeout = litellm_params.timeout
+    custom_guardrail_callback.hook_timeout_fallback = litellm_params.unreachable_fallback
 
 
 class InMemoryGuardrailHandler:

litellm/proxy/utils.py

@@ -276,6 +276,20 @@
 _T: Final = TypeVar("_T")
 
 
+async def _await_within_guardrail_timeout(callback: object, coro: Awaitable[_T], fail_open_result: _T) -> _T:
+    if not isinstance(callback, CustomGuardrail) or callback.hook_timeout is None:
+        return await coro
+    timeout: Final = callback.hook_timeout
+    try:
+        return await asyncio.wait_for(coro, timeout)
+    except asyncio.TimeoutError:
+        message: Final = f"Guardrail '{callback.guardrail_name}' did not finish within {timeout}s"
+        if callback.hook_timeout_fallback == "fail_open":
+            verbose_proxy_logger.critical("%s; unreachable_fallback=fail_open, skipping it", message)
+            return fail_open_result
+        raise litellm.Timeout(message=message, model="", llm_provider="guardrail") from None
+
+
 class _ViewCountRow(TypedDict):
     view_count: ReadOnly[int]
     view_names: ReadOnly[Sequence[str] | None]
@@ -1833,23 +1847,35 @@ async def _execute_guardrail_hook(
         target: Final = unified_guardrail if use_unified else callback
 
         if hook_type == "pre_call":
-            return await target.async_pre_call_hook(
-                user_api_key_dict=user_api_key_dict,
-                cache=self.call_details["user_api_key_cache"],
-                data=data,
-                call_type=call_type,
+            return await _await_within_guardrail_timeout(
+                callback,
+                target.async_pre_call_hook(
+                    user_api_key_dict=user_api_key_dict,
+                    cache=self.call_details["user_api_key_cache"],
+                    data=data,
+                    call_type=call_type,
+                ),
+                fail_open_result=None,
             )
         elif hook_type == "during_call":
-            return await target.async_moderation_hook(
-                data=data,
-                user_api_key_dict=user_api_key_dict,
-                call_type=call_type,
+            return await _await_within_guardrail_timeout(
+                callback,
+                target.async_moderation_hook(
+                    data=data,
+                    user_api_key_dict=user_api_key_dict,
+                    call_type=call_type,
+                ),
+                fail_open_result=None,
             )
         elif hook_type == "post_call":
-            return await target.async_post_call_success_hook(
-                user_api_key_dict=user_api_key_dict,
-                data=data,
-                response=response,
+            return await _await_within_guardrail_timeout(
+                callback,
+                target.async_post_call_success_hook(
+                    user_api_key_dict=user_api_key_dict,
+                    data=data,
+                    response=response,
+                ),
+                fail_open_result=None,
             )
         else:
             raise ValueError(f"Unknown hook_type: {hook_type}")
@@ -2706,6 +2732,7 @@ async def _run_guardrail_with_metrics(
         coro: Awaitable[_T],
         hook_type: str,
         request_data: Mapping[str, object],
+        fail_open_result: _T,
     ) -> _T:
         """
         Await `coro`, recording its latency and status to the
@@ -2718,7 +2745,7 @@ async def _run_guardrail_with_metrics(
         status = "success"
         error_type: str | None = None
         try:
-            return await coro
+            return await _await_within_guardrail_timeout(callback, coro, fail_open_result)
         except SensitiveDataRouteException:
             status = "intervened"
             raise
@@ -3001,6 +3028,7 @@ async def _run_during_call_guardrail(
                 ),
                 "during_call",
                 request_data=data,
+                fail_open_result=None,
             )
             return
         await self._run_guardrail_with_metrics(
@@ -3012,6 +3040,7 @@ async def _run_during_call_guardrail(
             ),
             "during_call",
             request_data=data,
+            fail_open_result=None,
         )
 
     async def failed_tracking_alert(
@@ -3549,6 +3578,7 @@ async def post_call_success_hook(
                         ),
                         "post_call",
                         request_data=data,
+                        fail_open_result=None,
                     )
                 else:
                     guardrail_response = await self._run_guardrail_with_metrics(
@@ -3560,6 +3590,7 @@ async def post_call_success_hook(
                         ),
                         "post_call",
                         request_data=data,
+                        fail_open_result=None,
                     )
 
                 if guardrail_response is not None:
@@ -3624,6 +3655,7 @@ async def _run_one(callback: CustomGuardrail) -> None:
                     ),
                     "post_call",
                     request_data=data,
+                    fail_open_result=None,
                 )
             else:
                 await self._run_guardrail_with_metrics(
@@ -3635,6 +3667,7 @@ async def _run_one(callback: CustomGuardrail) -> None:
                     ),
                     "post_call",
                     request_data=data,
+                    fail_open_result=None,
                 )
 
         results: Final = await asyncio.gather(
@@ -3699,6 +3732,7 @@ async def post_mcp_call_hook(
                 ),
                 "post_mcp_call",
                 request_data=request_data,
+                fail_open_result=response,
             )
         return response
 

litellm/types/guardrails.py

@@ -1060,7 +1060,8 @@ class BaseLitellmParams(ContentFilterConfigModel):  # works for new and patch up
         description=(
             "Behavior when a guardrail endpoint is unreachable due to network errors. "
             "Implemented by guardrail='generic_guardrail_api', 'agent_365', 'akto', 'vigil_guard', 'repelloai', 'headroom', 'compresr', and 'typesafe'. "
-            "'fail_closed' raises an error (default). 'fail_open' logs a critical error and allows the request to proceed."
+            "'fail_closed' raises an error (default). 'fail_open' logs a critical error and allows the request to proceed. "
+            "Every guardrail also honors it when its check exceeds timeout."
         ),
     )
 
@@ -1082,9 +1083,10 @@ class BaseLitellmParams(ContentFilterConfigModel):  # works for new and patch up
     timeout: float | None = Field(
         default=None,
         description=(
-            "Per-request timeout for the guardrail provider API call (seconds). "
-            "Accepts int, float, or numeric string; coerced to float on load. "
-            "Each guardrail handler chooses its own default when unset."
+            "Maximum seconds one guardrail check may take, covering every call it makes. "
+            "On expiry the check fails with a 408 timeout, or is skipped when unreachable_fallback='fail_open'. "
+            "Some handlers also apply it to their provider HTTP call. "
+            "Accepts int, float, or numeric string; coerced to float on load. Unset means no limit."
         ),
     )
 

tests/test_litellm/proxy/guardrails/test_guardrail_registry.py

@@ -1084,3 +1084,42 @@ def db_guardrail(word: str, mode: str) -> Guardrail:
     finally:
         for cb_list, snapshot in zip(lists, snapshots):
             cb_list[:] = snapshot
+
+
+class _ProviderOwnsTimeoutGuardrail(CustomGuardrail):
+    def __init__(self, **kwargs):
+        self.timeout = 10.0
+        super().__init__(**kwargs)
+
+
+@pytest.mark.parametrize(
+    "litellm_params, expected",
+    [
+        ({}, (None, "fail_closed")),
+        ({"timeout": "2.5", "unreachable_fallback": "FAIL_OPEN"}, (2.5, "fail_open")),
+    ],
+)
+def test_initialize_guardrail_sets_hook_timeout_without_touching_provider_timeout(litellm_params, expected):
+    from litellm.proxy.guardrails import guardrail_registry as registry_module
+
+    def _initializer(litellm_params, guardrail):
+        return _ProviderOwnsTimeoutGuardrail(
+            guardrail_name=guardrail["guardrail_name"],
+            event_hook=GuardrailEventHooks.pre_call,
+        )
+
+    registry_module.guardrail_initializer_registry["hook_timeout_test"] = _initializer
+    try:
+        handler = InMemoryGuardrailHandler()
+        result = handler.initialize_guardrail(
+            guardrail={
+                "guardrail_name": "hook-timeout",
+                "litellm_params": {"guardrail": "hook_timeout_test", "mode": "pre_call", **litellm_params},
+            },
+        )
+
+        stored = handler.guardrail_id_to_custom_guardrail[result["guardrail_id"]]
+        assert (stored.hook_timeout, stored.hook_timeout_fallback) == expected
+        assert stored.timeout == 10.0
+    finally:
+        registry_module.guardrail_initializer_registry.pop("hook_timeout_test", None)

tests/test_litellm/proxy/utils/proxy_logging/test_during_call_hook.py

@@ -24,6 +24,7 @@ def _clear_caps_cache():
 def _make_guardrail(name="g1", should_run=True, response=None):
     cb = MagicMock(spec=CustomGuardrail)
     cb.__class__ = CustomGuardrail
+    cb.hook_timeout = None
     cb.guardrail_name = name
     cb.event_hook = GuardrailEventHooks.during_call
     cb.use_native_during_call_hook = False