Skip to content
64 changes: 62 additions & 2 deletions litellm/integrations/custom_guardrail.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
import asyncio
import contextvars
import copy
import hashlib
import os
import secrets
from collections.abc import Mapping, Sequence
from collections.abc import Awaitable, Mapping, Sequence
from dataclasses import dataclass
from datetime import datetime
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, get_args
from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, TypeVar, get_args

from litellm._logging import verbose_logger
from litellm.caching import DualCache
Expand Down Expand Up @@ -51,6 +53,7 @@
GuardrailRaisedException,
ModifyResponseException,
SensitiveDataRouteException,
Timeout,
)

# Per-process secret tagging each recorded marker. The deployment hook only
Expand Down Expand Up @@ -167,6 +170,60 @@ def get_session_id_from_request_data(request_data: dict[str, Any]) -> str | None
return None


_T = TypeVar("_T")


@dataclass(frozen=True)
class GuardrailHookTimedOut:
guardrail_name: str | None
timeout: float
fail_open: bool

@property
def message(self) -> str:
return f"Guardrail '{self.guardrail_name}' did not finish within {self.timeout}s"


@dataclass(frozen=True)
class _HookRaisedTimeout:
error: TimeoutError | asyncio.TimeoutError


async def _capture_hook_timeout(coro: Awaitable[_T]) -> _T | _HookRaisedTimeout:
try:
return await coro
except (TimeoutError, asyncio.TimeoutError) as e:
return _HookRaisedTimeout(e)


async def await_within_hook_timeout(callback: object, coro: Awaitable[_T]) -> _T | GuardrailHookTimedOut:
"""Await a guardrail check, bounded by the callback's configured ``hook_timeout``."""
if not isinstance(callback, CustomGuardrail) or callback.hook_timeout is None:
return await coro
timeout: Final = callback.hook_timeout
try:
outcome: Final = await asyncio.wait_for(_capture_hook_timeout(coro), timeout)
except (TimeoutError, asyncio.TimeoutError):
return GuardrailHookTimedOut(
guardrail_name=callback.guardrail_name,
timeout=timeout,
fail_open=callback.hook_timeout_fallback == "fail_open",
)
if isinstance(outcome, _HookRaisedTimeout):
raise outcome.error
return outcome


def resolve_hook_timeout(outcome: _T | GuardrailHookTimedOut, fail_open_result: _T) -> _T:
"""Map a timed-out check to ``fail_open_result`` or a 408 ``Timeout``, per ``unreachable_fallback``."""
if not isinstance(outcome, GuardrailHookTimedOut):
return outcome
if outcome.fail_open:
verbose_logger.critical("%s; unreachable_fallback=fail_open, skipping it", outcome.message)
return fail_open_result
raise Timeout(message=outcome.message, model="", llm_provider="guardrail")
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved


class CustomGuardrail(CustomLogger):
# If True, during_call runs async_moderation_hook instead of the unified apply_guardrail path.
use_native_during_call_hook: ClassVar[bool] = False
Expand All @@ -176,6 +233,9 @@ class CustomGuardrail(CustomLogger):

records_own_guardrail_information: ClassVar[bool] = False

hook_timeout: float | None = None
hook_timeout_fallback: Literal["fail_closed", "fail_open"] = "fail_closed"

def __init_subclass__(cls, **kwargs: object) -> None: # kwargs-ok: forwarded to cooperative __init_subclass__ hooks
super().__init_subclass__(**kwargs)
own_apply_guardrail: Final[object] = cls.__dict__.get("apply_guardrail")
Expand Down
4 changes: 2 additions & 2 deletions litellm/integrations/prometheus.py
Original file line number Diff line number Diff line change
Expand Up @@ -3222,7 +3222,7 @@ def _record_guardrail_metrics(
Args:
guardrail_name: Name of the guardrail
latency_seconds: Execution latency in seconds
status: "success", "error", or "intervened"
status: "success", "error", "intervened", or "timeout"
error_type: Type of error if any, None otherwise
hook_type: "pre_call", "during_call", or "post_call"
"""
Expand All @@ -3243,7 +3243,7 @@ def _record_guardrail_metrics(
).inc()

# Record error count if there was an error
if status == "error" and error_type:
if status in ("error", "timeout") and error_type:
self.litellm_guardrail_errors_total.labels(
guardrail_name=guardrail_name,
error_type=error_type,
Expand Down
6 changes: 3 additions & 3 deletions litellm/proxy/_lazy_openapi_snapshot.json
Original file line number Diff line number Diff line change
Expand Up @@ -10377,12 +10377,12 @@
"type": "null"
}
],
"description": "Per-request timeout for the guardrail provider API call (seconds). Accepts int, float, or numeric string; coerced to float on load. Each guardrail handler chooses its own default when unset.",
"description": "Maximum seconds one non-streaming guardrail check may take, covering every call it makes. On expiry the check fails with a 408 timeout, or is skipped when unreachable_fallback='fail_open'. Streaming response hooks are not bounded by it. Some handlers also apply it to their provider HTTP call. Accepts int, float, or numeric string; coerced to float on load. Unset means no proxy-side limit; provider HTTP client defaults still apply.",
"title": "Timeout"
},
"unreachable_fallback": {
"default": "fail_closed",
"description": "Behavior when a guardrail endpoint is unreachable due to network errors. Implemented by guardrail='generic_guardrail_api', 'agent_365', 'akto', 'vigil_guard', 'repelloai', 'headroom', 'compresr', and 'typesafe'. 'fail_closed' raises an error (default). 'fail_open' logs a critical error and allows the request to proceed.",
"description": "Behavior when a guardrail endpoint is unreachable due to network errors. Implemented by guardrail='generic_guardrail_api', 'agent_365', 'akto', 'vigil_guard', 'repelloai', 'headroom', 'compresr', and 'typesafe'. 'fail_closed' raises an error (default). 'fail_open' logs a critical error and allows the request to proceed. Every guardrail also honors it when its check exceeds timeout.",
"enum": [
"fail_closed",
"fail_open"
Expand Down Expand Up @@ -13173,7 +13173,7 @@
"type": "null"
}
],
"description": "Per-request timeout for the guardrail provider API call (seconds). Accepts int, float, or numeric string; coerced to float on load. Each guardrail handler chooses its own default when unset.",
"description": "Maximum seconds one non-streaming guardrail check may take, covering every call it makes. On expiry the check fails with a 408 timeout, or is skipped when unreachable_fallback='fail_open'. Streaming response hooks are not bounded by it. Some handlers also apply it to their provider HTTP call. Accepts int, float, or numeric string; coerced to float on load. Unset means no proxy-side limit; provider HTTP client defaults still apply.",
"title": "Timeout"
},
"tool_selection_quality_check": {
Expand Down
2 changes: 2 additions & 0 deletions litellm/proxy/guardrails/guardrail_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -459,6 +459,8 @@ def _configure_callback_scoping(
"scanning, so no request content would ever be scanned. Remove one of the two."
)
_apply_configured_bool_overrides(custom_guardrail_callback, litellm_params)
custom_guardrail_callback.hook_timeout = litellm_params.timeout
custom_guardrail_callback.hook_timeout_fallback = litellm_params.unreachable_fallback


class InMemoryGuardrailHandler:
Expand Down
32 changes: 23 additions & 9 deletions litellm/proxy/policy_engine/pipeline_executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@
from litellm.integrations.custom_guardrail import (
CustomGuardrail,
ModifyResponseException,
await_within_hook_timeout,
resolve_hook_timeout,
)
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import (
Expand Down Expand Up @@ -528,11 +530,17 @@ async def _run_step(

try:
if mode == "pre_call":
response = await target.async_pre_call_hook(
user_api_key_dict=user_api_key_dict,
cache=None,
data=hook_input,
call_type=call_type,
response = resolve_hook_timeout(
await await_within_hook_timeout(
callback,
target.async_pre_call_hook(
user_api_key_dict=user_api_key_dict,
cache=None,
data=hook_input,
call_type=call_type,
),
),
fail_open_result=None,
Comment thread
cursor[bot] marked this conversation as resolved.
Resolved
)
if isinstance(callback, CustomGuardrail):
callback.mark_pre_call_hook_ran(data)
Expand All @@ -557,10 +565,16 @@ async def _run_step(
)
response = None
elif mode == "post_call":
response = await target.async_post_call_success_hook(
user_api_key_dict=user_api_key_dict,
data=data,
response=data.get("response"),
response = resolve_hook_timeout(
await await_within_hook_timeout(
callback,
target.async_post_call_success_hook(
user_api_key_dict=user_api_key_dict,
data=data,
response=data.get("response"),
),
),
fail_open_result=None,
)
else:
return ("error", None, f"Unsupported pipeline mode: {mode}", None)
Expand Down
73 changes: 52 additions & 21 deletions litellm/proxy/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,10 @@
)
from litellm.integrations.custom_guardrail import (
CustomGuardrail,
GuardrailHookTimedOut,
ModifyResponseException,
await_within_hook_timeout,
resolve_hook_timeout,
)
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.prometheus import PrometheusLogger
Expand Down Expand Up @@ -1818,7 +1821,7 @@ async def _execute_guardrail_hook(
response: Response object (for post_call hooks)

Returns:
Result from the guardrail execution
Result from the guardrail execution, or ``GuardrailHookTimedOut`` when it exceeded ``hook_timeout``
"""
# Use unified_guardrail if callback has apply_guardrail method
has_apply_guardrail: Final = "apply_guardrail" in type(callback).__dict__ and not getattr(
Expand All @@ -1833,23 +1836,32 @@ async def _execute_guardrail_hook(
target: Final = unified_guardrail if use_unified else callback

if hook_type == "pre_call":
return await target.async_pre_call_hook(
user_api_key_dict=user_api_key_dict,
cache=self.call_details["user_api_key_cache"],
data=data,
call_type=call_type,
return await await_within_hook_timeout(
callback,
target.async_pre_call_hook(
user_api_key_dict=user_api_key_dict,
cache=self.call_details["user_api_key_cache"],
data=data,
call_type=call_type,
),
)
elif hook_type == "during_call":
return await target.async_moderation_hook(
data=data,
user_api_key_dict=user_api_key_dict,
call_type=call_type,
return await await_within_hook_timeout(
callback,
target.async_moderation_hook(
data=data,
user_api_key_dict=user_api_key_dict,
call_type=call_type,
),
)
elif hook_type == "post_call":
return await target.async_post_call_success_hook(
user_api_key_dict=user_api_key_dict,
data=data,
response=response,
return await await_within_hook_timeout(
callback,
target.async_post_call_success_hook(
user_api_key_dict=user_api_key_dict,
data=data,
response=response,
),
)
else:
raise ValueError(f"Unknown hook_type: {hook_type}")
Expand Down Expand Up @@ -1941,7 +1953,7 @@ async def _process_guardrail_callback(
try:
# Check if load balancing should be used
if guardrail_name and self._should_use_guardrail_load_balancing(guardrail_name):
response = await self._execute_guardrail_with_load_balancing(
outcome = await self._execute_guardrail_with_load_balancing(
guardrail_name=guardrail_name,
hook_type="pre_call",
data=data,
Expand All @@ -1950,14 +1962,19 @@ async def _process_guardrail_callback(
)
else:
# Single guardrail - execute directly
response = await self._execute_guardrail_hook(
outcome = await self._execute_guardrail_hook(
callback=callback,
hook_type="pre_call",
data=data,
user_api_key_dict=user_api_key_dict,
call_type=call_type,
)

if isinstance(outcome, GuardrailHookTimedOut):
status = "timeout"
error_type = litellm.Timeout.__name__
response: Final = resolve_hook_timeout(outcome, fail_open_result=None)

# Process the response if one was returned
if response is not None:
data = await self.process_pre_call_hook_response(response=response, data=data, call_type=call_type)
Expand All @@ -1968,8 +1985,9 @@ async def _process_guardrail_callback(
status = "intervened"
raise
except Exception as e:
status = "error"
error_type = type(e).__name__
if status != "timeout":
status = "error"
error_type = type(e).__name__
enrich_http_exception_with_guardrail_context(e, callback)
# Re-raise the exception to maintain existing behavior
raise
Expand Down Expand Up @@ -2706,6 +2724,7 @@ async def _run_guardrail_with_metrics(
coro: Awaitable[_T],
hook_type: str,
request_data: Mapping[str, object],
fail_open_result: _T,
) -> _T:
"""
Await `coro`, recording its latency and status to the
Expand All @@ -2718,13 +2737,18 @@ async def _run_guardrail_with_metrics(
status = "success"
error_type: str | None = None
try:
return await coro
outcome: Final = await await_within_hook_timeout(callback, coro)
if isinstance(outcome, GuardrailHookTimedOut):
status = "timeout"
error_type = litellm.Timeout.__name__
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
return resolve_hook_timeout(outcome, fail_open_result)
except SensitiveDataRouteException:
status = "intervened"
raise
except Exception as e:
status = "error"
error_type = type(e).__name__
if status != "timeout":
status = "error"
error_type = type(e).__name__
enrich_http_exception_with_guardrail_context(e, callback)
_record_raising_guardrail(request_data, callback)
raise
Expand Down Expand Up @@ -3001,6 +3025,7 @@ async def _run_during_call_guardrail(
),
"during_call",
request_data=data,
fail_open_result=None,
)
return
await self._run_guardrail_with_metrics(
Expand All @@ -3012,6 +3037,7 @@ async def _run_during_call_guardrail(
),
"during_call",
request_data=data,
fail_open_result=None,
)

async def failed_tracking_alert(
Expand Down Expand Up @@ -3549,6 +3575,7 @@ async def post_call_success_hook(
),
"post_call",
request_data=data,
fail_open_result=None,
)
else:
guardrail_response = await self._run_guardrail_with_metrics(
Expand All @@ -3560,6 +3587,7 @@ async def post_call_success_hook(
),
"post_call",
request_data=data,
fail_open_result=None,
)

if guardrail_response is not None:
Expand Down Expand Up @@ -3624,6 +3652,7 @@ async def _run_one(callback: CustomGuardrail) -> None:
),
"post_call",
request_data=data,
fail_open_result=None,
)
else:
await self._run_guardrail_with_metrics(
Expand All @@ -3635,6 +3664,7 @@ async def _run_one(callback: CustomGuardrail) -> None:
),
"post_call",
request_data=data,
fail_open_result=None,
)

results: Final = await asyncio.gather(
Expand Down Expand Up @@ -3699,6 +3729,7 @@ async def post_mcp_call_hook(
),
"post_mcp_call",
request_data=request_data,
fail_open_result=response,
)
return response

Expand Down
Loading
Loading