Skip to content
44 changes: 26 additions & 18 deletions litellm/integrations/custom_guardrail.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import contextvars
import os
import secrets
from datetime import datetime
Expand Down Expand Up @@ -60,6 +61,10 @@
# proxy's metadata sanitizer.
_PRE_CALL_EXECUTED_TOKEN = secrets.token_hex(16)

_guardrail_self_recorded: contextvars.ContextVar[bool] = contextvars.ContextVar(
"litellm_guardrail_self_recorded", default=False
)


def _strict_guardrail_modes_enabled() -> bool:
"""Whether guardrail-mode validation raises (default) or logs a warning.
Expand Down Expand Up @@ -113,6 +118,7 @@ def __init__(
on_sensitive_data: Optional[str] = None,
sensitive_data_route_to_model: Optional[str] = None,
sticky_session_routing: bool = True,
run_in_parallel: bool = False,
**kwargs,
):
"""
Expand All @@ -131,6 +137,9 @@ def __init__(
on_sensitive_data: Action when sensitive data is detected. 'block' (default) or 'route'
sensitive_data_route_to_model: Model to route to when on_sensitive_data='route'
sticky_session_routing: When True, all subsequent requests in the session use the same model
run_in_parallel: When True, this pre_call or post_call guardrail runs concurrently with
other opted-in guardrails of the same hook. Only safe for block-only guardrails that
do not mutate the request or response.
"""
self.guardrail_name = guardrail_name
self.supported_event_hooks = supported_event_hooks
Expand All @@ -145,6 +154,7 @@ def __init__(
self.on_sensitive_data: Optional[str] = on_sensitive_data
self.sensitive_data_route_to_model: Optional[str] = sensitive_data_route_to_model
self.sticky_session_routing: bool = sticky_session_routing
self.run_in_parallel: bool = run_in_parallel

if supported_event_hooks:
## validate event_hook is in supported_event_hooks
Expand Down Expand Up @@ -856,6 +866,8 @@ def _append_guardrail_info(container: dict) -> None:
request_data["metadata"] = {}
_append_guardrail_info(request_data["metadata"])

_guardrail_self_recorded.set(True)

# Emit the otel guardrail span here, where every guardrail execution lands,
# rather than relying on a post-call hook that does not fire on every path
# (e.g. a pass-through request that passes its guardrails).
Expand Down Expand Up @@ -1138,8 +1150,12 @@ def log_guardrail_information(func):
(structured detections, tracing detail) than this decorator's
"allow"/"mask"/raw-response default. To avoid double-recording in that
case (which would emit two spans, two Datadog records, two spend-log
entries, etc.), snapshot the entry count before invocation: if the
wrapped function already appended its own entry, skip the auto-record.
entries, etc.), a context-local flag records whether the wrapped function
appended its own entry; if so, the auto-record is skipped. The flag is a
``ContextVar`` rather than a count of entries in the shared ``request_data``
so it stays correct when guardrails run concurrently (asyncio copies the
context into each gathered task): counting shared entries would let one
guardrail's append hide another guardrail's missing record.
"""
import functools
import inspect
Expand All @@ -1159,16 +1175,6 @@ def _infer_event_type_from_function_name(
return GuardrailEventHooks.post_call
return None

def _count_recorded_guardrail_entries(request_data: dict) -> int:
total = 0
for container_key in ("metadata", "litellm_metadata"):
container = request_data.get(container_key)
if isinstance(container, dict):
entries = container.get("standard_logging_guardrail_information")
if isinstance(entries, list):
total += len(entries)
return total

@functools.wraps(func)
async def async_wrapper(*args, **kwargs):
start_time = datetime.now() # Move start_time inside the wrapper
Expand All @@ -1182,10 +1188,10 @@ async def async_wrapper(*args, **kwargs):
original_inputs = kwargs.get("inputs")

logging_obj = kwargs.get("logging_obj")
entries_before = _count_recorded_guardrail_entries(request_data)
self_recorded_token = _guardrail_self_recorded.set(False)
try:
response = await func(*args, **kwargs)
if _count_recorded_guardrail_entries(request_data) > entries_before:
if _guardrail_self_recorded.get():
return response
return self._process_response(
response=response,
Expand All @@ -1197,7 +1203,7 @@ async def async_wrapper(*args, **kwargs):
original_inputs=original_inputs,
)
except Exception as e:
if _count_recorded_guardrail_entries(request_data) > entries_before:
if _guardrail_self_recorded.get():
raise
return self._process_error(
e=e,
Expand All @@ -1208,6 +1214,7 @@ async def async_wrapper(*args, **kwargs):
event_type=event_type,
)
finally:
_guardrail_self_recorded.reset(self_recorded_token)
_sync_guardrail_info_to_logging_obj(request_data, logging_obj)

@functools.wraps(func)
Expand All @@ -1223,10 +1230,10 @@ def sync_wrapper(*args, **kwargs):
original_inputs = kwargs.get("inputs")

logging_obj = kwargs.get("logging_obj")
entries_before = _count_recorded_guardrail_entries(request_data)
self_recorded_token = _guardrail_self_recorded.set(False)
try:
response = func(*args, **kwargs)
if _count_recorded_guardrail_entries(request_data) > entries_before:
if _guardrail_self_recorded.get():
return response
return self._process_response(
response=response,
Expand All @@ -1236,7 +1243,7 @@ def sync_wrapper(*args, **kwargs):
original_inputs=original_inputs,
)
except Exception as e:
if _count_recorded_guardrail_entries(request_data) > entries_before:
if _guardrail_self_recorded.get():
raise
return self._process_error(
e=e,
Expand All @@ -1245,6 +1252,7 @@ def sync_wrapper(*args, **kwargs):
event_type=event_type,
)
finally:
_guardrail_self_recorded.reset(self_recorded_token)
_sync_guardrail_info_to_logging_obj(request_data, logging_obj)

@functools.wraps(func)
Expand Down
3 changes: 3 additions & 0 deletions litellm/proxy/guardrails/guardrail_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -489,6 +489,9 @@ def initialize_guardrail(
"skip_tool_message_in_guardrail",
getattr(litellm_params, "skip_tool_message_in_guardrail", None),
)
configured_run_in_parallel = getattr(litellm_params, "run_in_parallel", None)
if configured_run_in_parallel is not None:
custom_guardrail_callback.run_in_parallel = bool(configured_run_in_parallel)

parsed_guardrail = Guardrail(
guardrail_id=guardrail.get("guardrail_id"),
Expand Down
136 changes: 136 additions & 0 deletions litellm/proxy/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -1397,6 +1397,14 @@ async def pre_call_hook(
self._process_guardrail_metadata(data)
return data

parallel_guardrails: tuple[CustomGuardrail, ...] = tuple(
cb
for cb in caps.resolved_callbacks
if isinstance(cb, CustomGuardrail)
and cb.run_in_parallel
and not (cb.guardrail_name and cb.guardrail_name in pipeline_managed)
)

deferred_route_exc: Optional[SensitiveDataRouteException] = None
for _callback in caps.resolved_callbacks:
start_time = time.time()
Expand All @@ -1406,6 +1414,9 @@ async def pre_call_hook(
if _callback.guardrail_name and _callback.guardrail_name in pipeline_managed:
continue

if _callback.run_in_parallel:
continue

result = await self._process_guardrail_callback(
callback=_callback,
data=data, # type: ignore
Expand Down Expand Up @@ -1462,6 +1473,14 @@ async def pre_call_hook(
if deferred_route_exc is not None and data is not None:
data = await self._handle_sensitive_data_route_exception(deferred_route_exc, data, user_api_key_dict)

if parallel_guardrails and data is not None:
await self._run_parallel_pre_call_guardrails(
guardrails=parallel_guardrails,
data=data,
user_api_key_dict=user_api_key_dict,
call_type=call_type,
)

if data is not None:
self._process_guardrail_metadata(data)

Expand All @@ -1474,6 +1493,47 @@ async def pre_call_hook(
except Exception as e:
raise e

async def _run_parallel_pre_call_guardrails(
self,
guardrails: tuple[CustomGuardrail, ...],
data: dict,
user_api_key_dict: UserAPIKeyAuth,
call_type: CallTypesLiteral,
) -> None:
"""
Run opted-in pre_call guardrails concurrently against one shared payload
snapshot. These guardrails are declared block-only, so any modified data
they return is discarded; they run for their blocking side effect (raising
to reject the request before it reaches the LLM). Every guardrail is
awaited to completion (``return_exceptions=True``) so a raise by one never
leaves the others running as unobserved background tasks. A guardrail that
blocks (any exception other than a reroute or passthrough) takes precedence
over one that only changes the request flow, so a fast reroute can never
let a slower block be bypassed; the request is rejected before it reaches
the LLM, preserving the pre-call barrier that ``during_call`` guardrails
cannot provide. Per-guardrail latency is recorded by
``_process_guardrail_callback``'s own metrics.
"""
results = await asyncio.gather(
*(
self._process_guardrail_callback(
callback=callback,
data=data,
user_api_key_dict=user_api_key_dict,
call_type=call_type,
event_type=GuardrailEventHooks.pre_call,
)
for callback in guardrails
),
return_exceptions=True,
)
raised = tuple(result for result in results if isinstance(result, BaseException))
blocking = next((exc for exc in raised if not _exception_changes_request_flow(exc)), None)
if blocking is not None:
raise blocking
if raised:
raise raised[0]

async def _handle_sensitive_data_route_exception(
self,
exc: SensitiveDataRouteException,
Expand Down Expand Up @@ -2274,9 +2334,16 @@ async def post_call_success_hook(
# Merge model-level guardrails before checking which guardrails to run
guardrail_data = _check_and_merge_model_level_guardrails(data=data, llm_router=llm_router)

parallel_guardrails: tuple[CustomGuardrail, ...] = tuple(
callback for callback in guardrail_callbacks if callback.run_in_parallel
)

for callback in guardrail_callbacks:
# Main - V2 Guardrails implementation

if callback.run_in_parallel:
continue

if (
callback.should_run_guardrail(
data=guardrail_data,
Expand Down Expand Up @@ -2313,6 +2380,15 @@ async def post_call_success_hook(
if guardrail_response is not None:
response = guardrail_response

if parallel_guardrails:
await self._run_parallel_post_call_guardrails(
guardrails=parallel_guardrails,
data=data,
guardrail_data=guardrail_data,
response=response,
user_api_key_dict=user_api_key_dict,
)

############ Handle CustomLogger ###############################
#################################################################

Expand All @@ -2326,6 +2402,66 @@ async def post_call_success_hook(
raise e
return response

async def _run_parallel_post_call_guardrails(
self,
guardrails: tuple[CustomGuardrail, ...],
data: dict,
guardrail_data: dict,
response: LLMResponseTypes,
user_api_key_dict: UserAPIKeyAuth,
) -> None:
"""
Run opted-in post_call guardrails concurrently against the response
produced by the sequential guardrails. These guardrails are declared
block-only, so any modified response they return is discarded; they run
for their blocking side effect (raising to reject the response before it
reaches the client). Every guardrail is awaited to completion
(``return_exceptions=True``) so a raise by one never leaves the others
running as unobserved background tasks. A guardrail that blocks (any
exception other than a passthrough) takes precedence over one that only
changes the response flow, so a fast passthrough can never let a slower
block be bypassed. Each per-guardrail coroutine sets ``guardrail_to_apply``
immediately before awaiting, and the unified hook pops it before its first
suspension point, so concurrent guardrails never race on that key.
"""
from litellm.types.guardrails import GuardrailEventHooks

async def _run_one(callback: CustomGuardrail) -> None:
if callback.should_run_guardrail(data=guardrail_data, event_type=GuardrailEventHooks.post_call) is not True:
return
if "apply_guardrail" in type(callback).__dict__:
data["guardrail_to_apply"] = callback
await self._run_guardrail_with_metrics(
callback,
unified_guardrail.async_post_call_success_hook(
user_api_key_dict=user_api_key_dict,
data=data,
response=response,
),
"post_call",
)
else:
await self._run_guardrail_with_metrics(
callback,
callback.async_post_call_success_hook(
user_api_key_dict=user_api_key_dict,
data=data,
response=response,
),
"post_call",
)

results = await asyncio.gather(
*(_run_one(callback) for callback in guardrails),
return_exceptions=True,
)
raised = tuple(result for result in results if isinstance(result, BaseException))
blocking = next((exc for exc in raised if not _exception_changes_request_flow(exc)), None)
if blocking is not None:
raise blocking
if raised:
raise raised[0]

async def post_call_response_headers_hook(
self,
data: dict,
Expand Down
11 changes: 11 additions & 0 deletions litellm/types/guardrails.py
Original file line number Diff line number Diff line change
Expand Up @@ -873,6 +873,17 @@ class BaseLitellmParams(ContentFilterConfigModel): # works for new and patch up
),
)

run_in_parallel: Optional[bool] = Field(
default=None,
description=(
"When True, this pre_call or post_call guardrail runs concurrently with other opted-in "
"guardrails of the same hook, after the sequential guardrails have run. Use only for "
"block-only guardrails that inspect and reject; do not enable it for guardrails that "
"modify the request or response (e.g. PII masking or sensitive-data routing), since "
"parallel runs share one snapshot and their mutations would race."
),
)

@field_validator(
"mode",
"default_action",
Expand Down
4 changes: 2 additions & 2 deletions ruff-strict-budget.json
Original file line number Diff line number Diff line change
Expand Up @@ -222,7 +222,7 @@
"limit": 38
},
"RET504": {
"limit": 721
"limit": 719
},
"RUF010": {
"limit": 874
Expand Down Expand Up @@ -324,7 +324,7 @@
"limit": 883
},
"UP006": {
"limit": 12792
"limit": 12789
},
"UP007": {
"limit": 2570
Expand Down
Loading
Loading