Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions litellm/proxy/common_request_processing.py
Original file line number Diff line number Diff line change
Expand Up @@ -839,6 +839,7 @@ async def common_processing_pre_call_logic(
"aget_run",
"acancel_run",
"adelete_run",
"apply_guardrail",
],
version: Optional[str] = None,
user_model: Optional[str] = None,
Expand Down
107 changes: 105 additions & 2 deletions litellm/proxy/guardrails/guardrail_endpoints.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
from typing import Any, Dict, List, Literal, Optional, Type, TypeVar, Union, cast
from urllib.parse import urlparse

from fastapi import APIRouter, Depends, HTTPException
from fastapi import APIRouter, Depends, HTTPException, Request
from pydantic import BaseModel

from litellm.proxy.common_utils.path_utils import safe_join
Expand Down Expand Up @@ -2190,6 +2190,7 @@ def execute_guardrail():
@router.post("/guardrails/apply_guardrail", response_model=ApplyGuardrailResponse)
@router.post("/apply_guardrail", response_model=ApplyGuardrailResponse)
async def apply_guardrail(
fastapi_request: Request,
request: ApplyGuardrailRequest,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
):
Expand All @@ -2198,7 +2199,27 @@ async def apply_guardrail(

This endpoint allows testing guardrails by applying them to custom text inputs.
"""
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
from litellm.litellm_core_utils.thread_pool_executor import (
executor as thread_pool_executor,
)
from litellm.proxy.proxy_server import (
general_settings,
proxy_config,
proxy_logging_obj,
version,
)
from litellm.proxy.utils import handle_exception_on_proxy
import traceback

data: dict = {
"guardrail_name": request.guardrail_name,
"input": [request.text],
"messages": request.messages or [],
"metadata": {"route": "/apply_guardrail"},
}
litellm_logging_obj = None
start_time = datetime.now(timezone.utc)

try:
active_guardrail: Optional[CustomGuardrail] = (
Expand All @@ -2212,6 +2233,31 @@ async def apply_guardrail(
detail=f"Guardrail '{request.guardrail_name}' not found. Please ensure the guardrail is configured in your LiteLLM proxy.",
)

request_processor = ProxyBaseLLMRequestProcessing(data=data)
data, litellm_logging_obj = (
await request_processor.common_processing_pre_call_logic(
request=fastapi_request,
general_settings=general_settings,
user_api_key_dict=user_api_key_dict,
version=version,
proxy_logging_obj=proxy_logging_obj,
proxy_config=proxy_config,
route_type="apply_guardrail",
)
)

# Langfuse reads input/output for pass_through_endpoint call_type only.
if litellm_logging_obj is not None:
litellm_logging_obj.call_type = "pass_through_endpoint"
litellm_logging_obj.model_call_details["call_type"] = (
"pass_through_endpoint"
)
litellm_logging_obj.model_call_details["messages"] = (
request.messages
if request.messages
else [{"role": "user", "content": request.text}]
)

request_data: dict = {}
if request.messages:
request_data["messages"] = request.messages
Expand All @@ -2237,12 +2283,69 @@ async def apply_guardrail(
)
response_text = guardrailed_inputs.get("texts", [])

return ApplyGuardrailResponse(
response = ApplyGuardrailResponse(
response_text=response_text[0] if response_text else request.text
)
response_for_logging = {
"response": response.model_dump(exclude_none=True),
}
Comment thread
cursor[bot] marked this conversation as resolved.
Resolved
except Exception as e:
try:
if litellm_logging_obj is not None and not isinstance(e, HTTPException):
await litellm_logging_obj.async_failure_handler(
exception=e,
traceback_exception=traceback.format_exc(),
)
thread_pool_executor.submit(
litellm_logging_obj.failure_handler,
e,
traceback.format_exc(),
)
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict,
original_exception=e,
request_data=data,
)
except Exception:
verbose_proxy_logger.exception("apply_guardrail: failure logging failed")
Comment thread
cursor[bot] marked this conversation as resolved.
Resolved
raise handle_exception_on_proxy(e)

# Success logging is outside the try/except so a hook error never
# triggers failure handlers for a successful guardrail run, and each
# logging call is wrapped defensively so callback failures don't
# discard the response the caller is waiting for.
try:
modified_response = await proxy_logging_obj.post_call_success_hook(
data=data,
user_api_key_dict=user_api_key_dict,
response=response,
)
Comment thread
Sameerlite marked this conversation as resolved.
Resolved
if isinstance(modified_response, ApplyGuardrailResponse):
response = modified_response
except Exception:
verbose_proxy_logger.exception("apply_guardrail: post_call_success_hook failed")

end_time = datetime.now(timezone.utc)
if litellm_logging_obj is not None:
try:
await litellm_logging_obj.async_success_handler(
result=response_for_logging,
start_time=start_time,
end_time=end_time,
cache_hit=False,
)
thread_pool_executor.submit(
litellm_logging_obj.success_handler,
response_for_logging,
start_time,
end_time,
False,
)
Comment thread
cursor[bot] marked this conversation as resolved.
Resolved
except Exception:
verbose_proxy_logger.exception("apply_guardrail: success logging failed")

return response


# Usage (dashboard) endpoints: overview, detail, logs
router.include_router(guardrails_usage_router)
85 changes: 83 additions & 2 deletions tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py
Original file line number Diff line number Diff line change
Expand Up @@ -1159,7 +1159,11 @@ async def test_apply_guardrail_not_found(mocker):

# Call endpoint and expect ProxyException
with pytest.raises(ProxyException) as exc_info:
await apply_guardrail(request=request, user_api_key_dict=mock_user_auth)
await apply_guardrail(
fastapi_request=mocker.Mock(),
request=request,
user_api_key_dict=mock_user_auth,
)

# Verify error details
assert str(exc_info.value.code) == "404"
Expand All @@ -1186,6 +1190,25 @@ async def test_apply_guardrail_execution_error(mocker):
"litellm.proxy.guardrails.guardrail_endpoints.GUARDRAIL_REGISTRY", mock_registry
)

mock_logging_obj = mocker.Mock()
mock_logging_obj.async_failure_handler = AsyncMock()
mock_logging_obj.model_call_details = {}
mock_processor = mocker.Mock()
mock_processor.common_processing_pre_call_logic = AsyncMock(
return_value=({"guardrail_name": "test-guardrail"}, mock_logging_obj)
)
mocker.patch(
"litellm.proxy.common_request_processing.ProxyBaseLLMRequestProcessing",
return_value=mock_processor,
)
mock_proxy_logging = mocker.Mock()
mock_proxy_logging.post_call_failure_hook = AsyncMock()
mocker.patch("litellm.proxy.proxy_server.proxy_logging_obj", mock_proxy_logging)
mocker.patch("litellm.proxy.proxy_server.general_settings", {})
mocker.patch("litellm.proxy.proxy_server.proxy_config", mocker.Mock())
mocker.patch("litellm.proxy.proxy_server.version", "test")
mocker.patch("litellm.litellm_core_utils.thread_pool_executor.executor")

# Create request
request = ApplyGuardrailRequest(
guardrail_name="test-guardrail", text="Test input text with forbidden content"
Expand All @@ -1196,12 +1219,70 @@ async def test_apply_guardrail_execution_error(mocker):

# Call endpoint and expect ProxyException
with pytest.raises(ProxyException) as exc_info:
await apply_guardrail(request=request, user_api_key_dict=mock_user_auth)
await apply_guardrail(
fastapi_request=mocker.Mock(),
request=request,
user_api_key_dict=mock_user_auth,
)

# Verify error is properly handled
assert "Bedrock guardrail failed" in str(exc_info.value.message)


@pytest.mark.asyncio
async def test_apply_guardrail_invokes_logging_pipeline(mocker):
mock_guardrail = mocker.Mock()
mock_guardrail.apply_guardrail = AsyncMock(return_value={"texts": ["masked"]})

mock_registry = mocker.Mock()
mock_registry.get_initialized_guardrail_callback.return_value = mock_guardrail
mocker.patch(
"litellm.proxy.guardrails.guardrail_endpoints.GUARDRAIL_REGISTRY", mock_registry
)

mock_logging_obj = mocker.Mock()
mock_logging_obj.async_success_handler = AsyncMock()
mock_logging_obj.model_call_details = {}
mock_processor = mocker.Mock()
mock_processor.common_processing_pre_call_logic = AsyncMock(
return_value=({"guardrail_name": "test-guardrail"}, mock_logging_obj)
)
mocker.patch(
"litellm.proxy.common_request_processing.ProxyBaseLLMRequestProcessing",
return_value=mock_processor,
)

mock_proxy_logging = mocker.Mock()
mock_proxy_logging.post_call_success_hook = AsyncMock()
mocker.patch("litellm.proxy.proxy_server.proxy_logging_obj", mock_proxy_logging)
mocker.patch("litellm.proxy.proxy_server.general_settings", {})
mocker.patch("litellm.proxy.proxy_server.proxy_config", mocker.Mock())
mocker.patch("litellm.proxy.proxy_server.version", "test")
mock_executor = mocker.Mock()
mocker.patch(
"litellm.litellm_core_utils.thread_pool_executor.executor", mock_executor
)

request = ApplyGuardrailRequest(
guardrail_name="test-guardrail", text="hello@example.com"
)
response = await apply_guardrail(
fastapi_request=mocker.Mock(),
request=request,
user_api_key_dict=UserAPIKeyAuth(),
)

assert response.response_text == "masked"
mock_processor.common_processing_pre_call_logic.assert_awaited_once()
mock_proxy_logging.post_call_success_hook.assert_awaited_once()
mock_logging_obj.async_success_handler.assert_awaited_once()
assert mock_logging_obj.call_type == "pass_through_endpoint"
mock_executor.submit.assert_called_once()
assert mock_logging_obj.async_success_handler.await_args.kwargs["result"] == {
"response": {"response_text": "masked"}
}


@pytest.mark.asyncio
async def test_get_guardrail_info_endpoint_config_guardrail(mocker):
"""
Expand Down
Loading