diff --git a/litellm/proxy/guardrails/guardrail_hooks/custom_code/custom_code_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/custom_code/custom_code_guardrail.py index 58502e309ef..22d4548aa99 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/custom_code/custom_code_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/custom_code/custom_code_guardrail.py @@ -41,6 +41,7 @@ async def apply_guardrail(inputs, request_data, input_type): from fastapi import HTTPException from litellm._logging import verbose_proxy_logger +from litellm.exceptions import ModifyResponseException from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, @@ -253,6 +254,9 @@ async def apply_guardrail( except HTTPException: # Re-raise HTTP exceptions (from block action) raise + except ModifyResponseException: + # Pre-call block uses passthrough; must not wrap as execution error (500) + raise except Exception as e: verbose_proxy_logger.error( f"Custom code guardrail '{self.guardrail_name}' execution error: {e}" diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py index d6d2e014948..d6065ef73f5 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py +++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py @@ -1202,7 +1202,7 @@ def _handle_conditional_match( ) verbose_proxy_logger.warning(error_msg) raise HTTPException( - status_code=403, + status_code=400, detail={ "error": error_msg, "category": category_name, @@ -1242,7 +1242,7 @@ def _handle_category_keyword_match( ) verbose_proxy_logger.warning(error_msg) raise HTTPException( - status_code=403, + status_code=400, detail={ "error": error_msg, "category": category_name, @@ -1285,7 +1285,7 @@ def _handle_pattern_match( error_msg = f"Content blocked: {pattern_name} pattern detected" verbose_proxy_logger.warning(error_msg) raise HTTPException( - status_code=403, + status_code=400, detail={"error": error_msg, "pattern": pattern_name}, ) elif action == ContentFilterAction.MASK: @@ -1325,7 +1325,7 @@ def _handle_blocked_word_match( error_msg += f" ({description})" verbose_proxy_logger.warning(error_msg) raise HTTPException( - status_code=403, + status_code=400, detail={ "error": error_msg, "keyword": keyword, @@ -1677,7 +1677,7 @@ def _apply_competitor_intent_policy( "ContentFilterGuardrail: competitor intent refuse - %s", intent_val ) raise HTTPException( - status_code=403, + status_code=400, detail={ "error": msg, "intent": intent_val, diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/test_eval.py b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/test_eval.py index 56398739b9b..aedc6acc810 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/test_eval.py +++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/test_eval.py @@ -59,7 +59,7 @@ def _run(checker, text: str) -> dict: checker.check(text) return {"decision": "ALLOW", "score": 0.0, "matched_topic": None} except HTTPException as e: - if e.status_code == 403: + if e.status_code == 400: detail: Dict[str, Any] = e.detail if isinstance(e.detail, dict) else {} return { "decision": "BLOCK", @@ -542,7 +542,7 @@ def check(self, text: str) -> str: if "BLOCK" in decision: raise HTTPException( - status_code=403, + status_code=400, detail={ "error": "Content blocked by LLM judge", "topic": "financial_advice", diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_competitor_intent.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_competitor_intent.py index 545b75fa06b..723d4b7db75 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_competitor_intent.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_competitor_intent.py @@ -226,7 +226,7 @@ async def test_apply_guardrail_with_competitor_intent_refuse(self): await guardrail.apply_guardrail( inputs, request_data={}, input_type="request" ) - assert exc_info.value.status_code == 403 + assert exc_info.value.status_code == 400 # Exact config from litellm/proxy/_new_secret_config.yaml (lines 27-53). diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_content_filter.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_content_filter.py index fb952d4b18b..bb079ea6580 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_content_filter.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_content_filter.py @@ -198,7 +198,7 @@ async def test_apply_guardrail_block(self): input_type="request", ) - assert exc_info.value.status_code == 403 + assert exc_info.value.status_code == 400 assert "us_ssn" in str(exc_info.value.detail) @pytest.mark.asyncio @@ -563,7 +563,7 @@ async def mock_stream(): ): pass - assert exc_info.value.status_code == 403 + assert exc_info.value.status_code == 400 assert "us_ssn" in str(exc_info.value.detail) @pytest.mark.asyncio @@ -1010,7 +1010,7 @@ async def test_apply_guardrail_blocks_on_any_blocked_keyword(self): input_type="request", ) - assert exc_info.value.status_code == 403 + assert exc_info.value.status_code == 400 assert "danger_word" in str(exc_info.value.detail) @pytest.mark.asyncio @@ -1298,7 +1298,7 @@ async def test_harm_toxic_abuse_blocks_abusive_input(self): input_type="request", ) - assert exc_info.value.status_code == 403 + assert exc_info.value.status_code == 400 detail = exc_info.value.detail if isinstance(detail, dict): assert detail.get("category") == "harm_toxic_abuse" @@ -1327,7 +1327,7 @@ async def test_harm_toxic_abuse_blocks_sht_ai(self): input_type="request", ) - assert exc_info.value.status_code == 403 + assert exc_info.value.status_code == 400 detail = exc_info.value.detail if isinstance(detail, dict): assert detail.get("category") == "harm_toxic_abuse" @@ -1375,7 +1375,7 @@ async def test_category_keywords_with_asterisks_match_actual_text(self): input_type="request", ) - assert exc_info.value.status_code == 403, f"Failed to block: '{test_input}'" + assert exc_info.value.status_code == 400, f"Failed to block: '{test_input}'" detail = exc_info.value.detail if isinstance(detail, dict): assert detail.get("category") == "harm_toxic_abuse" @@ -1443,7 +1443,7 @@ async def test_blocked_words_with_asterisks_custom(self): input_type="request", ) - assert exc_info.value.status_code == 403 + assert exc_info.value.status_code == 400 assert "te*st" in str(exc_info.value.detail) def test_check_category_keywords_asterisk_pattern_matching(self): @@ -1510,7 +1510,7 @@ async def test_nsfw_keywords_in_harm_toxic_abuse(self): input_type="request", ) - assert exc_info.value.status_code == 403, f"Failed to block: '{test_input}'" + assert exc_info.value.status_code == 400, f"Failed to block: '{test_input}'" detail = exc_info.value.detail if isinstance(detail, dict): assert detail.get("category") == "harm_toxic_abuse" @@ -1560,7 +1560,7 @@ async def test_solicitation_keywords_block_escort_requests(self): input_type="request", ) - assert exc_info.value.status_code == 403, f"Failed to block: '{test_input}'" + assert exc_info.value.status_code == 400, f"Failed to block: '{test_input}'" detail = exc_info.value.detail if isinstance(detail, dict): assert detail.get("category") == "harm_toxic_abuse" @@ -1646,7 +1646,7 @@ async def test_multilanguage_harm_toxic_abuse_spanish(self): ) assert ( - exc_info.value.status_code == 403 + exc_info.value.status_code == 400 ), f"Failed to block Spanish: '{test_input}'" @pytest.mark.asyncio @@ -1683,7 +1683,7 @@ async def test_multilanguage_harm_toxic_abuse_french(self): ) assert ( - exc_info.value.status_code == 403 + exc_info.value.status_code == 400 ), f"Failed to block French: '{test_input}'" @pytest.mark.asyncio @@ -1720,7 +1720,7 @@ async def test_multilanguage_harm_toxic_abuse_german(self): ) assert ( - exc_info.value.status_code == 403 + exc_info.value.status_code == 400 ), f"Failed to block German: '{test_input}'" @pytest.mark.asyncio @@ -1766,7 +1766,7 @@ async def test_multilanguage_harm_toxic_abuse_australian(self): ) assert ( - exc_info.value.status_code == 403 + exc_info.value.status_code == 400 ), f"Failed to block Australian: '{test_input}'" async def test_html_tags_in_messages_not_blocked(self): @@ -1942,7 +1942,7 @@ async def test_conditional_child_safety_category(self): request_data={}, input_type="request", ) - assert exc_info.value.status_code == 403 + assert exc_info.value.status_code == 400 assert "harmful_child_safety" in str(exc_info.value.detail) # Test case 2: Should BLOCK - identifier + block word combination @@ -1956,7 +1956,7 @@ async def test_conditional_child_safety_category(self): request_data={}, input_type="request", ) - assert exc_info.value.status_code == 403 + assert exc_info.value.status_code == 400 # Test case 3: Should BLOCK - explicit content + minors with pytest.raises(HTTPException) as exc_info: @@ -1967,7 +1967,7 @@ async def test_conditional_child_safety_category(self): request_data={}, input_type="request", ) - assert exc_info.value.status_code == 403 + assert exc_info.value.status_code == 400 # Test case 4: Should NOT block - identifier word alone (no block word) result = await guardrail.apply_guardrail( @@ -2009,7 +2009,7 @@ async def test_conditional_child_safety_category(self): request_data={}, input_type="request", ) - assert exc_info.value.status_code == 403 + assert exc_info.value.status_code == 400 @pytest.mark.asyncio async def test_conditional_category_sentence_boundaries(self): @@ -2093,7 +2093,7 @@ async def test_conditional_racial_bias_category(self): request_data={}, input_type="request", ) - assert exc_info.value.status_code == 403 + assert exc_info.value.status_code == 400 assert "bias_racial" in str(exc_info.value.detail) # Test case 2: Should BLOCK - identifier + dehumanizing language @@ -2107,7 +2107,7 @@ async def test_conditional_racial_bias_category(self): request_data={}, input_type="request", ) - assert exc_info.value.status_code == 403 + assert exc_info.value.status_code == 400 # Test case 3: Should BLOCK - supremacist content with pytest.raises(HTTPException) as exc_info: @@ -2120,7 +2120,7 @@ async def test_conditional_racial_bias_category(self): request_data={}, input_type="request", ) - assert exc_info.value.status_code == 403 + assert exc_info.value.status_code == 400 # Test case 4: Should BLOCK - elimination rhetoric with pytest.raises(HTTPException) as exc_info: @@ -2133,7 +2133,7 @@ async def test_conditional_racial_bias_category(self): request_data={}, input_type="request", ) - assert exc_info.value.status_code == 403 + assert exc_info.value.status_code == 400 # Test case 5: Should NOT block - identifier word alone (no block word) result = await guardrail.apply_guardrail( @@ -2171,7 +2171,7 @@ async def test_conditional_racial_bias_category(self): request_data={}, input_type="request", ) - assert exc_info.value.status_code == 403 + assert exc_info.value.status_code == 400 # Test case 9: Should NOT block - block word alone (no identifier) result = await guardrail.apply_guardrail( diff --git a/tests/test_litellm/proxy/guardrails/test_custom_code_security.py b/tests/test_litellm/proxy/guardrails/test_custom_code_security.py index 00cf3f317c9..f93ecfc3010 100644 --- a/tests/test_litellm/proxy/guardrails/test_custom_code_security.py +++ b/tests/test_litellm/proxy/guardrails/test_custom_code_security.py @@ -1,11 +1,12 @@ import pytest +from fastapi import HTTPException +from litellm.exceptions import ModifyResponseException from litellm.proxy.guardrails.guardrail_hooks.custom_code.custom_code_guardrail import ( CustomCodeCompilationError, CustomCodeGuardrail, ) - # str.mro() + generator gi_code + code.replace(co_names=...) + __setattr__ # to swap a function's bytecode and read http_get's real builtins dict. BYTECODE_REWRITE_PAYLOAD = ( @@ -153,6 +154,49 @@ async def test_async_guardrail_compiles_and_runs(): assert result["texts"][0] == "test" +@pytest.mark.asyncio +async def test_custom_code_pre_call_block_uses_passthrough(): + code = ( + "def apply_guardrail(inputs, request_data, input_type):\n" + ' return block("blocked by test")\n' + ) + guardrail = _compile(code) + + with pytest.raises(ModifyResponseException) as exc_info: + await guardrail.apply_guardrail( + inputs={"texts": ["test"]}, + request_data={"model": "test-model"}, + input_type="request", + ) + + assert exc_info.value.message == "blocked by test" + assert exc_info.value.model == "test-model" + assert exc_info.value.guardrail_name == "t" + + +@pytest.mark.asyncio +async def test_custom_code_post_call_block_raises_http_400(): + code = ( + "def apply_guardrail(inputs, request_data, input_type):\n" + ' return block("blocked by test")\n' + ) + guardrail = _compile(code) + + with pytest.raises(HTTPException) as exc_info: + await guardrail.apply_guardrail( + inputs={"texts": ["test"]}, + request_data={"model": "test-model"}, + input_type="response", + ) + + assert exc_info.value.status_code == 400 + assert exc_info.value.detail == { + "error": "blocked by test", + "guardrail": "t", + "detection_info": {}, + } + + def test_typical_sync_guardrail_still_works(): code = ( "def apply_guardrail(inputs, request_data, input_type):\n" diff --git a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py index 16c3c696519..058d5b0283b 100644 --- a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py +++ b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py @@ -10,6 +10,9 @@ import litellm from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.proxy.guardrails.guardrail_hooks.custom_code.custom_code_guardrail import ( + CustomCodeGuardrail, +) from litellm.proxy.policy_engine.pipeline_executor import PipelineExecutor from litellm.types.proxy.policy_engine.pipeline_types import ( GuardrailPipeline, @@ -85,6 +88,29 @@ async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type): return None +class PassthroughBlockGuardrail(CustomGuardrail): + """Mock guardrail that blocks using the legacy passthrough contract.""" + + def __init__(self, guardrail_name: str): + super().__init__( + guardrail_name=guardrail_name, + event_hook="pre_call", + default_on=True, + ) + self.calls = 0 + + def should_run_guardrail(self, data, event_type) -> bool: + return True + + async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type): + self.calls += 1 + self.raise_passthrough_exception( + violation_message="Content policy violation", + request_data=data, + detection_info={"source": "passthrough"}, + ) + + class PiiMaskingGuardrail(CustomGuardrail): """Mock guardrail that masks PII in messages and returns modified data.""" @@ -183,6 +209,105 @@ async def test_escalation_step1_fails_step2_blocks(): litellm.callbacks = original_callbacks +@pytest.mark.asyncio +async def test_passthrough_guardrail_failure_can_pipeline_block(): + """ + Pipeline: passthrough guardrail (on_fail: block) + Expected: passthrough ModifyResponseException is treated as policy fail, + and the pipeline terminal action is block. + """ + passthrough_guard = PassthroughBlockGuardrail(guardrail_name="passthrough-filter") + + pipeline = GuardrailPipeline( + mode="pre_call", + steps=[ + PipelineStep( + guardrail="passthrough-filter", + on_fail="block", + on_pass="allow", + ), + ], + ) + + original_callbacks = litellm.callbacks.copy() + litellm.callbacks = [passthrough_guard] + + try: + result = await PipelineExecutor.execute_steps( + steps=pipeline.steps, + mode=pipeline.mode, + data={ + "model": "fake-model", + "messages": [{"role": "user", "content": "bad content"}], + }, + user_api_key_dict=MagicMock(), + call_type="completion", + policy_name="content-safety", + ) + + assert passthrough_guard.calls == 1 + assert result.terminal_action == "block" + assert len(result.step_results) == 1 + assert result.step_results[0].guardrail_name == "passthrough-filter" + assert result.step_results[0].outcome == "fail" + assert result.step_results[0].action_taken == "block" + assert result.error_message == "Content policy violation" + finally: + litellm.callbacks = original_callbacks + + +@pytest.mark.asyncio +async def test_custom_code_guardrail_failure_can_pipeline_block(): + """ + Pipeline: custom code guardrail (on_fail: block) + Expected: custom code keeps its standalone passthrough block behavior, and + the pipeline converts that guardrail intervention into a block action. + """ + custom_guard = CustomCodeGuardrail( + guardrail_name="custom-code-filter", + custom_code=( + "def apply_guardrail(inputs, request_data, input_type):\n" + ' return block("SSN detected")\n' + ), + ) + + pipeline = GuardrailPipeline( + mode="pre_call", + steps=[ + PipelineStep( + guardrail="custom-code-filter", + on_fail="block", + on_pass="allow", + ), + ], + ) + + original_callbacks = litellm.callbacks.copy() + litellm.callbacks = [custom_guard] + + try: + result = await PipelineExecutor.execute_steps( + steps=pipeline.steps, + mode=pipeline.mode, + data={ + "model": "fake-model", + "messages": [{"role": "user", "content": "123-45-6789"}], + }, + user_api_key_dict=MagicMock(), + call_type="completion", + policy_name="content-safety", + ) + + assert result.terminal_action == "block" + assert len(result.step_results) == 1 + assert result.step_results[0].guardrail_name == "custom-code-filter" + assert result.step_results[0].outcome == "fail" + assert result.step_results[0].action_taken == "block" + assert result.error_message == "SSN detected" + finally: + litellm.callbacks = original_callbacks + + @pytest.mark.skipif(HTTPException is None, reason="fastapi not installed") @pytest.mark.asyncio async def test_early_allow_step1_passes_step2_skipped():