feat(add-new-block_code_execution-guardrail): prevent agent from executing code

litellm/policy_templates_backup.json

@@ -2816,5 +2816,136 @@
       "Singapore"
     ],
     "estimated_latency_ms": 1
+  },
+  {
+    "id": "claims-agent-safety",
+    "title": "Claims Agent Chatbot Safety",
+    "description": "Comprehensive safety guardrails for healthcare claims agent chatbots. Blocks fraud coaching (exaggeration, document forgery), PHI disclosure without authorization, prior-auth gaming (code manipulation, medical necessity misrepresentation), system override injection (prompt injection, role impersonation), and medical advice in claims context (diagnosis, treatment recommendations). Evaluated on 243 test cases with 100% precision and 100% recall across all 5 categories.",
+    "icon": "ShieldExclamationIcon",
+    "iconColor": "text-red-500",
+    "iconBg": "bg-red-50",
+    "guardrails": [
+      "claims-fraud-coaching-filter",
+      "claims-phi-disclosure-filter",
+      "claims-prior-auth-gaming-filter",
+      "claims-system-override-filter",
+      "claims-medical-advice-filter"
+    ],
+    "complexity": "High",
+    "guardrailDefinitions": [
+      {
+        "guardrail_name": "claims-fraud-coaching-filter",
+        "litellm_params": {
+          "guardrail": "litellm_content_filter",
+          "mode": "pre_call",
+          "categories": [
+            {
+              "category": "claims_fraud_coaching",
+              "category_file": "litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/categories/claims_fraud_coaching.yaml",
+              "enabled": true,
+              "action": "BLOCK",
+              "severity_threshold": "medium"
+            }
+          ]
+        },
+        "guardrail_info": {
+          "description": "Blocks fraud coaching including exaggeration of injuries, fabrication of claims, document forgery, and insurance fraud tactics"
+        }
+      },
+      {
+        "guardrail_name": "claims-phi-disclosure-filter",
+        "litellm_params": {
+          "guardrail": "litellm_content_filter",
+          "mode": "pre_call",
+          "categories": [
+            {
+              "category": "claims_phi_disclosure",
+              "category_file": "litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/categories/claims_phi_disclosure.yaml",
+              "enabled": true,
+              "action": "BLOCK",
+              "severity_threshold": "medium"
+            }
+          ]
+        },
+        "guardrail_info": {
+          "description": "Blocks unauthorized PHI disclosure, bulk data extraction, and HIPAA violations in claims context"
+        }
+      },
+      {
+        "guardrail_name": "claims-prior-auth-gaming-filter",
+        "litellm_params": {
+          "guardrail": "litellm_content_filter",
+          "mode": "pre_call",
+          "categories": [
+            {
+              "category": "claims_prior_auth_gaming",
+              "category_file": "litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/categories/claims_prior_auth_gaming.yaml",
+              "enabled": true,
+              "action": "BLOCK",
+              "severity_threshold": "medium"
+            }
+          ]
+        },
+        "guardrail_info": {
+          "description": "Blocks prior-authorization gaming including code manipulation, upcoding, medical necessity misrepresentation, and approval guarantee schemes"
+        }
+      },
+      {
+        "guardrail_name": "claims-system-override-filter",
+        "litellm_params": {
+          "guardrail": "litellm_content_filter",
+          "mode": "pre_call",
+          "categories": [
+            {
+              "category": "claims_system_override",
+              "category_file": "litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/categories/claims_system_override.yaml",
+              "enabled": true,
+              "action": "BLOCK",
+              "severity_threshold": "medium"
+            }
+          ]
+        },
+        "guardrail_info": {
+          "description": "Blocks system override injection, prompt manipulation, adjudication rule bypass, and unauthorized role impersonation (employer, TPA, broker)"
+        }
+      },
+      {
+        "guardrail_name": "claims-medical-advice-filter",
+        "litellm_params": {
+          "guardrail": "litellm_content_filter",
+          "mode": "pre_call",
+          "categories": [
+            {
+              "category": "claims_medical_advice",
+              "category_file": "litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/categories/claims_medical_advice.yaml",
+              "enabled": true,
+              "action": "BLOCK",
+              "severity_threshold": "medium"
+            }
+          ]
+        },
+        "guardrail_info": {
+          "description": "Blocks medical advice in claims context including diagnosis, treatment recommendations, medication guidance, and dosage questions"
+        }
+      }
+    ],
+    "templateData": {
+      "policy_name": "claims-agent-safety",
+      "description": "Comprehensive safety policy for healthcare claims agent chatbots. Covers fraud coaching, PHI disclosure, prior-auth gaming, system override injection, and medical advice. Evaluated on 243 test cases with 100% precision and 100% recall.",
+      "guardrails_add": [
+        "claims-fraud-coaching-filter",
+        "claims-phi-disclosure-filter",
+        "claims-prior-auth-gaming-filter",
+        "claims-system-override-filter",
+        "claims-medical-advice-filter"
+      ],
+      "guardrails_remove": []
+    },
+    "tags": [
+      "Healthcare",
+      "Claims",
+      "Content Safety"
+    ],
+    "estimated_latency_ms": 1
   }
 ]

litellm/proxy/guardrails/guardrail_endpoints.py

@@ -14,14 +14,14 @@
 from litellm.integrations.custom_guardrail import CustomGuardrail
 from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
-from litellm.proxy.guardrails.guardrail_registry import GuardrailRegistry
 from litellm.proxy.guardrails.guardrail_hooks.custom_code.code_validator import (
     CustomCodeValidationError,
     validate_custom_code,
 )
 from litellm.proxy.guardrails.guardrail_hooks.custom_code.primitives import (
     get_custom_code_primitives,
 )
+from litellm.proxy.guardrails.guardrail_registry import GuardrailRegistry
 from litellm.proxy.guardrails.usage_endpoints import router as guardrails_usage_router
 from litellm.types.guardrails import (
     PII_ENTITY_CATEGORIES_MAP,
@@ -1170,10 +1170,11 @@ def _build_field_dict(
     # Determine the field type from annotation
     field_type = _get_field_type_from_annotation(field_annotation)
 
-    # Check for custom UI type override
-    field_json_schema_extra = getattr(field, "json_schema_extra", {})
+    # Check for custom UI type override (ui_type preferred; "type" leaks into OpenAPI and breaks schema)
+    field_json_schema_extra = getattr(field, "json_schema_extra", {}) or {}
     if field_json_schema_extra and "ui_type" in field_json_schema_extra:
-        field_type = field_json_schema_extra["ui_type"].value
+        ut = field_json_schema_extra["ui_type"]
+        field_type = ut if isinstance(ut, str) else getattr(ut, "value", ut)
     elif field_json_schema_extra and "type" in field_json_schema_extra:
         field_type = field_json_schema_extra["type"]
 
@@ -1205,11 +1206,22 @@ def _build_field_dict(
     # Add options if they exist in json_schema_extra (this takes precedence)
     if field_json_schema_extra and "options" in field_json_schema_extra:
         field_dict["options"] = field_json_schema_extra["options"]
+    elif field_type == "select":
+        # For Literal types, populate options so the UI can render a dropdown
+        literal_options = _extract_literal_values(field_annotation)
+        if literal_options:
+            field_dict["options"] = literal_options
 
     # Add default value if it exists
     if field.default is not None and field.default is not ...:
         field_dict["default_value"] = field.default
 
+    # Copy min, max, step from json_schema_extra for number/percentage inputs
+    if field_json_schema_extra:
+        for key in ("min", "max", "step", "default_value"):
+            if key in field_json_schema_extra:
+                field_dict[key] = field_json_schema_extra[key]
+
     return field_dict
 
 
@@ -1485,6 +1497,7 @@ async def test_custom_code_guardrail(
     ```
     """
 
+
     if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN:
         raise HTTPException(
             status_code=403,

litellm/proxy/guardrails/guardrail_hooks/block_code_execution/__init__.py

@@ -0,0 +1,95 @@
+"""Block Code Execution guardrail: blocks or masks fenced code blocks by language."""
+
+from typing import TYPE_CHECKING, Any, List, Literal, Optional, Union, cast
+
+from litellm.types.guardrails import GuardrailEventHooks, SupportedGuardrailIntegrations
+
+from .block_code_execution import BlockCodeExecutionGuardrail
+
+if TYPE_CHECKING:
+    from litellm.types.guardrails import Guardrail, LitellmParams
+
+# Default: run on both request and response (and during_call is supported too)
+DEFAULT_EVENT_HOOKS = [
+    GuardrailEventHooks.pre_call.value,
+    GuardrailEventHooks.post_call.value,
+]
+
+
+def _get_param(
+    litellm_params: "LitellmParams",
+    guardrail: "Guardrail",
+    key: str,
+    default: Any = None,
+) -> Any:
+    """Get a param from litellm_params, with fallback to raw guardrail litellm_params (for extra fields not on LitellmParams)."""
+    value = getattr(litellm_params, key, default)
+    if value is not None:
+        return value
+    raw = guardrail.get("litellm_params")
+    if isinstance(raw, dict) and key in raw:
+        return raw[key]
+    return default
+
+
+def initialize_guardrail(
+    litellm_params: "LitellmParams",
+    guardrail: "Guardrail",
+) -> BlockCodeExecutionGuardrail:
+    """Initialize the Block Code Execution guardrail from config."""
+    import litellm
+
+    guardrail_name = guardrail.get("guardrail_name")
+    if not guardrail_name:
+        raise ValueError(
+            "Block Code Execution guardrail requires a guardrail_name"
+        )
+
+    blocked_languages: Optional[List[str]] = cast(
+        Optional[List[str]],
+        _get_param(litellm_params, guardrail, "blocked_languages"),
+    )
+    action = cast(
+        Literal["block", "mask"],
+        _get_param(litellm_params, guardrail, "action", "block"),
+    )
+    confidence_threshold = float(
+        cast(
+            Union[int, float, str],
+            _get_param(litellm_params, guardrail, "confidence_threshold", 0.5),
+        )
+    )
+    detect_execution_intent = bool(
+        _get_param(litellm_params, guardrail, "detect_execution_intent", True)
+    )
+    mode = _get_param(litellm_params, guardrail, "mode")
+    event_hook = cast(
+        Optional[Union[Literal["pre_call", "post_call", "during_call"], List[str]]],
+        mode if mode is not None else DEFAULT_EVENT_HOOKS,
+    )
+
+    instance = BlockCodeExecutionGuardrail(
+        guardrail_name=guardrail_name,
+        blocked_languages=blocked_languages,
+        action=action,
+        confidence_threshold=confidence_threshold,
+        detect_execution_intent=detect_execution_intent,
+        event_hook=event_hook,
+        default_on=bool(_get_param(litellm_params, guardrail, "default_on", False)),
+    )
+    litellm.logging_callback_manager.add_litellm_callback(instance)
+    return instance
+
+
+guardrail_initializer_registry = {
+    SupportedGuardrailIntegrations.BLOCK_CODE_EXECUTION.value: initialize_guardrail,
+}
+
+guardrail_class_registry = {
+    SupportedGuardrailIntegrations.BLOCK_CODE_EXECUTION.value: BlockCodeExecutionGuardrail,
+}
+
+__all__ = [
+    "BlockCodeExecutionGuardrail",
+    "initialize_guardrail",
+]