Litellm oss staging 02 16 2026

docs/my-website/docs/adding_provider/generic_guardrail_api.md

@@ -237,6 +237,7 @@ litellm_settings:
         mode: pre_call  # or post_call, during_call
         api_base: https://your-guardrail-api.com
         api_key: os.environ/YOUR_GUARDRAIL_API_KEY  # optional
+        unreachable_fallback: fail_closed  # default: fail_closed. Set to fail_open to proceed if the guardrail endpoint is unreachable (network errors, or HTTP 502/503/504 from an upstream proxy/LB).
         additional_provider_specific_params:
           # your custom parameters
           threshold: 0.8

litellm/integrations/prometheus.py

@@ -1055,16 +1055,16 @@ async def async_log_success_event(self, kwargs, response_obj, start_time, end_ti
             enum_values=enum_values,
         )
 
-        if (
-            standard_logging_payload["stream"] is True
-        ):  # log successful streaming requests from logging event hook.
-            _labels = prometheus_label_factory(
-                supported_enum_labels=self.get_labels_for_metric(
-                    metric_name="litellm_proxy_total_requests_metric"
-                ),
-                enum_values=enum_values,
-            )
-            self.litellm_proxy_total_requests_metric.labels(**_labels).inc()
+        # increment litellm_proxy_total_requests_metric for all successful requests
+        # (both streaming and non-streaming) in this single location to prevent
+        # double-counting that occurs when async_post_call_success_hook also increments
+        _labels = prometheus_label_factory(
+            supported_enum_labels=self.get_labels_for_metric(
+                metric_name="litellm_proxy_total_requests_metric"
+            ),
+            enum_values=enum_values,
+        )
+        self.litellm_proxy_total_requests_metric.labels(**_labels).inc()
 
     def _increment_token_metrics(
         self,
@@ -1086,13 +1086,6 @@ def _increment_token_metrics(
         ):
             _tags = standard_logging_payload["request_tags"]
 
-        _labels = prometheus_label_factory(
-            supported_enum_labels=self.get_labels_for_metric(
-                metric_name="litellm_proxy_total_requests_metric"
-            ),
-            enum_values=enum_values,
-        )
-
         _labels = prometheus_label_factory(
             supported_enum_labels=self.get_labels_for_metric(
                 metric_name="litellm_total_tokens_metric"
@@ -1655,49 +1648,12 @@ async def async_post_call_success_hook(
     ):
         """
         Proxy level tracking - triggered when the proxy responds with a success response to the client
-        """
-        try:
-            from litellm.litellm_core_utils.litellm_logging import (
-                StandardLoggingPayloadSetup,
-            )
-
-            if self._should_skip_metrics_for_invalid_key(
-                user_api_key_dict=user_api_key_dict
-            ):
-                return
 
-            _metadata = data.get("metadata", {}) or {}
-            enum_values = UserAPIKeyLabelValues(
-                end_user=user_api_key_dict.end_user_id,
-                hashed_api_key=user_api_key_dict.api_key,
-                api_key_alias=user_api_key_dict.key_alias,
-                requested_model=data.get("model", ""),
-                team=user_api_key_dict.team_id,
-                team_alias=user_api_key_dict.team_alias,
-                user=user_api_key_dict.user_id,
-                user_email=user_api_key_dict.user_email,
-                status_code="200",
-                route=user_api_key_dict.request_route,
-                tags=StandardLoggingPayloadSetup._get_request_tags(
-                    litellm_params=data,
-                    proxy_server_request=data.get("proxy_server_request", {}),
-                ),
-                client_ip=_metadata.get("requester_ip_address"),
-                user_agent=_metadata.get("user_agent"),
-            )
-            _labels = prometheus_label_factory(
-                supported_enum_labels=self.get_labels_for_metric(
-                    metric_name="litellm_proxy_total_requests_metric"
-                ),
-                enum_values=enum_values,
-            )
-            self.litellm_proxy_total_requests_metric.labels(**_labels).inc()
-
-        except Exception as e:
-            verbose_logger.exception(
-                "prometheus Layer Error(): Exception occured - {}".format(str(e))
-            )
-            pass
+        Note: litellm_proxy_total_requests_metric is NOT incremented here to avoid
+        double-counting. It is incremented in async_log_success_event which fires
+        for all successful requests (both streaming and non-streaming).
+        """
+        pass
 
     def _safe_get(self, obj: Any, key: str, default: Any = None) -> Any:
         """Get value from dict or Pydantic model."""

litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/__init__.py

@@ -18,6 +18,9 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail"
         additional_provider_specific_params=getattr(
             litellm_params, "additional_provider_specific_params", {}
         ),
+        unreachable_fallback=getattr(
+            litellm_params, "unreachable_fallback", "fail_closed"
+        ),
         guardrail_name=guardrail.get("guardrail_name", ""),
         event_hook=litellm_params.mode,
         default_on=litellm_params.default_on,

litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/example_config.yaml

@@ -14,6 +14,7 @@ litellm_settings:
         mode: pre_call  # Options: pre_call, post_call, during_call, [pre_call, post_call]
         api_key: os.environ/GENERIC_GUARDRAIL_API_KEY  # Optional if using Bearer auth
         api_base: http://localhost:8080  # Required. Endpoint /beta/litellm_basic_guardrail_api is automatically appended
+        unreachable_fallback: fail_closed  # Options: fail_closed (default, raise), fail_open (proceed if endpoint unreachable or upstream returns 502/503/504)
         default_on: false  # Set to true to apply to all requests by default
         additional_provider_specific_params:
           # Any additional parameters your guardrail API needs

litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py

@@ -9,9 +9,11 @@
 import os
 from typing import TYPE_CHECKING, Any, Dict, Literal, Optional
 
+import httpx
+
 from litellm._logging import verbose_proxy_logger
 from litellm._version import version as litellm_version
-from litellm.exceptions import GuardrailRaisedException
+from litellm.exceptions import GuardrailRaisedException, Timeout
 from litellm.integrations.custom_guardrail import (
     CustomGuardrail,
     log_guardrail_information,
@@ -34,17 +36,19 @@
 GUARDRAIL_NAME = "generic_guardrail_api"
 
 # Headers whose values are forwarded as-is (case-insensitive). Glob patterns supported (e.g. x-stainless-*, x-litellm*).
-_HEADER_VALUE_ALLOWLIST = frozenset({
-    "host",
-    "accept-encoding",
-    "connection",
-    "accept",
-    "content-type",
-    "user-agent",
-    "x-stainless-*",
-    "x-litellm-*",
-    "content-length",
-})
+_HEADER_VALUE_ALLOWLIST = frozenset(
+    {
+        "host",
+        "accept-encoding",
+        "connection",
+        "accept",
+        "content-type",
+        "user-agent",
+        "x-stainless-*",
+        "x-litellm-*",
+        "content-length",
+    }
+)
 
 # Placeholder for headers that exist but are not on the allowlist (we don't expose their value).
 _HEADER_PRESENT_PLACEHOLDER = "[present]"
@@ -166,6 +170,7 @@ def __init__(
         api_base: Optional[str] = None,
         api_key: Optional[str] = None,
         additional_provider_specific_params: Optional[Dict[str, Any]] = None,
+        unreachable_fallback: Literal["fail_closed", "fail_open"] = "fail_closed",
         **kwargs,
     ):
         self.async_handler = get_async_httpx_client(
@@ -196,6 +201,10 @@ def __init__(
             additional_provider_specific_params or {}
         )
 
+        self.unreachable_fallback: Literal["fail_closed", "fail_open"] = (
+            unreachable_fallback
+        )
+
         # Set supported event hooks
         if "supported_event_hooks" not in kwargs:
             kwargs["supported_event_hooks"] = [
@@ -259,6 +268,54 @@ def _extract_user_api_key_metadata(
 
         return result_metadata
 
+    def _fail_open_passthrough(
+        self,
+        *,
+        inputs: GenericGuardrailAPIInputs,
+        input_type: Literal["request", "response"],
+        logging_obj: Optional["LiteLLMLoggingObj"],
+        error: Exception,
+        http_status_code: Optional[int] = None,
+    ) -> GenericGuardrailAPIInputs:
+        status_suffix = f" http_status_code={http_status_code}" if http_status_code else ""
+        verbose_proxy_logger.critical(
+            "Generic Guardrail API unreachable (fail-open). Proceeding without guardrail.%s "
+            "guardrail_name=%s api_base=%s input_type=%s litellm_call_id=%s litellm_trace_id=%s",
+            status_suffix,
+            getattr(self, "guardrail_name", None),
+            getattr(self, "api_base", None),
+            input_type,
+            getattr(logging_obj, "litellm_call_id", None) if logging_obj else None,
+            getattr(logging_obj, "litellm_trace_id", None) if logging_obj else None,
+            exc_info=error,
+        )
+        # Keep flow going - treat as action=NONE (no modifications)
+        return_inputs: GenericGuardrailAPIInputs = {}
+        return_inputs.update(inputs)
+        return return_inputs
+
+    def _build_guardrail_return_inputs(
+        self,
+        *,
+        texts: list,
+        images: Any,
+        tools: Any,
+        guardrail_response: GenericGuardrailAPIResponse,
+    ) -> GenericGuardrailAPIInputs:
+        # Action is NONE or no modifications needed
+        return_inputs = GenericGuardrailAPIInputs(texts=texts)
+        if guardrail_response.texts:
+            return_inputs["texts"] = guardrail_response.texts
+        if guardrail_response.images:
+            return_inputs["images"] = guardrail_response.images
+        elif images:
+            return_inputs["images"] = images
+        if guardrail_response.tools:
+            return_inputs["tools"] = guardrail_response.tools
+        elif tools:
+            return_inputs["tools"] = tools
+        return return_inputs
+
     @log_guardrail_information
     async def apply_guardrail(
         self,
@@ -313,7 +370,9 @@ async def apply_guardrail(
 
         # Extract user API key metadata
         user_metadata = self._extract_user_api_key_metadata(request_data)
-        inbound_headers = _extract_inbound_headers(request_data=request_data, logging_obj=logging_obj)
+        inbound_headers = _extract_inbound_headers(
+            request_data=request_data, logging_obj=logging_obj
+        )
 
         # Create request payload
         guardrail_request = GenericGuardrailAPIRequest(
@@ -370,23 +429,64 @@ async def apply_guardrail(
                     should_wrap_with_default_message=False,
                 )
 
-            # Action is NONE or no modifications needed
-            return_inputs = GenericGuardrailAPIInputs(texts=texts)
-            if guardrail_response.texts:
-                return_inputs["texts"] = guardrail_response.texts
-            if guardrail_response.images:
-                return_inputs["images"] = guardrail_response.images
-            elif images:
-                return_inputs["images"] = images
-            if guardrail_response.tools:
-                return_inputs["tools"] = guardrail_response.tools
-            elif tools:
-                return_inputs["tools"] = tools
-            return return_inputs
+            return self._build_guardrail_return_inputs(
+                texts=texts,
+                images=images,
+                tools=tools,
+                guardrail_response=guardrail_response,
+            )
 
         except GuardrailRaisedException:
             # Re-raise guardrail exceptions as-is
             raise
+        except Timeout as e:
+            # AsyncHTTPHandler wraps httpx.TimeoutException into litellm.Timeout
+            if self.unreachable_fallback == "fail_open":
+                return self._fail_open_passthrough(
+                    inputs=inputs,
+                    input_type=input_type,
+                    logging_obj=logging_obj,
+                    error=e,
+                )
+
+            verbose_proxy_logger.error(
+                "Generic Guardrail API: failed to make request: %s", str(e)
+            )
+            raise Exception(f"Generic Guardrail API failed: {str(e)}")
+        except httpx.HTTPStatusError as e:
+            # Common reverse-proxy/LB failures can present as HTTP errors even when the backend is unreachable.
+            status_code = getattr(getattr(e, "response", None), "status_code", None)
+            if self.unreachable_fallback == "fail_open" and status_code in (
+                502,
+                503,
+                504,
+            ):
+                return self._fail_open_passthrough(
+                    inputs=inputs,
+                    input_type=input_type,
+                    logging_obj=logging_obj,
+                    error=e,
+                    http_status_code=status_code,
+                )
+
+            verbose_proxy_logger.error(
+                "Generic Guardrail API: failed to make request: %s", str(e)
+            )
+            raise Exception(f"Generic Guardrail API failed: {str(e)}")
+        except httpx.RequestError as e:
+            # Guardrail endpoint is unreachable (DNS/connect/timeout/etc)
+            if self.unreachable_fallback == "fail_open":
+                return self._fail_open_passthrough(
+                    inputs=inputs,
+                    input_type=input_type,
+                    logging_obj=logging_obj,
+                    error=e,
+                )
+
+            verbose_proxy_logger.error(
+                "Generic Guardrail API: failed to make request: %s", str(e)
+            )
+            raise Exception(f"Generic Guardrail API failed: {str(e)}")
         except Exception as e:
             verbose_proxy_logger.error(
                 "Generic Guardrail API: failed to make request: %s", str(e)

litellm/proxy/utils.py

@@ -455,18 +455,26 @@ def get_proxy_hook(self, hook: str) -> Optional[CustomLogger]:
     def _init_litellm_callbacks(self, llm_router: Optional[Router] = None):
         self._add_proxy_hooks(llm_router)
         litellm.logging_callback_manager.add_litellm_callback(self.service_logging_obj)  # type: ignore
-        for callback in litellm.callbacks:
+
+        # Track string callbacks and their initialized instances so we can
+        # replace them in-place, preventing duplicates (string + instance) in
+        # litellm.callbacks which caused double-counting of metrics.
+        string_callbacks_to_replace: Dict[int, CustomLogger] = {}
+
+        for idx, callback in enumerate(litellm.callbacks):
             if isinstance(callback, str):
-                callback = litellm.litellm_core_utils.litellm_logging._init_custom_logger_compatible_class(  # type: ignore
+                initialized_callback = litellm.litellm_core_utils.litellm_logging._init_custom_logger_compatible_class(
                     cast(_custom_logger_compatible_callbacks_literal, callback),
                     internal_usage_cache=self.internal_usage_cache.dual_cache,
                     llm_router=llm_router,
                 )
 
-                if callback is None:
-                    continue
+                if initialized_callback is not None:
+                    string_callbacks_to_replace[idx] = initialized_callback
 
-            litellm.logging_callback_manager.add_litellm_callback(callback)
+        # Replace string entries in litellm.callbacks with initialized instances
+        for idx, initialized_callback in string_callbacks_to_replace.items():
+            litellm.callbacks[idx] = initialized_callback
 
     async def update_request_status(
         self, litellm_call_id: str, status: Literal["success", "fail"]