fix: prevent double-counting of litellm_proxy_total_requests_metric

litellm/integrations/prometheus.py

@@ -1055,16 +1055,16 @@ async def async_log_success_event(self, kwargs, response_obj, start_time, end_ti
             enum_values=enum_values,
         )
 
-        if (
-            standard_logging_payload["stream"] is True
-        ):  # log successful streaming requests from logging event hook.
-            _labels = prometheus_label_factory(
-                supported_enum_labels=self.get_labels_for_metric(
-                    metric_name="litellm_proxy_total_requests_metric"
-                ),
-                enum_values=enum_values,
-            )
-            self.litellm_proxy_total_requests_metric.labels(**_labels).inc()
+        # increment litellm_proxy_total_requests_metric for all successful requests
+        # (both streaming and non-streaming) in this single location to prevent
+        # double-counting that occurs when async_post_call_success_hook also increments
+        _labels = prometheus_label_factory(
+            supported_enum_labels=self.get_labels_for_metric(
+                metric_name="litellm_proxy_total_requests_metric"
+            ),
+            enum_values=enum_values,
+        )
+        self.litellm_proxy_total_requests_metric.labels(**_labels).inc()
 
     def _increment_token_metrics(
         self,
@@ -1086,13 +1086,6 @@ def _increment_token_metrics(
         ):
             _tags = standard_logging_payload["request_tags"]
 
-        _labels = prometheus_label_factory(
-            supported_enum_labels=self.get_labels_for_metric(
-                metric_name="litellm_proxy_total_requests_metric"
-            ),
-            enum_values=enum_values,
-        )
-
         _labels = prometheus_label_factory(
             supported_enum_labels=self.get_labels_for_metric(
                 metric_name="litellm_total_tokens_metric"
@@ -1655,49 +1648,12 @@ async def async_post_call_success_hook(
     ):
         """
         Proxy level tracking - triggered when the proxy responds with a success response to the client
-        """
-        try:
-            from litellm.litellm_core_utils.litellm_logging import (
-                StandardLoggingPayloadSetup,
-            )
-
-            if self._should_skip_metrics_for_invalid_key(
-                user_api_key_dict=user_api_key_dict
-            ):
-                return
 
-            _metadata = data.get("metadata", {}) or {}
-            enum_values = UserAPIKeyLabelValues(
-                end_user=user_api_key_dict.end_user_id,
-                hashed_api_key=user_api_key_dict.api_key,
-                api_key_alias=user_api_key_dict.key_alias,
-                requested_model=data.get("model", ""),
-                team=user_api_key_dict.team_id,
-                team_alias=user_api_key_dict.team_alias,
-                user=user_api_key_dict.user_id,
-                user_email=user_api_key_dict.user_email,
-                status_code="200",
-                route=user_api_key_dict.request_route,
-                tags=StandardLoggingPayloadSetup._get_request_tags(
-                    litellm_params=data,
-                    proxy_server_request=data.get("proxy_server_request", {}),
-                ),
-                client_ip=_metadata.get("requester_ip_address"),
-                user_agent=_metadata.get("user_agent"),
-            )
-            _labels = prometheus_label_factory(
-                supported_enum_labels=self.get_labels_for_metric(
-                    metric_name="litellm_proxy_total_requests_metric"
-                ),
-                enum_values=enum_values,
-            )
-            self.litellm_proxy_total_requests_metric.labels(**_labels).inc()
-
-        except Exception as e:
-            verbose_logger.exception(
-                "prometheus Layer Error(): Exception occured - {}".format(str(e))
-            )
-            pass
+        Note: litellm_proxy_total_requests_metric is NOT incremented here to avoid
+        double-counting. It is incremented in async_log_success_event which fires
+        for all successful requests (both streaming and non-streaming).
+        """
+        pass
 
     def _safe_get(self, obj: Any, key: str, default: Any = None) -> Any:
         """Get value from dict or Pydantic model."""

litellm/proxy/utils.py

@@ -450,19 +450,33 @@ def get_proxy_hook(self, hook: str) -> Optional[CustomLogger]:
     def _init_litellm_callbacks(self, llm_router: Optional[Router] = None):
         self._add_proxy_hooks(llm_router)
         litellm.logging_callback_manager.add_litellm_callback(self.service_logging_obj)  # type: ignore
-        for callback in litellm.callbacks:
+
+        # Track string callbacks and their initialized instances so we can
+        # replace them in-place, preventing duplicates (string + instance) in
+        # litellm.callbacks which caused double-counting of metrics.
+        string_callbacks_to_replace: Dict[int, CustomLogger] = {}
+
+        for idx, callback in enumerate(litellm.callbacks):
             if isinstance(callback, str):
-                callback = litellm.litellm_core_utils.litellm_logging._init_custom_logger_compatible_class(  # type: ignore
+                initialized_callback = litellm.litellm_core_utils.litellm_logging._init_custom_logger_compatible_class(
                     cast(_custom_logger_compatible_callbacks_literal, callback),
                     internal_usage_cache=self.internal_usage_cache.dual_cache,
                     llm_router=llm_router,
                 )
 
-                if callback is None:
-                    continue
-
-            litellm.logging_callback_manager.add_litellm_callback(callback)
-
+                if initialized_callback is not None:
+                    string_callbacks_to_replace[idx] = initialized_callback
+            else:
+                # Only add non-string callbacks to the manager; string
+                # callbacks will be replaced in-place below and are already
+                # present in litellm.callbacks, so adding them to the manager
+                # (which appends to litellm.callbacks) would create duplicates.
+                litellm.logging_callback_manager.add_litellm_callback(callback)
+
+        # Replace string entries in litellm.callbacks with initialized instances
+        for idx, initialized_callback in string_callbacks_to_replace.items():
+            litellm.callbacks[idx] = initialized_callback        
+
     async def update_request_status(
         self, litellm_call_id: str, status: Literal["success", "fail"]
     ):

tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py

@@ -792,7 +792,8 @@ async def test_async_post_call_success_hook(prometheus_logger):
     """
     Test for the async_post_call_success_hook method
 
-    it should increment the litellm_proxy_total_requests_metric
+    litellm_proxy_total_requests_metric is NOT incremented here to avoid double-counting.
+    It is incremented in async_log_success_event instead.
     """
     # Mock the prometheus metric
     prometheus_logger.litellm_proxy_total_requests_metric = MagicMock()
@@ -817,23 +818,8 @@ async def test_async_post_call_success_hook(prometheus_logger):
         data=data, user_api_key_dict=user_api_key_dict, response=response
     )
 
-    # Assert total requests metric was incremented with correct labels
-    prometheus_logger.litellm_proxy_total_requests_metric.labels.assert_called_once_with(
-        end_user=None,
-        hashed_api_key="test_key",
-        api_key_alias="test_alias",
-        requested_model="gpt-3.5-turbo",
-        team="test_team",
-        team_alias="test_team_alias",
-        user="test_user",
-        status_code="200",
-        user_email=None,
-        route=user_api_key_dict.request_route,
-        model_id=None,
-        client_ip=None,
-        user_agent=None,
-    )
-    prometheus_logger.litellm_proxy_total_requests_metric.labels().inc.assert_called_once()
+    # Assert total requests metric was NOT incremented (moved to async_log_success_event)
+    prometheus_logger.litellm_proxy_total_requests_metric.labels.assert_not_called()
 
 
 def test_set_llm_deployment_success_metrics(prometheus_logger):

tests/enterprise/litellm_enterprise/integrations/test_prometheus.py

@@ -545,6 +545,9 @@ async def test_request_counter_semantic_validation(mock_prometheus_logger):
     CRITICAL TEST: Validates that request counters are incremented by 1, not by token count.
     This test specifically catches the bug where litellm_proxy_total_requests_metric
     is incorrectly incremented by total_tokens instead of 1.
+
+    The metric is now ONLY incremented in async_log_success_event (for both streaming
+    and non-streaming) to prevent double-counting.
     """
     from datetime import datetime, timedelta
     from unittest.mock import MagicMock
@@ -583,18 +586,18 @@ async def test_request_counter_semantic_validation(mock_prometheus_logger):
         },
     }
 
-    # Call the success event
+    # Call the success event - should increment for both streaming and non-streaming
     await mock_prometheus_logger.async_log_success_event(
         kwargs, None, kwargs["start_time"], kwargs["end_time"]
     )
 
-    # CRITICAL ASSERTION: Request counter should not be incremented
+    # CRITICAL ASSERTION: Request counter should be incremented by 1
     total_requests_metric = mock_prometheus_logger.litellm_proxy_total_requests_metric
     assert (
-        len(total_requests_metric.inc_calls) == 0
-    ), "Request metric should not be incremented"
+        len(total_requests_metric.inc_calls) == 1
+    ), "Request metric should be incremented once in async_log_success_event"
 
-    # Call the post-call logging hook
+    # Call the post-call logging hook - should NOT increment (to prevent double-counting)
     await mock_prometheus_logger.async_post_call_success_hook(
         data={},
         user_api_key_dict=UserAPIKeyAuth(
@@ -607,11 +610,11 @@ async def test_request_counter_semantic_validation(mock_prometheus_logger):
         response=MagicMock(),
     )
 
-    # CRITICAL ASSERTION: Request counter be incremented by 1
+    # CRITICAL ASSERTION: Request counter should still be 1 (not incremented again)
     total_requests_metric = mock_prometheus_logger.litellm_proxy_total_requests_metric
     assert (
         len(total_requests_metric.inc_calls) == 1
-    ), "Request metric should not be incremented"
+    ), "Request metric should not be incremented again in async_post_call_success_hook"
 
     # Check that ALL request counter increments are by 1 (not by token count)
     for inc_value in total_requests_metric.inc_calls:
@@ -684,8 +687,8 @@ async def test_multiple_requests_counter_semantics(mock_prometheus_logger):
     expected_total_tokens = num_requests * tokens_per_request  # 3 * 500 = 1500
 
     # With the bug, total_request_increments would be 1500 instead of 3
-    assert total_request_increments == 0, (
-        f"SEMANTIC BUG: Request counter total increments = 0, "
+    assert total_request_increments == num_requests, (
+        f"SEMANTIC BUG: Request counter total increments = {total_request_increments}, "
         f"expected {num_requests}. This suggests request counters are being incremented "
         f"by token counts instead of request counts."
     )