fix(router): clamp least_busy request counter at zero (salvage of #25393, credit @mango766)

litellm/router_strategy/least_busy.py

@@ -57,6 +57,9 @@ def log_success_event(self, kwargs, response_obj, start_time, end_time):
                 id = kwargs["litellm_params"].get("model_info", {}).get("id", None)
                 if model_group is None or id is None:
                     return
+                # Cache hits never ran log_pre_api_call (no increment); do not decrement (#39322).
+                if kwargs.get("cache_hit") is True:
+                    return
                 elif isinstance(id, int):
                     id = str(id)
 
@@ -66,7 +69,7 @@ def log_success_event(self, kwargs, response_obj, start_time, end_time):
                 request_count_value: Optional[int] = request_count_dict.get(id, 0)
                 if request_count_value is None:
                     return
-                request_count_dict[id] = request_count_value - 1
+                request_count_dict[id] = max(request_count_value - 1, 0)
                 self.router_cache.set_cache(key=request_count_api_key, value=request_count_dict)
 
                 ### TESTING ###
@@ -84,6 +87,9 @@ def log_failure_event(self, kwargs, response_obj, start_time, end_time):
                 id = kwargs["litellm_params"].get("model_info", {}).get("id", None)
                 if model_group is None or id is None:
                     return
+                # Cache hits never ran log_pre_api_call (no increment); do not decrement (#39322).
+                if kwargs.get("cache_hit") is True:
+                    return
                 elif isinstance(id, int):
                     id = str(id)
 
@@ -93,7 +99,7 @@ def log_failure_event(self, kwargs, response_obj, start_time, end_time):
                 request_count_value: Optional[int] = request_count_dict.get(id, 0)
                 if request_count_value is None:
                     return
-                request_count_dict[id] = request_count_value - 1
+                request_count_dict[id] = max(request_count_value - 1, 0)
                 self.router_cache.set_cache(key=request_count_api_key, value=request_count_dict)
 
                 ### TESTING ###
@@ -112,6 +118,9 @@ async def async_log_success_event(self, kwargs, response_obj, start_time, end_ti
                 id = kwargs["litellm_params"].get("model_info", {}).get("id", None)
                 if model_group is None or id is None:
                     return
+                # Cache hits never ran log_pre_api_call (no increment); do not decrement (#39322).
+                if kwargs.get("cache_hit") is True:
+                    return
                 elif isinstance(id, int):
                     id = str(id)
 
@@ -121,7 +130,7 @@ async def async_log_success_event(self, kwargs, response_obj, start_time, end_ti
                 request_count_value: Optional[int] = request_count_dict.get(id, 0)
                 if request_count_value is None:
                     return
-                request_count_dict[id] = request_count_value - 1
+                request_count_dict[id] = max(request_count_value - 1, 0)
                 await self.router_cache.async_set_cache(key=request_count_api_key, value=request_count_dict)
 
                 ### TESTING ###
@@ -139,6 +148,9 @@ async def async_log_failure_event(self, kwargs, response_obj, start_time, end_ti
                 id = kwargs["litellm_params"].get("model_info", {}).get("id", None)
                 if model_group is None or id is None:
                     return
+                # Cache hits never ran log_pre_api_call (no increment); do not decrement (#39322).
+                if kwargs.get("cache_hit") is True:
+                    return
                 elif isinstance(id, int):
                     id = str(id)
 
@@ -148,7 +160,7 @@ async def async_log_failure_event(self, kwargs, response_obj, start_time, end_ti
                 request_count_value: Optional[int] = request_count_dict.get(id, 0)
                 if request_count_value is None:
                     return
-                request_count_dict[id] = request_count_value - 1
+                request_count_dict[id] = max(request_count_value - 1, 0)
                 await self.router_cache.async_set_cache(key=request_count_api_key, value=request_count_dict)
 
                 ### TESTING ###

tests/test_litellm/router_strategy/test_least_busy_clamp.py

@@ -0,0 +1,44 @@
+def test_least_busy_decrement_clamps_at_zero():
+    assert max(0 - 1, 0) == 0
+    assert max(2 - 1, 0) == 1
+
+
+def test_cache_hit_skips_decrement():
+    """Response-cache hits must not decrement least_busy counters (#39322)."""
+    from unittest.mock import MagicMock
+
+    from litellm.router_strategy.least_busy import LeastBusyLoggingHandler
+
+    cache = MagicMock()
+    cache.get_cache.return_value = {"dep-a": 2}
+    handler = LeastBusyLoggingHandler(router_cache=cache)
+    kwargs = {
+        "cache_hit": True,
+        "litellm_params": {
+            "metadata": {"model_group": "g"},
+            "model_info": {"id": "dep-a"},
+        },
+    }
+    handler.log_success_event(kwargs, response_obj=None, start_time=None, end_time=None)
+    cache.set_cache.assert_not_called()
+
+
+def test_non_cache_hit_still_decrements_with_clamp():
+    from unittest.mock import MagicMock
+
+    from litellm.router_strategy.least_busy import LeastBusyLoggingHandler
+
+    cache = MagicMock()
+    cache.get_cache.return_value = {"dep-a": 0}
+    handler = LeastBusyLoggingHandler(router_cache=cache)
+    kwargs = {
+        "cache_hit": False,
+        "litellm_params": {
+            "metadata": {"model_group": "g"},
+            "model_info": {"id": "dep-a"},
+        },
+    }
+    handler.log_success_event(kwargs, response_obj=None, start_time=None, end_time=None)
+    cache.set_cache.assert_called_once()
+    value = cache.set_cache.call_args.kwargs.get("value")
+    assert value["dep-a"] == 0