diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 02bb66388ca3..6547eea9cd72 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -2680,7 +2680,7 @@ async def async_streaming_data_generator( # on disconnect, so the nested iterator hook (which only sees # GeneratorExit on GC) cannot own the refund. if not stream_completed: - proxy_logging_obj._release_max_parallel_requests_on_disconnect(user_api_key_dict) + proxy_logging_obj._release_max_parallel_requests_on_disconnect(user_api_key_dict, request_data) client_disconnected = True if not delivered_chunk: from litellm.proxy.spend_tracking.budget_reservation import ( diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index d60c17c744f1..22ea9fe176a9 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -7,6 +7,7 @@ import asyncio import binascii import os +import uuid from datetime import datetime from typing import ( TYPE_CHECKING, @@ -185,6 +186,69 @@ return results """ +PARALLEL_ACQUIRE_SCRIPT = """ +-- Atomic check-and-acquire for the max_parallel_requests concurrency gauge. +-- Each gauge key is a sorted set of per-request slot ids scored by acquire +-- time (Redis server clock). In-flight requests are counted by ZCARD after +-- pruning slots older than the slot TTL, so unlike the windowed RPM/TPM +-- counters the gauge is never reset while requests are in flight, a +-- rejected request never occupies a slot, and a slot leaked by a crashed +-- worker self-heals after the slot TTL even under continuous traffic. +-- +-- KEYS: one gauge zset key per descriptor. +-- ARGV: per-key triples (limit, slot_ttl_seconds, slot_id). +-- Success: { 0, in_flight_1, ... }. Over-limit: { 1, key_index, in_flight, limit }. +local time_reply = redis.call('TIME') +local now = tonumber(time_reply[1]) +for i = 1, #KEYS do + local limit = tonumber(ARGV[(i - 1) * 3 + 1]) + local slot_ttl = tonumber(ARGV[(i - 1) * 3 + 2]) + redis.call('ZREMRANGEBYSCORE', KEYS[i], '-inf', now - slot_ttl) + local in_flight = redis.call('ZCARD', KEYS[i]) + if in_flight + 1 > limit then + return { 1, i, in_flight, limit } + end +end +local results = { 0 } +for i = 1, #KEYS do + local slot_ttl = tonumber(ARGV[(i - 1) * 3 + 2]) + local slot_id = ARGV[(i - 1) * 3 + 3] + redis.call('ZADD', KEYS[i], now, slot_id) + redis.call('EXPIRE', KEYS[i], slot_ttl) + table.insert(results, redis.call('ZCARD', KEYS[i])) +end +return results +""" + +PARALLEL_RELEASE_SCRIPT = """ +-- Release one slot per gauge key by removing this request's slot id. +-- ZREM of an absent member (or key) is a no-op, so a release without a +-- matching acquire (proxy-side rejection, double-fired callback, slot +-- already expired) can never free a slot owned by another request. +-- KEYS: gauge zset keys. ARGV: per-key slot_id. +-- Returns the remaining in-flight count per key. +local results = {} +for i = 1, #KEYS do + redis.call('ZREM', KEYS[i], ARGV[i]) + table.insert(results, redis.call('ZCARD', KEYS[i])) +end +return results +""" + +PARALLEL_COUNT_SCRIPT = """ +-- Read the current in-flight count per gauge key (prunes expired slots +-- first so leaked slots do not inflate the reading). +-- KEYS: gauge zset keys. ARGV: per-key slot_ttl_seconds. +local time_reply = redis.call('TIME') +local now = tonumber(time_reply[1]) +local results = {} +for i = 1, #KEYS do + redis.call('ZREMRANGEBYSCORE', KEYS[i], '-inf', now - tonumber(ARGV[i])) + table.insert(results, redis.call('ZCARD', KEYS[i])) +end +return results +""" + TOKEN_INCREMENT_SCRIPT = """ local results = {} @@ -248,6 +312,19 @@ # mirror ``x-ratelimit-*`` headers into the SLP. Streaming exits # common_request_processing before ``async_post_call_success_hook`` runs. RATE_LIMIT_RESPONSE_KEY = "_litellm_proxy_rate_limit_response" +# Holds the acquisition the pre-call hook made for this request: the slot id +# plus the gauge counter keys it was registered under. The success/failure +# callbacks release only this exact acquisition: those callbacks also fire +# for requests rejected at pre-call (which never acquired a slot), and an +# id-less release would free a slot still owned by another in-flight request +# — every rejection would then raise effective concurrency above the +# configured limit. +MAX_PARALLEL_SLOT_ACQUIRED_KEY = "_litellm_max_parallel_slot_acquired" +# How long an acquired slot counts toward the in-flight total before it is +# considered leaked (worker crashed without any release callback firing) and +# pruned. Also the longest request duration the gauge can track: a request +# running longer than this stops occupying its slot. +PARALLEL_REQUEST_SLOT_TTL_SECONDS = 3600 # Stash keys live ONLY in metadata channels — never at the top level of the # request body. Top-level keys are forwarded as body params to upstream # providers, which reject unknown fields with 400/429 errors. @@ -258,6 +335,7 @@ TPM_RESERVATION_RELEASED_KEY, RATE_LIMIT_DESCRIPTORS_KEY, RATE_LIMIT_RESPONSE_KEY, + MAX_PARALLEL_SLOT_ACQUIRED_KEY, ) @@ -274,6 +352,17 @@ class RateLimitDescriptor(TypedDict): rate_limit: Optional[RateLimitDescriptorRateLimitObject] +class ParallelRequestGauge(TypedDict): + counter_key: str + limit: int + descriptor_key: str + + +class ParallelSlotAcquisition(TypedDict): + slot_id: str + counter_keys: list[str] + + class RateLimitStatus(TypedDict): code: str current_limit: int @@ -310,10 +399,22 @@ def __init__( self.check_and_increment_by_n_script = ( self.internal_usage_cache.dual_cache.redis_cache.async_register_script(CHECK_AND_INCREMENT_BY_N_SCRIPT) ) + self.parallel_acquire_script = self.internal_usage_cache.dual_cache.redis_cache.async_register_script( + PARALLEL_ACQUIRE_SCRIPT + ) + self.parallel_release_script = self.internal_usage_cache.dual_cache.redis_cache.async_register_script( + PARALLEL_RELEASE_SCRIPT + ) + self.parallel_count_script = self.internal_usage_cache.dual_cache.redis_cache.async_register_script( + PARALLEL_COUNT_SCRIPT + ) else: self.batch_rate_limiter_script = None self.token_increment_script = None self.check_and_increment_by_n_script = None + self.parallel_acquire_script = None + self.parallel_release_script = None + self.parallel_count_script = None self.window_size = int(os.getenv("LITELLM_RATE_LIMIT_WINDOW_SIZE", 60)) @@ -559,7 +660,6 @@ def is_cache_list_over_limit( counter_key = keys_to_fetch[i + 1] counter_value = cache_values[i + 1] requests_limit = key_metadata[window_key]["requests_limit"] - max_parallel_requests_limit = key_metadata[window_key]["max_parallel_requests_limit"] tokens_limit = key_metadata[window_key]["tokens_limit"] # Determine which limit to use for current_limit and limit_remaining @@ -568,9 +668,6 @@ def is_cache_list_over_limit( if counter_key.endswith(":requests"): current_limit = requests_limit rate_limit_type = "requests" - elif counter_key.endswith(":max_parallel_requests"): - current_limit = max_parallel_requests_limit - rate_limit_type = "max_parallel_requests" elif counter_key.endswith(":tokens"): current_limit = tokens_limit rate_limit_type = "tokens" @@ -694,6 +791,7 @@ async def should_rate_limit( parent_otel_span: Optional[Span] = None, read_only: bool = False, skip_tpm_check: bool = False, + parallel_slot_id: str | None = None, ) -> RateLimitResponse: """ Check if any of the rate limit descriptors should be rate limited. @@ -710,15 +808,122 @@ async def should_rate_limit( ``reserve_tpm_tokens`` reservation path should set this to avoid the +1-per-key Lua / in-memory increment double-charging the tokens counter. + + ``max_parallel_requests`` descriptors are enforced by the dedicated + concurrency-gauge path (``_check_parallel_request_gauges``), never by + the windowed counters. The gauge phase must stay AFTER the windowed + check so a windowed rejection never strands an acquired slot; the + reverse order would leak one gauge slot per RPM/TPM rejection. + ``parallel_slot_id`` names the slot an admission registers; callers + that enforce (not read_only) should pass the id they will later + release with — when omitted, a generated slot id is used and the slot + can only be reclaimed by TTL expiry. """ current_time = self._get_current_time() now = current_time.timestamp() now_int = int(now) # Convert to integer for Redis Lua script - # Collect all keys and their metadata upfront + keys_to_fetch, key_metadata, gauges = self._collect_windowed_keys_and_gauges( + descriptors=descriptors, + skip_tpm_check=skip_tpm_check, + ) + + windowed_response = RateLimitResponse(overall_code="OK", statuses=[]) + if keys_to_fetch: + ## CHECK IN-MEMORY CACHE + cache_values = await self.internal_usage_cache.async_batch_get_cache( + keys=keys_to_fetch, + parent_otel_span=parent_otel_span, + local_only=True, + ) + + if cache_values is not None: + rate_limit_response = self.is_cache_list_over_limit(keys_to_fetch, cache_values, key_metadata) + if rate_limit_response["overall_code"] == "OVER_LIMIT": + return rate_limit_response + + ## IF under limit in-memory, check Redis + if read_only: + # READ-ONLY MODE: Just read current values without incrementing + cache_values = await self.internal_usage_cache.async_batch_get_cache( + keys=keys_to_fetch, + parent_otel_span=parent_otel_span, + local_only=False, # Check Redis too + ) + + # For keys that don't exist yet, set them to 0 + if cache_values is None: + cache_values = [] + for _ in keys_to_fetch: + cache_values.append(str(now_int) if _.endswith(":window") else 0) + elif self.batch_rate_limiter_script is not None: + # NORMAL MODE: Increment counters in Redis + # Group keys by hash tag for Redis cluster compatibility + cache_values = await self._execute_redis_batch_rate_limiter_script( + keys_to_fetch=keys_to_fetch, + now_int=now_int, + ) + + # update in-memory cache with new values + for i in range(0, len(cache_values), 2): + window_key = keys_to_fetch[i] + counter_key = keys_to_fetch[i + 1] + window_value = cache_values[i] + counter_value = cache_values[i + 1] + await self.internal_usage_cache.async_set_cache( + key=counter_key, + value=counter_value, + ttl=self.window_size, + litellm_parent_otel_span=parent_otel_span, + local_only=True, + ) + await self.internal_usage_cache.async_set_cache( + key=window_key, + value=window_value, + ttl=self.window_size, + litellm_parent_otel_span=parent_otel_span, + local_only=True, + ) + else: + # NORMAL MODE: In-memory sliding window (no Redis) + cache_values = await self.in_memory_cache_sliding_window( + keys=keys_to_fetch, + now_int=now_int, + window_size=self.window_size, + ) + + windowed_response = self.is_cache_list_over_limit(keys_to_fetch, cache_values, key_metadata) + if windowed_response["overall_code"] == "OVER_LIMIT": + return windowed_response + + if not gauges: + return windowed_response + + gauge_response = await self._check_parallel_request_gauges( + gauges=gauges, + slot_id=parallel_slot_id or uuid.uuid4().hex, + parent_otel_span=parent_otel_span, + read_only=read_only, + ) + return RateLimitResponse( + overall_code=gauge_response["overall_code"], + statuses=[*windowed_response["statuses"], *gauge_response["statuses"]], + ) + + def _collect_windowed_keys_and_gauges( + self, + descriptors: list[RateLimitDescriptor], + skip_tpm_check: bool, + ) -> tuple[list[str], dict[str, dict[str, Any]], list[ParallelRequestGauge]]: + """ + Split descriptors into the windowed (window_key, counter_key) fetch + list with its per-window metadata, and the concurrency gauges for + descriptors carrying a max_parallel_requests limit. + """ keys_to_fetch: List[str] = [] - key_metadata = {} # Store metadata for each key + key_metadata: dict[str, dict[str, Any]] = {} + gauges: list[ParallelRequestGauge] = [] for descriptor in descriptors: descriptor_key = descriptor["key"] descriptor_value = descriptor["value"] @@ -732,6 +937,17 @@ async def should_rate_limit( window_key = f"{{{descriptor_key}:{descriptor_value}}}:window" + if max_parallel_requests_limit is not None: + gauges.append( + ParallelRequestGauge( + counter_key=self.create_rate_limit_keys( + descriptor_key, descriptor_value, "max_parallel_requests" + ), + limit=int(max_parallel_requests_limit), + descriptor_key=descriptor_key, + ) + ) + rate_limit_set = False if requests_limit is not None: rpm_key = self.create_rate_limit_keys(descriptor_key, descriptor_value, "requests") @@ -741,12 +957,6 @@ async def should_rate_limit( tpm_key = self.create_rate_limit_keys(descriptor_key, descriptor_value, "tokens") keys_to_fetch.extend([window_key, tpm_key]) rate_limit_set = True - if max_parallel_requests_limit is not None: - max_parallel_requests_key = self.create_rate_limit_keys( - descriptor_key, descriptor_value, "max_parallel_requests" - ) - keys_to_fetch.extend([window_key, max_parallel_requests_key]) - rate_limit_set = True if not rate_limit_set: continue @@ -754,77 +964,252 @@ async def should_rate_limit( key_metadata[window_key] = { "requests_limit": (int(requests_limit) if requests_limit is not None else None), "tokens_limit": int(tokens_limit) if tokens_limit is not None else None, - "max_parallel_requests_limit": ( - int(max_parallel_requests_limit) if max_parallel_requests_limit is not None else None - ), "window_size": int(window_size), "descriptor_key": descriptor_key, } + return keys_to_fetch, key_metadata, gauges + + def _gauge_status(self, gauge: ParallelRequestGauge, in_flight: int, code: str) -> RateLimitStatus: + return RateLimitStatus( + code=code, + current_limit=gauge["limit"], + limit_remaining=max(0, gauge["limit"] - in_flight), + rate_limit_type="max_parallel_requests", + descriptor_key=gauge["descriptor_key"], + ) + + def _gauge_in_flight_from_cache_value(self, raw_value: Any) -> int: + """ + In-flight count from a cached gauge value: a dict of slot_id -> + acquire timestamp when the in-memory registry is authoritative, or + the mirrored integer count from the last Redis script result. + """ + if raw_value is None: + return 0 + if isinstance(raw_value, dict): + cutoff = self._get_current_time().timestamp() - PARALLEL_REQUEST_SLOT_TTL_SECONDS + return sum(1 for ts in raw_value.values() if isinstance(ts, (int, float)) and ts >= cutoff) + return max(0, int(raw_value)) + + async def _check_parallel_request_gauges( + self, + gauges: list[ParallelRequestGauge], + slot_id: str, + parent_otel_span: Span | None = None, + read_only: bool = False, + ) -> RateLimitResponse: + """ + Enforce max_parallel_requests as a concurrency gauge over a per-slot + registry: each admitted request registers ``slot_id`` with its + acquire time, and admission requires in_flight + 1 <= limit over the + unexpired slots. Unlike the windowed RPM/TPM counters, the gauge is + never reset while requests are in flight, a rejected request never + occupies a slot, and a slot leaked by a crashed worker is pruned + after PARALLEL_REQUEST_SLOT_TTL_SECONDS even under continuous + traffic. Releases remove exactly this request's slot id, so a + double-fired or unmatched release can never free another request's + slot. + """ + gauge_keys = [gauge["counter_key"] for gauge in gauges] - ## CHECK IN-MEMORY CACHE - cache_values = await self.internal_usage_cache.async_batch_get_cache( - keys=keys_to_fetch, + if read_only: + if self.parallel_count_script is not None: + try: + raw_counts = await self.parallel_count_script( + keys=gauge_keys, + args=[PARALLEL_REQUEST_SLOT_TTL_SECONDS for _ in gauges], + ) + counts = [max(0, int(value)) for value in raw_counts] + except Exception as e: # noqa: BLE001 - any Redis/Lua failure degrades to the local mirror, never a 500 + verbose_proxy_logger.warning(f"parallel_count_script failed, using local mirror: {str(e)}") + counts = await self._read_local_gauge_counts(gauge_keys, parent_otel_span) + else: + counts = await self._read_local_gauge_counts(gauge_keys, parent_otel_span) + statuses = [] + overall_code = "OK" + for gauge, in_flight in zip(gauges, counts): + code = "OVER_LIMIT" if in_flight >= gauge["limit"] else "OK" + if code == "OVER_LIMIT": + overall_code = "OVER_LIMIT" + statuses.append(self._gauge_status(gauge, in_flight, code)) + return RateLimitResponse(overall_code=overall_code, statuses=statuses) + + local_counts = await self._read_local_gauge_counts(gauge_keys, parent_otel_span) + for gauge, in_flight in zip(gauges, local_counts): + if in_flight >= gauge["limit"]: + return RateLimitResponse( + overall_code="OVER_LIMIT", + statuses=[self._gauge_status(gauge, in_flight, "OVER_LIMIT")], + ) + + if self.parallel_acquire_script is not None: + try: + raw = await self.parallel_acquire_script( + keys=gauge_keys, + args=[ + arg for gauge in gauges for arg in (gauge["limit"], PARALLEL_REQUEST_SLOT_TTL_SECONDS, slot_id) + ], + ) + except Exception as e: # noqa: BLE001 - any Redis/Lua failure degrades to in-memory enforcement, never a 500 + verbose_proxy_logger.warning( + f"parallel_acquire_script failed, falling back to in-memory gauge: {str(e)}" + ) + async with self._check_and_increment_lock: + return await self._acquire_parallel_slots_in_memory(gauges, slot_id, parent_otel_span) + if int(raw[0]) == 1: + gauge = gauges[int(raw[1]) - 1] + return RateLimitResponse( + overall_code="OVER_LIMIT", + statuses=[self._gauge_status(gauge, int(raw[2]), "OVER_LIMIT")], + ) + statuses = [] + for gauge, in_flight in zip(gauges, raw[1:]): + await self.internal_usage_cache.async_set_cache( + key=gauge["counter_key"], + value=int(in_flight), + ttl=PARALLEL_REQUEST_SLOT_TTL_SECONDS, + litellm_parent_otel_span=parent_otel_span, + local_only=True, + ) + statuses.append(self._gauge_status(gauge, int(in_flight), "OK")) + return RateLimitResponse(overall_code="OK", statuses=statuses) + + async with self._check_and_increment_lock: + return await self._acquire_parallel_slots_in_memory(gauges, slot_id, parent_otel_span) + + async def _read_local_gauge_counts( + self, + gauge_keys: list[str], + parent_otel_span: Span | None = None, + ) -> list[int]: + values = await self.internal_usage_cache.async_batch_get_cache( + keys=gauge_keys, parent_otel_span=parent_otel_span, local_only=True, ) + if values is None: + return [0 for _ in gauge_keys] + return [self._gauge_in_flight_from_cache_value(value) for value in values] + + async def _acquire_parallel_slots_in_memory( + self, + gauges: list[ParallelRequestGauge], + slot_id: str, + parent_otel_span: Span | None = None, + ) -> RateLimitResponse: + """ + All-or-nothing in-memory slot-registry acquire. Caller holds the lock. + + A cached dict is the authoritative in-memory registry. A cached + integer is the count mirrored from the last successful Redis script + call: when Redis fails over to this path, that mirror still counts + the slots in flight on the Redis side, so it is carried forward as + an integer counter (not discarded as an empty registry, which would + briefly double the admitted concurrency during a Redis outage). + """ + now = self._get_current_time().timestamp() + cutoff = now - PARALLEL_REQUEST_SLOT_TTL_SECONDS + states: list[tuple[dict[str, float] | None, int]] = [] + for gauge in gauges: + raw_value = await self.internal_usage_cache.async_get_cache( + key=gauge["counter_key"], + litellm_parent_otel_span=parent_otel_span, + local_only=True, + ) + if isinstance(raw_value, dict): + registry: dict[str, float] | None = { + key: float(ts) for key, ts in raw_value.items() if isinstance(ts, (int, float)) and ts >= cutoff + } + in_flight = len(registry or {}) + elif raw_value is None: + registry = {} + in_flight = 0 + else: + registry = None + in_flight = max(0, int(raw_value)) + if in_flight + 1 > gauge["limit"]: + return RateLimitResponse( + overall_code="OVER_LIMIT", + statuses=[self._gauge_status(gauge, in_flight, "OVER_LIMIT")], + ) + states.append((registry, in_flight)) - if cache_values is not None: - rate_limit_response = self.is_cache_list_over_limit(keys_to_fetch, cache_values, key_metadata) - if rate_limit_response["overall_code"] == "OVER_LIMIT": - return rate_limit_response + statuses = [] + for gauge, (registry, in_flight) in zip(gauges, states): + new_value: Union[dict[str, float], int] = ( + {**registry, slot_id: now} if registry is not None else in_flight + 1 + ) + await self.internal_usage_cache.async_set_cache( + key=gauge["counter_key"], + value=new_value, + ttl=PARALLEL_REQUEST_SLOT_TTL_SECONDS, + litellm_parent_otel_span=parent_otel_span, + local_only=True, + ) + statuses.append(self._gauge_status(gauge, in_flight + 1, "OK")) + return RateLimitResponse(overall_code="OK", statuses=statuses) - ## IF under limit in-memory, check Redis - if read_only: - # READ-ONLY MODE: Just read current values without incrementing - cache_values = await self.internal_usage_cache.async_batch_get_cache( - keys=keys_to_fetch, - parent_otel_span=parent_otel_span, - local_only=False, # Check Redis too - ) - - # For keys that don't exist yet, set them to 0 - if cache_values is None: - cache_values = [] - for _ in keys_to_fetch: - cache_values.append(str(now_int) if _.endswith(":window") else 0) - elif self.batch_rate_limiter_script is not None: - # NORMAL MODE: Increment counters in Redis - # Group keys by hash tag for Redis cluster compatibility - cache_values = await self._execute_redis_batch_rate_limiter_script( - keys_to_fetch=keys_to_fetch, - now_int=now_int, - ) - - # update in-memory cache with new values - for i in range(0, len(cache_values), 2): - window_key = keys_to_fetch[i] - counter_key = keys_to_fetch[i + 1] - window_value = cache_values[i] - counter_value = cache_values[i + 1] - await self.internal_usage_cache.async_set_cache( + async def _release_parallel_request_slots( + self, + acquisition: ParallelSlotAcquisition, + parent_otel_span: Span | None = None, + ) -> None: + """ + Release the max_parallel_requests slots acquired at pre-call by + removing this request's slot id from every gauge it was registered + under. Removing an absent slot id is a no-op, so a release without a + matching acquire or a double-fired release can never free another + request's slot. The in-memory fallback decrements integer mirror + values (floored at 0) because the mirror carries no per-slot ids. + """ + counter_keys = acquisition["counter_keys"] + slot_id = acquisition["slot_id"] + if not counter_keys or not slot_id: + return + if self.parallel_release_script is not None: + try: + raw = await self.parallel_release_script( + keys=counter_keys, + args=[slot_id for _ in counter_keys], + ) + for counter_key, remaining in zip(counter_keys, raw): + await self.internal_usage_cache.async_set_cache( + key=counter_key, + value=max(0, int(remaining)), + ttl=PARALLEL_REQUEST_SLOT_TTL_SECONDS, + litellm_parent_otel_span=parent_otel_span, + local_only=True, + ) + return + except Exception as e: # noqa: BLE001 - any Redis/Lua failure degrades to the in-memory release, never a 500 + verbose_proxy_logger.warning( + f"parallel_release_script failed, falling back to in-memory release: {str(e)}" + ) + + async with self._check_and_increment_lock: + for counter_key in counter_keys: + raw_value = await self.internal_usage_cache.async_get_cache( key=counter_key, - value=counter_value, - ttl=self.window_size, litellm_parent_otel_span=parent_otel_span, local_only=True, ) + if isinstance(raw_value, dict): + if slot_id not in raw_value: + continue + new_value: Union[dict[str, float], int] = { + key: ts for key, ts in raw_value.items() if key != slot_id + } + elif raw_value is None: + continue + else: + new_value = max(0, int(raw_value) - 1) await self.internal_usage_cache.async_set_cache( - key=window_key, - value=window_value, - ttl=self.window_size, + key=counter_key, + value=new_value, + ttl=PARALLEL_REQUEST_SLOT_TTL_SECONDS, litellm_parent_otel_span=parent_otel_span, local_only=True, ) - else: - # NORMAL MODE: In-memory sliding window (no Redis) - cache_values = await self.in_memory_cache_sliding_window( - keys=keys_to_fetch, - now_int=now_int, - window_size=self.window_size, - ) - - rate_limit_response = self.is_cache_list_over_limit(keys_to_fetch, cache_values, key_metadata) - return rate_limit_response async def atomic_check_and_increment_by_n( self, @@ -2027,10 +2412,18 @@ async def async_pre_call_hook( # shrinking the effective TPM budget by N and causing # false-positive 429s under bursts. When reservation is disabled, # this pass enforces TPM directly from the post-call counters. + parallel_counter_keys = [ + self.create_rate_limit_keys(d["key"], d["value"], "max_parallel_requests") + for d in descriptors + if (d.get("rate_limit") or {}).get("max_parallel_requests") is not None + ] + parallel_slot_id = uuid.uuid4().hex if parallel_counter_keys else None + response = await self.should_rate_limit( descriptors=descriptors, parent_otel_span=user_api_key_dict.parent_otel_span, skip_tpm_check=self.tpm_reservation_enabled, + parallel_slot_id=parallel_slot_id, ) if response["overall_code"] == "OVER_LIMIT": @@ -2049,6 +2442,15 @@ async def async_pre_call_hook( key=RATE_LIMIT_RESPONSE_KEY, value=response, ) + if parallel_slot_id is not None: + self._stash_value_in_metadata_channels( + data=data, + key=MAX_PARALLEL_SLOT_ACQUIRED_KEY, + value={ + "slot_id": parallel_slot_id, + "counter_keys": parallel_counter_keys, + }, + ) # ---------------------------------------------------------------- # TPM token reservation @@ -2108,6 +2510,13 @@ async def async_pre_call_hook( ) if tpm_response["overall_code"] == "OVER_LIMIT": + acquisition = self._get_parallel_slot_acquisition(kwargs=data) + if acquisition is not None: + await self._release_parallel_request_slots( + acquisition=acquisition, + parent_otel_span=user_api_key_dict.parent_otel_span, + ) + self._clear_parallel_slot_marker(data) self._handle_rate_limit_error( response=tpm_response, descriptors=descriptors, @@ -2480,6 +2889,50 @@ def _is_reservation_released( """True if a prior callback already refunded this request's reservation.""" return bool(cls._lookup_stashed_value(kwargs, standard_logging_metadata, TPM_RESERVATION_RELEASED_KEY)) + @classmethod + def _get_parallel_slot_acquisition( + cls, + kwargs: Any, + standard_logging_metadata: dict[str, Any] | None = None, + ) -> ParallelSlotAcquisition | None: + """The slot acquisition this request's pre-call hook made, if any.""" + candidate = cls._lookup_stashed_value(kwargs, standard_logging_metadata, MAX_PARALLEL_SLOT_ACQUIRED_KEY) + if not isinstance(candidate, dict): + return None + slot_id = candidate.get("slot_id") + counter_keys = candidate.get("counter_keys") + if not isinstance(slot_id, str) or not slot_id: + return None + if not isinstance(counter_keys, list) or not counter_keys: + return None + if not all(isinstance(key, str) and key for key in counter_keys): + return None + return ParallelSlotAcquisition(slot_id=slot_id, counter_keys=counter_keys) + + @staticmethod + def _clear_parallel_slot_marker(data: Any) -> None: + """ + Remove the acquired-slot marker from every metadata channel a sibling + callback might read, so one release per acquire is an invariant even + when multiple callbacks fire for the same request. + """ + if not isinstance(data, dict): + return + for channel in ("metadata", "litellm_metadata"): + channel_dict = data.get(channel) + if isinstance(channel_dict, dict): + channel_dict.pop(MAX_PARALLEL_SLOT_ACQUIRED_KEY, None) + litellm_params = data.get("litellm_params") + if isinstance(litellm_params, dict): + lp_metadata = litellm_params.get("metadata") + if isinstance(lp_metadata, dict): + lp_metadata.pop(MAX_PARALLEL_SLOT_ACQUIRED_KEY, None) + slo = data.get("standard_logging_object") + if isinstance(slo, dict): + slo_meta = slo.get("metadata") + if isinstance(slo_meta, dict): + slo_meta.pop(MAX_PARALLEL_SLOT_ACQUIRED_KEY, None) + @staticmethod def _mark_reservation_released(data: Any) -> None: """ @@ -2621,7 +3074,6 @@ def _build_success_event_pipeline_operations( standard_logging_object = kwargs.get("standard_logging_object") or {} standard_logging_metadata = standard_logging_object.get("metadata") or {} - user_api_key = standard_logging_metadata.get("user_api_key_hash") model_group = get_model_group_from_litellm_kwargs(kwargs) # Get total tokens from response @@ -2658,20 +3110,6 @@ def _build_success_event_pipeline_operations( pipeline_operations: List[RedisPipelineIncrementOperation] = [] - # max_parallel_requests is its own counter (api-key only) — always decrement. - if user_api_key: - pipeline_operations.append( - RedisPipelineIncrementOperation( - key=self.create_rate_limit_keys( - key="api_key", - value=user_api_key, - rate_limit_type="max_parallel_requests", - ), - increment_value=-1, - ttl=self.window_size, - ) - ) - # ---------------------------------------------------------------- # TPM reconciliation # Per-scope behavior: @@ -2719,6 +3157,19 @@ async def async_log_success_event(self, kwargs, response_obj, start_time, end_ti try: verbose_proxy_logger.debug("INSIDE parallel request limiter ASYNC SUCCESS LOGGING") + standard_logging_object = kwargs.get("standard_logging_object") or {} + standard_logging_metadata = standard_logging_object.get("metadata") or {} + acquisition = self._get_parallel_slot_acquisition( + kwargs=kwargs, + standard_logging_metadata=standard_logging_metadata, + ) + if acquisition is not None: + await self._release_parallel_request_slots( + acquisition=acquisition, + parent_otel_span=litellm_parent_otel_span, + ) + self._clear_parallel_slot_marker(kwargs) + pipeline_operations = self._build_success_event_pipeline_operations( kwargs=kwargs, response_obj=response_obj, @@ -2855,22 +3306,19 @@ async def async_log_failure_event(self, kwargs, response_obj, start_time, end_ti litellm_parent_otel_span: Union[Span, None] = _get_parent_otel_span_from_kwargs(kwargs) standard_logging_object = kwargs.get("standard_logging_object") or {} standard_logging_metadata = standard_logging_object.get("metadata") or {} - user_api_key = standard_logging_metadata.get("user_api_key_hash") pipeline_operations: List[RedisPipelineIncrementOperation] = [] - if user_api_key: - pipeline_operations.append( - RedisPipelineIncrementOperation( - key=self.create_rate_limit_keys( - key="api_key", - value=user_api_key, - rate_limit_type="max_parallel_requests", - ), - increment_value=-1, - ttl=self.window_size, - ) + acquisition = self._get_parallel_slot_acquisition( + kwargs=kwargs, + standard_logging_metadata=standard_logging_metadata, + ) + if acquisition is not None: + await self._release_parallel_request_slots( + acquisition=acquisition, + parent_otel_span=litellm_parent_otel_span, ) + self._clear_parallel_slot_marker(kwargs) # Skip the reservation refund if async_post_call_failure_hook # already released it (proxy-level rejection that also bubbles up @@ -2920,40 +3368,35 @@ async def async_log_failure_event(self, kwargs, response_obj, start_time, end_ti except Exception as e: verbose_proxy_logger.exception(f"Error in rate limit failure event: {str(e)}") - async def async_release_max_parallel_requests_on_disconnect(self, user_api_key_dict: UserAPIKeyAuth) -> None: + async def async_release_max_parallel_requests_on_disconnect( + self, + user_api_key_dict: UserAPIKeyAuth, + request_data: dict | None = None, + ) -> None: """ Release the api-key ``max_parallel_requests`` slot that - ``async_pre_call_hook`` reserved, for a request that ended without + ``async_pre_call_hook`` acquired, for a request that ended without either logging callback firing. - The +1 is normally undone by ``async_log_success_event`` (natural + The slot is normally released by ``async_log_success_event`` (natural stream completion) or ``async_log_failure_event`` (LLM error). When a client cancels a stream mid-flight, the cancellation surfaces as ``asyncio.CancelledError`` / ``GeneratorExit`` and neither callback - runs, so without this the counter leaks one slot per cancelled stream - until the key wedges at its limit. - """ - if not user_api_key_dict.api_key or user_api_key_dict.max_parallel_requests is None: + runs, so without this the slot leaks per cancelled stream until its + TTL prunes it. ``request_data`` carries the stashed acquisition; + its presence (not the key object's current max_parallel_requests + configuration, which can change mid-request) decides whether there + is anything to release. + """ + acquisition = self._get_parallel_slot_acquisition(kwargs=request_data) + if acquisition is None: return - await self.internal_usage_cache.dual_cache.async_increment_cache_pipeline( - increment_list=[ - RedisPipelineIncrementOperation( - key=self.create_rate_limit_keys( - key="api_key", - value=user_api_key_dict.api_key, - rate_limit_type="max_parallel_requests", - ), - increment_value=-1, - # Refresh the window TTL on the decrement, matching the - # failure path. max_parallel_requests is a concurrency - # gauge, not a rolling-window count, so the key must - # outlive in-flight requests rather than expire mid-stream. - ttl=self.window_size, - ) - ], - litellm_parent_otel_span=None, + await self._release_parallel_request_slots( + acquisition=acquisition, + parent_otel_span=None, ) + self._clear_parallel_slot_marker(request_data) async def async_post_call_success_hook(self, data: dict, user_api_key_dict: UserAPIKeyAuth, response): """ @@ -3002,17 +3445,29 @@ async def async_post_call_failure_hook( traceback_str: Optional[str] = None, ) -> None: """ - Release any TPM reservation when the request is rejected after the - pre-call hook reserved tokens but before the LLM call ran (e.g. a - downstream guardrail/auth hook raised). Without this, those - reservations are stranded — async_log_failure_event is a litellm - completion-level callback and never fires for proxy-side rejections. - - Idempotent via TPM_RESERVATION_RELEASED_KEY: if both this hook and + Release the parallel-request slot and any TPM reservation when the + request is rejected after the pre-call hook acquired them but before + the LLM call ran (e.g. a downstream guardrail/auth hook raised). + Without this, those resources are stranded — async_log_failure_event + is a litellm completion-level callback and never fires for proxy-side + rejections, so a leaked slot would occupy the gauge for the full + PARALLEL_REQUEST_SLOT_TTL_SECONDS. + + Idempotent: the slot release clears the acquisition marker (and slot + removal is a no-op ZREM on a second run), and the TPM refund is + guarded by TPM_RESERVATION_RELEASED_KEY — if both this hook and async_log_failure_event end up running in the same flow, only the - first refund applies. + first release/refund applies. """ try: + acquisition = self._get_parallel_slot_acquisition(kwargs=request_data) + if acquisition is not None: + await self._release_parallel_request_slots( + acquisition=acquisition, + parent_otel_span=user_api_key_dict.parent_otel_span, + ) + self._clear_parallel_slot_marker(request_data) + if self._is_reservation_released(kwargs=request_data): return reserved_tokens = self._get_reserved_tokens_from_kwargs(kwargs=request_data) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 7e6ca5a01088..b557f405f8d3 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -7372,7 +7372,7 @@ async def async_data_generator( # disconnect, so it fires reliably regardless of needs_iterator_wrap # (a nested iterator hook would only see GeneratorExit on GC). if not stream_completed: - proxy_logging_obj._release_max_parallel_requests_on_disconnect(user_api_key_dict) + proxy_logging_obj._release_max_parallel_requests_on_disconnect(user_api_key_dict, request_data) client_disconnected = True raise except Exception as e: diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 9f36e7293303..ac67ac611388 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -2583,7 +2583,11 @@ def _fire_deferred_stream_logging(request_data: dict) -> None: logging_obj._deferred_stream_complete_args = None asyncio.create_task(_deferred_cb(*_args)) - def _release_max_parallel_requests_on_disconnect(self, user_api_key_dict: UserAPIKeyAuth) -> None: + def _release_max_parallel_requests_on_disconnect( + self, + user_api_key_dict: UserAPIKeyAuth, + request_data: dict | None = None, + ) -> None: """ Release the api-key max_parallel_requests slot when a streaming response is cancelled mid-flight (client disconnect). Neither the @@ -2603,14 +2607,16 @@ def _release_max_parallel_requests_on_disconnect(self, user_api_key_dict: UserAP if not isinstance(limiter, _PROXY_MaxParallelRequestsHandler_v3): return try: - asyncio.create_task(limiter.async_release_max_parallel_requests_on_disconnect(user_api_key_dict)) + asyncio.create_task( + limiter.async_release_max_parallel_requests_on_disconnect(user_api_key_dict, request_data) + ) except RuntimeError: # No running event loop (e.g. interpreter/loop shutdown); the # counter's window TTL will reclaim the slot. verbose_proxy_logger.warning( "parallel_request_limiter_v3: could not schedule " "max_parallel_requests release on disconnect; no running " - "event loop. Slot will be reclaimed when its window TTL expires" + "event loop. Slot will be reclaimed when its TTL expires" ) def _init_response_taking_too_long_task(self, data: Optional[dict] = None): diff --git a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py index e7d2909263af..c76e1a60afd0 100644 --- a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py @@ -17,6 +17,10 @@ from litellm import Router from litellm.caching.caching import DualCache from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.hooks.parallel_request_limiter_v3 import ( + MAX_PARALLEL_SLOT_ACQUIRED_KEY, + PARALLEL_REQUEST_SLOT_TTL_SECONDS, +) from litellm.proxy.hooks.parallel_request_limiter_v3 import ( _PROXY_MaxParallelRequestsHandler_v3 as _PROXY_MaxParallelRequestsHandler, ) @@ -566,10 +570,9 @@ async def mock_increment_pipeline(increment_list, **kwargs): # Verify that the correct token count was used based on the rate limit type assert ( - len(captured_operations) == 2 - ), "Should have 2 operations: max_parallel_requests decrement and TPM increment" + len(captured_operations) == 1 + ), "Should have 1 operation: the TPM increment (parallel slots are released via the gauge, not the pipeline)" - # Find the TPM increment operation (not the max_parallel_requests decrement) tpm_operation = None for op in captured_operations: if op["key"].endswith(":tokens"): @@ -655,7 +658,10 @@ async def mock_increment_pipeline(increment_list, **kwargs): @pytest.mark.asyncio async def test_async_log_failure_event_v3(): """ - Simple test for async_log_failure_event - should decrement max_parallel_requests by 1 + async_log_failure_event releases exactly this request's slot id: the + first release removes it, and repeated or unknown-slot releases are + no-ops that can never free another request's slot (releasing more than + was acquired is what previously let concurrency exceed the limit). """ _api_key = "sk-12345" _api_key = hash_token(_api_key) @@ -663,33 +669,246 @@ async def test_async_log_failure_event_v3(): parallel_request_handler = _PROXY_MaxParallelRequestsHandler( internal_usage_cache=InternalUsageCache(local_cache) ) + counter_key = f"{{api_key:{_api_key}}}:max_parallel_requests" - # Mock kwargs with user_api_key via standard_logging_object - mock_kwargs = { - "standard_logging_object": {"metadata": {"user_api_key_hash": _api_key}} - } + await _seed_max_parallel_requests_slots(local_cache, counter_key, ["slot-a", "slot-b"]) - # Capture pipeline operations - captured_ops = [] + def kwargs_with_slot(slot_id): + return { + "metadata": { + MAX_PARALLEL_SLOT_ACQUIRED_KEY: { + "slot_id": slot_id, + "counter_keys": [counter_key], + } + }, + "standard_logging_object": {"metadata": {"user_api_key_hash": _api_key}}, + } - async def mock_pipeline(increment_list, **kwargs): - captured_ops.extend(increment_list) + async def in_flight(): + return parallel_request_handler._gauge_in_flight_from_cache_value( + await local_cache.async_get_cache(key=counter_key) + ) - parallel_request_handler.internal_usage_cache.dual_cache.async_increment_cache_pipeline = ( - mock_pipeline + await parallel_request_handler.async_log_failure_event( + kwargs=kwargs_with_slot("slot-a"), response_obj=None, start_time=None, end_time=None ) + assert await in_flight() == 1 + + for slot_id in ("slot-a", "slot-unknown", "slot-a"): + await parallel_request_handler.async_log_failure_event( + kwargs=kwargs_with_slot(slot_id), response_obj=None, start_time=None, end_time=None + ) + assert await in_flight() == 1 - # Call async_log_failure_event await parallel_request_handler.async_log_failure_event( - kwargs=mock_kwargs, response_obj=None, start_time=None, end_time=None + kwargs=kwargs_with_slot("slot-b"), response_obj=None, start_time=None, end_time=None + ) + assert await in_flight() == 0 + + +@pytest.mark.asyncio +async def test_failure_event_without_acquired_slot_does_not_release_v3(): + """ + Failure callbacks also fire for requests rejected at pre-call, which never + acquired a parallel slot. Releasing on those frees a slot still owned by + another in-flight request, so every 429 would raise effective concurrency + above the configured limit. Without the acquired-slot marker the gauge + must stay untouched. + """ + _api_key = hash_token("sk-12345") + local_cache = DualCache() + handler = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(local_cache) + ) + counter_key = f"{{api_key:{_api_key}}}:max_parallel_requests" + + await _seed_max_parallel_requests_slots( + local_cache, counter_key, ["slot-a", "slot-b", "slot-c"] + ) + + await handler.async_log_failure_event( + kwargs={ + "standard_logging_object": {"metadata": {"user_api_key_hash": _api_key}} + }, + response_obj=None, + start_time=None, + end_time=None, + ) + assert ( + handler._gauge_in_flight_from_cache_value( + await local_cache.async_get_cache(key=counter_key) + ) + == 3 + ) + + +@pytest.mark.asyncio +async def test_max_parallel_requests_not_reset_by_window_roll_v3(): + """ + max_parallel_requests is a concurrency gauge, not a windowed counter: the + rate-limit window rolling over must not reset it while requests are still + in flight. Previously the gauge shared the sliding-window reset with + RPM/TPM, so every window roll forgot all in-flight requests and admitted + a fresh batch of `limit` on top of what was still running. + """ + controller = TimeController() + local_cache = DualCache() + handler = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(local_cache), + time_provider=controller.now, + ) + _api_key = hash_token("sk-12345") + user_api_key_dict = UserAPIKeyAuth(api_key=_api_key, max_parallel_requests=2) + + for _ in range(2): + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=local_cache, + data={"model": "gpt-3.5-turbo"}, + call_type="", + ) + + controller.advance(handler.window_size + 1) + + with pytest.raises(HTTPException) as exc_info: + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=local_cache, + data={"model": "gpt-3.5-turbo"}, + call_type="", + ) + assert exc_info.value.status_code == 429 + assert "max_parallel_requests" in exc_info.value.detail + + +@pytest.mark.asyncio +async def test_rejected_request_does_not_consume_parallel_slot_v3(): + """ + A 429-rejected request must not occupy a parallel-request slot: nothing + ever releases a slot for a request that was never admitted, so the old + increment-then-check behavior wedged the gauge above the limit and + rejected requests that should have been admitted after a release. + """ + local_cache = DualCache() + handler = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(local_cache) + ) + _api_key = hash_token("sk-12345") + user_api_key_dict = UserAPIKeyAuth(api_key=_api_key, max_parallel_requests=1) + + admitted_data: Dict[str, Any] = {"model": "gpt-3.5-turbo"} + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=local_cache, + data=admitted_data, + call_type="", + ) + acquisition = admitted_data["metadata"][MAX_PARALLEL_SLOT_ACQUIRED_KEY] + assert isinstance(acquisition, dict) + assert isinstance(acquisition["slot_id"], str) and acquisition["slot_id"] + assert acquisition["counter_keys"] == [f"{{api_key:{_api_key}}}:max_parallel_requests"] + + for _ in range(3): + with pytest.raises(HTTPException): + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=local_cache, + data={"model": "gpt-3.5-turbo"}, + call_type="", + ) + + await handler.async_log_failure_event( + kwargs={ + "metadata": {MAX_PARALLEL_SLOT_ACQUIRED_KEY: acquisition}, + "standard_logging_object": {"metadata": {"user_api_key_hash": _api_key}}, + }, + response_obj=None, + start_time=None, + end_time=None, + ) + + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=local_cache, + data={"model": "gpt-3.5-turbo"}, + call_type="", + ) + + +@pytest.mark.asyncio +async def test_parallel_gauge_uses_atomic_redis_script_v3(): + """ + With Redis available, gauge admission goes through the atomic + check-and-acquire script (limit, slot TTL, and this request's slot id as + args), the returned in-flight count is mirrored into the local cache, + and an over-limit script result maps to a 429 without occupying a slot. + """ + local_cache = DualCache() + handler = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(local_cache) ) + _api_key = hash_token("sk-12345") + user_api_key_dict = UserAPIKeyAuth(api_key=_api_key, max_parallel_requests=5) + counter_key = f"{{api_key:{_api_key}}}:max_parallel_requests" + + captured_calls = [] - # Verify correct operation was created - assert len(captured_ops) == 1 - op = captured_ops[0] - assert op["key"] == f"{{api_key:{_api_key}}}:max_parallel_requests" - assert op["increment_value"] == -1 - assert op["ttl"] == 60 # default window size + async def fake_acquire(keys, args): + captured_calls.append((list(keys), list(args))) + return [0, 3] + + handler.parallel_acquire_script = fake_acquire + + data: Dict[str, Any] = {"model": "gpt-3.5-turbo"} + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=local_cache, + data=data, + call_type="", + ) + stashed_acquisition = data["metadata"][MAX_PARALLEL_SLOT_ACQUIRED_KEY] + assert isinstance(stashed_acquisition, dict) + stashed_slot_id = stashed_acquisition["slot_id"] + assert isinstance(stashed_slot_id, str) and stashed_slot_id + assert stashed_acquisition["counter_keys"] == [counter_key] + assert captured_calls == [ + ([counter_key], [5, PARALLEL_REQUEST_SLOT_TTL_SECONDS, stashed_slot_id]) + ] + assert ( + await handler.internal_usage_cache.async_get_cache( + key=counter_key, litellm_parent_otel_span=None, local_only=True + ) + == 3 + ) + gauge_statuses = [ + s + for s in data["litellm_proxy_rate_limit_response"]["statuses"] + if s["rate_limit_type"] == "max_parallel_requests" + ] + assert gauge_statuses == [ + { + "code": "OK", + "current_limit": 5, + "limit_remaining": 2, + "rate_limit_type": "max_parallel_requests", + "descriptor_key": "api_key", + } + ] + + async def fake_acquire_over_limit(keys, args): + return [1, 1, 5, 5] + + handler.parallel_acquire_script = fake_acquire_over_limit + + with pytest.raises(HTTPException) as exc_info: + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=local_cache, + data={"model": "gpt-3.5-turbo"}, + call_type="", + ) + assert exc_info.value.status_code == 429 + assert "max_parallel_requests" in exc_info.value.detail @pytest.mark.asyncio @@ -3227,27 +3446,28 @@ def test_get_key_mcp_rpm_limit_precedence(): assert get_team_mcp_rpm_limit(none_set) is None -async def _seed_max_parallel_requests_counter( - dual_cache: DualCache, counter_key: str, window_size: int +_TEST_SLOT_ID = "slot-disconnect-test" + + +async def _seed_max_parallel_requests_slots( + dual_cache: DualCache, counter_key: str, slot_ids: List[str] ) -> None: - await dual_cache.async_increment_cache_pipeline( - increment_list=[ - RedisPipelineIncrementOperation( - key=counter_key, increment_value=1, ttl=window_size - ) - ] + await dual_cache.async_set_cache( + key=counter_key, + value={slot_id: time.time() for slot_id in slot_ids}, + local_only=True, ) async def _build_seeded_limiter(): - """Build a v3 limiter whose api-key counter already holds the pre-call +1.""" + """Build a v3 limiter whose api-key slot registry already holds the pre-call slot.""" api_key = hash_token("sk-disconnect") cache = DualCache() limiter = _PROXY_MaxParallelRequestsHandler( internal_usage_cache=InternalUsageCache(cache) ) counter_key = f"{{api_key:{api_key}}}:max_parallel_requests" - await _seed_max_parallel_requests_counter(cache, counter_key, limiter.window_size) + await _seed_max_parallel_requests_slots(cache, counter_key, [_TEST_SLOT_ID]) user_api_key_dict = UserAPIKeyAuth(api_key=api_key, max_parallel_requests=2) return limiter, cache, counter_key, user_api_key_dict @@ -3286,14 +3506,370 @@ async def test_release_max_parallel_requests_on_disconnect_v3(): user_api_key_dict = UserAPIKeyAuth(api_key=_api_key, max_parallel_requests=2) counter_key = f"{{api_key:{_api_key}}}:max_parallel_requests" - await _seed_max_parallel_requests_counter( - local_cache, counter_key, handler.window_size + await _seed_max_parallel_requests_slots(local_cache, counter_key, [_TEST_SLOT_ID]) + assert handler._gauge_in_flight_from_cache_value( + await local_cache.async_get_cache(key=counter_key) + ) == 1 + + await handler.async_release_max_parallel_requests_on_disconnect( + user_api_key_dict, + request_data={ + "metadata": { + MAX_PARALLEL_SLOT_ACQUIRED_KEY: { + "slot_id": _TEST_SLOT_ID, + "counter_keys": [counter_key], + } + } + }, ) - assert await local_cache.async_get_cache(key=counter_key) == 1 - await handler.async_release_max_parallel_requests_on_disconnect(user_api_key_dict) + assert handler._gauge_in_flight_from_cache_value( + await local_cache.async_get_cache(key=counter_key) + ) == 0 - assert await local_cache.async_get_cache(key=counter_key) == 0 + +@pytest.mark.asyncio +async def test_release_on_disconnect_works_when_key_config_changed_v3(): + """ + The disconnect release must be driven by the stashed acquisition, not the + key object's current max_parallel_requests configuration: if the limit is + cleared on the key while a request is in flight, the acquired slot still + has to be released or it lingers until TTL pruning. + """ + _api_key = hash_token("sk-12345") + local_cache = DualCache() + handler = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(local_cache) + ) + counter_key = f"{{api_key:{_api_key}}}:max_parallel_requests" + await _seed_max_parallel_requests_slots(local_cache, counter_key, [_TEST_SLOT_ID]) + + await handler.async_release_max_parallel_requests_on_disconnect( + UserAPIKeyAuth(api_key=_api_key, max_parallel_requests=None), + request_data={ + "metadata": { + MAX_PARALLEL_SLOT_ACQUIRED_KEY: { + "slot_id": _TEST_SLOT_ID, + "counter_keys": [counter_key], + } + } + }, + ) + assert handler._gauge_in_flight_from_cache_value( + await local_cache.async_get_cache(key=counter_key) + ) == 0 + + +@pytest.mark.asyncio +async def test_post_call_failure_hook_releases_parallel_slot_v3(): + """ + A proxy-level rejection raised by a downstream hook after the rate + limiter's pre-call hook acquired a slot (guardrail, budget check) must + release that slot via async_post_call_failure_hook: + async_log_failure_event never fires for proxy-side rejections, so + without this the slot lingers for the full slot TTL and moderate + rejection rates wedge the key at its limit. The release must also be + idempotent with a later failure callback in the same flow. + """ + _api_key = hash_token("sk-12345") + local_cache = DualCache() + handler = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(local_cache) + ) + user_api_key_dict = UserAPIKeyAuth(api_key=_api_key, max_parallel_requests=1) + counter_key = f"{{api_key:{_api_key}}}:max_parallel_requests" + + admitted_data: Dict[str, Any] = {"model": "gpt-3.5-turbo"} + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=local_cache, + data=admitted_data, + call_type="", + ) + assert handler._gauge_in_flight_from_cache_value( + await local_cache.async_get_cache(key=counter_key) + ) == 1 + + await handler.async_post_call_failure_hook( + request_data=admitted_data, + original_exception=Exception("guardrail rejected the request"), + user_api_key_dict=user_api_key_dict, + ) + assert handler._gauge_in_flight_from_cache_value( + await local_cache.async_get_cache(key=counter_key) + ) == 0 + + await handler.async_log_failure_event( + kwargs={ + "metadata": admitted_data["metadata"], + "standard_logging_object": {"metadata": {"user_api_key_hash": _api_key}}, + }, + response_obj=None, + start_time=None, + end_time=None, + ) + assert handler._gauge_in_flight_from_cache_value( + await local_cache.async_get_cache(key=counter_key) + ) == 0 + + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=local_cache, + data={"model": "gpt-3.5-turbo"}, + call_type="", + ) + + +@pytest.mark.asyncio +async def test_success_event_releases_parallel_slot_v3(monkeypatch): + """ + A successful completion must release exactly the slot its pre-call + acquired, freeing capacity for the next request; without it every + completed request would keep occupying the gauge until TTL pruning. + """ + _api_key = hash_token("sk-12345") + local_cache = DualCache() + handler = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(local_cache) + ) + monkeypatch.setattr(handler, "get_rate_limit_type", lambda: "total") + user_api_key_dict = UserAPIKeyAuth(api_key=_api_key, max_parallel_requests=1) + counter_key = f"{{api_key:{_api_key}}}:max_parallel_requests" + + admitted_data: Dict[str, Any] = {"model": "gpt-3.5-turbo"} + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=local_cache, + data=admitted_data, + call_type="", + ) + assert handler._gauge_in_flight_from_cache_value( + await local_cache.async_get_cache(key=counter_key) + ) == 1 + + await handler.async_log_success_event( + kwargs={ + "metadata": admitted_data["metadata"], + "standard_logging_object": {"metadata": {"user_api_key_hash": _api_key}}, + }, + response_obj=ModelResponse( + usage=Usage(prompt_tokens=5, completion_tokens=5, total_tokens=10) + ), + start_time=datetime.now(), + end_time=datetime.now(), + ) + assert handler._gauge_in_flight_from_cache_value( + await local_cache.async_get_cache(key=counter_key) + ) == 0 + + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=local_cache, + data={"model": "gpt-3.5-turbo"}, + call_type="", + ) + + +@pytest.mark.asyncio +async def test_read_only_gauge_check_counts_without_acquiring_v3(): + """ + read_only callers (e.g. the context-compaction pre-check) must observe + the in-flight count via the count script without registering a slot, and + a count-script failure must degrade to the local mirror instead of + raising. + """ + _api_key = hash_token("sk-12345") + local_cache = DualCache() + handler = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(local_cache) + ) + counter_key = f"{{api_key:{_api_key}}}:max_parallel_requests" + descriptors = [ + { + "key": "api_key", + "value": _api_key, + "rate_limit": {"max_parallel_requests": 5}, + } + ] + + captured_calls = [] + + async def fake_count(keys, args): + captured_calls.append((list(keys), list(args))) + return [3] + + handler.parallel_count_script = fake_count + + response = await handler.should_rate_limit(descriptors=descriptors, read_only=True) + assert captured_calls == [ + ([counter_key], [PARALLEL_REQUEST_SLOT_TTL_SECONDS]) + ] + assert response["overall_code"] == "OK" + assert response["statuses"] == [ + { + "code": "OK", + "current_limit": 5, + "limit_remaining": 2, + "rate_limit_type": "max_parallel_requests", + "descriptor_key": "api_key", + } + ] + assert await local_cache.async_get_cache(key=counter_key) is None + + async def failing_count(keys, args): + raise ConnectionError("redis unavailable") + + handler.parallel_count_script = failing_count + await _seed_max_parallel_requests_slots( + local_cache, counter_key, ["s1", "s2", "s3", "s4", "s5"] + ) + response = await handler.should_rate_limit(descriptors=descriptors, read_only=True) + assert response["overall_code"] == "OVER_LIMIT" + assert response["statuses"][0]["rate_limit_type"] == "max_parallel_requests" + + +@pytest.mark.asyncio +async def test_redis_release_script_updates_local_mirror_v3(): + """ + With Redis available, releases go through the release script with this + request's slot id per gauge key, and the returned in-flight counts are + mirrored into the local cache so the local first-pass check stays fresh. + """ + _api_key = hash_token("sk-12345") + local_cache = DualCache() + handler = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(local_cache) + ) + counter_key = f"{{api_key:{_api_key}}}:max_parallel_requests" + + captured_calls = [] + + async def fake_release(keys, args): + captured_calls.append((list(keys), list(args))) + return [2] + + handler.parallel_release_script = fake_release + + await handler.async_log_failure_event( + kwargs={ + "metadata": { + MAX_PARALLEL_SLOT_ACQUIRED_KEY: { + "slot_id": "slot-redis-test", + "counter_keys": [counter_key], + } + }, + "standard_logging_object": {"metadata": {"user_api_key_hash": _api_key}}, + }, + response_obj=None, + start_time=None, + end_time=None, + ) + assert captured_calls == [([counter_key], ["slot-redis-test"])] + assert await local_cache.async_get_cache(key=counter_key) == 2 + + +@pytest.mark.asyncio +async def test_tpm_over_limit_rejection_releases_parallel_slot_v3(monkeypatch): + """ + When the TPM reservation phase rejects a request AFTER the gauge slot was + acquired earlier in the same pre-call hook, the slot must be released + before the 429 is raised; otherwise every TPM rejection would leak a + slot until TTL pruning. + """ + monkeypatch.delenv("LITELLM_TPM_TOKEN_RESERVATION_ENABLED", raising=False) + _api_key = hash_token("sk-12345") + local_cache = DualCache() + handler = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(local_cache) + ) + user_api_key_dict = UserAPIKeyAuth( + api_key=_api_key, max_parallel_requests=5, tpm_limit=100 + ) + counter_key = f"{{api_key:{_api_key}}}:max_parallel_requests" + + async def over_limit_reservation(descriptors, estimated_tokens, parent_otel_span=None): + return { + "overall_code": "OVER_LIMIT", + "statuses": [ + { + "code": "OVER_LIMIT", + "current_limit": 100, + "limit_remaining": 0, + "rate_limit_type": "tokens", + "descriptor_key": "api_key", + } + ], + } + + monkeypatch.setattr(handler, "reserve_tpm_tokens", over_limit_reservation) + + with pytest.raises(HTTPException) as exc_info: + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=local_cache, + data={"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "hi"}]}, + call_type="", + ) + assert exc_info.value.status_code == 429 + assert handler._gauge_in_flight_from_cache_value( + await local_cache.async_get_cache(key=counter_key) + ) == 0 + + +@pytest.mark.asyncio +async def test_in_memory_fallback_respects_mirrored_redis_count_v3(): + """ + When Redis scripting fails after having worked, the local cache holds the + integer in-flight count mirrored from the last successful script call. + The in-memory fallback must treat that count as real occupancy (and + release must decrement it, floored at 0), not start over from an empty + registry, which would double the admitted concurrency during a Redis + outage. + """ + _api_key = hash_token("sk-12345") + local_cache = DualCache() + handler = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(local_cache) + ) + user_api_key_dict = UserAPIKeyAuth(api_key=_api_key, max_parallel_requests=5) + counter_key = f"{{api_key:{_api_key}}}:max_parallel_requests" + + async def failing_script(keys, args): + raise ConnectionError("redis unavailable") + + handler.parallel_acquire_script = failing_script + handler.parallel_release_script = failing_script + + await local_cache.async_set_cache(key=counter_key, value=5, local_only=True) + with pytest.raises(HTTPException) as exc_info: + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=local_cache, + data={"model": "gpt-3.5-turbo"}, + call_type="", + ) + assert exc_info.value.status_code == 429 + + await local_cache.async_set_cache(key=counter_key, value=4, local_only=True) + admitted_data: Dict[str, Any] = {"model": "gpt-3.5-turbo"} + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=local_cache, + data=admitted_data, + call_type="", + ) + assert await local_cache.async_get_cache(key=counter_key) == 5 + + await handler.async_log_failure_event( + kwargs={ + "metadata": admitted_data["metadata"], + "standard_logging_object": {"metadata": {"user_api_key_hash": _api_key}}, + }, + response_obj=None, + start_time=None, + end_time=None, + ) + assert await local_cache.async_get_cache(key=counter_key) == 4 @pytest.mark.asyncio @@ -3338,7 +3914,9 @@ async def test_async_streaming_data_generator_releases_counter_on_disconnect_v3( from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing limiter, cache, counter_key, user_api_key_dict = await _build_seeded_limiter() - assert await cache.async_get_cache(key=counter_key) == 1 + assert limiter._gauge_in_flight_from_cache_value( + await cache.async_get_cache(key=counter_key) + ) == 1 proxy_logging_obj = ProxyLogging(user_api_key_cache=DualCache()) proxy_logging_obj.proxy_hook_mapping["parallel_request_limiter"] = limiter @@ -3354,7 +3932,15 @@ async def upstream(): gen = ProxyBaseLLMRequestProcessing.async_sse_data_generator( response=upstream(), user_api_key_dict=user_api_key_dict, - request_data={"model": "claude-test"}, + request_data={ + "model": "claude-test", + "metadata": { + MAX_PARALLEL_SLOT_ACQUIRED_KEY: { + "slot_id": _TEST_SLOT_ID, + "counter_keys": [counter_key], + } + }, + }, proxy_logging_obj=proxy_logging_obj, ) await gen.__anext__() @@ -3365,7 +3951,9 @@ async def upstream(): await gen.aclose() await _drain_release_task() - assert await cache.async_get_cache(key=counter_key) == 0 + assert limiter._gauge_in_flight_from_cache_value( + await cache.async_get_cache(key=counter_key) + ) == 0 @pytest.mark.parametrize("disconnect", ["cancel", "aclose"]) @@ -3399,7 +3987,15 @@ async def upstream(): gen = proxy_server.async_data_generator( response=upstream(), user_api_key_dict=user_api_key_dict, - request_data={"model": "gpt-test"}, + request_data={ + "model": "gpt-test", + "metadata": { + MAX_PARALLEL_SLOT_ACQUIRED_KEY: { + "slot_id": _TEST_SLOT_ID, + "counter_keys": [counter_key], + } + }, + }, ) await gen.__anext__() if disconnect == "cancel": @@ -3408,7 +4004,9 @@ async def upstream(): else: await gen.aclose() await _drain_release_task() - assert await cache.async_get_cache(key=counter_key) == 0 + assert limiter._gauge_in_flight_from_cache_value( + await cache.async_get_cache(key=counter_key) + ) == 0 finally: if saved_hook is not None: proxy_logging_obj.proxy_hook_mapping["parallel_request_limiter"] = ( @@ -3452,12 +4050,22 @@ async def upstream(): gen = proxy_server.async_data_generator( response=upstream(), user_api_key_dict=user_api_key_dict, - request_data={"model": "gpt-test"}, + request_data={ + "model": "gpt-test", + "metadata": { + MAX_PARALLEL_SLOT_ACQUIRED_KEY: { + "slot_id": _TEST_SLOT_ID, + "counter_keys": [counter_key], + } + }, + }, ) await gen.__anext__() await gen.aclose() await _drain_release_task() - assert await cache.async_get_cache(key=counter_key) == 0 + assert limiter._gauge_in_flight_from_cache_value( + await cache.async_get_cache(key=counter_key) + ) == 0 finally: if saved_hook is not None: proxy_logging_obj.proxy_hook_mapping["parallel_request_limiter"] = (