Skip to content
Merged
Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
7afd657
fix(batches): skip unnecessary batch input file reads
Sameerlite May 28, 2026
b391f77
fix(batch-rate-limiter): prevent user metadata flag from bypassing mo…
cursoragent May 28, 2026
4f9cedd
fix(batch_rate_limiter): enforce model access check before honoring s…
cursoragent May 28, 2026
f41ec3e
fix(batch_rate_limiter): wildcard model bypass + fail-open embedded m…
cursoragent May 28, 2026
c976d84
perf(batch_rate_limiter): reuse rate-limit descriptors across skip ch…
mateo-berri May 30, 2026
4cda2c5
Merge remote-tracking branch 'origin/litellm_internal_staging' into l…
mateo-berri May 30, 2026
82c1a36
test(batch_rate_limiter): cover skip-path and file-fetch helpers
mateo-berri May 30, 2026
523a984
fix(batch_rate_limiter): resolve provider skip from trusted deploymen…
mateo-berri May 30, 2026
a115df8
fix(batch_rate_limiter): guard model-embedded credential lookup on ll…
mateo-berri May 30, 2026
1e71bdd
test(batch_rate_limiter): drive real no-skip fetch path and pin wildc…
mateo-berri May 30, 2026
9d5ea6f
fix(batch_rate_limiter): drop client-controlled skip flag to close qu…
mateo-berri May 30, 2026
00cd3ed
fix(batch_rate_limiter): match per-model skip on file-bound model only
mateo-berri May 30, 2026
7a7e24b
fix(batch_rate_limiter): drop forgeable per-model skip to close quota…
mateo-berri May 30, 2026
f0e7c45
fix(batch_rate_limiter): warn when no-op per-model skip key is config…
mateo-berri May 30, 2026
56d7069
test(batch_rate_limiter): patch llm_router so model-embedded credenti…
mateo-berri May 30, 2026
c44fafa
fix(batch_rate_limiter): resolve provider skip from file-bound model
mateo-berri Jun 1, 2026
092bd71
Merge remote-tracking branch 'origin/litellm_internal_staging' into l…
mateo-berri Jun 1, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
288 changes: 276 additions & 12 deletions litellm/proxy/hooks/batch_rate_limiter.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,20 +17,21 @@
- async_log_success_event() fires on GET /v1/batches/{id} (batch completion)
"""

from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union
from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Tuple, Union

from fastapi import HTTPException
from pydantic import BaseModel

import litellm
from litellm._logging import verbose_proxy_logger
from litellm.batches.batch_utils import (
_extract_file_access_credentials,
_get_batch_job_input_file_usage,
_get_file_content_as_dictionary,
_get_models_from_batch_input_file_content,
)
from litellm.integrations.custom_logger import CustomLogger
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy._types import SpecialModelNames, UserAPIKeyAuth

if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
Expand Down Expand Up @@ -98,6 +99,243 @@ def __init__(
self.internal_usage_cache = internal_usage_cache
self.parallel_request_limiter = parallel_request_limiter

def _get_batch_routing_model(self, data: Dict) -> Optional[str]:
"""Resolve the deployment/model used for this batch from request data."""
model = data.get("model")
if isinstance(model, str) and model:
return model

input_file_id = data.get("input_file_id")
if not isinstance(input_file_id, str) or not input_file_id:
return None

from litellm.proxy.openai_files_endpoints.common_utils import (
_is_base64_encoded_unified_file_id,
decode_model_from_file_id,
get_models_from_unified_file_id,
)

model_from_file_id = decode_model_from_file_id(input_file_id)
if model_from_file_id:
return model_from_file_id

unified_file_id = _is_base64_encoded_unified_file_id(input_file_id)
if unified_file_id:
target_model_names = get_models_from_unified_file_id(unified_file_id)
if target_model_names:
return target_model_names[0]

return None

def _resolve_batch_provider(self, batch_model: Optional[str]) -> Optional[str]:
"""Resolve the provider from the deployment that serves ``batch_model``.

The provider is read from trusted router credentials rather than the
user-supplied ``custom_llm_provider`` request field, so a caller cannot
spoof a skip-listed provider to bypass batch rate limiting.
"""
if not batch_model:
return None

from litellm.proxy.openai_files_endpoints.common_utils import (
get_credentials_for_model,
)
from litellm.proxy.proxy_server import llm_router

if llm_router is None:
return None

try:
credentials = get_credentials_for_model(
llm_router=llm_router,
model_id=batch_model,
operation_context="batch input file read (rate limiting)",
)
except HTTPException:
return None

provider = credentials.get("custom_llm_provider")
return provider if isinstance(provider, str) and provider else None

def _matches_skip_list(self, value: str, skip_list: List[str]) -> bool:
if not skip_list:
return False
for entry in skip_list:
if not isinstance(entry, str) or not entry:
continue
if value == entry or value.startswith(f"{entry}/"):
return True
return False

def _create_batch_rate_limit_descriptors(
self,
user_api_key_dict: UserAPIKeyAuth,
data: Dict,
) -> List["RateLimitDescriptor"]:
return self.parallel_request_limiter._create_rate_limit_descriptors(
user_api_key_dict=user_api_key_dict,
data=data,
rpm_limit_type=None,
tpm_limit_type=None,
model_has_failures=False,
)

def _should_skip_batch_input_file_processing(
self,
data: Dict,
user_api_key_dict: UserAPIKeyAuth,
) -> Tuple[bool, Optional[List["RateLimitDescriptor"]]]:
"""
Skip downloading batch input files when configured or when there is
nothing to enforce (no applicable rate limits and no model allowlist).

When the caller's key has a model allowlist to enforce, no skip path
is honored: the JSONL must still be downloaded so
``_enforce_batch_file_model_access`` can validate every ``body.model``
entry. Otherwise a restricted key could smuggle unauthorized models
into the file via an admin-configured skip (global disable,
per-model, per-provider).

Returns ``(should_skip, descriptors)`` where ``descriptors`` is the
rate-limit descriptor list computed for the no-limits check, so the
caller can reuse it for counter enforcement without recomputing.
"""
if self._key_requires_batch_model_access_check(user_api_key_dict):
return False, None

from litellm.proxy.proxy_server import general_settings

if general_settings.get("disable_batch_input_file_rate_limiting") is True:
return True, None

litellm_metadata = data.get("litellm_metadata") or {}
if litellm_metadata.get("skip_batch_input_file_rate_limiting") is True:
Comment thread
Sameerlite marked this conversation as resolved.
Resolved
Comment thread
veria-ai[bot] marked this conversation as resolved.
Resolved
return True, None

batch_model = self._get_batch_routing_model(data)
skip_models = (
general_settings.get("skip_batch_input_file_rate_limiting_for_models") or []
)
if batch_model and self._matches_skip_list(batch_model, skip_models):
Comment thread
veria-ai[bot] marked this conversation as resolved.
Resolved
verbose_proxy_logger.debug(
f"Skipping batch input file processing for model={batch_model}"
)
return True, None

skip_providers = (
general_settings.get("skip_batch_input_file_rate_limiting_for_providers")
or []
)
if skip_providers:
batch_provider = self._resolve_batch_provider(batch_model)
if batch_provider and batch_provider in skip_providers:
verbose_proxy_logger.debug(
f"Skipping batch input file processing for provider={batch_provider}"
)
return True, None

descriptors = self._create_batch_rate_limit_descriptors(
user_api_key_dict=user_api_key_dict,
data=data,
)
if not self._has_applicable_batch_rate_limits(descriptors):
verbose_proxy_logger.debug(
"Skipping batch input file processing: no rate limits configured"
)
return True, None

return False, descriptors

@staticmethod
def _key_requires_batch_model_access_check(
user_api_key_dict: UserAPIKeyAuth,
) -> bool:
"""True when the key may only call a subset of models (JSONL must be checked)."""
models = user_api_key_dict.models or []
if "*" in models:
return False
if SpecialModelNames.all_proxy_models.value in models:
return False
if user_api_key_dict.access_group_ids:
return True
if not models:
return False
return True
Comment thread
cursor[bot] marked this conversation as resolved.
Resolved

@staticmethod
def _has_applicable_batch_rate_limits(
descriptors: List["RateLimitDescriptor"],
) -> bool:
for descriptor in descriptors:
rate_limit = descriptor.get("rate_limit") or {}
if (
rate_limit.get("requests_per_unit") is not None
or rate_limit.get("tokens_per_unit") is not None
or rate_limit.get("max_parallel_requests") is not None
):
return True
return False

def _resolve_batch_input_file_fetch_params(
self,
file_id: str,
custom_llm_provider: str,
data: Dict,
) -> Tuple[str, Dict[str, Any]]:
"""
Map proxy-facing file IDs to provider file IDs and credentials.

Model-embedded IDs (``file-<base64>``) are not unified managed-file IDs;
without decoding them, ``afile_content`` is called with the encoded ID
and the upstream provider returns 404.
"""
from litellm.proxy.openai_files_endpoints.common_utils import (
decode_model_from_file_id,
get_credentials_for_model,
get_original_file_id,
)
from litellm.proxy.proxy_server import llm_router

fetch_kwargs: Dict[str, Any] = {
"custom_llm_provider": custom_llm_provider,
}

model_from_file_id = decode_model_from_file_id(file_id)
if model_from_file_id:
if llm_router is not None:
try:
credentials = get_credentials_for_model(
llm_router=llm_router,
model_id=model_from_file_id,
operation_context="batch input file read (rate limiting)",
)
fetch_kwargs.update(_extract_file_access_credentials(credentials))
fetch_kwargs["model"] = model_from_file_id
provider = credentials.get("custom_llm_provider")
if provider:
fetch_kwargs["custom_llm_provider"] = provider
except HTTPException:
pass
return get_original_file_id(file_id), fetch_kwargs
Comment thread
Sameerlite marked this conversation as resolved.
Resolved
Comment thread
cursor[bot] marked this conversation as resolved.
Resolved

request_model = data.get("model")
if isinstance(request_model, str) and request_model and llm_router is not None:
try:
credentials = get_credentials_for_model(
llm_router=llm_router,
model_id=request_model,
operation_context="batch input file read (rate limiting)",
)
fetch_kwargs.update(_extract_file_access_credentials(credentials))
fetch_kwargs["model"] = request_model
provider = credentials.get("custom_llm_provider")
if provider:
fetch_kwargs["custom_llm_provider"] = provider
except HTTPException:
pass

return file_id, fetch_kwargs

def _raise_rate_limit_error(
self,
status: "RateLimitStatus",
Expand Down Expand Up @@ -163,6 +401,7 @@ async def _check_and_increment_batch_counters(
user_api_key_dict: UserAPIKeyAuth,
data: Dict,
batch_usage: BatchFileUsage,
descriptors: Optional[List["RateLimitDescriptor"]] = None,
) -> None:
"""
Atomically check + increment rate-limit counters by the batch amounts.
Expand All @@ -171,14 +410,15 @@ async def _check_and_increment_batch_counters(
case no counter is modified. Backed by `atomic_check_and_increment_by_n`
which uses a Redis Lua script when available (multi-process atomic) and
falls back to a per-process asyncio.Lock + in-memory operation.

``descriptors`` may be passed in by the pre-call hook to reuse the list
already computed when deciding whether to skip file processing.
"""
descriptors = self.parallel_request_limiter._create_rate_limit_descriptors(
user_api_key_dict=user_api_key_dict,
data=data,
rpm_limit_type=None,
tpm_limit_type=None,
model_has_failures=False,
)
if descriptors is None:
descriptors = self._create_batch_rate_limit_descriptors(
user_api_key_dict=user_api_key_dict,
data=data,
)

increment: Dict[Literal["requests", "tokens"], int] = {
"requests": batch_usage.request_count,
Expand Down Expand Up @@ -211,6 +451,7 @@ async def count_input_file_usage(
file_id: str,
custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
user_api_key_dict: Optional[UserAPIKeyAuth] = None,
data: Optional[Dict] = None,
) -> BatchFileUsage:
"""
Count number of requests and tokens in a batch input file.
Expand Down Expand Up @@ -238,14 +479,27 @@ async def count_input_file_usage(
user_api_key_dict=user_api_key_dict,
)
else:
provider_file_id, fetch_kwargs = (
self._resolve_batch_input_file_fetch_params(
file_id=file_id,
custom_llm_provider=custom_llm_provider,
data=data or {},
)
)
# For non-managed files, use the standard litellm.afile_content
file_content = await litellm.afile_content(
file_id=file_id,
custom_llm_provider=custom_llm_provider,
file_id=provider_file_id,
user_api_key_dict=user_api_key_dict,
**fetch_kwargs,
)

file_content_as_dict = _get_file_content_as_dictionary(file_content.content)
file_content_bytes = getattr(file_content, "content", None)
if not isinstance(file_content_bytes, bytes):
raise ValueError(
f"Expected bytes content from file retrieval for {file_id}, "
f"got {type(file_content_bytes)}"
)
file_content_as_dict = _get_file_content_as_dictionary(file_content_bytes)

# Validate every model named in the batch JSONL against the
# caller's per-key model allowlist. Without this, a caller
Expand Down Expand Up @@ -441,6 +695,14 @@ async def async_pre_call_hook(
)
return data

should_skip, batch_rate_limit_descriptors = (
self._should_skip_batch_input_file_processing(
data=data, user_api_key_dict=user_api_key_dict
)
)
if should_skip:
return data

# Get custom_llm_provider for token counting
custom_llm_provider = data.get("custom_llm_provider", "openai")

Expand All @@ -452,6 +714,7 @@ async def async_pre_call_hook(
file_id=input_file_id,
custom_llm_provider=custom_llm_provider,
user_api_key_dict=user_api_key_dict,
data=data,
)

verbose_proxy_logger.debug(
Expand All @@ -469,6 +732,7 @@ async def async_pre_call_hook(
user_api_key_dict=user_api_key_dict,
data=data,
batch_usage=batch_usage,
descriptors=batch_rate_limit_descriptors,
)

verbose_proxy_logger.debug(
Expand Down
Loading
Loading