Skip to content
Merged
Show file tree
Hide file tree
Changes from 8 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions litellm/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -2204,6 +2204,16 @@
# admins may write it: when present it replaces the standard RPM/TPM checks for
# batch submissions.
BATCH_ENQUEUED_TOKEN_LIMIT_METADATA_KEY: Final = "batch_enqueued_token_limit"
MAX_BATCH_FILE_RECORDS_KEY: Final = "max_batch_file_records"
MAX_BATCH_FILE_UPLOADS_PER_DAY_KEY: Final = "max_batch_file_uploads_per_day"
MAX_FILE_DOWNLOADS_PER_MINUTE_KEY: Final = "max_file_downloads_per_minute"
ADMIN_ONLY_BATCH_LIMIT_METADATA_KEYS: Final = (
BATCH_ENQUEUED_TOKEN_LIMIT_METADATA_KEY,
MAX_BATCH_FILE_RECORDS_KEY,
MAX_BATCH_FILE_UPLOADS_PER_DAY_KEY,
MAX_FILE_DOWNLOADS_PER_MINUTE_KEY,
)
FILE_USAGE_MAX_TRACKED_COUNTERS: Final = 20_000

# Shared read-only empty mapping, for defaulting optional Mapping parameters without
# constructing a fresh mutable dict at each call site.
Expand Down
15 changes: 15 additions & 0 deletions litellm/proxy/_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -2887,6 +2887,21 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
None,
description="max batch input file size in MB for /v1/files uploads with purpose=batch, if a file is larger than this size it will be rejected before being forwarded to the provider",
)
max_batch_file_records: int | None = Field(
None,
gt=0,
description="max records (non-blank lines) per batch input file for /v1/files uploads with purpose=batch, applied per key. A key's metadata can override it and a team's metadata adds a team cap on top, both set by a proxy admin; the lower of the key's value and the team's value wins. Unset means no limit",
)
max_batch_file_uploads_per_day: int | None = Field(
None,
gt=0,
description="max /v1/files uploads with purpose=batch per key (per user for JWT callers) per UTC day. A key's metadata can override it and a team's metadata adds a shared team cap, both set by a proxy admin. Unset means no limit",
)
max_file_downloads_per_minute: int | None = Field(
None,
gt=0,
description="max GET /v1/files/{file_id}/content calls per key (per user for JWT callers) per file per minute. A key's metadata can override it and a team's metadata adds a shared team cap, both set by a proxy admin. Unset means no limit",
)
max_file_size_mb: int | None = Field(
None,
description="max file size in MB for /v1/files uploads, for any purpose, if a file is larger than this size it will be rejected before being forwarded to the provider",
Expand Down
30 changes: 17 additions & 13 deletions litellm/proxy/auth/auth_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
from litellm import Router, constants, provider_list
from litellm._logging import verbose_proxy_logger
from litellm.constants import (
BATCH_ENQUEUED_TOKEN_LIMIT_METADATA_KEY,
ADMIN_ONLY_BATCH_LIMIT_METADATA_KEYS,
EMPTY_MAPPING,
INVALID_VIRTUAL_KEY_ERROR_MARKER,
MINIMUM_CUSTOM_KEY_LENGTH,
Expand Down Expand Up @@ -1226,8 +1226,8 @@ def enforce_output_token_estimates_are_admin_only(
)


class BatchEnqueuedTokenLimitRequest(Protocol):
"""The shape of any management request that can carry a batch enqueued-token limit."""
class BatchLimitRequest(Protocol):
"""The shape of any management request that can carry an admin-only batch limit in its metadata."""

@property
def metadata(self) -> Mapping[str, object] | None: ...
Expand All @@ -1236,32 +1236,36 @@ def metadata(self) -> Mapping[str, object] | None: ...
def model_fields_set(self) -> Collection[str]: ...


def enforce_batch_enqueued_token_limit_is_admin_only(
data: BatchEnqueuedTokenLimitRequest,
def enforce_batch_limits_are_admin_only(
data: BatchLimitRequest,
existing_metadata: Mapping[str, object] | None,
user_api_key_dict: UserAPIKeyAuth,
entity: Literal["key", "team"],
) -> None:
"""Only a proxy admin may change a key or team's batch enqueued-token limit.
"""Only a proxy admin may change a key or team's batch limits.

When set, ``batch_enqueued_token_limit`` replaces the standard RPM/TPM checks
for batch submissions, so a holder-writable copy would let a caller lift their
own batch quota. Gated on the resulting value rather than on presence, so a
form resending the stored value stays a no-op.
Every key in ``ADMIN_ONLY_BATCH_LIMIT_METADATA_KEYS`` caps what the holder
may do with batches, so a holder-writable copy would let a caller lift their
own quota. Gated on the resulting value rather than on presence, so a form
resending the stored value stays a no-op.
"""
if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value:
return
stored: Final[Mapping[str, object]] = existing_metadata or EMPTY_MAPPING
requested: Final[Mapping[str, object]] = (
(data.metadata or EMPTY_MAPPING) if "metadata" in data.model_fields_set else stored
)
if requested.get(BATCH_ENQUEUED_TOKEN_LIMIT_METADATA_KEY) == stored.get(BATCH_ENQUEUED_TOKEN_LIMIT_METADATA_KEY):
changed: Final = next(
(key for key in ADMIN_ONLY_BATCH_LIMIT_METADATA_KEYS if requested.get(key) != stored.get(key)),
None,
)
if changed is None:
return
raise HTTPException(
status_code=403,
detail={
"error": f"Only proxy admins can set {BATCH_ENQUEUED_TOKEN_LIMIT_METADATA_KEY} on a {entity}. "
"It replaces the standard rate limit checks for batch submissions."
"error": f"Only proxy admins can set {changed} on a {entity}. "
"It limits what the holder can do with batches, so the holder cannot raise it."
},
)

Expand Down
4 changes: 4 additions & 0 deletions litellm/proxy/management_endpoints/internal_user_endpoints.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
get_team_object,
get_user_object,
)
from litellm.proxy.auth.auth_utils import enforce_batch_limits_are_admin_only
from litellm.proxy.auth.password_policy import (
validate_password_not_breached,
validate_password_policy,
Expand Down Expand Up @@ -587,6 +588,9 @@ async def new_user(
detail=f"Only proxy admins can create administrative users (proxy_admin, proxy_admin_viewer). Attempted to create user with role: {data.user_role}. Your role: {user_api_key_dict.user_role}",
)

if data.auto_create_key and isinstance(user_api_key_dict, UserAPIKeyAuth):
enforce_batch_limits_are_admin_only(data, None, user_api_key_dict, "key")
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved

_check_permissions_caller_permission(
data=data,
user_api_key_dict=user_api_key_dict,
Expand Down
10 changes: 5 additions & 5 deletions litellm/proxy/management_endpoints/key_management_endpoints.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@
)
from litellm.proxy.auth.auth_utils import (
abbreviate_api_key,
enforce_batch_enqueued_token_limit_is_admin_only,
enforce_batch_limits_are_admin_only,
enforce_output_token_estimates_are_admin_only,
)
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
Expand Down Expand Up @@ -1215,7 +1215,7 @@ async def _common_key_generation_helper(
user_api_key_dict=user_api_key_dict,
entity="key",
)
enforce_batch_enqueued_token_limit_is_admin_only(
enforce_batch_limits_are_admin_only(
data=data,
existing_metadata=None,
user_api_key_dict=user_api_key_dict,
Expand Down Expand Up @@ -2754,7 +2754,7 @@ async def _process_single_key_update(
existing_metadata=existing_key_row.metadata, # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # LiteLLM_VerificationToken.metadata is a bare dict
)

enforce_batch_enqueued_token_limit_is_admin_only(
enforce_batch_limits_are_admin_only(
data=update_key_request,
existing_metadata=existing_key_row.metadata,
user_api_key_dict=user_api_key_dict,
Expand Down Expand Up @@ -3202,7 +3202,7 @@ async def _validate_update_key_data(
user_api_key_dict=user_api_key_dict,
entity="key",
)
enforce_batch_enqueued_token_limit_is_admin_only(
enforce_batch_limits_are_admin_only(
data=data,
existing_metadata=_existing_metadata if isinstance(_existing_metadata, dict) else None,
user_api_key_dict=user_api_key_dict,
Expand Down Expand Up @@ -5595,7 +5595,7 @@ async def _execute_virtual_key_regeneration(
user_api_key_dict=user_api_key_dict,
entity="key",
)
enforce_batch_enqueued_token_limit_is_admin_only(
enforce_batch_limits_are_admin_only(
data=data,
existing_metadata=_existing_key_metadata if isinstance(_existing_key_metadata, dict) else None,
user_api_key_dict=user_api_key_dict,
Expand Down
6 changes: 3 additions & 3 deletions litellm/proxy/management_endpoints/team_endpoints.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@
invalidate_team_member_spend_state,
)
from litellm.proxy.auth.auth_utils import (
enforce_batch_enqueued_token_limit_is_admin_only,
enforce_batch_limits_are_admin_only,
enforce_output_token_estimates_are_admin_only,
)
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
Expand Down Expand Up @@ -1488,7 +1488,7 @@ async def new_team(
user_api_key_dict=user_api_key_dict,
entity="team",
)
enforce_batch_enqueued_token_limit_is_admin_only(
enforce_batch_limits_are_admin_only(
data=data,
existing_metadata=None,
user_api_key_dict=user_api_key_dict,
Expand Down Expand Up @@ -2274,7 +2274,7 @@ async def update_team(
user_api_key_dict=user_api_key_dict,
entity="team",
)
enforce_batch_enqueued_token_limit_is_admin_only(
enforce_batch_limits_are_admin_only(
data=data,
existing_metadata=_existing_team_metadata if isinstance(_existing_team_metadata, dict) else None,
user_api_key_dict=user_api_key_dict,
Expand Down
3 changes: 3 additions & 0 deletions litellm/proxy/management_helpers/bulk_user_creation.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
UserAPIKeyAuth,
)
from litellm.proxy.auth.auth_checks import invalidate_team_member_spend_state
from litellm.proxy.auth.auth_utils import enforce_batch_limits_are_admin_only
from litellm.proxy.auth.litellm_license import LicenseCheck
from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
Expand Down Expand Up @@ -213,6 +214,8 @@ def _row_error(item: BulkNewUserItem, user_api_key_dict: UserAPIKeyAuth) -> str
try:
validate_budget_duration(item.budget_duration)
_check_permissions_caller_permission(data=item, user_api_key_dict=user_api_key_dict)
if item.auto_create_key:
enforce_batch_limits_are_admin_only(item, None, user_api_key_dict, "key")
except Exception as exc: # noqa: BLE001 # any validation failure is reported on this row only
return _error_message(exc)
return None
Expand Down
51 changes: 41 additions & 10 deletions litellm/proxy/openai_files_endpoints/batch_file_validation.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from typing_extensions import assert_never

from litellm.proxy._types import ProxyException
from litellm.proxy.openai_files_endpoints.file_usage_caps import FileUsageLimit, describe_limit_source

_MB: Final = 1024 * 1024

Expand All @@ -33,6 +34,11 @@ class BatchFileTooLarge:
limit_mb: int


@dataclass(frozen=True, slots=True)
class BatchFileTooManyRecords:
limit: FileUsageLimit


@dataclass(frozen=True, slots=True)
class BatchFileWrongExtension:
filename: str
Expand Down Expand Up @@ -62,6 +68,7 @@ class BatchFileMissingLineKey:

BatchFileValidationFailure = (
BatchFileTooLarge
| BatchFileTooManyRecords
| BatchFileWrongExtension
| BatchFileEmpty
| BatchFileInvalidJsonLine
Expand Down Expand Up @@ -99,7 +106,23 @@ def _check_line(line_number: int, raw_line: bytes, line_shape: BatchLineShape) -
return BatchFileMissingLineKey(line_number=line_number, key=missing, line_shape=line_shape)


def _scan_lines(file_source: bytes | BinaryIO, line_shape: BatchLineShape) -> BatchFileValidationFailure | None:
def _check_record(
record_number: int,
line_number: int,
raw_line: bytes,
line_shape: BatchLineShape,
max_records: FileUsageLimit | None,
) -> BatchFileValidationFailure | None:
if max_records is not None and record_number > max_records.value:
return BatchFileTooManyRecords(limit=max_records)
return _check_line(line_number, raw_line, line_shape)


def _scan_lines(
file_source: bytes | BinaryIO,
line_shape: BatchLineShape,
max_records: FileUsageLimit | None,
) -> BatchFileValidationFailure | None:
content_lines: Final = (
(line_number, raw_line)
for line_number, raw_line in enumerate(_iter_lines(file_source), start=1)
Expand All @@ -108,30 +131,27 @@ def _scan_lines(file_source: bytes | BinaryIO, line_shape: BatchLineShape) -> Ba
first_line: Final = next(content_lines, None)
if first_line is None:
return BatchFileEmpty()
return next(
(
failure
for line_number, raw_line in chain((first_line,), content_lines)
for failure in (_check_line(line_number, raw_line, line_shape),)
if failure is not None
),
None,
failures: Final = (
_check_record(record_number, line_number, raw_line, line_shape, max_records)
for record_number, (line_number, raw_line) in enumerate(chain((first_line,), content_lines), start=1)
)
return next((failure for failure in failures if failure is not None), None)


def check_batch_file_upload(
filename: str | None,
file_source: bytes | BinaryIO,
max_batch_file_size_mb: int | None,
line_shape: BatchLineShape = BATCH_LINE_SHAPE,
max_records: FileUsageLimit | None = None,
) -> BatchFileValidationFailure | None:
if filename is None or not filename.lower().endswith(".jsonl"):
return BatchFileWrongExtension(filename=filename or "")
if max_batch_file_size_mb is not None and max_batch_file_size_mb > 0:
size_bytes: Final = _file_size_bytes(file_source)
if size_bytes > max_batch_file_size_mb * _MB:
return BatchFileTooLarge(size_bytes=size_bytes, limit_mb=max_batch_file_size_mb)
scan_failure: Final = _scan_lines(file_source, line_shape)
scan_failure: Final = _scan_lines(file_source, line_shape, max_records)
if not isinstance(file_source, bytes):
file_source.seek(0)
return scan_failure
Expand All @@ -149,6 +169,17 @@ def raise_batch_file_validation_failure(failure: BatchFileValidationFailure) ->
param="file",
code=413,
)
case BatchFileTooManyRecords(limit=limit):
raise ProxyException(
message=(
f"Batch input file has more than {limit.value} records, which exceeds the "
f"{limit.setting} of {limit.value} set {describe_limit_source(limit.source)}. "
"The file was not forwarded to the provider."
),
type="invalid_request_error",
param="file",
code=413,
)
case BatchFileWrongExtension(filename=filename):
raise ProxyException(
message=(
Expand Down
Loading
Loading