Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 47 additions & 3 deletions litellm/batches/batch_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -387,6 +387,27 @@ def _get_batch_job_total_usage_from_file_content(
)


def _get_models_from_batch_input_file_content(
file_content_dictionary: List[dict],
) -> List[str]:
"""Extract the distinct ``body.model`` values from a batch *input* file.

Used by the proxy's batch pre-call hook to enforce that the caller is
authorized for every model named inside the JSONL — not just the one
on the outer request — so the proxy's per-key model allowlist isn't
bypassed by smuggling expensive models into the batch file.
"""
models: List[str] = []
seen: set = set()
for _item in file_content_dictionary:
body = _item.get("body") or {}
model = body.get("model")
if model and model not in seen:
seen.add(model)
models.append(model)
return models


def _get_batch_job_input_file_usage(
file_content_dictionary: List[dict],
custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
Expand All @@ -403,11 +424,34 @@ def _get_batch_job_input_file_usage(
for _item in file_content_dictionary:
body = _item.get("body", {})
model = body.get("model", model_name or "")
messages = body.get("messages", [])

# Chat completion payloads.
messages = body.get("messages")
if messages:
item_tokens = token_counter(model=model, messages=messages)
prompt_tokens += item_tokens
prompt_tokens += token_counter(model=model, messages=messages)
continue

# Text completion payloads (`prompt` may be a string or a list of
# strings/tokens).
prompt = body.get("prompt")
if prompt:
if isinstance(prompt, list):
for chunk in prompt:
if isinstance(chunk, str):
prompt_tokens += token_counter(model=model, text=chunk)
elif isinstance(prompt, str):
prompt_tokens += token_counter(model=model, text=prompt)
continue

# Embedding payloads (`input` may be a string or a list of strings).
input_data = body.get("input")
if input_data:
if isinstance(input_data, list):
for chunk in input_data:
if isinstance(chunk, str):
prompt_tokens += token_counter(model=model, text=chunk)
elif isinstance(input_data, str):
prompt_tokens += token_counter(model=model, text=input_data)
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved

return Usage(
total_tokens=prompt_tokens + completion_tokens,
Expand Down
56 changes: 56 additions & 0 deletions litellm/proxy/hooks/batch_rate_limiter.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
from litellm.batches.batch_utils import (
_get_batch_job_input_file_usage,
_get_file_content_as_dictionary,
_get_models_from_batch_input_file_content,
)
from litellm.integrations.custom_logger import CustomLogger
from litellm.proxy._types import UserAPIKeyAuth
Expand Down Expand Up @@ -288,6 +289,17 @@ async def count_input_file_usage(

file_content_as_dict = _get_file_content_as_dictionary(file_content.content)

# Validate every model named in the batch JSONL against the
# caller's per-key model allowlist. Without this, a caller
# could smuggle restricted/expensive models inside the file
# and the upstream provider would execute the batch under
# the proxy's shared API key.
if user_api_key_dict is not None:
await self._enforce_batch_file_model_access(
user_api_key_dict=user_api_key_dict,
file_content_as_dict=file_content_as_dict,
)

input_file_usage = _get_batch_job_input_file_usage(
file_content_dictionary=file_content_as_dict,
custom_llm_provider=custom_llm_provider,
Expand All @@ -304,6 +316,50 @@ async def count_input_file_usage(
)
raise

async def _enforce_batch_file_model_access(
self,
user_api_key_dict: UserAPIKeyAuth,
file_content_as_dict: List[dict],
) -> None:
"""Reject the batch if the caller is not authorized for every
``body.model`` named inside the JSONL.

Reuses ``can_key_call_model`` so the same allowlist semantics
(wildcards, access groups, ``all-proxy-models``, team aliases)
the proxy enforces on `/chat/completions` apply here.
"""
from litellm.proxy.auth.auth_checks import can_key_call_model
from litellm.proxy.proxy_server import llm_router

models = _get_models_from_batch_input_file_content(file_content_as_dict)
if not models:
return

llm_model_list = llm_router.model_list if llm_router is not None else None
for model in models:
try:
await can_key_call_model(
model=model,
llm_model_list=llm_model_list,
valid_token=user_api_key_dict,
llm_router=llm_router,
)
except HTTPException:
raise
except Exception as e:
# `can_key_call_model` raises ProxyException on denial;
# re-shape to a 403 so the batch endpoint returns a
# consistent rejection without leaking internal types.
raise HTTPException(
status_code=403,
detail={
"error": (
"Batch input file references a model the caller is "
f"not authorized to use: model={model}, reason={str(e)}"
)
},
)

async def _fetch_managed_file_content(
self,
file_id: str,
Expand Down
230 changes: 230 additions & 0 deletions tests/test_litellm/proxy/hooks/test_batch_file_validation.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,230 @@
"""
VERIA-39 regression tests:

- The batch input-file token counter must measure embeddings (`input`)
and text-completion (`prompt`) payloads, not only chat (`messages`).
- The batch rate-limiter pre-call hook must reject batch files that name
models the caller is not authorized to use.
"""

from unittest.mock import AsyncMock, MagicMock, patch

import pytest
from fastapi import HTTPException

from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth


# ---------------------------------------------------------------------------
# Token counter — covers all three batch payload shapes
# ---------------------------------------------------------------------------


def test_token_counter_counts_chat_messages():
from litellm.batches.batch_utils import _get_batch_job_input_file_usage

usage = _get_batch_job_input_file_usage(
file_content_dictionary=[
{
"body": {
"model": "gpt-4o-mini",
"messages": [{"role": "user", "content": "hello"}],
}
}
]
)
assert usage.prompt_tokens > 0


def test_token_counter_counts_text_completion_prompt():
"""Pre-fix this returned 0 tokens (the function only inspected
`messages`), letting `prompt`-style batches slip past TPM limits."""
from litellm.batches.batch_utils import _get_batch_job_input_file_usage

usage = _get_batch_job_input_file_usage(
file_content_dictionary=[
{"body": {"model": "gpt-3.5-turbo-instruct", "prompt": "hello world"}}
]
)
assert usage.prompt_tokens > 0


def test_token_counter_counts_embedding_input_string():
from litellm.batches.batch_utils import _get_batch_job_input_file_usage

usage = _get_batch_job_input_file_usage(
file_content_dictionary=[
{"body": {"model": "text-embedding-3-small", "input": "hello world"}}
]
)
assert usage.prompt_tokens > 0


def test_token_counter_counts_embedding_input_list():
from litellm.batches.batch_utils import _get_batch_job_input_file_usage

usage = _get_batch_job_input_file_usage(
file_content_dictionary=[
{
"body": {
"model": "text-embedding-3-small",
"input": ["hello", "world"],
}
}
]
)
assert usage.prompt_tokens > 0


def test_token_counter_counts_text_completion_prompt_list():
from litellm.batches.batch_utils import _get_batch_job_input_file_usage

usage = _get_batch_job_input_file_usage(
file_content_dictionary=[
{
"body": {
"model": "gpt-3.5-turbo-instruct",
"prompt": ["alpha", "beta"],
}
}
]
)
assert usage.prompt_tokens > 0


# ---------------------------------------------------------------------------
# Model extractor
# ---------------------------------------------------------------------------


def test_model_extractor_returns_distinct_models():
from litellm.batches.batch_utils import _get_models_from_batch_input_file_content

models = _get_models_from_batch_input_file_content(
[
{"body": {"model": "gpt-4o", "messages": []}},
{"body": {"model": "gpt-4o", "messages": []}}, # duplicate
{"body": {"model": "gpt-4o-mini", "messages": []}},
{"body": {}}, # missing model
]
)
assert models == ["gpt-4o", "gpt-4o-mini"]


# ---------------------------------------------------------------------------
# Pre-call hook model validation
# ---------------------------------------------------------------------------


@pytest.mark.asyncio
async def test_pre_call_rejects_unauthorized_model_in_batch_file():
"""Pre-fix the hook only validated the outer `model` parameter and
forwarded the file as-is. With this fix, a model named inside the
JSONL that the caller cannot use must trigger a 403."""
from litellm.proxy.hooks.batch_rate_limiter import _PROXY_BatchRateLimiter

rate_limiter = _PROXY_BatchRateLimiter(
internal_usage_cache=MagicMock(),
parallel_request_limiter=MagicMock(),
)

# Simulated decoded batch file: caller is restricted to gpt-3.5
# but the JSONL points at gpt-4o.
file_dict = [
{"body": {"model": "gpt-4o", "messages": [{"role": "user", "content": "x"}]}}
]

user = UserAPIKeyAuth(
api_key="sk-restricted",
user_id="alice",
models=["gpt-3.5-turbo"],
user_role=LitellmUserRoles.INTERNAL_USER.value,
)

# `can_key_call_model` raises a ProxyException for non-allowed models.
async def _raise_unauthorized(**kwargs):
raise Exception(
f"Key not allowed to access model. This key only has access to models={kwargs['valid_token'].models}"
)

with (
patch(
"litellm.proxy.auth.auth_checks.can_key_call_model",
new=AsyncMock(side_effect=_raise_unauthorized),
),
patch("litellm.proxy.proxy_server.llm_router", None),
):
with pytest.raises(HTTPException) as exc:
await rate_limiter._enforce_batch_file_model_access(
user_api_key_dict=user,
file_content_as_dict=file_dict,
)

assert exc.value.status_code == 403
assert "gpt-4o" in str(exc.value.detail)


@pytest.mark.asyncio
async def test_pre_call_allows_authorized_model_in_batch_file():
"""If every model in the JSONL is on the caller's allowlist, the hook
must not raise."""
from litellm.proxy.hooks.batch_rate_limiter import _PROXY_BatchRateLimiter

rate_limiter = _PROXY_BatchRateLimiter(
internal_usage_cache=MagicMock(),
parallel_request_limiter=MagicMock(),
)

file_dict = [
{
"body": {
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "x"}],
}
}
]

user = UserAPIKeyAuth(
api_key="sk-ok",
user_id="alice",
models=["gpt-3.5-turbo"],
user_role=LitellmUserRoles.INTERNAL_USER.value,
)

with (
patch(
"litellm.proxy.auth.auth_checks.can_key_call_model",
new=AsyncMock(return_value=True),
),
patch("litellm.proxy.proxy_server.llm_router", None),
):
# Should not raise
await rate_limiter._enforce_batch_file_model_access(
user_api_key_dict=user,
file_content_as_dict=file_dict,
)


@pytest.mark.asyncio
async def test_pre_call_skips_check_when_no_models_present():
"""Files without any `body.model` (corrupt or empty) must not 500;
the rate limiter logs a warning elsewhere and proceeds."""
from litellm.proxy.hooks.batch_rate_limiter import _PROXY_BatchRateLimiter

rate_limiter = _PROXY_BatchRateLimiter(
internal_usage_cache=MagicMock(),
parallel_request_limiter=MagicMock(),
)
user = UserAPIKeyAuth(api_key="sk-ok", user_id="alice")

# Should not raise even though `can_key_call_model` is the default
# (would fail). The early-return on empty models keeps the call out
# entirely.
await rate_limiter._enforce_batch_file_model_access(
user_api_key_dict=user,
file_content_as_dict=[],
)
await rate_limiter._enforce_batch_file_model_access(
user_api_key_dict=user,
file_content_as_dict=[{"body": {}}],
)
Loading