Repository navigation
fix(batches): count non-chat tokens, validate batch-file model access (VERIA-39) #27015
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
krrish-berri-2
merged 2 commits into
BerriAI:litellm_internal_staging
from
stuxf:fix/batch-file-validation
May 2, 2026
Merged
Changes from 1 commit
Commits
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
230 changes: 230 additions & 0 deletions
230
tests/test_litellm/proxy/hooks/test_batch_file_validation.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,230 @@ | ||
| """ | ||
| VERIA-39 regression tests: | ||
|
|
||
| - The batch input-file token counter must measure embeddings (`input`) | ||
| and text-completion (`prompt`) payloads, not only chat (`messages`). | ||
| - The batch rate-limiter pre-call hook must reject batch files that name | ||
| models the caller is not authorized to use. | ||
| """ | ||
|
|
||
| from unittest.mock import AsyncMock, MagicMock, patch | ||
|
|
||
| import pytest | ||
| from fastapi import HTTPException | ||
|
|
||
| from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth | ||
|
|
||
|
|
||
| # --------------------------------------------------------------------------- | ||
| # Token counter — covers all three batch payload shapes | ||
| # --------------------------------------------------------------------------- | ||
|
|
||
|
|
||
| def test_token_counter_counts_chat_messages(): | ||
| from litellm.batches.batch_utils import _get_batch_job_input_file_usage | ||
|
|
||
| usage = _get_batch_job_input_file_usage( | ||
| file_content_dictionary=[ | ||
| { | ||
| "body": { | ||
| "model": "gpt-4o-mini", | ||
| "messages": [{"role": "user", "content": "hello"}], | ||
| } | ||
| } | ||
| ] | ||
| ) | ||
| assert usage.prompt_tokens > 0 | ||
|
|
||
|
|
||
| def test_token_counter_counts_text_completion_prompt(): | ||
| """Pre-fix this returned 0 tokens (the function only inspected | ||
| `messages`), letting `prompt`-style batches slip past TPM limits.""" | ||
| from litellm.batches.batch_utils import _get_batch_job_input_file_usage | ||
|
|
||
| usage = _get_batch_job_input_file_usage( | ||
| file_content_dictionary=[ | ||
| {"body": {"model": "gpt-3.5-turbo-instruct", "prompt": "hello world"}} | ||
| ] | ||
| ) | ||
| assert usage.prompt_tokens > 0 | ||
|
|
||
|
|
||
| def test_token_counter_counts_embedding_input_string(): | ||
| from litellm.batches.batch_utils import _get_batch_job_input_file_usage | ||
|
|
||
| usage = _get_batch_job_input_file_usage( | ||
| file_content_dictionary=[ | ||
| {"body": {"model": "text-embedding-3-small", "input": "hello world"}} | ||
| ] | ||
| ) | ||
| assert usage.prompt_tokens > 0 | ||
|
|
||
|
|
||
| def test_token_counter_counts_embedding_input_list(): | ||
| from litellm.batches.batch_utils import _get_batch_job_input_file_usage | ||
|
|
||
| usage = _get_batch_job_input_file_usage( | ||
| file_content_dictionary=[ | ||
| { | ||
| "body": { | ||
| "model": "text-embedding-3-small", | ||
| "input": ["hello", "world"], | ||
| } | ||
| } | ||
| ] | ||
| ) | ||
| assert usage.prompt_tokens > 0 | ||
|
|
||
|
|
||
| def test_token_counter_counts_text_completion_prompt_list(): | ||
| from litellm.batches.batch_utils import _get_batch_job_input_file_usage | ||
|
|
||
| usage = _get_batch_job_input_file_usage( | ||
| file_content_dictionary=[ | ||
| { | ||
| "body": { | ||
| "model": "gpt-3.5-turbo-instruct", | ||
| "prompt": ["alpha", "beta"], | ||
| } | ||
| } | ||
| ] | ||
| ) | ||
| assert usage.prompt_tokens > 0 | ||
|
|
||
|
|
||
| # --------------------------------------------------------------------------- | ||
| # Model extractor | ||
| # --------------------------------------------------------------------------- | ||
|
|
||
|
|
||
| def test_model_extractor_returns_distinct_models(): | ||
| from litellm.batches.batch_utils import _get_models_from_batch_input_file_content | ||
|
|
||
| models = _get_models_from_batch_input_file_content( | ||
| [ | ||
| {"body": {"model": "gpt-4o", "messages": []}}, | ||
| {"body": {"model": "gpt-4o", "messages": []}}, # duplicate | ||
| {"body": {"model": "gpt-4o-mini", "messages": []}}, | ||
| {"body": {}}, # missing model | ||
| ] | ||
| ) | ||
| assert models == ["gpt-4o", "gpt-4o-mini"] | ||
|
|
||
|
|
||
| # --------------------------------------------------------------------------- | ||
| # Pre-call hook model validation | ||
| # --------------------------------------------------------------------------- | ||
|
|
||
|
|
||
| @pytest.mark.asyncio | ||
| async def test_pre_call_rejects_unauthorized_model_in_batch_file(): | ||
| """Pre-fix the hook only validated the outer `model` parameter and | ||
| forwarded the file as-is. With this fix, a model named inside the | ||
| JSONL that the caller cannot use must trigger a 403.""" | ||
| from litellm.proxy.hooks.batch_rate_limiter import _PROXY_BatchRateLimiter | ||
|
|
||
| rate_limiter = _PROXY_BatchRateLimiter( | ||
| internal_usage_cache=MagicMock(), | ||
| parallel_request_limiter=MagicMock(), | ||
| ) | ||
|
|
||
| # Simulated decoded batch file: caller is restricted to gpt-3.5 | ||
| # but the JSONL points at gpt-4o. | ||
| file_dict = [ | ||
| {"body": {"model": "gpt-4o", "messages": [{"role": "user", "content": "x"}]}} | ||
| ] | ||
|
|
||
| user = UserAPIKeyAuth( | ||
| api_key="sk-restricted", | ||
| user_id="alice", | ||
| models=["gpt-3.5-turbo"], | ||
| user_role=LitellmUserRoles.INTERNAL_USER.value, | ||
| ) | ||
|
|
||
| # `can_key_call_model` raises a ProxyException for non-allowed models. | ||
| async def _raise_unauthorized(**kwargs): | ||
| raise Exception( | ||
| f"Key not allowed to access model. This key only has access to models={kwargs['valid_token'].models}" | ||
| ) | ||
|
|
||
| with ( | ||
| patch( | ||
| "litellm.proxy.auth.auth_checks.can_key_call_model", | ||
| new=AsyncMock(side_effect=_raise_unauthorized), | ||
| ), | ||
| patch("litellm.proxy.proxy_server.llm_router", None), | ||
| ): | ||
| with pytest.raises(HTTPException) as exc: | ||
| await rate_limiter._enforce_batch_file_model_access( | ||
| user_api_key_dict=user, | ||
| file_content_as_dict=file_dict, | ||
| ) | ||
|
|
||
| assert exc.value.status_code == 403 | ||
| assert "gpt-4o" in str(exc.value.detail) | ||
|
|
||
|
|
||
| @pytest.mark.asyncio | ||
| async def test_pre_call_allows_authorized_model_in_batch_file(): | ||
| """If every model in the JSONL is on the caller's allowlist, the hook | ||
| must not raise.""" | ||
| from litellm.proxy.hooks.batch_rate_limiter import _PROXY_BatchRateLimiter | ||
|
|
||
| rate_limiter = _PROXY_BatchRateLimiter( | ||
| internal_usage_cache=MagicMock(), | ||
| parallel_request_limiter=MagicMock(), | ||
| ) | ||
|
|
||
| file_dict = [ | ||
| { | ||
| "body": { | ||
| "model": "gpt-3.5-turbo", | ||
| "messages": [{"role": "user", "content": "x"}], | ||
| } | ||
| } | ||
| ] | ||
|
|
||
| user = UserAPIKeyAuth( | ||
| api_key="sk-ok", | ||
| user_id="alice", | ||
| models=["gpt-3.5-turbo"], | ||
| user_role=LitellmUserRoles.INTERNAL_USER.value, | ||
| ) | ||
|
|
||
| with ( | ||
| patch( | ||
| "litellm.proxy.auth.auth_checks.can_key_call_model", | ||
| new=AsyncMock(return_value=True), | ||
| ), | ||
| patch("litellm.proxy.proxy_server.llm_router", None), | ||
| ): | ||
| # Should not raise | ||
| await rate_limiter._enforce_batch_file_model_access( | ||
| user_api_key_dict=user, | ||
| file_content_as_dict=file_dict, | ||
| ) | ||
|
|
||
|
|
||
| @pytest.mark.asyncio | ||
| async def test_pre_call_skips_check_when_no_models_present(): | ||
| """Files without any `body.model` (corrupt or empty) must not 500; | ||
| the rate limiter logs a warning elsewhere and proceeds.""" | ||
| from litellm.proxy.hooks.batch_rate_limiter import _PROXY_BatchRateLimiter | ||
|
|
||
| rate_limiter = _PROXY_BatchRateLimiter( | ||
| internal_usage_cache=MagicMock(), | ||
| parallel_request_limiter=MagicMock(), | ||
| ) | ||
| user = UserAPIKeyAuth(api_key="sk-ok", user_id="alice") | ||
|
|
||
| # Should not raise even though `can_key_call_model` is the default | ||
| # (would fail). The early-return on empty models keeps the call out | ||
| # entirely. | ||
| await rate_limiter._enforce_batch_file_model_access( | ||
| user_api_key_dict=user, | ||
| file_content_as_dict=[], | ||
| ) | ||
| await rate_limiter._enforce_batch_file_model_access( | ||
| user_api_key_dict=user, | ||
| file_content_as_dict=[{"body": {}}], | ||
| ) |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.