Skip to content
Merged
Show file tree
Hide file tree
Changes from 29 commits
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
6c97e96
tests: add e2e tests for spend, budgets and llms
mubashir1osmani Jun 18, 2026
fd11147
style: make chained comparison of status_code clearer
mateo-berri Jun 18, 2026
e255848
remove e2e_tests folder
mubashir1osmani Jun 19, 2026
98142ff
test: add spend tracking tests
mubashir1osmani Jun 19, 2026
400e3ac
fix: p0 issues, added types and shared functions for each test suite
mubashir1osmani Jun 19, 2026
387608e
style: carry clearer status_code comparison into renamed e2e dir
mubashir1osmani Jun 19, 2026
91dad56
refactor: migrate to gateway client
mubashir1osmani Jun 19, 2026
91a1642
fix: add new tests, split gateway
mubashir1osmani Jun 20, 2026
1d83aba
test(e2e): add live batches suite across providers and routing scenarios
Sameerlite Jun 22, 2026
737f4cf
test(batches): cover real cost tracking on completed batch retrieve
Sameerlite Jun 22, 2026
0082d56
test(e2e): assert managed vs raw file and batch id shapes per routing…
Sameerlite Jun 22, 2026
5f73dfa
test(e2e): assert full response shape of each batches and files endpoint
Sameerlite Jun 22, 2026
621ab85
test(e2e): only accept transitional statuses for a freshly created batch
Sameerlite Jun 22, 2026
2fd4500
test(prompt-factory): make test_convert_url deterministic with a data…
mateo-berri Jun 22, 2026
3524a48
fix: pass through async image data urls
cursoragent Jun 22, 2026
ccffea1
fix(image-handling): short-circuit data URLs in async path too
mateo-berri Jun 22, 2026
c311599
Fix: openai batches lifecycle
Sameerlite Jun 25, 2026
66df228
Fix: add e2e azure openai tests
Sameerlite Jun 25, 2026
e18af65
Fix e2e for vertex ai
Sameerlite Jun 25, 2026
bee2eed
Add all models for testing
Sameerlite Jun 25, 2026
eccb8a7
test(managed-files): assert idempotent upsert in store_unified_file_id
mateo-berri Jun 25, 2026
b617fb5
Merge branch 'litellm_internal_staging' into litellm_e2e_batches
Sameerlite Jun 26, 2026
a04e3c9
test(batches): reconcile vertex_ai native batch-id comment with fallb…
mateo-berri Jun 26, 2026
870056e
fix(test-config): keep rust-ocr models in model_list by moving files_…
mateo-berri Jun 26, 2026
f0e749b
fix(test-config): move batch models after OCR block to keep merge wit…
mateo-berri Jun 26, 2026
c3b5b80
Merge branch 'litellm_internal_staging' into litellm_e2e_batches
Sameerlite Jun 29, 2026
1b150e4
fix(batches): use '24hrs' completion window and allow managed-files l…
Sameerlite Jun 29, 2026
8f90ea3
style: ruff format transformation.py and endpoints.py
Sameerlite Jun 29, 2026
5a86253
fix(e2e/batches): set Azure raw_model to gpt-4.1-mini-batch to match …
Sameerlite Jun 29, 2026
5081cb6
fix(vertex-ai/batches): correct completion_window to 24h per Literal …
Sameerlite Jun 29, 2026
5e82b7b
test(vertex-ai/batches): align completion_window assertion to 24h
Sameerlite Jun 29, 2026
a92bd75
chore: merge litellm_internal_staging into litellm_e2e_batches
mateo-berri Jul 2, 2026
c06d611
fix: update managed file metadata on upsert
cursoragent Jul 2, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions enterprise/litellm_enterprise/proxy/hooks/managed_files.py
Original file line number Diff line number Diff line change
Expand Up @@ -140,8 +140,9 @@ async def store_unified_file_id(
f"storage_url={db_data.get('storage_url')}"
)

result = await self.prisma_client.db.litellm_managedfiletable.create(
data=db_data
result = await self.prisma_client.db.litellm_managedfiletable.upsert(
where={"unified_file_id": file_id},
data={"create": db_data, "update": {}},
Comment thread
Sameerlite marked this conversation as resolved.
Resolved
)
verbose_logger.debug(
f"LiteLLM Managed File object with id={file_id} stored in db: {result}"
Expand Down
6 changes: 6 additions & 0 deletions litellm/litellm_core_utils/prompt_templates/image_handling.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,9 @@ def _process_image_response(response: Response, url: str) -> str:


async def async_convert_url_to_base64(url: str) -> str:
if url.startswith("data:") and ";base64," in url:
return url

# If MAX_IMAGE_URL_DOWNLOAD_SIZE_MB is 0, block all image downloads
if MAX_IMAGE_URL_DOWNLOAD_SIZE_MB == 0:
raise litellm.ImageFetchError(
Expand All @@ -95,6 +98,9 @@ async def async_convert_url_to_base64(url: str) -> str:


def convert_url_to_base64(url: str) -> str:
if url.startswith("data:") and ";base64," in url:
return url
Comment thread
mateo-berri marked this conversation as resolved.
Resolved

# If MAX_IMAGE_URL_DOWNLOAD_SIZE_MB is 0, block all image downloads
if MAX_IMAGE_URL_DOWNLOAD_SIZE_MB == 0:
raise litellm.ImageFetchError(
Expand Down
2 changes: 1 addition & 1 deletion litellm/proxy/batches_endpoints/endpoints.py
Original file line number Diff line number Diff line change
Expand Up @@ -625,7 +625,7 @@ async def list_batches(
route_type="alist_batches",
)

# Try to use managed objects table for listing batches (returns encoded IDs)
# Try to use managed objects table for listing batches (returns encoded IDs).
managed_files_obj = proxy_logging_obj.get_proxy_hook("managed_files")
if managed_files_obj is not None and hasattr(managed_files_obj, "list_user_batches"):
verbose_proxy_logger.debug("Using managed objects table for batch listing")
Expand Down
4 changes: 2 additions & 2 deletions litellm/proxy/proxy_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -15441,9 +15441,10 @@ async def get_routes():
app.include_router(image_router)
app.include_router(fine_tuning_router)
app.include_router(credential_router)
app.include_router(batches_router)
app.include_router(openai_files_router)
app.include_router(llm_passthrough_router)
app.include_router(pass_through_router)
app.include_router(batches_router)
app.include_router(health_router)
app.include_router(key_management_router)
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
app.include_router(internal_user_router)
Expand All @@ -15458,7 +15459,6 @@ async def get_routes():
app.include_router(debugging_endpoints_router)
app.include_router(rust_control_plane_router)
app.include_router(ui_crud_endpoints_router)
app.include_router(openai_files_router)
app.include_router(team_callback_router)
app.include_router(budget_management_router)
app.include_router(model_management_router)
Expand Down
49 changes: 49 additions & 0 deletions tests/batches_tests/test_batches_logging_unit_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -255,6 +255,55 @@ async def test_batch_retrieve_cost_tracking_with_completed_batch_no_explicit_cos
assert mock_batch.usage == expected_usage


@pytest.mark.asyncio
async def test_handle_completed_batch_computes_real_cost_from_output_file(
sample_file_content_dict,
):
"""Integration: a completed batch's cost and usage are computed from its output
file via the real cost-calc chain (only the file download is stubbed). This is
the function the retrieve handler invokes on completion; a dropped output line, a
wrong token sum, or mispriced model fails this test.
"""
from litellm.batches.batch_utils import _handle_completed_batch
from litellm.types.utils import LiteLLMBatch

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This has been added as a replacement for gap of e2e testing for batch polling and cost tracking


batch = LiteLLMBatch(
id="batch-real-cost-123",
object="batch",
endpoint="/v1/chat/completions",
input_file_id="file-input-123",
completion_window="24h",
status="completed",
output_file_id="file-output-123",
created_at=1234567890,
)

with patch(
"litellm.batches.batch_utils._get_batch_output_file_content_as_dictionary",
new=AsyncMock(return_value=sample_file_content_dict),
):
cost, usage, models = await _handle_completed_batch(
batch=batch, custom_llm_provider="openai"
)

pricing = litellm.model_cost["gpt-4o-mini-2024-07-18"]
expected_cost = (
42 * pricing["input_cost_per_token_batches"]
+ 20 * pricing["output_cost_per_token_batches"]
)

assert cost == pytest.approx(expected_cost)
assert cost > 0
assert (
cost
< 42 * pricing["input_cost_per_token"] + 20 * pricing["output_cost_per_token"]
)
assert usage.prompt_tokens == 42
assert usage.completion_tokens == 20
assert usage.total_tokens == 62
assert models == ["gpt-4o-mini-2024-07-18", "gpt-4o-mini-2024-07-18"]


@pytest.mark.asyncio
async def test_batch_retrieve_cost_tracking_with_explicit_cost_data():
"""
Expand Down
79 changes: 79 additions & 0 deletions tests/e2e/batches/COVERAGE.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
# Batches Test Coverage Matrix

Live e2e coverage of the Batches API over a real proxy, real provider keys, and
real cost. Synchronous tier only: a batch's completion window is 24h, so these
tests never wait for `completed`. They assert the proxy accepts, routes, retrieves,
Comment thread
Sameerlite marked this conversation as resolved.
Resolved
cancels, and lists a batch; everything created is deleted on teardown.

## Provider x operation

Only supported cells are tested. The capability table in `capabilities.py` holds one
row per supported (provider, scenario) pair, so there are no skipped cells in the
parametrized run.

| Provider | create | retrieve | cancel | list | file backing |
|-----------|--------|----------|--------|------|--------------|
| OpenAI | yes | yes | yes | yes | OpenAI Files |
| Azure | yes | yes | yes | yes | Azure Files |
| Vertex AI | yes | yes | yes | yes | GCS bucket |
| Bedrock | yes | yes | no (limited upstream) | no | S3 bucket |
| Anthropic | no | yes (env-gated) | no | no | Anthropic Files |

Bedrock cancel is unreliable upstream and list is unsupported, so both are gated off
(`can_cancel=False`, `can_list=False`). Anthropic cannot create/cancel/list through
litellm, so it has a standalone retrieve test that skips unless `ANTHROPIC_BATCH_ID`
points at a real Anthropic batch.

## Routing scenarios (per `litellm/proxy/batches_endpoints/endpoints.py`)

Each create-capable provider runs all four. The test asserts the returned file id
and batch id carry the shape that scenario must produce (`matches_id_shape`):

| Scenario | How the batch is routed | File id | Batch id |
|----------|-------------------------|---------|----------|
| `encoded` | upload with `?model=` -> model-encoded file id -> create with just that id | model-encoded | model-encoded |
| `unified` | upload with `target_model_names=` -> unified managed file id -> create with that id | managed | managed |
| `model_param` | raw file (provider-fallback upload) -> create with `model` in the body | raw | model-encoded |
| `provider_fallback` | raw file -> `POST /{provider}/v1/batches`, env creds, no model | raw | raw (native provider shape) |

"managed" ids base64-decode to a `litellm_proxy` marker; "model-encoded" ids keep the
provider prefix and base64-encode `litellm:<id>;model,<model>`; "raw" ids are the
provider's native ids. Asserting these catches a proxy that returns a raw id where it
should manage it, or vice versa. On top of the id shape, a misroute to the wrong
provider also fails create (the file id / model do not belong there), and the
`provider_fallback` raw batch id is additionally checked against the provider's native
shape (`raw_id_matches_provider`).

## Key model restriction

`test_batch_key_model_access_denied` mints a key restricted to one model
(`resources.key(models=[...])`) and proves the proxy returns 403
`key_model_access_denied` both when that key uploads a file for a disallowed model
(files endpoint) and when it creates a batch for a disallowed model (batches
endpoint).

## Per-endpoint output assertions

Each endpoint's full response is validated, not just the id. File upload asserts
`object=="file"`, `purpose=="batch"`, a positive `bytes`, a status, and a created-at.
Batch create / retrieve assert `object=="batch"`, `endpoint=="/v1/chat/completions"`,
`completion_window=="24h"`, a non-empty `input_file_id`, and a created-at; retrieve
additionally cross-checks that `id` and `input_file_id` match the created batch.
Cancel asserts the same id, `object=="batch"`, and a cancelling/cancelled status. List
asserts the `object=="list"` envelope and that the created batch is present as a batch.
File delete asserts `object=="file"` and `deleted==True`.

## This suite's files

| File | Covers |
|------|--------|
| `batch_client.py` | typed file upload/download + batch create/retrieve/cancel/list/delete over the shared Gateway; denial helpers |
| `capabilities.py` | the provider x scenario matrix + id-shape classifiers + per-provider raw-id assertion |
| `test_batches_e2e.py` | parametrized lifecycle with per-endpoint output assertions, file upload/delete outputs, key-model-access denial, anthropic retrieve |

## Out of scope (intentionally)

Driving a batch to `completed`, cost tracking on completion, and the DB write-back
are not covered here; the 24h window makes them unfit for a synchronous gate. That
logic belongs in a DI-stubbed proxy integration test under `tests/test_litellm/proxy/`
where the provider client is injected to return `completed` deterministically.
165 changes: 165 additions & 0 deletions tests/e2e/batches/batch_client.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,165 @@
"""Client for the batches e2e suite: file upload/download and the batch
operations (create / retrieve / cancel / list) over the shared Gateway.
`create_batch` returns the raw HTTP outcome (StreamingResponse) so a 403 model
access denial and a provider-native batch body both surface; the test parses
BatchObject from the body. A `provider` arg routes a call to /{provider}/v1/...,
which the provider-fallback scenario needs (its ids are raw, not model-encoded).
The request/response models are co-located here because only this suite uses them.
"""

from __future__ import annotations

from dataclasses import dataclass

from pydantic import BaseModel

from e2e_gateway import Gateway, build_gateway
from e2e_http import (
FileUploadForm,
NoBody,
Result,
StreamingResponse,
UnknownApiError,
)


class FileObject(BaseModel):
id: str
object: str | None = None
purpose: str | None = None
bytes: int | None = None
status: str | None = None
created_at: int | None = None


class BatchObject(BaseModel):
id: str
object: str | None = None
status: str
endpoint: str | None = None
input_file_id: str | None = None
output_file_id: str | None = None
completion_window: str | None = None
created_at: int | None = None
model: str | None = None


class BatchList(BaseModel):
object: str | None = None
data: list[BatchObject] = []


class FileDeleteResponse(BaseModel):
id: str
object: str | None = None
deleted: bool


class BatchCreateBody(BaseModel):
input_file_id: str
endpoint: str = "/v1/chat/completions"
completion_window: str = "24h"
model: str | None = None


class ModelQuery(BaseModel):
model: str | None = None


def is_model_access_denied(resp: StreamingResponse) -> bool:
"""True if the proxy rejected the call because the key may not access the model."""
return resp.status_code == 403 and "key_model_access_denied" in resp.body


def is_result_access_denied[R: BaseModel](result: Result[R]) -> bool:
match result:
case UnknownApiError(status_code=403, body=body):
return "key_model_access_denied" in body
case _:
return False


@dataclass(frozen=True, slots=True)
class BatchClient:
gateway: Gateway

def upload_file(
self,
*,
content: bytes,
form: FileUploadForm,
key: str,
model: str | None = None,
provider: str | None = None,
) -> Result[FileObject]:
return self.gateway.transport.upload(
_files_path(provider),
headers=self.gateway.transport.bearer(key),
form=form,
filename="batch_input.jsonl",
content=content,
params=ModelQuery(model=model),
response_type=FileObject,
)

def create_batch(
self, *, body: BatchCreateBody, key: str, provider: str | None = None
) -> StreamingResponse:
return self.gateway.transport.send(
_batches_path(provider),
headers=self.gateway.transport.bearer(key),
json=body,
)

def retrieve_batch(
self, batch_id: str, *, key: str, provider: str | None = None
) -> Result[BatchObject]:
return self.gateway.transport.get(
f"{_batches_path(provider)}/{batch_id}",
headers=self.gateway.transport.bearer(key),
params=NoBody(),
response_type=BatchObject,
)

def cancel_batch(
self, batch_id: str, *, key: str, provider: str | None = None
) -> Result[BatchObject]:
return self.gateway.transport.post(
f"{_batches_path(provider)}/{batch_id}/cancel",
headers=self.gateway.transport.bearer(key),
json=NoBody(),
response_type=BatchObject,
)

def list_batches(
self, *, key: str, provider: str | None = None
) -> Result[BatchList]:
return self.gateway.transport.get(
_batches_path(provider),
headers=self.gateway.transport.bearer(key),
params=NoBody(),
response_type=BatchList,
)

def delete_file(
self, file_id: str, *, key: str, provider: str | None = None
) -> Result[FileDeleteResponse]:
return self.gateway.transport.delete(
f"{_files_path(provider)}/{file_id}",
headers=self.gateway.transport.bearer(key),
json=NoBody(),
response_type=FileDeleteResponse,
)


def _files_path(provider: str | None) -> str:
return f"/{provider}/v1/files" if provider else "/v1/files"


def _batches_path(provider: str | None) -> str:
return f"/{provider}/v1/batches" if provider else "/v1/batches"


def build_client() -> BatchClient:
return BatchClient(gateway=build_gateway())
Loading
Loading