Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion tests/e2e/CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ When contributing to this directory, please first discuss the change you wish to

## Setup

The suites run against a live proxy, so bring one up first by running the litellm proxy locally. Point it at a config that prewires the example models the suites use (`gpt-5.5`, `claude-haiku-4-5`, `gemini-2.5-flash`, `openai-text-embedding-3-small`) with keys from your `.env`, and enables prompt storage, a redis cache, and the fast budget rescheduler the quota suites rely on. If your test needs another model, a pricing override, or a guardrail declared up front, add it to that config and read it back in the test rather than hardcoding values
The suites run against a live proxy, so bring one up first by running the litellm proxy locally. Point it at a config that prewires the example models the suites use (`gpt-5.5`, `claude-haiku-4-5`, `gemini-2.5-flash`, `openai-text-embedding-3-small`) with keys from your `.env`, and enables prompt storage, a redis cache, the fast budget rescheduler the quota suites rely on, and `router_settings.optional_pre_call_checks: ["prompt_caching"]`, which the router suite's prompt-cache affinity test reads back from `GET /router/settings` and fails without. If your test needs another model, a pricing override, or a guardrail declared up front, add it to that config and read it back in the test rather than hardcoding values

## Running the tests locally

Expand Down
50 changes: 23 additions & 27 deletions tests/e2e/e2e_http.py
Original file line number Diff line number Diff line change
Expand Up @@ -104,12 +104,7 @@ class UnknownApiError(BaseModel):


type Result[R: BaseModel] = (
Success[R]
| NetworkError
| UnauthorizedError
| RateLimitedError
| ValidationError
| UnknownApiError
Success[R] | NetworkError | UnauthorizedError | RateLimitedError | ValidationError | UnknownApiError
)


Expand Down Expand Up @@ -222,22 +217,19 @@ def require_successful_call(result: StreamingResponse) -> None:
if the proxy can't make a call it's expected to, the test must fail."""
if result.ok:
return
pytest.fail(
f"upstream call failed (status {result.status_code}); body={result.body[:300]}"
)
pytest.fail(f"upstream call failed (status {result.status_code}); body={result.body[:300]}")


def assert_client_error(result: StreamingResponse, context: str) -> None:
assert 400 <= result.status_code < 500, (
f"{context}: expected 4xx, got {result.status_code}: {result.body[:300]}"
)
assert 400 <= result.status_code < 500, f"{context}: expected 4xx, got {result.status_code}: {result.body[:300]}"


def assert_auth_denied(result: StreamingResponse, context: str) -> None:
assert result.status_code in (401, 403), (
f"{context}: expected 401/403, got {result.status_code}: {result.body[:300]}"
)


def _headers(headers: BaseModel) -> dict[str, str]:
dumped: dict[str, object] = headers.model_dump(by_alias=True, exclude_none=True)
return {key: str(value) for key, value in dumped.items()}
Expand Down Expand Up @@ -293,9 +285,7 @@ def request_with_retry[T: RetryableResponse](
return issue()


def _classify[R: BaseModel](
resp: requests.Response, response_type: type[R]
) -> Result[R]:
def _classify[R: BaseModel](resp: requests.Response, response_type: type[R]) -> Result[R]:
if resp.status_code == 401:
return UnauthorizedError(body=resp.text)
if resp.status_code == 429:
Expand Down Expand Up @@ -440,9 +430,7 @@ def put[R: BaseModel](
return _classify(resp, response_type)


def probe(
url: URL, *, headers: BaseModel, params: BaseModel, timeout: float = 30.0
) -> ProbeResult:
def probe(url: URL, *, headers: BaseModel, params: BaseModel, timeout: float = 30.0) -> ProbeResult:
try:
resp = request_with_retry(
lambda: requests.get(
Expand Down Expand Up @@ -547,9 +535,7 @@ def send(
return _streaming_outcome(resp, stream)


def stream(
url: URL, *, headers: BaseModel, json: BaseModel, timeout: float = 60.0
) -> StreamingResponse:
def stream(url: URL, *, headers: BaseModel, json: BaseModel, timeout: float = 60.0) -> StreamingResponse:
"""Streaming (SSE) call: consumes the stream counting events, and captures the
x-litellm-call-id + content-type headers. Body is elided."""
return send(url, headers=headers, json=json, stream=True, timeout=timeout)
Expand Down Expand Up @@ -638,9 +624,7 @@ def stream_binary(
)


def download(
url: URL, *, headers: BaseModel, timeout: float = 60.0
) -> StreamingResponse:
def download(url: URL, *, headers: BaseModel, timeout: float = 60.0) -> StreamingResponse:
"""Raw GET for file content (/v1/files/{id}/content): provider-native bytes, no
schema. Returns the decoded body and the x-litellm-call-id header."""
try:
Expand Down Expand Up @@ -677,9 +661,7 @@ def forward(
mode. No retries, no redirects, no schema: the proxy owns retry policy and
the recorded bundle must hold exactly what the provider returned."""
try:
resp = requests.request(
method, url, headers=headers, data=body, timeout=timeout, allow_redirects=False
)
resp = requests.request(method, url, headers=headers, data=body, timeout=timeout, allow_redirects=False)
except requests.RequestException as exc:
return NetworkError(message=str(exc))
return RawResponse(
Expand Down Expand Up @@ -739,6 +721,20 @@ def _stream_steps(resp: requests.Response) -> Generator[StreamStep, None, None]:
resp.close()


def open_stream(url: URL, *, headers: BaseModel, json: BaseModel, timeout: float = 60.0) -> StreamHead | NetworkError:
"""POST a streaming request and return the moment its response head arrives,
leaving the body unread behind ``StreamHead.steps``. For a test that must keep
one request in flight while it sends others: the head carries the routing
headers (x-litellm-model-id), and draining ``steps`` ends the request."""
return forward_stream(
"POST",
str(url),
headers={**_headers(headers), "Content-Type": "application/json"},
body=json.model_dump_json(by_alias=True, exclude_none=True).encode(),
timeout=timeout,
)


def forward_stream(
method: str,
url: str,
Expand Down
44 changes: 40 additions & 4 deletions tests/e2e/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,7 @@ class ImageUrl(BaseModel):
class TextContentPart(BaseModel):
type: str = "text"
text: str
cache_control: "CacheControl | None" = None


class ImageContentPart(BaseModel):
Expand Down Expand Up @@ -266,21 +267,41 @@ class ChatBody(BaseModel):
cache: dict[str, bool] | None = {"no-cache": True}


RoutingStrategy = Literal[
"simple-shuffle",
"least-busy",
"usage-based-routing-v2",
"latency-based-routing",
"cost-based-routing",
]


class RouterSettingsOverride(BaseModel):
"""Router settings a test scopes below the global config: sent per request as
`router_settings_override` in a /chat/completions body (the reliability suite's
fallback and retry knobs) or stored on a key as `router_settings` at
/key/generate (the auto-router suite's tag filtering switch). Serialized
exclude_none, so an override sets only the knobs a test exercises. Each
fallbacks map is model_name -> the ordered fallback model_names to try."""
fallback, retry, routing-strategy, and deadline knobs) or stored on a key as
`router_settings` at /key/generate (the auto-router suite's tag filtering
switch). Serialized exclude_none, so an override sets only the knobs a test
exercises. Each fallbacks map is model_name -> the ordered fallback model_names
to try; `timeout` is the per-request upstream deadline in seconds."""

fallbacks: list[dict[str, list[str]]] | None = None
context_window_fallbacks: list[dict[str, list[str]]] | None = None
content_policy_fallbacks: list[dict[str, list[str]]] | None = None
num_retries: int | None = None
routing_strategy: RoutingStrategy | None = None
timeout: float | None = None
enable_tag_filtering: bool | None = None


class DeploymentExtraBody(BaseModel):
"""`litellm_params.extra_body` of a deployment whose upstream is another LiteLLM
proxy: forwarded verbatim in every request body, so the inner proxy honors the
same per-request router knobs an end user could send it."""

router_settings_override: RouterSettingsOverride | None = None


class ReliabilityChatBody(ChatBody):
"""A /chat/completions body carrying a per-request router_settings_override.
Composes ChatBody (no attribute repetition) and adds the override; serialized
Expand Down Expand Up @@ -713,6 +734,18 @@ class ModelInfoResponse(BaseModel):
data: list[ModelInfoEntry] = []


class RouterCurrentValues(BaseModel):
"""The `current_values` block of GET /router/settings: the router knobs the
proxy is actually running with (only the ones a test preconditions on)."""

routing_strategy: str | None = None
optional_pre_call_checks: list[str] = []


class RouterSettingsResponse(BaseModel):
current_values: RouterCurrentValues


class CostMapEntry(BaseModel):
model_config = ConfigDict(extra="ignore")
litellm_provider: str | None = None
Expand Down Expand Up @@ -805,6 +838,9 @@ class LiteLLMParamsBody(BaseModel):
tags: list[str] | None = None
mock_response: str | None = None
timeout: float | None = None
max_retries: int | None = None
cooldown_time: float | None = None
extra_body: DeploymentExtraBody | None = None
tpm: int | None = None
weight: int | None = None

Expand Down
30 changes: 18 additions & 12 deletions tests/e2e/proxy_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,8 @@
ModelUpdateBody,
OcrBody,
OcrResponse,
RouterCurrentValues,
RouterSettingsResponse,
SpendLogRow,
SpendLogs,
SpendLogsPage,
Expand Down Expand Up @@ -133,9 +135,7 @@ def await_servable(
last_result: Result[ModelsListResponse] | None = None
while True:
t = now()
phase_deadline = (
started + timeout if first_seen_at is None else first_seen_at + db_sync_seconds
)
phase_deadline = started + timeout if first_seen_at is None else first_seen_at + db_sync_seconds
remaining = phase_deadline - t
if remaining <= 0:
if (
Expand All @@ -148,9 +148,7 @@ def await_servable(

poll_timeout = min(request_timeout, remaining)
last_result = list_models(poll_timeout)
listed = isinstance(last_result, Success) and any(
entry.id == model_name for entry in last_result.data.data
)
listed = isinstance(last_result, Success) and any(entry.id == model_name for entry in last_result.data.data)
t = now()
if not listed:
first_seen_at = None
Expand All @@ -163,9 +161,7 @@ def await_servable(
elif t - first_seen_at >= db_sync_seconds:
return Servable()

phase_deadline = (
started + timeout if first_seen_at is None else first_seen_at + db_sync_seconds
)
phase_deadline = started + timeout if first_seen_at is None else first_seen_at + db_sync_seconds
wait = min(interval, phase_deadline - now())
if wait > 0:
sleep(wait)
Expand Down Expand Up @@ -253,6 +249,18 @@ def model_info(self) -> list[ModelInfoEntry]:
)
).data

def router_settings(self) -> RouterCurrentValues:
"""The router knobs the proxy is running with, for a test whose behavior
needs one of them switched on in the proxy config."""
return unwrap(
self.transport.get(
"/router/settings",

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Split route misclassified In split deployments, this management request goes to the data plane and fails before prompt-cache routing is tested

Note: If this suggestion doesn't match your team's coding style, reply to this and let me know. I'll remember it for next time!

headers=self.transport.master,
params=NoBody(),
response_type=RouterSettingsResponse,
)
).current_values

def model_cost_map(self) -> dict[str, CostMapEntry]:
return unwrap(
self.transport.get(
Expand All @@ -271,9 +279,7 @@ def list_files(self, key: str) -> Result[FileListResponse]:
response_type=FileListResponse,
)

def list_fine_tuning_jobs(
self, key: str, params: FineTuningJobsParams
) -> Result[FineTuningJobsResponse]:
def list_fine_tuning_jobs(self, key: str, params: FineTuningJobsParams) -> Result[FineTuningJobsResponse]:
return self.transport.get(
"/v1/fine_tuning/jobs",
headers=self.transport.bearer(key),
Expand Down
Loading
Loading