diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index b155a1a7024..ae2bf3ce754 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -6,7 +6,8 @@ Code-style rules for writing tests under `tests/e2e/`. The harness already encod Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family or behavior area. If you add a new folder, you must add a line here describing what kind of tests belong in it, so the layout stays self-describing. `gateway/` is the exception: it holds proxy configuration only and never tests -- `llm_translation/` - LLM endpoint and provider-translation behavior: passthrough, custom pricing, OCR +- `llm_translation/` - LLM endpoint and provider-translation behavior: passthrough, custom pricing, OCR, and the non-chat inference endpoints (`/v1/responses`, `/v1/messages`, `/embeddings`, `/v1/rerank`, `/v1/audio/speech`, `/v1/images/generations`), each against a deployment the test creates via `/model/new` and deletes on teardown +- `access_control/` - the gateway's authorization and error-shape contract: per-key model allow-lists, route-group permissions (`allowed_routes`), and unknown-model validation - `embeddings/` - the `/embeddings` endpoint across providers - `batches/` - the `/batches` endpoint (placeholder until the first test lands) - `realtime/` - realtime websocket sessions, including the pipecat audio path diff --git a/tests/e2e/access_control/access_control_client.py b/tests/e2e/access_control/access_control_client.py new file mode 100644 index 00000000000..d7bc9c280aa --- /dev/null +++ b/tests/e2e/access_control/access_control_client.py @@ -0,0 +1,56 @@ +"""Client for the access-control e2e suite.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from e2e_gateway import Gateway, build_gateway +from e2e_http import StreamingResponse +from models import ( + ChatBody, + ChatMessage, + KeyGenerateBody, + LiteLLMParamsBody, + ModelInfoBody, + ModelNewBody, +) + +MODEL_ACCESS_DENIED_MARKER = "key_model_access_denied" +ROUTE_NOT_ALLOWED_MARKER = "not allowed to call this route" + + +@dataclass(frozen=True, slots=True) +class AccessControlClient: + gateway: Gateway + + def llm_only_key(self) -> str: + return self.gateway.generate_key( + KeyGenerateBody(models=[], allowed_routes=["llm_api_routes"]) + ) + + def delete_key(self, key: str) -> None: + self.gateway.delete_key(key) + + def chat_status(self, key: str, model: str, content: str) -> StreamingResponse: + return self.gateway.transport.send( + "/chat/completions", + headers=self.gateway.transport.bearer(key), + json=ChatBody( + model=model, messages=[ChatMessage(role="user", content=content)] + ), + ) + + def create_model_status(self, key: str, model_name: str) -> StreamingResponse: + return self.gateway.transport.send( + "/model/new", + headers=self.gateway.transport.bearer(key), + json=ModelNewBody( + model_name=model_name, + litellm_params=LiteLLMParamsBody(model="openai/gpt-4o-mini"), + model_info=ModelInfoBody(id=model_name), + ), + ) + + +def build_client() -> AccessControlClient: + return AccessControlClient(gateway=build_gateway()) diff --git a/tests/e2e/access_control/conftest.py b/tests/e2e/access_control/conftest.py new file mode 100644 index 00000000000..9f4a00fe06f --- /dev/null +++ b/tests/e2e/access_control/conftest.py @@ -0,0 +1,10 @@ +"""Access-control suite client fixture; lifecycle/skip/marker live in the parent conftest.""" + +import pytest + +from access_control_client import AccessControlClient, build_client + + +@pytest.fixture(scope="session") +def client() -> AccessControlClient: + return build_client() diff --git a/tests/e2e/access_control/test_access_control_e2e.py b/tests/e2e/access_control/test_access_control_e2e.py new file mode 100644 index 00000000000..ce649fa2400 --- /dev/null +++ b/tests/e2e/access_control/test_access_control_e2e.py @@ -0,0 +1,83 @@ +"""Live e2e: the gateway's authorization and error-shape contract. + +A virtual key may only call models in its allow-list and route groups in its +allowed_routes; both denials are a 403 raised before any provider is touched. A +syntactically valid request naming a non-existent model is a 400 with a JSON body, +never forwarded and never a 5xx. Migrated from +litellm-regression-tests/tests/test_access_control.py: the source asserted 401 for +the disallowed-model case against an older proxy, but the current contract +(auth_checks.py) is a 403 key_model_access_denied, and the unknown-route check is +replaced by a stronger route-permission check (an llm-only key rejected from a +management route). +""" + +from __future__ import annotations + +import json + +import pytest + +from access_control_client import ( + AccessControlClient, + MODEL_ACCESS_DENIED_MARKER, + ROUTE_NOT_ALLOWED_MARKER, +) +from e2e_config import unique_marker +from lifecycle import ResourceManager + +pytestmark = pytest.mark.e2e + +ALLOWED_MODEL = "gemini-2.5-flash" +DISALLOWED_MODEL = "gpt-5.5" + + +def _is_json(body: str) -> bool: + try: + json.loads(body) + return True + except ValueError: + return False + + +class TestAccessControl: + def test_disallowed_model_is_denied_403( + self, client: AccessControlClient, resources: ResourceManager + ) -> None: + key = resources.key(models=[ALLOWED_MODEL]) + result = client.chat_status( + key, DISALLOWED_MODEL, f"capital of France? {unique_marker()}" + ) + assert result.status_code == 403, ( + f"key limited to {ALLOWED_MODEL!r} calling {DISALLOWED_MODEL!r} must be " + f"denied 403, got {result.status_code}: {result.body[:300]}" + ) + assert MODEL_ACCESS_DENIED_MARKER in result.body, ( + f"403 body must be a model-access denial, got: {result.body[:300]}" + ) + + def test_llm_only_key_forbidden_from_management_route_403( + self, client: AccessControlClient, resources: ResourceManager + ) -> None: + key = client.llm_only_key() + resources.defer(lambda: client.delete_key(key)) + result = client.create_model_status(key, f"e2e-forbidden-{unique_marker()}") + assert result.status_code == 403, ( + f"llm-only key calling a management route must be denied 403, got " + f"{result.status_code}: {result.body[:300]}" + ) + assert ROUTE_NOT_ALLOWED_MARKER in result.body, ( + f"403 body must be a route-permission denial, got: {result.body[:300]}" + ) + + def test_unknown_model_returns_400( + self, client: AccessControlClient, resources: ResourceManager + ) -> None: + key = resources.key() + result = client.chat_status( + key, f"nonexistent-model-{unique_marker()}", "hi this is a test" + ) + assert result.status_code == 400, ( + f"unknown model must be rejected 400 before forwarding, got " + f"{result.status_code}: {result.body[:300]}" + ) + assert _is_json(result.body), f"400 body must be valid JSON: {result.body[:300]}" diff --git a/tests/e2e/llm_translation/conftest.py b/tests/e2e/llm_translation/conftest.py index fbf008cf085..014e056d06d 100644 --- a/tests/e2e/llm_translation/conftest.py +++ b/tests/e2e/llm_translation/conftest.py @@ -7,9 +7,15 @@ import pytest +from endpoints_client import EndpointsClient, build_endpoints_client from passthrough_client import PassthroughClient, build_client @pytest.fixture(scope="session") def client() -> PassthroughClient: return build_client() + + +@pytest.fixture(scope="session") +def endpoints_client() -> EndpointsClient: + return build_endpoints_client() diff --git a/tests/e2e/llm_translation/endpoints_client.py b/tests/e2e/llm_translation/endpoints_client.py new file mode 100644 index 00000000000..508aa3e9fc6 --- /dev/null +++ b/tests/e2e/llm_translation/endpoints_client.py @@ -0,0 +1,219 @@ +"""Client for the non-chat inference endpoints (responses, messages, rerank, +embeddings, audio speech, image generation). + +Each test registers the deployment it needs through /model/new (deleted on +teardown), so nothing is hardcoded into the gateway config, then drives the +endpoint with `send` and parses the provider-native body with a suite-local model +so the assertion is on real content, not just a 200. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from pydantic import BaseModel + +from e2e_gateway import Gateway, build_gateway +from e2e_http import NoBody, StreamingResponse, is_ok, unwrap +from models import ( + ChatMessage, + LiteLLMParamsBody, + ModelDeleteBody, + ModelInfoBody, + ModelNewBody, + ModelNewResponse, +) + + +class ResponsesRequest(BaseModel): + model: str + input: str + instructions: str | None = None + + +class MessagesRequest(BaseModel): + model: str + max_tokens: int + messages: list[ChatMessage] + + +class EmbeddingsRequest(BaseModel): + model: str + input: str + + +class RerankRequest(BaseModel): + model: str + query: str + documents: list[str] + top_n: int + + +class SpeechRequest(BaseModel): + model: str + input: str + voice: str + + +class ImageRequest(BaseModel): + model: str + prompt: str + n: int = 1 + size: str = "1024x1024" + + +class ResponsesOutputContent(BaseModel): + type: str | None = None + text: str | None = None + + +class ResponsesOutputItem(BaseModel): + type: str | None = None + content: list[ResponsesOutputContent] = [] + + +class ResponsesResult(BaseModel): + id: str | None = None + status: str | None = None + model: str | None = None + output: list[ResponsesOutputItem] = [] + + @property + def text(self) -> str: + return "".join( + content.text or "" for item in self.output for content in item.content + ) + + +class AnthropicContentBlock(BaseModel): + type: str | None = None + text: str | None = None + + +class MessagesResult(BaseModel): + id: str | None = None + role: str | None = None + model: str | None = None + content: list[AnthropicContentBlock] = [] + + @property + def text(self) -> str: + return "".join(block.text or "" for block in self.content) + + +class EmbeddingItem(BaseModel): + embedding: list[float] = [] + + +class EmbeddingsResult(BaseModel): + data: list[EmbeddingItem] = [] + + @property + def first_vector(self) -> tuple[float, ...]: + return tuple(self.data[0].embedding) if self.data else () + + +class RerankItem(BaseModel): + index: int | None = None + relevance_score: float | None = None + + +class RerankResult(BaseModel): + results: list[RerankItem] = [] + + +class ImageItem(BaseModel): + url: str | None = None + b64_json: str | None = None + + +class ImagesResult(BaseModel): + data: list[ImageItem] = [] + + +@dataclass(frozen=True, slots=True) +class EndpointsClient: + gateway: Gateway + + def create_model(self, model_name: str, litellm_params: LiteLLMParamsBody) -> str: + """Register a deployment under `model_name` (id == model_name) and return the + model_id. add_deployment runs synchronously in /model/new, so the model is + callable as soon as this returns.""" + return unwrap( + self.gateway.transport.post( + "/model/new", + headers=self.gateway.transport.master, + json=ModelNewBody( + model_name=model_name, + litellm_params=litellm_params, + model_info=ModelInfoBody(id=model_name), + ), + response_type=ModelNewResponse, + ) + ).model_id + + def delete_model(self, model_id: str) -> None: + result = self.gateway.transport.post( + "/model/delete", + headers=self.gateway.transport.master, + json=ModelDeleteBody(id=model_id), + response_type=NoBody, + ) + if not is_ok(result): + import warnings + warnings.warn(f"delete_model({model_id!r}) failed: {result}", stacklevel=2) + + def _send(self, path: str, key: str, body: BaseModel) -> StreamingResponse: + return self.gateway.transport.send( + path, headers=self.gateway.transport.bearer(key), json=body + ) + + def responses(self, key: str, model: str, text: str) -> StreamingResponse: + return self._send( + "/v1/responses", + key, + ResponsesRequest( + model=model, input=text, instructions="You are a helpful assistant" + ), + ) + + def messages( + self, key: str, model: str, text: str, *, max_tokens: int = 64 + ) -> StreamingResponse: + return self._send( + "/v1/messages", + key, + MessagesRequest( + model=model, + max_tokens=max_tokens, + messages=[ChatMessage(role="user", content=text)], + ), + ) + + def embeddings(self, key: str, model: str, text: str) -> StreamingResponse: + return self._send("/embeddings", key, EmbeddingsRequest(model=model, input=text)) + + def rerank( + self, key: str, model: str, query: str, documents: list[str], top_n: int + ) -> StreamingResponse: + return self._send( + "/v1/rerank", + key, + RerankRequest(model=model, query=query, documents=documents, top_n=top_n), + ) + + def audio_speech( + self, key: str, model: str, text: str, *, voice: str = "alloy" + ) -> StreamingResponse: + return self._send( + "/v1/audio/speech", key, SpeechRequest(model=model, input=text, voice=voice) + ) + + def images(self, key: str, model: str, prompt: str) -> StreamingResponse: + return self._send( + "/v1/images/generations", key, ImageRequest(model=model, prompt=prompt) + ) + + +def build_endpoints_client() -> EndpointsClient: + return EndpointsClient(gateway=build_gateway()) diff --git a/tests/e2e/llm_translation/test_audio_speech_e2e.py b/tests/e2e/llm_translation/test_audio_speech_e2e.py new file mode 100644 index 00000000000..f7a04d94cb3 --- /dev/null +++ b/tests/e2e/llm_translation/test_audio_speech_e2e.py @@ -0,0 +1,40 @@ +"""Live e2e: POST /v1/audio/speech returns audio. + +Registers an OpenAI text-to-speech deployment at runtime and asserts the response +is an audio body (binary, not JSON). Migrated from +litellm-regression-tests/tests/test_inference_endpoints.py. +""" + +from __future__ import annotations + +import pytest + +from e2e_config import unique_marker +from e2e_http import require_successful_call +from endpoints_client import EndpointsClient +from lifecycle import ResourceManager +from models import LiteLLMParamsBody + +pytestmark = pytest.mark.e2e + + +class TestAudioSpeech: + def test_audio_speech_returns_audio( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + model = f"e2e-speech-{unique_marker()}" + model_id = endpoints_client.create_model( + model, + LiteLLMParamsBody( + model="openai/gpt-4o-mini-tts", api_key="os.environ/OPENAI_API_KEY" + ), + ) + resources.defer(lambda: endpoints_client.delete_model(model_id)) + key = resources.key() + + result = endpoints_client.audio_speech(key, model, "Hello!") + require_successful_call(result) + assert "audio" in (result.content_type or ""), ( + f"/audio/speech content-type is not audio: {result.content_type!r}" + ) + assert result.body, "/audio/speech returned an empty body" diff --git a/tests/e2e/llm_translation/test_custom_pricing_e2e.py b/tests/e2e/llm_translation/test_custom_pricing_e2e.py index 58faab61aac..7894b447be9 100644 --- a/tests/e2e/llm_translation/test_custom_pricing_e2e.py +++ b/tests/e2e/llm_translation/test_custom_pricing_e2e.py @@ -1,52 +1,48 @@ -"""Live e2e: a model's custom per-token pricing is loaded, billed, and isolated. +"""Live e2e: a deployment's custom per-token pricing is loaded, billed, and isolated. -The gateway config declares ``custom-priced-flash`` (gemini-2.5-flash underneath) -with input/output rates deliberately far above the canonical gemini price, read -back here from the same config file. Three behaviors are checked independently: +Each test registers the deployment(s) it needs through /model/new (deleted on +teardown) instead of relying on a statically configured model, so the check is +self-contained and never inherits pricing another suite or a stale config left on +the shared proxy. custom-priced-flash sets input/output rates deliberately far +above the canonical gemini price; the isolation sibling shares the same +gemini/gemini-2.5-flash backend but sets no override. Three behaviors are checked +independently: - billing: a real call's logged cost breakdown charges input and output tokens at the custom rates, each component checked separately (a base-rate bill lands ~100x lower; a swapped input/output rate passes a total-only check but not this) -- reporting: /model/info surfaces those rates for the model -- isolation: gemini-2.5-flash shares the same underlying gemini/gemini-2.5-flash - but sets no override, so it must keep its own price; an override that leaks into - the shared cost map misprices it. A regression that reintroduces that leak makes - the sibling's rate match the custom one and fails the isolation check. +- reporting: /model/info surfaces those rates for the deployment +- isolation: the sibling keeps its own price; an override that leaks into the + shared backend cost map (LIT-3897) misprices it, making the sibling's rate match + the custom one and failing the isolation check """ import time -from dataclasses import dataclass -from pathlib import Path import pytest -import yaml from pydantic import BaseModel, RootModel from e2e_config import unique_marker +from e2e_gateway import Gateway from e2e_http import Success, unwrap -from models import ChatBody, ChatMessage, CustomPricing, ModelInfoEntry, SpendLogsParams -from passthrough_client import PassthroughClient +from endpoints_client import EndpointsClient +from lifecycle import ResourceManager +from models import ( + ChatBody, + ChatMessage, + LiteLLMParamsBody, + ModelInfoEntry, + SpendLogsParams, +) pytestmark = pytest.mark.e2e -CUSTOM_MODEL = "custom-priced-flash" -BASE_MODEL = "gemini-2.5-flash" -CONFIG_PATH = Path(__file__).resolve().parents[1] / "gateway" / "litellm-config.yml" - - -@dataclass(frozen=True, slots=True) -class _Rates: - input_per_token: float - output_per_token: float - - -class _ConfiguredModel(BaseModel): - model_name: str - litellm_params: CustomPricing - - -class _GatewayConfig(BaseModel): - model_list: list[_ConfiguredModel] +BACKEND_MODEL = "gemini/gemini-2.5-flash" +GEMINI_API_KEY = "os.environ/GEMINI_API_KEY" +# Deliberately ~100x above canonical gemini-2.5-flash (input 3e-7 / output 2.5e-6) +# so an override that is ignored or under-applied bills at the base rate and fails. +CUSTOM_INPUT_RATE = 5e-05 +CUSTOM_OUTPUT_RATE = 1e-04 class _CostBreakdown(BaseModel): @@ -74,40 +70,60 @@ def _approx_equal(actual: float, expected: float) -> bool: return abs(actual - expected) <= max(1e-9, abs(expected) * 1e-2) -def _configured_pricing(model_name: str) -> _Rates: - """The custom rates declared for `model_name` in the gateway config the proxy - runs with - the source of truth the billed and reported prices are checked - against.""" - config = _GatewayConfig.model_validate(yaml.safe_load(CONFIG_PATH.read_text())) - for entry in config.model_list: - if entry.model_name == model_name: - pricing = entry.litellm_params - assert pricing.input_cost_per_token and pricing.output_cost_per_token, ( - f"{model_name} declares no custom per-token rates in {CONFIG_PATH.name}" - ) - return _Rates(pricing.input_cost_per_token, pricing.output_cost_per_token) - pytest.fail(f"{model_name} not found in {CONFIG_PATH.name}") +def _provision( + endpoints_client: EndpointsClient, + resources: ResourceManager, + prefix: str, + *, + input_cost_per_token: float | None, + output_cost_per_token: float | None, +) -> str: + """Register a fresh gemini/gemini-2.5-flash deployment (deleted on teardown) and + return its model name. With the cost fields set the deployment carries a custom + pricing override; with them None it is a plain sibling on the same backend. The + marker keeps the name unique so concurrent runs on the shared proxy never + collide.""" + model_name = f"{prefix}-{unique_marker()}" + model_id = endpoints_client.create_model( + model_name, + LiteLLMParamsBody( + model=BACKEND_MODEL, + api_key=GEMINI_API_KEY, + input_cost_per_token=input_cost_per_token, + output_cost_per_token=output_cost_per_token, + ), + ) + resources.defer(lambda: endpoints_client.delete_model(model_id)) + return model_name + + +def _provision_custom_priced( + endpoints_client: EndpointsClient, resources: ResourceManager +) -> str: + return _provision( + endpoints_client, + resources, + "custom-priced-flash", + input_cost_per_token=CUSTOM_INPUT_RATE, + output_cost_per_token=CUSTOM_OUTPUT_RATE, + ) -def _model_info_entry( - entries: list[ModelInfoEntry], model_name: str -) -> ModelInfoEntry: +def _model_info_entry(entries: list[ModelInfoEntry], model_name: str) -> ModelInfoEntry: for entry in entries: if entry.model_name == model_name: return entry pytest.fail(f"{model_name} absent from /model/info; the override did not load") -def _poll_breakdown_row( - client: PassthroughClient, key: str, response_id: str | None -) -> _SpendRow: +def _poll_breakdown_row(gateway: Gateway, key: str, response_id: str | None) -> _SpendRow: """Poll /spend/logs until the call's row lands with a cost breakdown (rows flush ~60s behind the call via proxy_batch_write_at).""" - deadline = time.monotonic() + client.gateway.poll_timeout + deadline = time.monotonic() + gateway.poll_timeout while time.monotonic() < deadline: - result = client.gateway.transport.get( + result = gateway.transport.get( "/spend/logs", - headers=client.gateway.transport.master, + headers=gateway.transport.master, params=SpendLogsParams(api_key=key), response_type=_SpendRows, ) @@ -128,85 +144,99 @@ def _poll_breakdown_row( return row if priced and response_id is None: return priced[0] - time.sleep(client.gateway.poll_interval) + time.sleep(gateway.poll_interval) pytest.fail("no spend row with a cost breakdown landed before the deadline") -def test_custom_pricing_is_billed_at_configured_rate( - client: PassthroughClient, scoped_key: str -) -> None: - rates = _configured_pricing(CUSTOM_MODEL) - - chat = unwrap( - client.gateway.chat( - scoped_key, - ChatBody( - model=CUSTOM_MODEL, - messages=[ - ChatMessage( - role="user", content=f"reply with one word {unique_marker()}" - ) - ], - max_tokens=16, - ), +class TestCustomPricing: + def test_custom_pricing_is_billed_at_configured_rate( + self, + endpoints_client: EndpointsClient, + resources: ResourceManager, + scoped_key: str, + ) -> None: + model = _provision_custom_priced(endpoints_client, resources) + + chat = unwrap( + endpoints_client.gateway.chat( + scoped_key, + ChatBody( + model=model, + messages=[ + ChatMessage( + role="user", content=f"reply with one word {unique_marker()}" + ) + ], + max_tokens=16, + ), + ) ) - ) - - row = _poll_breakdown_row(client, scoped_key, chat.id) - assert row.metadata and row.metadata.cost_breakdown # guaranteed by the poll - breakdown = row.metadata.cost_breakdown - prompt = row.prompt_tokens or 0 - completion = row.completion_tokens or 0 - assert prompt > 0 and completion > 0, f"call tokens not logged on the row: {row}" + row = _poll_breakdown_row(endpoints_client.gateway, scoped_key, chat.id) + assert row.metadata and row.metadata.cost_breakdown # guaranteed by the poll + breakdown = row.metadata.cost_breakdown - input_cost = breakdown.input_cost - output_cost = breakdown.output_cost - assert input_cost is not None and output_cost is not None, ( - f"row cost breakdown missing input/output cost: {breakdown}" - ) - assert _approx_equal(input_cost, prompt * rates.input_per_token), ( - f"input_cost {input_cost} != {prompt} tokens * {rates.input_per_token} " - f"= {prompt * rates.input_per_token}" - ) - assert _approx_equal(output_cost, completion * rates.output_per_token), ( - f"output_cost {output_cost} != {completion} tokens * {rates.output_per_token} " - f"= {completion * rates.output_per_token}" - ) + prompt = row.prompt_tokens or 0 + completion = row.completion_tokens or 0 + assert prompt > 0 and completion > 0, f"call tokens not logged on the row: {row}" + input_cost = breakdown.input_cost + output_cost = breakdown.output_cost + assert input_cost is not None and output_cost is not None, ( + f"row cost breakdown missing input/output cost: {breakdown}" + ) + assert _approx_equal(input_cost, prompt * CUSTOM_INPUT_RATE), ( + f"input_cost {input_cost} != {prompt} tokens * {CUSTOM_INPUT_RATE} " + f"= {prompt * CUSTOM_INPUT_RATE}" + ) + assert _approx_equal(output_cost, completion * CUSTOM_OUTPUT_RATE), ( + f"output_cost {output_cost} != {completion} tokens * {CUSTOM_OUTPUT_RATE} " + f"= {completion * CUSTOM_OUTPUT_RATE}" + ) -def test_model_info_reports_custom_pricing(client: PassthroughClient) -> None: - rates = _configured_pricing(CUSTOM_MODEL) - entry = _model_info_entry(client.gateway.model_info(), CUSTOM_MODEL) + def test_model_info_reports_custom_pricing( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + model = _provision_custom_priced(endpoints_client, resources) + entry = _model_info_entry(endpoints_client.gateway.model_info(), model) - assert entry.litellm_params.input_cost_per_token == rates.input_per_token, ( - f"/model/info litellm_params input rate " - f"{entry.litellm_params.input_cost_per_token} != configured " - f"{rates.input_per_token}" - ) - assert entry.litellm_params.output_cost_per_token == rates.output_per_token, ( - f"/model/info litellm_params output rate " - f"{entry.litellm_params.output_cost_per_token} != configured " - f"{rates.output_per_token}" - ) + assert entry.litellm_params.input_cost_per_token == CUSTOM_INPUT_RATE, ( + f"/model/info litellm_params input rate " + f"{entry.litellm_params.input_cost_per_token} != configured {CUSTOM_INPUT_RATE}" + ) + assert entry.litellm_params.output_cost_per_token == CUSTOM_OUTPUT_RATE, ( + f"/model/info litellm_params output rate " + f"{entry.litellm_params.output_cost_per_token} != configured {CUSTOM_OUTPUT_RATE}" + ) + def test_custom_pricing_is_isolated_from_sibling_deployment( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + # Register the override first so its rate is in the backend cost map before + # the sibling resolves; a leak (LIT-3897) would then poison the sibling. + custom = _provision_custom_priced(endpoints_client, resources) + sibling = _provision( + endpoints_client, + resources, + "base-flash", + input_cost_per_token=None, + output_cost_per_token=None, + ) -def test_custom_pricing_is_isolated_from_sibling_deployment( - client: PassthroughClient, -) -> None: - entries = {entry.model_name: entry for entry in client.gateway.model_info()} - custom = entries.get(CUSTOM_MODEL) - base = entries.get(BASE_MODEL) - assert custom is not None, f"{CUSTOM_MODEL} absent from /model/info" - assert base is not None, f"{BASE_MODEL} absent from /model/info" - - # custom-priced-flash overrides pricing; gemini-2.5-flash shares the same - # underlying gemini/gemini-2.5-flash but sets no override, so it must keep its - # own price. Equal rates mean the override leaked into the shared cost map. - assert ( - base.model_info.input_cost_per_token != custom.model_info.input_cost_per_token - ), ( - f"{BASE_MODEL} input rate {base.model_info.input_cost_per_token} matches " - f"{CUSTOM_MODEL}'s override {custom.model_info.input_cost_per_token}; " - f"per-deployment custom pricing is not isolated" - ) + entries = {entry.model_name: entry for entry in endpoints_client.gateway.model_info()} + custom_entry = entries.get(custom) + sibling_entry = entries.get(sibling) + assert custom_entry is not None, f"{custom} absent from /model/info" + assert sibling_entry is not None, f"{sibling} absent from /model/info" + + # custom-priced-flash overrides pricing; the sibling shares the same + # gemini/gemini-2.5-flash backend but sets no override, so it must keep its + # own price. Equal rates mean the override leaked into the shared cost map. + assert ( + sibling_entry.model_info.input_cost_per_token + != custom_entry.model_info.input_cost_per_token + ), ( + f"{sibling} input rate {sibling_entry.model_info.input_cost_per_token} matches " + f"{custom}'s override {custom_entry.model_info.input_cost_per_token}; " + f"per-deployment custom pricing is not isolated" + ) diff --git a/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py b/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py new file mode 100644 index 00000000000..56f2de8bd4f --- /dev/null +++ b/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py @@ -0,0 +1,42 @@ +"""Live e2e: POST /embeddings returns a real vector. + +Registers an OpenAI embedding deployment at runtime and asserts a non-empty, +non-zero vector came back. Migrated from +litellm-regression-tests/tests/test_inference_endpoints.py; the LIT-3167 guard in +tests/e2e/embeddings/ covers the Gemini embedding path. +""" + +from __future__ import annotations + +import pytest + +from e2e_config import unique_marker +from e2e_http import require_successful_call +from endpoints_client import EmbeddingsResult, EndpointsClient +from lifecycle import ResourceManager +from models import LiteLLMParamsBody + +pytestmark = pytest.mark.e2e + + +class TestEmbeddingsEndpoint: + def test_embeddings_returns_vector( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + model = f"e2e-embeddings-{unique_marker()}" + model_id = endpoints_client.create_model( + model, + LiteLLMParamsBody( + model="openai/text-embedding-3-small", api_key="os.environ/OPENAI_API_KEY" + ), + ) + resources.defer(lambda: endpoints_client.delete_model(model_id)) + key = resources.key() + + result = endpoints_client.embeddings(key, model, "Say this is a test!") + require_successful_call(result) + parsed = EmbeddingsResult.model_validate_json(result.body) + assert parsed.first_vector, f"/embeddings returned no vector: {result.body[:300]}" + assert any(component != 0.0 for component in parsed.first_vector), ( + f"embedding vector is all zeros: {result.body[:300]}" + ) diff --git a/tests/e2e/llm_translation/test_image_generation_e2e.py b/tests/e2e/llm_translation/test_image_generation_e2e.py new file mode 100644 index 00000000000..4d2211f3be4 --- /dev/null +++ b/tests/e2e/llm_translation/test_image_generation_e2e.py @@ -0,0 +1,42 @@ +"""Live e2e: POST /v1/images/generations returns an image. + +Registers an OpenAI image deployment at runtime and asserts the response carries a +generated image (url or base64). Migrated from +litellm-regression-tests/tests/test_inference_endpoints.py. +""" + +from __future__ import annotations + +import pytest + +from e2e_config import unique_marker +from e2e_http import require_successful_call +from endpoints_client import EndpointsClient, ImagesResult +from lifecycle import ResourceManager +from models import LiteLLMParamsBody + +pytestmark = pytest.mark.e2e + + +class TestImageGeneration: + def test_image_generation_returns_image( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + model = f"e2e-image-{unique_marker()}" + model_id = endpoints_client.create_model( + model, + LiteLLMParamsBody( + model="openai/gpt-image-1-mini", api_key="os.environ/OPENAI_API_KEY" + ), + ) + resources.defer(lambda: endpoints_client.delete_model(model_id)) + key = resources.key() + + result = endpoints_client.images(key, model, "Draw a cute cat") + require_successful_call(result) + parsed = ImagesResult.model_validate_json(result.body) + assert parsed.data, f"/images/generations returned no data: {result.body[:300]}" + first = parsed.data[0] + assert first.b64_json or first.url, ( + f"generated image has neither b64_json nor url: {result.body[:300]}" + ) diff --git a/tests/e2e/llm_translation/test_messages_e2e.py b/tests/e2e/llm_translation/test_messages_e2e.py new file mode 100644 index 00000000000..b0a48f22118 --- /dev/null +++ b/tests/e2e/llm_translation/test_messages_e2e.py @@ -0,0 +1,39 @@ +"""Live e2e: POST /v1/messages (Anthropic Messages API) returns a real completion. + +Registers an Anthropic deployment at runtime, drives the Messages endpoint through +the gateway, and asserts an assistant message with text came back. Migrated from +litellm-regression-tests/tests/test_inference_endpoints.py. +""" + +from __future__ import annotations + +import pytest + +from e2e_config import unique_marker +from e2e_http import require_successful_call +from endpoints_client import EndpointsClient, MessagesResult +from lifecycle import ResourceManager +from models import LiteLLMParamsBody + +pytestmark = pytest.mark.e2e + + +class TestAnthropicMessages: + def test_messages_returns_completion( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + model = f"e2e-messages-{unique_marker()}" + model_id = endpoints_client.create_model( + model, + LiteLLMParamsBody( + model="anthropic/claude-haiku-4-5", api_key="os.environ/ANTHROPIC_API_KEY" + ), + ) + resources.defer(lambda: endpoints_client.delete_model(model_id)) + key = resources.key() + + result = endpoints_client.messages(key, model, "reply with one word") + require_successful_call(result) + parsed = MessagesResult.model_validate_json(result.body) + assert parsed.role == "assistant", f"unexpected role: {result.body[:300]}" + assert parsed.text.strip(), f"/v1/messages returned no text: {result.body[:300]}" diff --git a/tests/e2e/llm_translation/test_rerank_e2e.py b/tests/e2e/llm_translation/test_rerank_e2e.py new file mode 100644 index 00000000000..4b30ac1ea5c --- /dev/null +++ b/tests/e2e/llm_translation/test_rerank_e2e.py @@ -0,0 +1,49 @@ +"""Live e2e: POST /v1/rerank ranks documents by relevance. + +Registers a Cohere rerank deployment at runtime and asserts the endpoint returns +scored results within the requested top_n. Migrated from +litellm-regression-tests/tests/test_inference_endpoints.py. +""" + +from __future__ import annotations + +import pytest + +from e2e_config import unique_marker +from e2e_http import require_successful_call +from endpoints_client import EndpointsClient, RerankResult +from lifecycle import ResourceManager +from models import LiteLLMParamsBody + +pytestmark = pytest.mark.e2e + +DOCUMENTS = [ + "Carson City is the capital city of the American state of Nevada.", + "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean.", + "Washington, D.C. is the capital of the United States.", + "Capital punishment has existed in the United States since before it was a country.", +] + + +class TestRerank: + def test_rerank_scores_top_n( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + model = f"e2e-rerank-{unique_marker()}" + model_id = endpoints_client.create_model( + model, + LiteLLMParamsBody(model="cohere/rerank-v3.5", api_key="os.environ/COHERE_API_KEY"), + ) + resources.defer(lambda: endpoints_client.delete_model(model_id)) + key = resources.key() + + result = endpoints_client.rerank( + key, model, "What is the capital of the United States?", DOCUMENTS, top_n=3 + ) + require_successful_call(result) + parsed = RerankResult.model_validate_json(result.body) + assert parsed.results, f"/rerank returned no results: {result.body[:300]}" + assert len(parsed.results) <= 3, f"top_n=3 not honored: {result.body[:300]}" + assert parsed.results[0].relevance_score is not None, ( + f"top rerank result has no relevance_score: {result.body[:300]}" + ) diff --git a/tests/e2e/llm_translation/test_responses_e2e.py b/tests/e2e/llm_translation/test_responses_e2e.py new file mode 100644 index 00000000000..743de79880f --- /dev/null +++ b/tests/e2e/llm_translation/test_responses_e2e.py @@ -0,0 +1,36 @@ +"""Live e2e: POST /v1/responses returns a real completion. + +Registers an OpenAI deployment at runtime, drives the Responses API through the +gateway, and asserts output text came back. Migrated from +litellm-regression-tests/tests/test_inference_endpoints.py. +""" + +from __future__ import annotations + +import pytest + +from e2e_config import unique_marker +from e2e_http import require_successful_call +from endpoints_client import EndpointsClient, ResponsesResult +from lifecycle import ResourceManager +from models import LiteLLMParamsBody + +pytestmark = pytest.mark.e2e + + +class TestResponses: + def test_responses_returns_completion( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + model = f"e2e-responses-{unique_marker()}" + model_id = endpoints_client.create_model( + model, + LiteLLMParamsBody(model="openai/gpt-4o-mini", api_key="os.environ/OPENAI_API_KEY"), + ) + resources.defer(lambda: endpoints_client.delete_model(model_id)) + key = resources.key() + + result = endpoints_client.responses(key, model, "reply with one word") + require_successful_call(result) + parsed = ResponsesResult.model_validate_json(result.body) + assert parsed.text.strip(), f"/responses returned no output text: {result.body[:300]}" diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 0d0352df9af..f45ffef6ed7 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -34,6 +34,7 @@ class KeyGenerateBody(BaseModel): budget_limits: list[BudgetWindow] | None = None tpm_limit: int | None = None rpm_limit: int | None = None + allowed_routes: list[str] | None = None class KeyGenerateResponse(BaseModel): @@ -263,3 +264,41 @@ class ModelInfoEntry(BaseModel): class ModelInfoResponse(BaseModel): data: list[ModelInfoEntry] = [] + + +# ---------- model management ---------- + + +class LiteLLMParamsBody(BaseModel): + """POST /model/new litellm_params: `model` is the only required field; `api_key` + et al may be an `os.environ/FOO` reference the proxy resolves at call time. + `input_cost_per_token`/`output_cost_per_token` register a per-deployment custom + pricing override; left None (and dropped from the body) the deployment keeps the + backend's canonical rate.""" + + model: str + api_key: str | None = None + api_base: str | None = None + api_version: str | None = None + input_cost_per_token: float | None = None + output_cost_per_token: float | None = None + + +class ModelInfoBody(BaseModel): + id: str + + +class ModelNewBody(BaseModel): + model_config = ConfigDict(protected_namespaces=()) + model_name: str + litellm_params: LiteLLMParamsBody + model_info: ModelInfoBody + + +class ModelNewResponse(BaseModel): + model_config = ConfigDict(protected_namespaces=()) + model_id: str + + +class ModelDeleteBody(BaseModel): + id: str