diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index 5716155361d0..958db6f9f5bc 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -95,9 +95,12 @@ def _prepare_ocr_request( api_key=api_key, ) + _is_doc_intelligence = custom_llm_provider == "azure_ai" and ( + "doc-intelligence" in model.lower() or "documentintelligence" in model.lower() + ) if dynamic_api_key: api_key = dynamic_api_key - if dynamic_api_base: + if dynamic_api_base and not _is_doc_intelligence: api_base = dynamic_api_base ocr_provider_config = ProviderConfigManager.get_provider_ocr_config( diff --git a/tests/e2e/e2e_gateway.py b/tests/e2e/e2e_gateway.py index 055d06d1c799..d7b639d85915 100644 --- a/tests/e2e/e2e_gateway.py +++ b/tests/e2e/e2e_gateway.py @@ -28,6 +28,9 @@ CustomerDeleteBody, EmbedBody, EmbedResponse, + FileListResponse, + FineTuningJobsParams, + FineTuningJobsResponse, KeyDeleteBody, KeyGenerateBody, KeyGenerateResponse, @@ -119,6 +122,24 @@ def model_info(self) -> list[ModelInfoEntry]: ) ).data + def list_files(self, key: str) -> Result[FileListResponse]: + return self.transport.get( + "/v1/files", + headers=self.transport.bearer(key), + params=NoBody(), + response_type=FileListResponse, + ) + + def list_fine_tuning_jobs( + self, key: str, params: FineTuningJobsParams + ) -> Result[FineTuningJobsResponse]: + return self.transport.get( + "/v1/fine_tuning/jobs", + headers=self.transport.bearer(key), + params=params, + response_type=FineTuningJobsResponse, + ) + def create_model( self, model_name: str, diff --git a/tests/e2e/llm_translation/realtime/REALTIME_COVERAGE_MATRIX.md b/tests/e2e/llm_translation/realtime/REALTIME_COVERAGE_MATRIX.md new file mode 100644 index 000000000000..57963fcbc62d --- /dev/null +++ b/tests/e2e/llm_translation/realtime/REALTIME_COVERAGE_MATRIX.md @@ -0,0 +1,62 @@ +# Realtime e2e coverage + +Live tests for the proxy realtime websocket endpoint (`/v1/realtime`). One +GA-speaking websocket client drives every provider; the proxy normalizes each +provider's stream into the OpenAI GA event schema, so the same assertions hold +across providers and only the model alias changes. + +## What is asserted + +For each configured provider, `test_text_conversation` checks the session +lifecycle (`session.created`, then `session.update` echoed by `session.updated`), +the canonical response sequence (`response.created`, `response.output_item.added`, +through `response.done`), that the streamed deltas reconstruct a non-empty +transcript, and that `response.done` carries normalized usage. + +`test_tool_call_round_trip` checks the full tool path: the model emits a +normalized `response.function_call_arguments.done` with valid JSON arguments and +a matching `function_call` output item, the test sends a `function_call_output` +back, and the follow-up response incorporates the result (the temperature 72 +appears). + +`test_realtime_pipecat_e2e` is a realism layer that drives the same providers +through pipecat's GA `OpenAIRealtimeLLMService` (base_url pointed at the proxy) +rather than speaking the protocol by hand. Its assertions are coarse (the tool +callback fired, assistant text was produced); the raw-websocket suite is the +source of truth. It skips unless `pipecat-ai` is installed +(`uv pip install "pipecat-ai[openai]"`). + +## Provisioning + +The suite registers every provider's realtime deployment through `/model/new` at +session start (the `realtime_models` fixture) and deletes them on teardown, so it +never depends on a static or misconfigured gateway `model_list`. Each deployment +is created with `model_info.mode: realtime` and marker-unique names, and its +`litellm_params` point the credentials at `os.environ/*` refs the gateway resolves +at call time. The provider table below is the source of truth; edit `PROVIDERS` in +`realtime_client.py` to change a model or add one. + +| provider | model alias | upstream model | +|----------|-------------|----------------| +| openai | `openai-realtime` | `openai/gpt-realtime-2` | +| azure | `azure-realtime` | `azure/gpt-realtime-2` (GA protocol) | +| gemini | `gemini-realtime` | `gemini/gemini-3.1-flash-live-preview` | +| vertex_ai | `vertex-realtime` | `vertex_ai/gemini-live-2.5-flash-preview-native-audio-09-2025` | + +Every provider is provisioned and asserted; the suite never skips a provider. Per +`tests/e2e/CLAUDE.md` the only sanctioned skip is the whole-suite proxy-liveness +skip, so a provider whose credentials or upstream realtime model are missing on the +gateway is a hard failure, not a skip. Give the gateway each provider's credentials +to turn its tests green. + +## Running + +Start a proxy with the provider keys set in its environment (the suite registers +the deployments itself), then + +``` +uv run pytest tests/e2e/llm_translation/realtime/ -v +``` + +The whole suite skips only when no proxy answers `GET /health/liveliness` at +`LITELLM_PROXY_URL` (default `http://localhost:4000`). diff --git a/tests/e2e/llm_translation/realtime/conftest.py b/tests/e2e/llm_translation/realtime/conftest.py new file mode 100644 index 000000000000..15cd789664eb --- /dev/null +++ b/tests/e2e/llm_translation/realtime/conftest.py @@ -0,0 +1,37 @@ +"""Realtime suite's `client` and `realtime_models` fixtures. + +The shared lifecycle (resources/scoped_key), proxy liveness skip, and e2e marker +live in the parent tests/e2e/conftest.py. RealtimeClient holds the shared Gateway, +so the `resources` fixture cleans up keys this suite creates. + +`realtime_models` registers every provider's realtime deployment through /model/new +at session start and deletes them at teardown, so the suite provisions the models it +uses through the management endpoints instead of depending on a static (or +misconfigured) gateway model_list. +""" + +from collections.abc import Iterator + +import pytest + +from realtime_client import PROVIDERS, RealtimeClient, build_client + + +@pytest.fixture(scope="session") +def client() -> RealtimeClient: + return build_client() + + +@pytest.fixture(scope="session") +def realtime_models(client: RealtimeClient) -> Iterator[dict[str, str]]: + """Provision each provider's realtime deployment via /model/new and yield a + provider-id -> model-name map the tests connect with; delete them on teardown. + Every provider is provisioned (never skipped): a provider whose credentials or + upstream model are missing on the gateway hard-fails its test, per the suite's + fail-on-behavior contract in tests/e2e/CLAUDE.md.""" + records = tuple((provider.id, *client.provision(provider)) for provider in PROVIDERS) + try: + yield {provider_id: model_name for provider_id, model_name, _ in records} + finally: + for _, _, model_id in records: + client.gateway.delete_model(model_id) diff --git a/tests/e2e/realtime/fixtures/weather_question_24k.wav b/tests/e2e/llm_translation/realtime/fixtures/weather_question_24k.wav similarity index 100% rename from tests/e2e/realtime/fixtures/weather_question_24k.wav rename to tests/e2e/llm_translation/realtime/fixtures/weather_question_24k.wav diff --git a/tests/e2e/realtime/pipecat_service.py b/tests/e2e/llm_translation/realtime/pipecat_service.py similarity index 100% rename from tests/e2e/realtime/pipecat_service.py rename to tests/e2e/llm_translation/realtime/pipecat_service.py diff --git a/tests/e2e/realtime/realtime_client.py b/tests/e2e/llm_translation/realtime/realtime_client.py similarity index 71% rename from tests/e2e/realtime/realtime_client.py rename to tests/e2e/llm_translation/realtime/realtime_client.py index 07dfd76108bb..756b68441be7 100644 --- a/tests/e2e/realtime/realtime_client.py +++ b/tests/e2e/llm_translation/realtime/realtime_client.py @@ -11,19 +11,19 @@ from __future__ import annotations import time -from collections.abc import Generator +from collections.abc import Generator, Mapping from contextlib import contextmanager from dataclasses import dataclass from typing import Any, TypeVar from urllib.parse import urlencode -import pytest from pydantic import BaseModel, ConfigDict from websockets.sync.client import connect from websockets.sync.connection import Connection -from e2e_config import PROXY_BASE_URL +from e2e_config import PROXY_BASE_URL, unique_marker from e2e_gateway import Gateway, build_gateway +from models import LiteLLMParamsBody _M = TypeVar("_M", bound=BaseModel) @@ -41,25 +41,68 @@ def realtime_ws_url(model: str) -> str: @dataclass(frozen=True, slots=True) class RealtimeProvider: + """A realtime provider the suite exercises. `litellm_params` is the deployment + the suite registers through /model/new (the gateway resolves the os.environ/* + credential refs), so the suite is self-contained and never depends on a static + gateway model_list. Every provider here is provisioned and asserted: per + tests/e2e/CLAUDE.md the suite never skips a provider, so a provider whose + credentials or upstream realtime model are missing on the gateway is a hard + failure, not a skip.""" + id: str - model: str + alias: str + litellm_params: LiteLLMParamsBody PROVIDERS = ( - RealtimeProvider("openai", "openai-realtime"), - RealtimeProvider("azure", "azure-realtime"), - RealtimeProvider("gemini", "gemini-realtime"), - RealtimeProvider("vertex_ai", "vertex-realtime"), - # RealtimeProvider("bedrock", "bedrock-realtime"), # TODO: Enable this when Bedrock is passing - RealtimeProvider("xai", "xai-realtime"), + RealtimeProvider( + "openai", + "openai-realtime", + LiteLLMParamsBody( + model="openai/gpt-realtime-2", + api_key="os.environ/OPENAI_API_KEY", + ), + ), + RealtimeProvider( + "azure", + "azure-realtime", + LiteLLMParamsBody( + model="azure/gpt-realtime", + api_key="os.environ/AZURE_API_KEY", + api_version="2025-08-28", + realtime_protocol="GA", + ), + ), + RealtimeProvider( + "gemini", + "gemini-realtime", + LiteLLMParamsBody( + model="gemini/gemini-3.1-flash-live-preview", + api_key="os.environ/GEMINI_API_KEY", + ), + ), + RealtimeProvider( + "vertex_ai", + "vertex-realtime", + LiteLLMParamsBody( + model="vertex_ai/gemini-live-2.5-flash-preview-native-audio-09-2025", + vertex_location="us-central1", + vertex_credentials="os.environ/VERTEXAI_CREDENTIALS", + ), + ), + # RealtimeProvider("bedrock", "bedrock-realtime", ...) # TODO: Enable when Bedrock is passing ) -def skip_if_unconfigured( - provider: RealtimeProvider, configured: frozenset[str] -) -> None: - if provider.model not in configured: - pytest.skip(f"{provider.model} not configured on proxy") +def realtime_model(provider: RealtimeProvider, provisioned: Mapping[str, str]) -> str: + """Return the provisioned deployment name for this provider. Every provider in + PROVIDERS is provisioned at session start, so a missing entry is a harness bug, + never an environment skip - the suite hard-fails instead (see tests/e2e/CLAUDE.md).""" + model = provisioned.get(provider.id) + assert model is not None, ( + f"{provider.id} was not provisioned; the realtime_models fixture is broken" + ) + return model # ---- sent events ------------------------------------------------------- @@ -280,12 +323,17 @@ def collect_until( class RealtimeClient: gateway: Gateway - def configured_models(self) -> frozenset[str]: - return frozenset( - entry.model_name - for entry in self.gateway.model_info() - if entry.model_info.mode == "realtime" + def provision(self, provider: RealtimeProvider) -> tuple[str, str]: + """Register this provider's realtime deployment through /model/new and return + (model_name, model_id). The name is marker-unique so it never collides with a + same-named deployment already on the shared proxy, and mode=realtime makes it + show up as a realtime model on /model/info. add_deployment runs synchronously, + so the deployment is connectable as soon as this returns.""" + model_name = f"{provider.alias}-{unique_marker()}" + model_id = self.gateway.create_model( + model_name, provider.litellm_params, mode="realtime" ) + return model_name, model_id @contextmanager def connect( diff --git a/tests/e2e/realtime/test_realtime_e2e.py b/tests/e2e/llm_translation/realtime/test_realtime_e2e.py similarity index 92% rename from tests/e2e/realtime/test_realtime_e2e.py rename to tests/e2e/llm_translation/realtime/test_realtime_e2e.py index 013569001412..6aaffdd208ef 100644 --- a/tests/e2e/realtime/test_realtime_e2e.py +++ b/tests/e2e/llm_translation/realtime/test_realtime_e2e.py @@ -31,7 +31,7 @@ SessionUpdate, function_call_item, parse_last, - skip_if_unconfigured, + realtime_model, transcript, user_message, ) @@ -62,12 +62,12 @@ class WeatherResult(BaseModel): def test_text_conversation( client: RealtimeClient, scoped_key: str, - configured_models: frozenset[str], + realtime_models: dict[str, str], provider: RealtimeProvider, ) -> None: - skip_if_unconfigured(provider, configured_models) + model = realtime_model(provider, realtime_models) - with client.connect(key=scoped_key, model=provider.model) as session: + with client.connect(key=scoped_key, model=model) as session: created = session.collect_until("session.created", timeout=20) assert created[-1].type == "session.created" @@ -99,12 +99,12 @@ def test_text_conversation( def test_tool_call_round_trip( client: RealtimeClient, scoped_key: str, - configured_models: frozenset[str], + realtime_models: dict[str, str], provider: RealtimeProvider, ) -> None: - skip_if_unconfigured(provider, configured_models) + model = realtime_model(provider, realtime_models) - with client.connect(key=scoped_key, model=provider.model) as session: + with client.connect(key=scoped_key, model=model) as session: session.collect_until("session.created", timeout=20) session.send( SessionUpdate( diff --git a/tests/e2e/realtime/test_realtime_pipecat_audio_e2e.py b/tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e.py similarity index 95% rename from tests/e2e/realtime/test_realtime_pipecat_audio_e2e.py rename to tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e.py index d9d7c744f66a..31c038b4e029 100644 --- a/tests/e2e/realtime/test_realtime_pipecat_audio_e2e.py +++ b/tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e.py @@ -31,7 +31,7 @@ PROVIDERS, RealtimeProvider, _ws_base_url, - skip_if_unconfigured, + realtime_model, ) pytestmark = pytest.mark.e2e @@ -189,13 +189,13 @@ async def get_weather(params: FunctionCallParams) -> None: @pytest.mark.parametrize("provider", PROVIDER_PARAMS) def test_pipecat_server_vad( scoped_key: str, - configured_models: frozenset[str], + realtime_models: dict[str, str], provider: RealtimeProvider, ) -> None: """Session is configured with server-VAD; bot must respond to a text prompt.""" - skip_if_unconfigured(provider, configured_models) + model = realtime_model(provider, realtime_models) - tool_called, got_text, _ = asyncio.run(_run_pipeline(scoped_key, provider.model)) + tool_called, got_text, _ = asyncio.run(_run_pipeline(scoped_key, model)) assert tool_called, "get_weather tool was not invoked" assert got_text, "no assistant text frames produced" @@ -204,16 +204,16 @@ def test_pipecat_server_vad( @pytest.mark.parametrize("provider", PROVIDER_PARAMS) def test_pipecat_audio_output( scoped_key: str, - configured_models: frozenset[str], + realtime_models: dict[str, str], provider: RealtimeProvider, ) -> None: """Bot must produce at least one non-empty TTS audio frame.""" - skip_if_unconfigured(provider, configured_models) + model = realtime_model(provider, realtime_models) _, got_text, audio_bytes = asyncio.run( _run_pipeline( scoped_key, - provider.model, + model, prompt="Say hello in one short sentence.", timeout=30.0, ) @@ -328,7 +328,7 @@ async def _run() -> None: @pytest.mark.parametrize("provider", PROVIDER_PARAMS) def test_pipecat_server_vad_audio_input( scoped_key: str, - configured_models: frozenset[str], + realtime_models: dict[str, str], provider: RealtimeProvider, ) -> None: """Stream a real PCM16 WAV fixture; server VAD must detect speech end and respond. @@ -337,12 +337,11 @@ def test_pipecat_server_vad_audio_input( → server-VAD turn detection → response.create (auto) → assistant reply. No LLMRunFrame is sent — the response must be triggered entirely by VAD. """ - if not WEATHER_WAV.exists(): - pytest.skip(f"audio fixture not found: {WEATHER_WAV}") - skip_if_unconfigured(provider, configured_models) + assert WEATHER_WAV.exists(), f"audio fixture not found: {WEATHER_WAV}" + model = realtime_model(provider, realtime_models) got_text, audio_bytes = asyncio.run( - _run_audio_input_pipeline(scoped_key, provider.model) + _run_audio_input_pipeline(scoped_key, model) ) assert got_text, "server VAD did not trigger a response (no assistant text)" diff --git a/tests/e2e/realtime/test_realtime_pipecat_e2e.py b/tests/e2e/llm_translation/realtime/test_realtime_pipecat_e2e.py similarity index 97% rename from tests/e2e/realtime/test_realtime_pipecat_e2e.py rename to tests/e2e/llm_translation/realtime/test_realtime_pipecat_e2e.py index 1068c54fdec5..799958ef4e3c 100644 --- a/tests/e2e/realtime/test_realtime_pipecat_e2e.py +++ b/tests/e2e/llm_translation/realtime/test_realtime_pipecat_e2e.py @@ -29,7 +29,7 @@ PROVIDERS, RealtimeProvider, _ws_base_url, - skip_if_unconfigured, + realtime_model, ) pytestmark = pytest.mark.e2e @@ -123,12 +123,12 @@ async def get_weather(params: FunctionCallParams) -> None: @pytest.mark.parametrize("provider", PROVIDER_PARAMS) def test_pipecat_tool_smoke( scoped_key: str, - configured_models: frozenset[str], + realtime_models: dict[str, str], provider: RealtimeProvider, ) -> None: - skip_if_unconfigured(provider, configured_models) + model = realtime_model(provider, realtime_models) - tool_called, produced_text = asyncio.run(_run_pipeline(scoped_key, provider.model)) + tool_called, produced_text = asyncio.run(_run_pipeline(scoped_key, model)) assert tool_called, "pipecat did not invoke the get_weather callback" assert produced_text, "pipecat produced no assistant text frames" diff --git a/tests/e2e/llm_translation/test_ocr_rust_e2e.py b/tests/e2e/llm_translation/test_ocr_rust_e2e.py index 361bb5126a78..921010e5eaef 100644 --- a/tests/e2e/llm_translation/test_ocr_rust_e2e.py +++ b/tests/e2e/llm_translation/test_ocr_rust_e2e.py @@ -86,16 +86,18 @@ def litellm_params(self) -> LiteLLMParamsBody: @dataclass(frozen=True, slots=True) class VertexOcr: + """Vertex AI OCR (Mistral publisher). Only the location (not a secret) is set; + the project and credentials are left unset so the gateway resolves VERTEXAI_PROJECT + and VERTEXAI_CREDENTIALS from its own environment by name, keeping every secret on + the gateway like the azure_ai cases above. This is deliberate: the OCR path reads + vertex_project verbatim from litellm_params and never unwraps an `os.environ/*` + ref, so passing one would put the literal string in the request URL.""" + model: str location: str def litellm_params(self) -> LiteLLMParamsBody: - return LiteLLMParamsBody( - model=self.model, - vertex_project="os.environ/VERTEXAI_PROJECT", - vertex_location=self.location, - vertex_credentials="os.environ/VERTEXAI_CREDENTIALS", - ) + return LiteLLMParamsBody(model=self.model, vertex_location=self.location) @dataclass(frozen=True, slots=True) @@ -113,7 +115,7 @@ class _OcrCase: ), _OcrCase( "azure-ai", - AzureAiOcr("azure_ai/mistral-document-ai-2505"), + AzureAiOcr("azure_ai/mistral-document-ai-2512"), OcrDocument(type="document_url", document_url=TEST_PDF_URL), ), _OcrCase( @@ -126,11 +128,6 @@ class _OcrCase: VertexOcr("vertex_ai/mistral-ocr-2505", "us-central1"), OcrDocument(type="document_url", document_url=TEST_PDF_URL), ), - _OcrCase( - "vertex-deepseek", - VertexOcr("vertex_ai/deepseek-ocr-maas", "global"), - OcrDocument(type="image_url", image_url=TEST_IMAGE_URL), - ), ) _CASE_IDS = tuple(case.suffix for case in RUST_OCR_CASES) @@ -155,3 +152,5 @@ def test_rust_ocr_response( response = unwrap(endpoints_client.gateway.ocr(key, OcrBody(model=model, document=case.document))) _assert_ocr_document(response) + + diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 0490db286ea9..f5467854b117 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -376,6 +376,7 @@ class LiteLLMParamsBody(BaseModel): api_key: str | None = None api_base: str | None = None api_version: str | None = None + realtime_protocol: str | None = None aws_region_name: str | None = None vertex_project: str | None = None vertex_location: str | None = None diff --git a/tests/e2e/realtime/REALTIME_COVERAGE_MATRIX.md b/tests/e2e/realtime/REALTIME_COVERAGE_MATRIX.md deleted file mode 100644 index 8624475d0dea..000000000000 --- a/tests/e2e/realtime/REALTIME_COVERAGE_MATRIX.md +++ /dev/null @@ -1,55 +0,0 @@ -# Realtime e2e coverage - -Live tests for the proxy realtime websocket endpoint (`/v1/realtime`). One -GA-speaking websocket client drives every provider; the proxy normalizes each -provider's stream into the OpenAI GA event schema, so the same assertions hold -across providers and only the model alias changes. - -## What is asserted - -For each configured provider, `test_text_conversation` checks the session -lifecycle (`session.created`, then `session.update` echoed by `session.updated`), -the canonical response sequence (`response.created`, `response.output_item.added`, -through `response.done`), that the streamed deltas reconstruct a non-empty -transcript, and that `response.done` carries normalized usage. - -`test_tool_call_round_trip` checks the full tool path: the model emits a -normalized `response.function_call_arguments.done` with valid JSON arguments and -a matching `function_call` output item, the test sends a `function_call_output` -back, and the follow-up response incorporates the result (the temperature 72 -appears). - -`test_realtime_pipecat_e2e` is a realism layer that drives the same providers -through pipecat's GA `OpenAIRealtimeLLMService` (base_url pointed at the proxy) -rather than speaking the protocol by hand. Its assertions are coarse (the tool -callback fired, assistant text was produced); the raw-websocket suite is the -source of truth. It skips unless `pipecat-ai` is installed -(`uv pip install "pipecat-ai[openai]"`). - -## Provider status - -| provider | model alias | status | -|----------|-------------|--------| -| openai | `openai-realtime` | covered (in gateway config) | -| gemini | `gemini-realtime` | covered (in gateway config; needs Gemini Live API access) | -| azure | `azure-realtime` | gap: add to gateway config + AZURE creds | -| vertex_ai | `vertex-realtime` | gap: add to gateway config + Vertex creds | -| bedrock | `bedrock-realtime` | gap: add to gateway config + AWS creds | -| xai | `xai-realtime` | gap: add to gateway config + XAI_API_KEY | - -A provider whose alias is not present in the proxy's `/model/info` skips (skip on -environment). To enable one, add a `model_info.mode: realtime` entry under that -alias to `tests/e2e/gateway/litellm-config.yml` and give the proxy the -provider's credentials; the test then runs with no code change. - -## Running - -Start a proxy with the gateway config and the provider keys set in its -environment, then - -``` -uv run pytest tests/e2e/realtime/ -v -``` - -Tests skip when no proxy answers `GET /health/liveliness` at `LITELLM_PROXY_URL` -(default `http://localhost:4000`). diff --git a/tests/e2e/realtime/conftest.py b/tests/e2e/realtime/conftest.py deleted file mode 100644 index 4a5c4837a1a0..000000000000 --- a/tests/e2e/realtime/conftest.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Realtime suite's `client` fixture. - -The shared lifecycle (resources/scoped_key), proxy liveness skip, and e2e marker -live in the parent tests/e2e/conftest.py. RealtimeClient holds the shared -Gateway, so the `resources` fixture cleans up keys this suite creates. -""" - -import pytest - -from realtime_client import RealtimeClient, build_client - - -@pytest.fixture(scope="session") -def client() -> RealtimeClient: - return build_client() - - -@pytest.fixture(scope="session") -def configured_models(client: RealtimeClient) -> frozenset[str]: - return client.configured_models()