From 0b1d328dc2af3dbb3594fe41a42e07badbd771eb Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Wed, 8 Jul 2026 16:00:06 -0700 Subject: [PATCH 1/6] fix: rust ocr tests finally pass --- litellm/ocr/main.py | 7 +++--- tests/e2e/docker-compose.yml | 10 ++++---- .../e2e/llm_translation/test_ocr_rust_e2e.py | 23 +++++++++---------- tests/e2e/models.py | 1 + 4 files changed, 22 insertions(+), 19 deletions(-) diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index 5716155361d0..d9f72c63dbec 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -188,12 +188,13 @@ def _rust_bridge_api_base( prepared_request: _PreparedOCRRequest, resolve_secret: Callable[[str], str | None], ) -> str | None: - if prepared_request.api_base is not None: - return prepared_request.api_base if prepared_request.custom_llm_provider == "azure_ai": model = prepared_request.model.lower() if "doc-intelligence" in model or "documentintelligence" in model: - return resolve_secret("AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT") + return resolve_secret("AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT") or prepared_request.api_base + if prepared_request.api_base is not None: + return prepared_request.api_base + if prepared_request.custom_llm_provider == "azure_ai": return resolve_secret("AZURE_AI_API_BASE") return None diff --git a/tests/e2e/docker-compose.yml b/tests/e2e/docker-compose.yml index cdf5d6cbf6ff..a3a7f6a3c1d6 100644 --- a/tests/e2e/docker-compose.yml +++ b/tests/e2e/docker-compose.yml @@ -1,10 +1,10 @@ -# local setup to run e2e tests +# local setup to run e2e tests, do not commit when you make a PR configs: litellm_config: content: | general_settings: - master_key: os.environ/LITELLM_MASTER_KEY - database_url: os.environ/DATABASE_URL + master_key: sk-1234 + database_url: postgres://litellm@litellm:5432/litellm store_prompts_in_spend_logs: true litellm_settings: @@ -48,7 +48,7 @@ configs: services: litellm: - image: ghcr.io/berriai/litellm:main-latest + image: docker.litellm.ai/berriai/litellm:1.92.0-rc.1 depends_on: db: condition: service_healthy @@ -60,6 +60,8 @@ services: DATABASE_URL: postgresql://litellm:litellm@db:5432/litellm UI_USERNAME: admin UI_PASSWORD: sk-1234 + LITELLM_USE_RUST_OCR: 1 + # TEMP override to verify the azure-ai Mistral OCR Foundry base; remove before commit. ports: - "4000:4000" configs: diff --git a/tests/e2e/llm_translation/test_ocr_rust_e2e.py b/tests/e2e/llm_translation/test_ocr_rust_e2e.py index 361bb5126a78..921010e5eaef 100644 --- a/tests/e2e/llm_translation/test_ocr_rust_e2e.py +++ b/tests/e2e/llm_translation/test_ocr_rust_e2e.py @@ -86,16 +86,18 @@ def litellm_params(self) -> LiteLLMParamsBody: @dataclass(frozen=True, slots=True) class VertexOcr: + """Vertex AI OCR (Mistral publisher). Only the location (not a secret) is set; + the project and credentials are left unset so the gateway resolves VERTEXAI_PROJECT + and VERTEXAI_CREDENTIALS from its own environment by name, keeping every secret on + the gateway like the azure_ai cases above. This is deliberate: the OCR path reads + vertex_project verbatim from litellm_params and never unwraps an `os.environ/*` + ref, so passing one would put the literal string in the request URL.""" + model: str location: str def litellm_params(self) -> LiteLLMParamsBody: - return LiteLLMParamsBody( - model=self.model, - vertex_project="os.environ/VERTEXAI_PROJECT", - vertex_location=self.location, - vertex_credentials="os.environ/VERTEXAI_CREDENTIALS", - ) + return LiteLLMParamsBody(model=self.model, vertex_location=self.location) @dataclass(frozen=True, slots=True) @@ -113,7 +115,7 @@ class _OcrCase: ), _OcrCase( "azure-ai", - AzureAiOcr("azure_ai/mistral-document-ai-2505"), + AzureAiOcr("azure_ai/mistral-document-ai-2512"), OcrDocument(type="document_url", document_url=TEST_PDF_URL), ), _OcrCase( @@ -126,11 +128,6 @@ class _OcrCase: VertexOcr("vertex_ai/mistral-ocr-2505", "us-central1"), OcrDocument(type="document_url", document_url=TEST_PDF_URL), ), - _OcrCase( - "vertex-deepseek", - VertexOcr("vertex_ai/deepseek-ocr-maas", "global"), - OcrDocument(type="image_url", image_url=TEST_IMAGE_URL), - ), ) _CASE_IDS = tuple(case.suffix for case in RUST_OCR_CASES) @@ -155,3 +152,5 @@ def test_rust_ocr_response( response = unwrap(endpoints_client.gateway.ocr(key, OcrBody(model=model, document=case.document))) _assert_ocr_document(response) + + diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 0490db286ea9..f5467854b117 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -376,6 +376,7 @@ class LiteLLMParamsBody(BaseModel): api_key: str | None = None api_base: str | None = None api_version: str | None = None + realtime_protocol: str | None = None aws_region_name: str | None = None vertex_project: str | None = None vertex_location: str | None = None From a239ab19a90b4a05e4dffbe70581da0f6ae95704 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Wed, 8 Jul 2026 16:01:38 -0700 Subject: [PATCH 2/6] fix: move realtime dir --- .../realtime/REALTIME_COVERAGE_MATRIX.md | 62 ++++++++++++ .../e2e/llm_translation/realtime/conftest.py | 37 +++++++ .../fixtures/weather_question_24k.wav | Bin .../realtime/pipecat_service.py | 0 .../realtime/realtime_client.py | 90 ++++++++++++++---- .../realtime/test_realtime_e2e.py | 14 +-- .../test_realtime_pipecat_audio_e2e.py | 23 +++-- .../realtime/test_realtime_pipecat_e2e.py | 8 +- .../e2e/realtime/REALTIME_COVERAGE_MATRIX.md | 55 ----------- tests/e2e/realtime/conftest.py | 20 ---- 10 files changed, 191 insertions(+), 118 deletions(-) create mode 100644 tests/e2e/llm_translation/realtime/REALTIME_COVERAGE_MATRIX.md create mode 100644 tests/e2e/llm_translation/realtime/conftest.py rename tests/e2e/{ => llm_translation}/realtime/fixtures/weather_question_24k.wav (100%) rename tests/e2e/{ => llm_translation}/realtime/pipecat_service.py (100%) rename tests/e2e/{ => llm_translation}/realtime/realtime_client.py (70%) rename tests/e2e/{ => llm_translation}/realtime/test_realtime_e2e.py (92%) rename tests/e2e/{ => llm_translation}/realtime/test_realtime_pipecat_audio_e2e.py (95%) rename tests/e2e/{ => llm_translation}/realtime/test_realtime_pipecat_e2e.py (97%) delete mode 100644 tests/e2e/realtime/REALTIME_COVERAGE_MATRIX.md delete mode 100644 tests/e2e/realtime/conftest.py diff --git a/tests/e2e/llm_translation/realtime/REALTIME_COVERAGE_MATRIX.md b/tests/e2e/llm_translation/realtime/REALTIME_COVERAGE_MATRIX.md new file mode 100644 index 000000000000..57963fcbc62d --- /dev/null +++ b/tests/e2e/llm_translation/realtime/REALTIME_COVERAGE_MATRIX.md @@ -0,0 +1,62 @@ +# Realtime e2e coverage + +Live tests for the proxy realtime websocket endpoint (`/v1/realtime`). One +GA-speaking websocket client drives every provider; the proxy normalizes each +provider's stream into the OpenAI GA event schema, so the same assertions hold +across providers and only the model alias changes. + +## What is asserted + +For each configured provider, `test_text_conversation` checks the session +lifecycle (`session.created`, then `session.update` echoed by `session.updated`), +the canonical response sequence (`response.created`, `response.output_item.added`, +through `response.done`), that the streamed deltas reconstruct a non-empty +transcript, and that `response.done` carries normalized usage. + +`test_tool_call_round_trip` checks the full tool path: the model emits a +normalized `response.function_call_arguments.done` with valid JSON arguments and +a matching `function_call` output item, the test sends a `function_call_output` +back, and the follow-up response incorporates the result (the temperature 72 +appears). + +`test_realtime_pipecat_e2e` is a realism layer that drives the same providers +through pipecat's GA `OpenAIRealtimeLLMService` (base_url pointed at the proxy) +rather than speaking the protocol by hand. Its assertions are coarse (the tool +callback fired, assistant text was produced); the raw-websocket suite is the +source of truth. It skips unless `pipecat-ai` is installed +(`uv pip install "pipecat-ai[openai]"`). + +## Provisioning + +The suite registers every provider's realtime deployment through `/model/new` at +session start (the `realtime_models` fixture) and deletes them on teardown, so it +never depends on a static or misconfigured gateway `model_list`. Each deployment +is created with `model_info.mode: realtime` and marker-unique names, and its +`litellm_params` point the credentials at `os.environ/*` refs the gateway resolves +at call time. The provider table below is the source of truth; edit `PROVIDERS` in +`realtime_client.py` to change a model or add one. + +| provider | model alias | upstream model | +|----------|-------------|----------------| +| openai | `openai-realtime` | `openai/gpt-realtime-2` | +| azure | `azure-realtime` | `azure/gpt-realtime-2` (GA protocol) | +| gemini | `gemini-realtime` | `gemini/gemini-3.1-flash-live-preview` | +| vertex_ai | `vertex-realtime` | `vertex_ai/gemini-live-2.5-flash-preview-native-audio-09-2025` | + +Every provider is provisioned and asserted; the suite never skips a provider. Per +`tests/e2e/CLAUDE.md` the only sanctioned skip is the whole-suite proxy-liveness +skip, so a provider whose credentials or upstream realtime model are missing on the +gateway is a hard failure, not a skip. Give the gateway each provider's credentials +to turn its tests green. + +## Running + +Start a proxy with the provider keys set in its environment (the suite registers +the deployments itself), then + +``` +uv run pytest tests/e2e/llm_translation/realtime/ -v +``` + +The whole suite skips only when no proxy answers `GET /health/liveliness` at +`LITELLM_PROXY_URL` (default `http://localhost:4000`). diff --git a/tests/e2e/llm_translation/realtime/conftest.py b/tests/e2e/llm_translation/realtime/conftest.py new file mode 100644 index 000000000000..15cd789664eb --- /dev/null +++ b/tests/e2e/llm_translation/realtime/conftest.py @@ -0,0 +1,37 @@ +"""Realtime suite's `client` and `realtime_models` fixtures. + +The shared lifecycle (resources/scoped_key), proxy liveness skip, and e2e marker +live in the parent tests/e2e/conftest.py. RealtimeClient holds the shared Gateway, +so the `resources` fixture cleans up keys this suite creates. + +`realtime_models` registers every provider's realtime deployment through /model/new +at session start and deletes them at teardown, so the suite provisions the models it +uses through the management endpoints instead of depending on a static (or +misconfigured) gateway model_list. +""" + +from collections.abc import Iterator + +import pytest + +from realtime_client import PROVIDERS, RealtimeClient, build_client + + +@pytest.fixture(scope="session") +def client() -> RealtimeClient: + return build_client() + + +@pytest.fixture(scope="session") +def realtime_models(client: RealtimeClient) -> Iterator[dict[str, str]]: + """Provision each provider's realtime deployment via /model/new and yield a + provider-id -> model-name map the tests connect with; delete them on teardown. + Every provider is provisioned (never skipped): a provider whose credentials or + upstream model are missing on the gateway hard-fails its test, per the suite's + fail-on-behavior contract in tests/e2e/CLAUDE.md.""" + records = tuple((provider.id, *client.provision(provider)) for provider in PROVIDERS) + try: + yield {provider_id: model_name for provider_id, model_name, _ in records} + finally: + for _, _, model_id in records: + client.gateway.delete_model(model_id) diff --git a/tests/e2e/realtime/fixtures/weather_question_24k.wav b/tests/e2e/llm_translation/realtime/fixtures/weather_question_24k.wav similarity index 100% rename from tests/e2e/realtime/fixtures/weather_question_24k.wav rename to tests/e2e/llm_translation/realtime/fixtures/weather_question_24k.wav diff --git a/tests/e2e/realtime/pipecat_service.py b/tests/e2e/llm_translation/realtime/pipecat_service.py similarity index 100% rename from tests/e2e/realtime/pipecat_service.py rename to tests/e2e/llm_translation/realtime/pipecat_service.py diff --git a/tests/e2e/realtime/realtime_client.py b/tests/e2e/llm_translation/realtime/realtime_client.py similarity index 70% rename from tests/e2e/realtime/realtime_client.py rename to tests/e2e/llm_translation/realtime/realtime_client.py index 07dfd76108bb..dbe9c8324319 100644 --- a/tests/e2e/realtime/realtime_client.py +++ b/tests/e2e/llm_translation/realtime/realtime_client.py @@ -11,19 +11,19 @@ from __future__ import annotations import time -from collections.abc import Generator +from collections.abc import Generator, Mapping from contextlib import contextmanager from dataclasses import dataclass from typing import Any, TypeVar from urllib.parse import urlencode -import pytest from pydantic import BaseModel, ConfigDict from websockets.sync.client import connect from websockets.sync.connection import Connection -from e2e_config import PROXY_BASE_URL +from e2e_config import PROXY_BASE_URL, unique_marker from e2e_gateway import Gateway, build_gateway +from models import LiteLLMParamsBody _M = TypeVar("_M", bound=BaseModel) @@ -41,25 +41,70 @@ def realtime_ws_url(model: str) -> str: @dataclass(frozen=True, slots=True) class RealtimeProvider: + """A realtime provider the suite exercises. `litellm_params` is the deployment + the suite registers through /model/new (the gateway resolves the os.environ/* + credential refs), so the suite is self-contained and never depends on a static + gateway model_list. Every provider here is provisioned and asserted: per + tests/e2e/CLAUDE.md the suite never skips a provider, so a provider whose + credentials or upstream realtime model are missing on the gateway is a hard + failure, not a skip.""" + id: str - model: str + alias: str + litellm_params: LiteLLMParamsBody PROVIDERS = ( - RealtimeProvider("openai", "openai-realtime"), - RealtimeProvider("azure", "azure-realtime"), - RealtimeProvider("gemini", "gemini-realtime"), - RealtimeProvider("vertex_ai", "vertex-realtime"), - # RealtimeProvider("bedrock", "bedrock-realtime"), # TODO: Enable this when Bedrock is passing - RealtimeProvider("xai", "xai-realtime"), + RealtimeProvider( + "openai", + "openai-realtime", + LiteLLMParamsBody( + model="openai/gpt-realtime-2", + api_key="os.environ/OPENAI_API_KEY", + ), + ), + RealtimeProvider( + "azure", + "azure-realtime", + LiteLLMParamsBody( + model="azure/gpt-realtime-2", + api_key="os.environ/AZURE_API_KEY", + api_base="os.environ/AZURE_API_BASE", + api_version="2025-08-28", + realtime_protocol="GA", + ), + ), + RealtimeProvider( + "gemini", + "gemini-realtime", + LiteLLMParamsBody( + model="gemini/gemini-3.1-flash-live-preview", + api_key="os.environ/GEMINI_API_KEY", + ), + ), + RealtimeProvider( + "vertex_ai", + "vertex-realtime", + LiteLLMParamsBody( + model="vertex_ai/gemini-live-2.5-flash-preview-native-audio-09-2025", + vertex_project="os.environ/VERTEXAI_PROJECT", + vertex_location="us-central1", + vertex_credentials="os.environ/VERTEXAI_CREDENTIALS", + ), + ), + # RealtimeProvider("bedrock", "bedrock-realtime", ...) # TODO: Enable when Bedrock is passing ) -def skip_if_unconfigured( - provider: RealtimeProvider, configured: frozenset[str] -) -> None: - if provider.model not in configured: - pytest.skip(f"{provider.model} not configured on proxy") +def realtime_model(provider: RealtimeProvider, provisioned: Mapping[str, str]) -> str: + """Return the provisioned deployment name for this provider. Every provider in + PROVIDERS is provisioned at session start, so a missing entry is a harness bug, + never an environment skip - the suite hard-fails instead (see tests/e2e/CLAUDE.md).""" + model = provisioned.get(provider.id) + assert model is not None, ( + f"{provider.id} was not provisioned; the realtime_models fixture is broken" + ) + return model # ---- sent events ------------------------------------------------------- @@ -280,12 +325,17 @@ def collect_until( class RealtimeClient: gateway: Gateway - def configured_models(self) -> frozenset[str]: - return frozenset( - entry.model_name - for entry in self.gateway.model_info() - if entry.model_info.mode == "realtime" + def provision(self, provider: RealtimeProvider) -> tuple[str, str]: + """Register this provider's realtime deployment through /model/new and return + (model_name, model_id). The name is marker-unique so it never collides with a + same-named deployment already on the shared proxy, and mode=realtime makes it + show up as a realtime model on /model/info. add_deployment runs synchronously, + so the deployment is connectable as soon as this returns.""" + model_name = f"{provider.alias}-{unique_marker()}" + model_id = self.gateway.create_model( + model_name, provider.litellm_params, mode="realtime" ) + return model_name, model_id @contextmanager def connect( diff --git a/tests/e2e/realtime/test_realtime_e2e.py b/tests/e2e/llm_translation/realtime/test_realtime_e2e.py similarity index 92% rename from tests/e2e/realtime/test_realtime_e2e.py rename to tests/e2e/llm_translation/realtime/test_realtime_e2e.py index 013569001412..6aaffdd208ef 100644 --- a/tests/e2e/realtime/test_realtime_e2e.py +++ b/tests/e2e/llm_translation/realtime/test_realtime_e2e.py @@ -31,7 +31,7 @@ SessionUpdate, function_call_item, parse_last, - skip_if_unconfigured, + realtime_model, transcript, user_message, ) @@ -62,12 +62,12 @@ class WeatherResult(BaseModel): def test_text_conversation( client: RealtimeClient, scoped_key: str, - configured_models: frozenset[str], + realtime_models: dict[str, str], provider: RealtimeProvider, ) -> None: - skip_if_unconfigured(provider, configured_models) + model = realtime_model(provider, realtime_models) - with client.connect(key=scoped_key, model=provider.model) as session: + with client.connect(key=scoped_key, model=model) as session: created = session.collect_until("session.created", timeout=20) assert created[-1].type == "session.created" @@ -99,12 +99,12 @@ def test_text_conversation( def test_tool_call_round_trip( client: RealtimeClient, scoped_key: str, - configured_models: frozenset[str], + realtime_models: dict[str, str], provider: RealtimeProvider, ) -> None: - skip_if_unconfigured(provider, configured_models) + model = realtime_model(provider, realtime_models) - with client.connect(key=scoped_key, model=provider.model) as session: + with client.connect(key=scoped_key, model=model) as session: session.collect_until("session.created", timeout=20) session.send( SessionUpdate( diff --git a/tests/e2e/realtime/test_realtime_pipecat_audio_e2e.py b/tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e.py similarity index 95% rename from tests/e2e/realtime/test_realtime_pipecat_audio_e2e.py rename to tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e.py index d9d7c744f66a..31c038b4e029 100644 --- a/tests/e2e/realtime/test_realtime_pipecat_audio_e2e.py +++ b/tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e.py @@ -31,7 +31,7 @@ PROVIDERS, RealtimeProvider, _ws_base_url, - skip_if_unconfigured, + realtime_model, ) pytestmark = pytest.mark.e2e @@ -189,13 +189,13 @@ async def get_weather(params: FunctionCallParams) -> None: @pytest.mark.parametrize("provider", PROVIDER_PARAMS) def test_pipecat_server_vad( scoped_key: str, - configured_models: frozenset[str], + realtime_models: dict[str, str], provider: RealtimeProvider, ) -> None: """Session is configured with server-VAD; bot must respond to a text prompt.""" - skip_if_unconfigured(provider, configured_models) + model = realtime_model(provider, realtime_models) - tool_called, got_text, _ = asyncio.run(_run_pipeline(scoped_key, provider.model)) + tool_called, got_text, _ = asyncio.run(_run_pipeline(scoped_key, model)) assert tool_called, "get_weather tool was not invoked" assert got_text, "no assistant text frames produced" @@ -204,16 +204,16 @@ def test_pipecat_server_vad( @pytest.mark.parametrize("provider", PROVIDER_PARAMS) def test_pipecat_audio_output( scoped_key: str, - configured_models: frozenset[str], + realtime_models: dict[str, str], provider: RealtimeProvider, ) -> None: """Bot must produce at least one non-empty TTS audio frame.""" - skip_if_unconfigured(provider, configured_models) + model = realtime_model(provider, realtime_models) _, got_text, audio_bytes = asyncio.run( _run_pipeline( scoped_key, - provider.model, + model, prompt="Say hello in one short sentence.", timeout=30.0, ) @@ -328,7 +328,7 @@ async def _run() -> None: @pytest.mark.parametrize("provider", PROVIDER_PARAMS) def test_pipecat_server_vad_audio_input( scoped_key: str, - configured_models: frozenset[str], + realtime_models: dict[str, str], provider: RealtimeProvider, ) -> None: """Stream a real PCM16 WAV fixture; server VAD must detect speech end and respond. @@ -337,12 +337,11 @@ def test_pipecat_server_vad_audio_input( → server-VAD turn detection → response.create (auto) → assistant reply. No LLMRunFrame is sent — the response must be triggered entirely by VAD. """ - if not WEATHER_WAV.exists(): - pytest.skip(f"audio fixture not found: {WEATHER_WAV}") - skip_if_unconfigured(provider, configured_models) + assert WEATHER_WAV.exists(), f"audio fixture not found: {WEATHER_WAV}" + model = realtime_model(provider, realtime_models) got_text, audio_bytes = asyncio.run( - _run_audio_input_pipeline(scoped_key, provider.model) + _run_audio_input_pipeline(scoped_key, model) ) assert got_text, "server VAD did not trigger a response (no assistant text)" diff --git a/tests/e2e/realtime/test_realtime_pipecat_e2e.py b/tests/e2e/llm_translation/realtime/test_realtime_pipecat_e2e.py similarity index 97% rename from tests/e2e/realtime/test_realtime_pipecat_e2e.py rename to tests/e2e/llm_translation/realtime/test_realtime_pipecat_e2e.py index 1068c54fdec5..799958ef4e3c 100644 --- a/tests/e2e/realtime/test_realtime_pipecat_e2e.py +++ b/tests/e2e/llm_translation/realtime/test_realtime_pipecat_e2e.py @@ -29,7 +29,7 @@ PROVIDERS, RealtimeProvider, _ws_base_url, - skip_if_unconfigured, + realtime_model, ) pytestmark = pytest.mark.e2e @@ -123,12 +123,12 @@ async def get_weather(params: FunctionCallParams) -> None: @pytest.mark.parametrize("provider", PROVIDER_PARAMS) def test_pipecat_tool_smoke( scoped_key: str, - configured_models: frozenset[str], + realtime_models: dict[str, str], provider: RealtimeProvider, ) -> None: - skip_if_unconfigured(provider, configured_models) + model = realtime_model(provider, realtime_models) - tool_called, produced_text = asyncio.run(_run_pipeline(scoped_key, provider.model)) + tool_called, produced_text = asyncio.run(_run_pipeline(scoped_key, model)) assert tool_called, "pipecat did not invoke the get_weather callback" assert produced_text, "pipecat produced no assistant text frames" diff --git a/tests/e2e/realtime/REALTIME_COVERAGE_MATRIX.md b/tests/e2e/realtime/REALTIME_COVERAGE_MATRIX.md deleted file mode 100644 index 8624475d0dea..000000000000 --- a/tests/e2e/realtime/REALTIME_COVERAGE_MATRIX.md +++ /dev/null @@ -1,55 +0,0 @@ -# Realtime e2e coverage - -Live tests for the proxy realtime websocket endpoint (`/v1/realtime`). One -GA-speaking websocket client drives every provider; the proxy normalizes each -provider's stream into the OpenAI GA event schema, so the same assertions hold -across providers and only the model alias changes. - -## What is asserted - -For each configured provider, `test_text_conversation` checks the session -lifecycle (`session.created`, then `session.update` echoed by `session.updated`), -the canonical response sequence (`response.created`, `response.output_item.added`, -through `response.done`), that the streamed deltas reconstruct a non-empty -transcript, and that `response.done` carries normalized usage. - -`test_tool_call_round_trip` checks the full tool path: the model emits a -normalized `response.function_call_arguments.done` with valid JSON arguments and -a matching `function_call` output item, the test sends a `function_call_output` -back, and the follow-up response incorporates the result (the temperature 72 -appears). - -`test_realtime_pipecat_e2e` is a realism layer that drives the same providers -through pipecat's GA `OpenAIRealtimeLLMService` (base_url pointed at the proxy) -rather than speaking the protocol by hand. Its assertions are coarse (the tool -callback fired, assistant text was produced); the raw-websocket suite is the -source of truth. It skips unless `pipecat-ai` is installed -(`uv pip install "pipecat-ai[openai]"`). - -## Provider status - -| provider | model alias | status | -|----------|-------------|--------| -| openai | `openai-realtime` | covered (in gateway config) | -| gemini | `gemini-realtime` | covered (in gateway config; needs Gemini Live API access) | -| azure | `azure-realtime` | gap: add to gateway config + AZURE creds | -| vertex_ai | `vertex-realtime` | gap: add to gateway config + Vertex creds | -| bedrock | `bedrock-realtime` | gap: add to gateway config + AWS creds | -| xai | `xai-realtime` | gap: add to gateway config + XAI_API_KEY | - -A provider whose alias is not present in the proxy's `/model/info` skips (skip on -environment). To enable one, add a `model_info.mode: realtime` entry under that -alias to `tests/e2e/gateway/litellm-config.yml` and give the proxy the -provider's credentials; the test then runs with no code change. - -## Running - -Start a proxy with the gateway config and the provider keys set in its -environment, then - -``` -uv run pytest tests/e2e/realtime/ -v -``` - -Tests skip when no proxy answers `GET /health/liveliness` at `LITELLM_PROXY_URL` -(default `http://localhost:4000`). diff --git a/tests/e2e/realtime/conftest.py b/tests/e2e/realtime/conftest.py deleted file mode 100644 index 4a5c4837a1a0..000000000000 --- a/tests/e2e/realtime/conftest.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Realtime suite's `client` fixture. - -The shared lifecycle (resources/scoped_key), proxy liveness skip, and e2e marker -live in the parent tests/e2e/conftest.py. RealtimeClient holds the shared -Gateway, so the `resources` fixture cleans up keys this suite creates. -""" - -import pytest - -from realtime_client import RealtimeClient, build_client - - -@pytest.fixture(scope="session") -def client() -> RealtimeClient: - return build_client() - - -@pytest.fixture(scope="session") -def configured_models(client: RealtimeClient) -> frozenset[str]: - return client.configured_models() From 23e608abffbea5eb8d57293e492acd5dffa504f7 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Wed, 8 Jul 2026 17:19:59 -0700 Subject: [PATCH 3/6] fix(realtime): normalize azure realtime api_base to host for Foundry endpoints The azure realtime handler appended the realtime path to api_base verbatim, so a Foundry base carrying a project path (.../api/projects/) produced an invalid realtime URL and the websocket handshake hung. Normalize api_base to scheme and host before building the realtime path so both Azure OpenAI and Foundry bases connect Point the e2e realtime azure deployment at the GA gpt-realtime model and stop passing the os.environ refs the realtime path never unwraps, resolving them from the gateway env by name instead. Drop the local docker-compose scaffolding from the tree --- litellm/llms/azure/realtime/handler.py | 5 ++++- tests/e2e/docker-compose.yml | 10 ++++------ tests/e2e/llm_translation/realtime/realtime_client.py | 4 +--- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/litellm/llms/azure/realtime/handler.py b/litellm/llms/azure/realtime/handler.py index 86c1ed51b680..33c0374a4631 100644 --- a/litellm/llms/azure/realtime/handler.py +++ b/litellm/llms/azure/realtime/handler.py @@ -57,8 +57,11 @@ def _construct_url( beta/default: "wss://.../openai/realtime?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview" GA/v1: "wss://.../openai/v1/realtime?model=gpt-realtime-deployment" """ - from urllib.parse import urlencode + from urllib.parse import urlencode, urlsplit, urlunsplit + split_base = urlsplit(api_base) + if split_base.netloc: + api_base = urlunsplit((split_base.scheme, split_base.netloc, "", "", "")) api_base = api_base.replace("https://", "wss://") # Determine path based on realtime_protocol (case-insensitive) diff --git a/tests/e2e/docker-compose.yml b/tests/e2e/docker-compose.yml index a3a7f6a3c1d6..cdf5d6cbf6ff 100644 --- a/tests/e2e/docker-compose.yml +++ b/tests/e2e/docker-compose.yml @@ -1,10 +1,10 @@ -# local setup to run e2e tests, do not commit when you make a PR +# local setup to run e2e tests configs: litellm_config: content: | general_settings: - master_key: sk-1234 - database_url: postgres://litellm@litellm:5432/litellm + master_key: os.environ/LITELLM_MASTER_KEY + database_url: os.environ/DATABASE_URL store_prompts_in_spend_logs: true litellm_settings: @@ -48,7 +48,7 @@ configs: services: litellm: - image: docker.litellm.ai/berriai/litellm:1.92.0-rc.1 + image: ghcr.io/berriai/litellm:main-latest depends_on: db: condition: service_healthy @@ -60,8 +60,6 @@ services: DATABASE_URL: postgresql://litellm:litellm@db:5432/litellm UI_USERNAME: admin UI_PASSWORD: sk-1234 - LITELLM_USE_RUST_OCR: 1 - # TEMP override to verify the azure-ai Mistral OCR Foundry base; remove before commit. ports: - "4000:4000" configs: diff --git a/tests/e2e/llm_translation/realtime/realtime_client.py b/tests/e2e/llm_translation/realtime/realtime_client.py index dbe9c8324319..756b68441be7 100644 --- a/tests/e2e/llm_translation/realtime/realtime_client.py +++ b/tests/e2e/llm_translation/realtime/realtime_client.py @@ -67,9 +67,8 @@ class RealtimeProvider: "azure", "azure-realtime", LiteLLMParamsBody( - model="azure/gpt-realtime-2", + model="azure/gpt-realtime", api_key="os.environ/AZURE_API_KEY", - api_base="os.environ/AZURE_API_BASE", api_version="2025-08-28", realtime_protocol="GA", ), @@ -87,7 +86,6 @@ class RealtimeProvider: "vertex-realtime", LiteLLMParamsBody( model="vertex_ai/gemini-live-2.5-flash-preview-native-audio-09-2025", - vertex_project="os.environ/VERTEXAI_PROJECT", vertex_location="us-central1", vertex_credentials="os.environ/VERTEXAI_CREDENTIALS", ), From 0ad562b46a7803d7efea0de5adb4494a76b2304f Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Wed, 8 Jul 2026 18:24:09 -0700 Subject: [PATCH 4/6] test(e2e): add Gateway.list_files and list_fine_tuning_jobs for the discovery suite The discovery endpoints suite calls client.gateway.list_files and list_fine_tuning_jobs, which did not exist on Gateway, so both tests errored with AttributeError before reaching the proxy. Add the two GET wrappers using the existing FileListResponse / FineTuningJobsResponse models --- tests/e2e/e2e_gateway.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/e2e/e2e_gateway.py b/tests/e2e/e2e_gateway.py index 055d06d1c799..d7b639d85915 100644 --- a/tests/e2e/e2e_gateway.py +++ b/tests/e2e/e2e_gateway.py @@ -28,6 +28,9 @@ CustomerDeleteBody, EmbedBody, EmbedResponse, + FileListResponse, + FineTuningJobsParams, + FineTuningJobsResponse, KeyDeleteBody, KeyGenerateBody, KeyGenerateResponse, @@ -119,6 +122,24 @@ def model_info(self) -> list[ModelInfoEntry]: ) ).data + def list_files(self, key: str) -> Result[FileListResponse]: + return self.transport.get( + "/v1/files", + headers=self.transport.bearer(key), + params=NoBody(), + response_type=FileListResponse, + ) + + def list_fine_tuning_jobs( + self, key: str, params: FineTuningJobsParams + ) -> Result[FineTuningJobsResponse]: + return self.transport.get( + "/v1/fine_tuning/jobs", + headers=self.transport.bearer(key), + params=params, + response_type=FineTuningJobsResponse, + ) + def create_model( self, model_name: str, From 836f2d8c1bbf9ee7b1f67bba24a2844546a9273b Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Wed, 8 Jul 2026 19:07:08 -0700 Subject: [PATCH 5/6] revert(realtime): drop azure realtime api_base host-normalization The azure realtime handshake failure was a config issue, not a litellm bug: the realtime base was set to the Azure AI Foundry project endpoint (.../api/projects/

), but the OpenAI-compatible realtime route lives at the resource root. litellm correctly appends the realtime path to whatever base it is given, so pointing the realtime deployment at the resource root is the fix and no core change is needed --- litellm/llms/azure/realtime/handler.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/litellm/llms/azure/realtime/handler.py b/litellm/llms/azure/realtime/handler.py index 33c0374a4631..86c1ed51b680 100644 --- a/litellm/llms/azure/realtime/handler.py +++ b/litellm/llms/azure/realtime/handler.py @@ -57,11 +57,8 @@ def _construct_url( beta/default: "wss://.../openai/realtime?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview" GA/v1: "wss://.../openai/v1/realtime?model=gpt-realtime-deployment" """ - from urllib.parse import urlencode, urlsplit, urlunsplit + from urllib.parse import urlencode - split_base = urlsplit(api_base) - if split_base.netloc: - api_base = urlunsplit((split_base.scheme, split_base.netloc, "", "", "")) api_base = api_base.replace("https://", "wss://") # Determine path based on realtime_protocol (case-insensitive) From 07218cb2394f953a8ad925614743a2077b90f0aa Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Wed, 8 Jul 2026 19:25:39 -0700 Subject: [PATCH 6/6] fix(ocr): route azure_ai doc-intelligence to its own endpoint at the source get_llm_provider inherits AZURE_AI_API_BASE into api_base for every azure_ai/* OCR model, but Azure Document Intelligence is a separate resource reached via AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT, so doc-intelligence requests went to the wrong host. Stop inheriting the azure_ai base for doc-intelligence models so api_base stays unset and both the rust bridge and the python get_complete_url fall back to the document-intelligence endpoint. This drops the earlier _rust_bridge_api_base reorder, which only covered the rust path and let the env silently override an explicit api_base --- litellm/ocr/main.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index d9f72c63dbec..958db6f9f5bc 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -95,9 +95,12 @@ def _prepare_ocr_request( api_key=api_key, ) + _is_doc_intelligence = custom_llm_provider == "azure_ai" and ( + "doc-intelligence" in model.lower() or "documentintelligence" in model.lower() + ) if dynamic_api_key: api_key = dynamic_api_key - if dynamic_api_base: + if dynamic_api_base and not _is_doc_intelligence: api_base = dynamic_api_base ocr_provider_config = ProviderConfigManager.get_provider_ocr_config( @@ -188,13 +191,12 @@ def _rust_bridge_api_base( prepared_request: _PreparedOCRRequest, resolve_secret: Callable[[str], str | None], ) -> str | None: - if prepared_request.custom_llm_provider == "azure_ai": - model = prepared_request.model.lower() - if "doc-intelligence" in model or "documentintelligence" in model: - return resolve_secret("AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT") or prepared_request.api_base if prepared_request.api_base is not None: return prepared_request.api_base if prepared_request.custom_llm_provider == "azure_ai": + model = prepared_request.model.lower() + if "doc-intelligence" in model or "documentintelligence" in model: + return resolve_secret("AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT") return resolve_secret("AZURE_AI_API_BASE") return None