fix(realtime): azure Foundry realtime endpoints + self-provision e2e realtime suite

litellm/ocr/main.py

@@ -95,9 +95,12 @@ def _prepare_ocr_request(
         api_key=api_key,
     )
 
+    _is_doc_intelligence = custom_llm_provider == "azure_ai" and (
+        "doc-intelligence" in model.lower() or "documentintelligence" in model.lower()
+    )
     if dynamic_api_key:
         api_key = dynamic_api_key
-    if dynamic_api_base:
+    if dynamic_api_base and not _is_doc_intelligence:
         api_base = dynamic_api_base
 
     ocr_provider_config = ProviderConfigManager.get_provider_ocr_config(

tests/e2e/e2e_gateway.py

@@ -28,6 +28,9 @@
     CustomerDeleteBody,
     EmbedBody,
     EmbedResponse,
+    FileListResponse,
+    FineTuningJobsParams,
+    FineTuningJobsResponse,
     KeyDeleteBody,
     KeyGenerateBody,
     KeyGenerateResponse,
@@ -119,6 +122,24 @@ def model_info(self) -> list[ModelInfoEntry]:
             )
         ).data
 
+    def list_files(self, key: str) -> Result[FileListResponse]:
+        return self.transport.get(
+            "/v1/files",
+            headers=self.transport.bearer(key),
+            params=NoBody(),
+            response_type=FileListResponse,
+        )
+
+    def list_fine_tuning_jobs(
+        self, key: str, params: FineTuningJobsParams
+    ) -> Result[FineTuningJobsResponse]:
+        return self.transport.get(
+            "/v1/fine_tuning/jobs",
+            headers=self.transport.bearer(key),
+            params=params,
+            response_type=FineTuningJobsResponse,
+        )
+
     def create_model(
         self,
         model_name: str,

tests/e2e/llm_translation/realtime/REALTIME_COVERAGE_MATRIX.md

@@ -0,0 +1,62 @@
+# Realtime e2e coverage
+
+Live tests for the proxy realtime websocket endpoint (`/v1/realtime`). One
+GA-speaking websocket client drives every provider; the proxy normalizes each
+provider's stream into the OpenAI GA event schema, so the same assertions hold
+across providers and only the model alias changes.
+
+## What is asserted
+
+For each configured provider, `test_text_conversation` checks the session
+lifecycle (`session.created`, then `session.update` echoed by `session.updated`),
+the canonical response sequence (`response.created`, `response.output_item.added`,
+through `response.done`), that the streamed deltas reconstruct a non-empty
+transcript, and that `response.done` carries normalized usage.
+
+`test_tool_call_round_trip` checks the full tool path: the model emits a
+normalized `response.function_call_arguments.done` with valid JSON arguments and
+a matching `function_call` output item, the test sends a `function_call_output`
+back, and the follow-up response incorporates the result (the temperature 72
+appears).
+
+`test_realtime_pipecat_e2e` is a realism layer that drives the same providers
+through pipecat's GA `OpenAIRealtimeLLMService` (base_url pointed at the proxy)
+rather than speaking the protocol by hand. Its assertions are coarse (the tool
+callback fired, assistant text was produced); the raw-websocket suite is the
+source of truth. It skips unless `pipecat-ai` is installed
+(`uv pip install "pipecat-ai[openai]"`).
+
+## Provisioning
+
+The suite registers every provider's realtime deployment through `/model/new` at
+session start (the `realtime_models` fixture) and deletes them on teardown, so it
+never depends on a static or misconfigured gateway `model_list`. Each deployment
+is created with `model_info.mode: realtime` and marker-unique names, and its
+`litellm_params` point the credentials at `os.environ/*` refs the gateway resolves
+at call time. The provider table below is the source of truth; edit `PROVIDERS` in
+`realtime_client.py` to change a model or add one.
+
+| provider | model alias | upstream model |
+|----------|-------------|----------------|
+| openai | `openai-realtime` | `openai/gpt-realtime-2` |
+| azure | `azure-realtime` | `azure/gpt-realtime-2` (GA protocol) |
+| gemini | `gemini-realtime` | `gemini/gemini-3.1-flash-live-preview` |
+| vertex_ai | `vertex-realtime` | `vertex_ai/gemini-live-2.5-flash-preview-native-audio-09-2025` |
+
+Every provider is provisioned and asserted; the suite never skips a provider. Per
+`tests/e2e/CLAUDE.md` the only sanctioned skip is the whole-suite proxy-liveness
+skip, so a provider whose credentials or upstream realtime model are missing on the
+gateway is a hard failure, not a skip. Give the gateway each provider's credentials
+to turn its tests green.
+
+## Running
+
+Start a proxy with the provider keys set in its environment (the suite registers
+the deployments itself), then
+
+```
+uv run pytest tests/e2e/llm_translation/realtime/ -v
+```
+
+The whole suite skips only when no proxy answers `GET /health/liveliness` at
+`LITELLM_PROXY_URL` (default `http://localhost:4000`).

tests/e2e/llm_translation/realtime/conftest.py

@@ -0,0 +1,37 @@
+"""Realtime suite's `client` and `realtime_models` fixtures.
+
+The shared lifecycle (resources/scoped_key), proxy liveness skip, and e2e marker
+live in the parent tests/e2e/conftest.py. RealtimeClient holds the shared Gateway,
+so the `resources` fixture cleans up keys this suite creates.
+
+`realtime_models` registers every provider's realtime deployment through /model/new
+at session start and deletes them at teardown, so the suite provisions the models it
+uses through the management endpoints instead of depending on a static (or
+misconfigured) gateway model_list.
+"""
+
+from collections.abc import Iterator
+
+import pytest
+
+from realtime_client import PROVIDERS, RealtimeClient, build_client
+
+
+@pytest.fixture(scope="session")
+def client() -> RealtimeClient:
+    return build_client()
+
+
+@pytest.fixture(scope="session")
+def realtime_models(client: RealtimeClient) -> Iterator[dict[str, str]]:
+    """Provision each provider's realtime deployment via /model/new and yield a
+    provider-id -> model-name map the tests connect with; delete them on teardown.
+    Every provider is provisioned (never skipped): a provider whose credentials or
+    upstream model are missing on the gateway hard-fails its test, per the suite's
+    fail-on-behavior contract in tests/e2e/CLAUDE.md."""
+    records = tuple((provider.id, *client.provision(provider)) for provider in PROVIDERS)
+    try:
+        yield {provider_id: model_name for provider_id, model_name, _ in records}
+    finally:
+        for _, _, model_id in records:
+            client.gateway.delete_model(model_id)

tests/e2e/realtime/realtime_client.py → tests/e2e/llm_translation/realtime/realtime_client.py

@@ -11,19 +11,19 @@
 from __future__ import annotations
 
 import time
-from collections.abc import Generator
+from collections.abc import Generator, Mapping
 from contextlib import contextmanager
 from dataclasses import dataclass
 from typing import Any, TypeVar
 from urllib.parse import urlencode
 
-import pytest
 from pydantic import BaseModel, ConfigDict
 from websockets.sync.client import connect
 from websockets.sync.connection import Connection
 
-from e2e_config import PROXY_BASE_URL
+from e2e_config import PROXY_BASE_URL, unique_marker
 from e2e_gateway import Gateway, build_gateway
+from models import LiteLLMParamsBody
 
 _M = TypeVar("_M", bound=BaseModel)
 
@@ -41,25 +41,68 @@ def realtime_ws_url(model: str) -> str:
 
 @dataclass(frozen=True, slots=True)
 class RealtimeProvider:
+    """A realtime provider the suite exercises. `litellm_params` is the deployment
+    the suite registers through /model/new (the gateway resolves the os.environ/*
+    credential refs), so the suite is self-contained and never depends on a static
+    gateway model_list. Every provider here is provisioned and asserted: per
+    tests/e2e/CLAUDE.md the suite never skips a provider, so a provider whose
+    credentials or upstream realtime model are missing on the gateway is a hard
+    failure, not a skip."""
+
     id: str
-    model: str
+    alias: str
+    litellm_params: LiteLLMParamsBody
 
 
 PROVIDERS = (
-    RealtimeProvider("openai", "openai-realtime"),
-    RealtimeProvider("azure", "azure-realtime"),
-    RealtimeProvider("gemini", "gemini-realtime"),
-    RealtimeProvider("vertex_ai", "vertex-realtime"),
-    # RealtimeProvider("bedrock", "bedrock-realtime"), # TODO: Enable this when Bedrock is passing
-    RealtimeProvider("xai", "xai-realtime"),
+    RealtimeProvider(
+        "openai",
+        "openai-realtime",
+        LiteLLMParamsBody(
+            model="openai/gpt-realtime-2",
+            api_key="os.environ/OPENAI_API_KEY",
+        ),
+    ),
+    RealtimeProvider(
+        "azure",
+        "azure-realtime",
+        LiteLLMParamsBody(
+            model="azure/gpt-realtime",
+            api_key="os.environ/AZURE_API_KEY",
+            api_version="2025-08-28",
+            realtime_protocol="GA",
+        ),
+    ),
+    RealtimeProvider(
+        "gemini",
+        "gemini-realtime",
+        LiteLLMParamsBody(
+            model="gemini/gemini-3.1-flash-live-preview",
+            api_key="os.environ/GEMINI_API_KEY",
+        ),
+    ),
+    RealtimeProvider(
+        "vertex_ai",
+        "vertex-realtime",
+        LiteLLMParamsBody(
+            model="vertex_ai/gemini-live-2.5-flash-preview-native-audio-09-2025",
+            vertex_location="us-central1",
+            vertex_credentials="os.environ/VERTEXAI_CREDENTIALS",
+        ),
+    ),
+    # RealtimeProvider("bedrock", "bedrock-realtime", ...) # TODO: Enable when Bedrock is passing
 )
 
 
-def skip_if_unconfigured(
-    provider: RealtimeProvider, configured: frozenset[str]
-) -> None:
-    if provider.model not in configured:
-        pytest.skip(f"{provider.model} not configured on proxy")
+def realtime_model(provider: RealtimeProvider, provisioned: Mapping[str, str]) -> str:
+    """Return the provisioned deployment name for this provider. Every provider in
+    PROVIDERS is provisioned at session start, so a missing entry is a harness bug,
+    never an environment skip - the suite hard-fails instead (see tests/e2e/CLAUDE.md)."""
+    model = provisioned.get(provider.id)
+    assert model is not None, (
+        f"{provider.id} was not provisioned; the realtime_models fixture is broken"
+    )
+    return model
 
 
 # ---- sent events -------------------------------------------------------
@@ -280,12 +323,17 @@ def collect_until(
 class RealtimeClient:
     gateway: Gateway
 
-    def configured_models(self) -> frozenset[str]:
-        return frozenset(
-            entry.model_name
-            for entry in self.gateway.model_info()
-            if entry.model_info.mode == "realtime"
+    def provision(self, provider: RealtimeProvider) -> tuple[str, str]:
+        """Register this provider's realtime deployment through /model/new and return
+        (model_name, model_id). The name is marker-unique so it never collides with a
+        same-named deployment already on the shared proxy, and mode=realtime makes it
+        show up as a realtime model on /model/info. add_deployment runs synchronously,
+        so the deployment is connectable as soon as this returns."""
+        model_name = f"{provider.alias}-{unique_marker()}"
+        model_id = self.gateway.create_model(
+            model_name, provider.litellm_params, mode="realtime"
         )
+        return model_name, model_id
 
     @contextmanager
     def connect(

tests/e2e/realtime/test_realtime_e2e.py → tests/e2e/llm_translation/realtime/test_realtime_e2e.py

@@ -31,7 +31,7 @@
     SessionUpdate,
     function_call_item,
     parse_last,
-    skip_if_unconfigured,
+    realtime_model,
     transcript,
     user_message,
 )
@@ -62,12 +62,12 @@ class WeatherResult(BaseModel):
 def test_text_conversation(
     client: RealtimeClient,
     scoped_key: str,
-    configured_models: frozenset[str],
+    realtime_models: dict[str, str],
     provider: RealtimeProvider,
 ) -> None:
-    skip_if_unconfigured(provider, configured_models)
+    model = realtime_model(provider, realtime_models)
 
-    with client.connect(key=scoped_key, model=provider.model) as session:
+    with client.connect(key=scoped_key, model=model) as session:
         created = session.collect_until("session.created", timeout=20)
         assert created[-1].type == "session.created"
 
@@ -99,12 +99,12 @@ def test_text_conversation(
 def test_tool_call_round_trip(
     client: RealtimeClient,
     scoped_key: str,
-    configured_models: frozenset[str],
+    realtime_models: dict[str, str],
     provider: RealtimeProvider,
 ) -> None:
-    skip_if_unconfigured(provider, configured_models)
+    model = realtime_model(provider, realtime_models)
 
-    with client.connect(key=scoped_key, model=provider.model) as session:
+    with client.connect(key=scoped_key, model=model) as session:
         session.collect_until("session.created", timeout=20)
         session.send(
             SessionUpdate(

tests/e2e/realtime/test_realtime_pipecat_audio_e2e.py → tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e.py

@@ -31,7 +31,7 @@
     PROVIDERS,
     RealtimeProvider,
     _ws_base_url,
-    skip_if_unconfigured,
+    realtime_model,
 )
 
 pytestmark = pytest.mark.e2e
@@ -189,13 +189,13 @@ async def get_weather(params: FunctionCallParams) -> None:
 @pytest.mark.parametrize("provider", PROVIDER_PARAMS)
 def test_pipecat_server_vad(
     scoped_key: str,
-    configured_models: frozenset[str],
+    realtime_models: dict[str, str],
     provider: RealtimeProvider,
 ) -> None:
     """Session is configured with server-VAD; bot must respond to a text prompt."""
-    skip_if_unconfigured(provider, configured_models)
+    model = realtime_model(provider, realtime_models)
 
-    tool_called, got_text, _ = asyncio.run(_run_pipeline(scoped_key, provider.model))
+    tool_called, got_text, _ = asyncio.run(_run_pipeline(scoped_key, model))
 
     assert tool_called, "get_weather tool was not invoked"
     assert got_text, "no assistant text frames produced"
@@ -204,16 +204,16 @@ def test_pipecat_server_vad(
 @pytest.mark.parametrize("provider", PROVIDER_PARAMS)
 def test_pipecat_audio_output(
     scoped_key: str,
-    configured_models: frozenset[str],
+    realtime_models: dict[str, str],
     provider: RealtimeProvider,
 ) -> None:
     """Bot must produce at least one non-empty TTS audio frame."""
-    skip_if_unconfigured(provider, configured_models)
+    model = realtime_model(provider, realtime_models)
 
     _, got_text, audio_bytes = asyncio.run(
         _run_pipeline(
             scoped_key,
-            provider.model,
+            model,
             prompt="Say hello in one short sentence.",
             timeout=30.0,
         )
@@ -328,7 +328,7 @@ async def _run() -> None:
 @pytest.mark.parametrize("provider", PROVIDER_PARAMS)
 def test_pipecat_server_vad_audio_input(
     scoped_key: str,
-    configured_models: frozenset[str],
+    realtime_models: dict[str, str],
     provider: RealtimeProvider,
 ) -> None:
     """Stream a real PCM16 WAV fixture; server VAD must detect speech end and respond.
@@ -337,12 +337,11 @@ def test_pipecat_server_vad_audio_input(
     → server-VAD turn detection → response.create (auto) → assistant reply.
     No LLMRunFrame is sent — the response must be triggered entirely by VAD.
     """
-    if not WEATHER_WAV.exists():
-        pytest.skip(f"audio fixture not found: {WEATHER_WAV}")
-    skip_if_unconfigured(provider, configured_models)
+    assert WEATHER_WAV.exists(), f"audio fixture not found: {WEATHER_WAV}"
+    model = realtime_model(provider, realtime_models)
 
     got_text, audio_bytes = asyncio.run(
-        _run_audio_input_pipeline(scoped_key, provider.model)
+        _run_audio_input_pipeline(scoped_key, model)
     )
 
     assert got_text, "server VAD did not trigger a response (no assistant text)"