From e7c3f7bc75df020071df14f790c5cc4f1c4986ed Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 15 Jul 2026 16:11:45 -0700 Subject: [PATCH 1/3] test(e2e): cover /chat/completions on Azure OpenAI, streaming and non-streaming --- .../coverage_registry/llm_conversational.yaml | 1 + tests/e2e/docker-compose.yml | 6 ++ tests/e2e/e2e_http.py | 38 ++++++---- .../test_chat_completions_regression_e2e.py | 73 ++++++++++++++++++- tests/e2e/models.py | 15 ++++ 5 files changed, 114 insertions(+), 19 deletions(-) diff --git a/tests/e2e/coverage_registry/llm_conversational.yaml b/tests/e2e/coverage_registry/llm_conversational.yaml index be8a291c6fcb..40d766e7ce55 100644 --- a/tests/e2e/coverage_registry/llm_conversational.yaml +++ b/tests/e2e/coverage_registry/llm_conversational.yaml @@ -29,6 +29,7 @@ - {id: llm.chat_completions.vertex.vision.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: vertex, capability: vision, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Gemini vision"} - {id: llm.chat_completions.vertex.prompt_cache_5m.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: vertex, capability: prompt_cache_5m, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Vertex Gemini prompt caching"} - {id: llm.chat_completions.azure_openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: azure_openai, capability: basic, streaming: nonstream, assertions: [works], source: "proxy_server.py:8455", rationale: "P0 route; Azure OpenAI deployments"} +- {id: llm.chat_completions.azure_openai.basic.stream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: azure_openai, capability: basic, streaming: stream, assertions: [works], source: "proxy_server.py:8455", rationale: "Streaming over Azure OpenAI"} - {id: llm.chat_completions.azure_openai.tool_use.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: azure_openai, capability: tool_use, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Azure OpenAI function_calling"} - {id: llm.chat_completions.azure_foundry.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: azure_foundry, capability: basic, streaming: nonstream, assertions: [works], source: "proxy_server.py:8455", rationale: "Azure Foundry (azure_ai); newer, smoke"} - {id: llm.messages.anthropic.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: anthropic, capability: basic, streaming: nonstream, assertions: [works], source: "anthropic_endpoints/endpoints.py:64", rationale: "Core endpoint; Anthropic Messages native"} diff --git a/tests/e2e/docker-compose.yml b/tests/e2e/docker-compose.yml index e2bb6ca89332..4e036b204621 100644 --- a/tests/e2e/docker-compose.yml +++ b/tests/e2e/docker-compose.yml @@ -66,6 +66,12 @@ configs: model: openai/text-embedding-3-small api_key: os.environ/OPENAI_API_KEY + - model_name: azure-gpt-5.4-mini + litellm_params: + model: azure/gpt-5.4-mini + api_base: os.environ/AZURE_API_BASE + api_key: os.environ/AZURE_API_KEY + services: litellm: image: ghcr.io/berriai/litellm:main-latest diff --git a/tests/e2e/e2e_http.py b/tests/e2e/e2e_http.py index ff296969079e..53d3c52d61a8 100644 --- a/tests/e2e/e2e_http.py +++ b/tests/e2e/e2e_http.py @@ -121,6 +121,7 @@ class StreamingResponse(BaseModel): headers: dict[str, str] = {} body: str chunks: int = 0 # streamed events (0 for non-streaming) + events: tuple[str, ...] = () # SSE "data:" payloads, prefix stripped (streaming only) # First in-stream error event, if any. A streamed call commits its HTTP 200 # before the upstream completes, so upstream failures (e.g. insufficient # quota) arrive as SSE error events inside an otherwise-successful response; @@ -293,20 +294,22 @@ def _streaming_outcome(resp: requests.Response, stream: bool) -> StreamingRespon headers=headers, body=resp.text, ) - lines = cast("Iterator[bytes]", resp.iter_lines()) - chunks = 0 - stream_error: str | None = None - for line in lines: - if not line: - continue - chunks += 1 - if stream_error is None and ( - line.startswith(b"event: error") - or b'"type":"error"' in line - or b'"type": "error"' in line - or line.startswith(b'data: {"error"') - ): - stream_error = line.decode(errors="replace")[:300] + raw_lines = tuple( + line.decode(errors="replace") + for line in cast("Iterator[bytes]", resp.iter_lines()) + if line + ) + stream_error = next( + ( + line[:300] + for line in raw_lines + if line.startswith("event: error") + or '"type":"error"' in line + or '"type": "error"' in line + or line.startswith('data: {"error"') + ), + None, + ) return StreamingResponse( status_code=resp.status_code, call_id=call_id, @@ -314,7 +317,12 @@ def _streaming_outcome(resp: requests.Response, stream: bool) -> StreamingRespon content_type=content_type, headers=headers, body="", - chunks=chunks, + chunks=len(raw_lines), + events=tuple( + line.removeprefix("data:").strip() + for line in raw_lines + if line.startswith("data:") + ), stream_error=stream_error, ) diff --git a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py index 5cc4ff308fac..efcd6fddc2f0 100644 --- a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py +++ b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py @@ -4,9 +4,11 @@ releases: a clean 200 came back but with no real completion. A status check alone would not have caught it, so each case here asserts the product promise - a non-empty assistant message and a real model name in the body - across the -three providers wired into the gateway config (OpenAI, Anthropic, Gemini). A -regression that empties the completion for any provider fails that provider's -row here. +providers wired into the gateway config (OpenAI, Anthropic, Gemini, Azure +OpenAI). A regression that empties the completion for any provider fails that +provider's row here. The Azure OpenAI streaming case applies the same standard +to the SSE path: every data event must parse as a chat.completion.chunk and the +deltas must reassemble into real text, not just count as a 200 with chunks. """ from __future__ import annotations @@ -15,15 +17,18 @@ from e2e_config import unique_marker from e2e_http import unwrap -from models import ChatBody, ChatMessage +from models import ChatBody, ChatMessage, ChatStreamChunk from passthrough_client import PassthroughClient pytestmark = pytest.mark.e2e +AZURE_CHAT_MODEL = "azure-gpt-5.4-mini" + CHAT_MODELS: tuple[tuple[str, str], ...] = ( ("gpt-5.5", "openai"), ("claude-haiku-4-5", "anthropic"), ("gemini-2.5-flash", "gemini"), + (AZURE_CHAT_MODEL, "azure_openai"), ) @@ -37,6 +42,7 @@ class TestChatCompletionsRegression: "llm.chat_completions.openai.basic.nonstream.works", "llm.chat_completions.anthropic.basic.nonstream.works", "llm.chat_completions.vertex.basic.nonstream.works", + "llm.chat_completions.azure_openai.basic.nonstream.works", exercised_on=[], ) def test_chat_returns_real_completion( @@ -68,3 +74,62 @@ def test_chat_returns_real_completion( assert ( message is not None and message.content and message.content.strip() ), f"{model} ({route}): 200 with an empty completion (#28991): {response}" + + @pytest.mark.covers("llm.chat_completions.azure_openai.basic.stream.works") + def test_azure_openai_stream_returns_real_completion( + self, client: PassthroughClient, scoped_key: str + ) -> None: + result = client.gateway.chat_stream( + scoped_key, + ChatBody( + model=AZURE_CHAT_MODEL, + messages=[ + ChatMessage( + role="user", + content=f"reply with one word {unique_marker()}", + ) + ], + max_tokens=512, + stream=True, + ), + ) + + assert result.ok, ( + f"{AZURE_CHAT_MODEL}: stream failed with status " + f"{result.status_code}: {result.body[:300]}" + ) + assert result.is_streaming, ( + f"{AZURE_CHAT_MODEL}: expected text/event-stream, got " + f"{result.content_type}: {result.body[:300]}" + ) + assert result.stream_error is None, ( + f"{AZURE_CHAT_MODEL}: 200 stream carried an error event: " + f"{result.stream_error}" + ) + assert result.events, f"{AZURE_CHAT_MODEL}: stream carried no SSE data events" + assert result.events[-1] == "[DONE]", ( + f"{AZURE_CHAT_MODEL}: stream did not terminate with [DONE]: " + f"{result.events[-1][:200]}" + ) + + chunks = [ + ChatStreamChunk.model_validate_json(event) for event in result.events[:-1] + ] + assert chunks, f"{AZURE_CHAT_MODEL}: stream held only the [DONE] sentinel" + assert all( + chunk.object == "chat.completion.chunk" for chunk in chunks + ), f"{AZURE_CHAT_MODEL}: malformed chunk object types: {result.events[:5]}" + assert any( + chunk.model for chunk in chunks + ), f"{AZURE_CHAT_MODEL}: no chunk carried a model name: {result.events[:5]}" + + content = "".join( + choice.delta.content or "" + for chunk in chunks + for choice in chunk.choices + if choice.delta is not None + ) + assert content.strip(), ( + f"{AZURE_CHAT_MODEL}: stream chunks reassembled to an empty " + f"completion (#28991): {result.events[:5]}" + ) diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 4140967f3e0c..6e3bd8ad5547 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -191,6 +191,21 @@ class ChatResponse(BaseModel): service_tier: str | None = None +class ChatStreamDelta(BaseModel): + content: str | None = None + + +class ChatStreamChoice(BaseModel): + delta: ChatStreamDelta | None = None + + +class ChatStreamChunk(BaseModel): + id: str | None = None + object: str | None = None + model: str | None = None + choices: list[ChatStreamChoice] = [] + + class EmbedBody(BaseModel): model: str input: str From 55f58cde8c4f28b782b48f3466db8fdf48deba32 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 15 Jul 2026 16:26:43 -0700 Subject: [PATCH 2/3] test(e2e): swap Azure OpenAI coverage to the GPT-5.6 tiers behind env vars --- tests/e2e/docker-compose.yml | 16 +++++++- tests/e2e/e2e_config.py | 9 ++++ .../test_chat_completions_regression_e2e.py | 41 ++++++++++--------- 3 files changed, 44 insertions(+), 22 deletions(-) diff --git a/tests/e2e/docker-compose.yml b/tests/e2e/docker-compose.yml index 4e036b204621..3ba8be370574 100644 --- a/tests/e2e/docker-compose.yml +++ b/tests/e2e/docker-compose.yml @@ -66,9 +66,21 @@ configs: model: openai/text-embedding-3-small api_key: os.environ/OPENAI_API_KEY - - model_name: azure-gpt-5.4-mini + - model_name: azure-${E2E_AZURE_SOL_MODEL:-gpt-5.6-sol} litellm_params: - model: azure/gpt-5.4-mini + model: azure/${E2E_AZURE_SOL_MODEL:-gpt-5.6-sol} + api_base: os.environ/AZURE_API_BASE + api_key: os.environ/AZURE_API_KEY + + - model_name: azure-${E2E_AZURE_TERRA_MODEL:-gpt-5.6-terra} + litellm_params: + model: azure/${E2E_AZURE_TERRA_MODEL:-gpt-5.6-terra} + api_base: os.environ/AZURE_API_BASE + api_key: os.environ/AZURE_API_KEY + + - model_name: azure-${E2E_AZURE_LUNA_MODEL:-gpt-5.6-luna} + litellm_params: + model: azure/${E2E_AZURE_LUNA_MODEL:-gpt-5.6-luna} api_base: os.environ/AZURE_API_BASE api_key: os.environ/AZURE_API_KEY diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index 6e6c30709de5..6eb040adc9e4 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -27,6 +27,15 @@ CHEAP_ANTHROPIC_MODEL = os.environ.get("E2E_CHEAP_ANTHROPIC_MODEL", "claude-haiku-4-5") CHEAP_OPENAI_MODEL = os.environ.get("E2E_CHEAP_OPENAI_MODEL", "gpt-5.5") +AZURE_CHAT_MODELS = tuple( + f"azure-{os.environ.get(var, default)}" + for var, default in ( + ("E2E_AZURE_SOL_MODEL", "gpt-5.6-sol"), + ("E2E_AZURE_TERRA_MODEL", "gpt-5.6-terra"), + ("E2E_AZURE_LUNA_MODEL", "gpt-5.6-luna"), + ) +) + # Jaeger query API of the compose stack's OTEL trace destination (the `jaeger` # service in docker-compose.yml maps it to host 16686). Trace-completeness tests # read exported spans back through it. diff --git a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py index efcd6fddc2f0..1231527eb3ad 100644 --- a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py +++ b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py @@ -4,31 +4,31 @@ releases: a clean 200 came back but with no real completion. A status check alone would not have caught it, so each case here asserts the product promise - a non-empty assistant message and a real model name in the body - across the -providers wired into the gateway config (OpenAI, Anthropic, Gemini, Azure -OpenAI). A regression that empties the completion for any provider fails that -provider's row here. The Azure OpenAI streaming case applies the same standard -to the SSE path: every data event must parse as a chat.completion.chunk and the -deltas must reassemble into real text, not just count as a 200 with chunks. +providers wired into the gateway config (OpenAI, Anthropic, Gemini, and the +Azure OpenAI GPT capability tiers from e2e_config.AZURE_CHAT_MODELS, deployment +names swappable via the E2E_AZURE_*_MODEL environment variables). A regression +that empties the completion for any provider fails that provider's row here. +The Azure OpenAI streaming cases apply the same standard to the SSE path: every +data event must parse as a chat.completion.chunk and the deltas must reassemble +into real text, not just count as a 200 with chunks. """ from __future__ import annotations import pytest -from e2e_config import unique_marker +from e2e_config import AZURE_CHAT_MODELS, unique_marker from e2e_http import unwrap from models import ChatBody, ChatMessage, ChatStreamChunk from passthrough_client import PassthroughClient pytestmark = pytest.mark.e2e -AZURE_CHAT_MODEL = "azure-gpt-5.4-mini" - CHAT_MODELS: tuple[tuple[str, str], ...] = ( ("gpt-5.5", "openai"), ("claude-haiku-4-5", "anthropic"), ("gemini-2.5-flash", "gemini"), - (AZURE_CHAT_MODEL, "azure_openai"), + *((model, "azure_openai") for model in AZURE_CHAT_MODELS), ) @@ -75,14 +75,15 @@ def test_chat_returns_real_completion( message is not None and message.content and message.content.strip() ), f"{model} ({route}): 200 with an empty completion (#28991): {response}" + @pytest.mark.parametrize("model", AZURE_CHAT_MODELS) @pytest.mark.covers("llm.chat_completions.azure_openai.basic.stream.works") def test_azure_openai_stream_returns_real_completion( - self, client: PassthroughClient, scoped_key: str + self, client: PassthroughClient, scoped_key: str, model: str ) -> None: result = client.gateway.chat_stream( scoped_key, ChatBody( - model=AZURE_CHAT_MODEL, + model=model, messages=[ ChatMessage( role="user", @@ -95,33 +96,33 @@ def test_azure_openai_stream_returns_real_completion( ) assert result.ok, ( - f"{AZURE_CHAT_MODEL}: stream failed with status " + f"{model}: stream failed with status " f"{result.status_code}: {result.body[:300]}" ) assert result.is_streaming, ( - f"{AZURE_CHAT_MODEL}: expected text/event-stream, got " + f"{model}: expected text/event-stream, got " f"{result.content_type}: {result.body[:300]}" ) assert result.stream_error is None, ( - f"{AZURE_CHAT_MODEL}: 200 stream carried an error event: " + f"{model}: 200 stream carried an error event: " f"{result.stream_error}" ) - assert result.events, f"{AZURE_CHAT_MODEL}: stream carried no SSE data events" + assert result.events, f"{model}: stream carried no SSE data events" assert result.events[-1] == "[DONE]", ( - f"{AZURE_CHAT_MODEL}: stream did not terminate with [DONE]: " + f"{model}: stream did not terminate with [DONE]: " f"{result.events[-1][:200]}" ) chunks = [ ChatStreamChunk.model_validate_json(event) for event in result.events[:-1] ] - assert chunks, f"{AZURE_CHAT_MODEL}: stream held only the [DONE] sentinel" + assert chunks, f"{model}: stream held only the [DONE] sentinel" assert all( chunk.object == "chat.completion.chunk" for chunk in chunks - ), f"{AZURE_CHAT_MODEL}: malformed chunk object types: {result.events[:5]}" + ), f"{model}: malformed chunk object types: {result.events[:5]}" assert any( chunk.model for chunk in chunks - ), f"{AZURE_CHAT_MODEL}: no chunk carried a model name: {result.events[:5]}" + ), f"{model}: no chunk carried a model name: {result.events[:5]}" content = "".join( choice.delta.content or "" @@ -130,6 +131,6 @@ def test_azure_openai_stream_returns_real_completion( if choice.delta is not None ) assert content.strip(), ( - f"{AZURE_CHAT_MODEL}: stream chunks reassembled to an empty " + f"{model}: stream chunks reassembled to an empty " f"completion (#28991): {result.events[:5]}" ) From 187cab50495508bd97ca761cfa0072272a474169 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 16 Jul 2026 15:41:00 -0700 Subject: [PATCH 3/3] test(e2e): guard Azure reasoning_effort=none via base_model and xfail the dual token-param bug Adds two customer-regression rows to the Azure chat e2e suite. The first sends reasoning_effort=none to a custom-named deployment (gpt-5.6-sol-e2e, swappable via E2E_AZURE_CUSTOM_MODEL) whose capabilities resolve through base_model, asserting the request completes with zero reasoning tokens on a prompt that reasons at default effort, so both a gate 400 (GH #31243, SDK fix in PR #28490) and a silently dropped param fail the row. The second, a strict xfail until GH #31614 is fixed, sends a client max_completion_tokens to a gpt-4o deployment carrying a config-level max_tokens default; the proxy forwards both and Azure rejects the pair --- .../coverage_registry/llm_conversational.yaml | 2 + tests/e2e/docker-compose.yml | 16 +++ tests/e2e/e2e_config.py | 6 + .../test_chat_completions_regression_e2e.py | 103 +++++++++++++++++- tests/e2e/models.py | 6 + 5 files changed, 132 insertions(+), 1 deletion(-) diff --git a/tests/e2e/coverage_registry/llm_conversational.yaml b/tests/e2e/coverage_registry/llm_conversational.yaml index 40d766e7ce55..1788dc411fac 100644 --- a/tests/e2e/coverage_registry/llm_conversational.yaml +++ b/tests/e2e/coverage_registry/llm_conversational.yaml @@ -31,6 +31,8 @@ - {id: llm.chat_completions.azure_openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: azure_openai, capability: basic, streaming: nonstream, assertions: [works], source: "proxy_server.py:8455", rationale: "P0 route; Azure OpenAI deployments"} - {id: llm.chat_completions.azure_openai.basic.stream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: azure_openai, capability: basic, streaming: stream, assertions: [works], source: "proxy_server.py:8455", rationale: "Streaming over Azure OpenAI"} - {id: llm.chat_completions.azure_openai.tool_use.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: azure_openai, capability: tool_use, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Azure OpenAI function_calling"} +- {id: llm.chat_completions.azure_openai.thinking.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: azure_openai, capability: thinking, streaming: nonstream, assertions: [works], source: "llms/azure/chat/gpt_5_transformation.py", rationale: "reasoning_effort=none on a custom-named deployment must resolve capabilities via base_model and reach Azure with reasoning disabled (GH #31243; SDK gate fixed by PR #28490, proxy base_model registration is the second guard; test fails if either layer 400s or silently drops the param)"} +- {id: llm.chat_completions.azure_openai.basic.nonstream.token_param_dedup, module: llm, tier: P1, subject_endpoint: chat_completions, route: azure_openai, capability: basic, streaming: nonstream, assertions: [works], source: "llms/azure/chat/transformation.py", rationale: "Config-level max_tokens default plus client max_completion_tokens must not forward both to Azure, which 400s on the pair; open bug GH #31614, covering test is xfail until a fix lands", fail_before_fix: proven} - {id: llm.chat_completions.azure_foundry.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: azure_foundry, capability: basic, streaming: nonstream, assertions: [works], source: "proxy_server.py:8455", rationale: "Azure Foundry (azure_ai); newer, smoke"} - {id: llm.messages.anthropic.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: anthropic, capability: basic, streaming: nonstream, assertions: [works], source: "anthropic_endpoints/endpoints.py:64", rationale: "Core endpoint; Anthropic Messages native"} - {id: llm.messages.anthropic.basic.stream.works, module: llm, tier: P0, subject_endpoint: messages, route: anthropic, capability: basic, streaming: stream, assertions: [works], source: "anthropic_endpoints/endpoints.py:64", rationale: "Streaming Messages API"} diff --git a/tests/e2e/docker-compose.yml b/tests/e2e/docker-compose.yml index 3ba8be370574..7af10c91fb73 100644 --- a/tests/e2e/docker-compose.yml +++ b/tests/e2e/docker-compose.yml @@ -84,6 +84,22 @@ configs: api_base: os.environ/AZURE_API_BASE api_key: os.environ/AZURE_API_KEY + - model_name: azure-${E2E_AZURE_CUSTOM_MODEL:-gpt-5.6-sol-e2e} + litellm_params: + model: azure/${E2E_AZURE_CUSTOM_MODEL:-gpt-5.6-sol-e2e} + api_base: os.environ/AZURE_API_BASE + api_key: os.environ/AZURE_API_KEY + drop_params: false + model_info: + base_model: ${E2E_AZURE_CUSTOM_BASE_MODEL:-azure/gpt-5.6-sol} + + - model_name: azure-${E2E_AZURE_GPT4O_MODEL:-gpt-4o} + litellm_params: + model: azure/${E2E_AZURE_GPT4O_MODEL:-gpt-4o} + api_base: os.environ/AZURE_API_BASE + api_key: os.environ/AZURE_API_KEY + max_tokens: 512 + services: litellm: image: ghcr.io/berriai/litellm:main-latest diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index 6eb040adc9e4..4d2f8fa36e4b 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -36,6 +36,12 @@ ) ) +AZURE_CUSTOM_NAME_CHAT_MODEL = ( + f"azure-{os.environ.get('E2E_AZURE_CUSTOM_MODEL', 'gpt-5.6-sol-e2e')}" +) + +AZURE_GPT4O_CHAT_MODEL = f"azure-{os.environ.get('E2E_AZURE_GPT4O_MODEL', 'gpt-4o')}" + # Jaeger query API of the compose stack's OTEL trace destination (the `jaeger` # service in docker-compose.yml maps it to host 16686). Trace-completeness tests # read exported spans back through it. diff --git a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py index 1231527eb3ad..62a29b46942b 100644 --- a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py +++ b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py @@ -11,13 +11,28 @@ The Azure OpenAI streaming cases apply the same standard to the SSE path: every data event must parse as a chat.completion.chunk and the deltas must reassemble into real text, not just count as a 200 with chunks. + +Two cases guard specific customer-reported Azure regressions beyond the happy +path. GH #31243: reasoning_effort='none' against a custom-named deployment must +resolve model capabilities through base_model and reach Azure with reasoning +actually disabled; the prompt is chosen to spend reasoning tokens at default +effort, so the test fails on a gate 400 (the SDK-level bug PR #28490 fixed) and +also on a silently dropped param (which drop_params=true would otherwise mask). +GH #31614 (still open, marked xfail): a config-level max_tokens default +combined with a client-sent max_completion_tokens forwards both parameters to +Azure, which rejects the pair; the strict xfail flips when a fix lands. """ from __future__ import annotations import pytest -from e2e_config import AZURE_CHAT_MODELS, unique_marker +from e2e_config import ( + AZURE_CHAT_MODELS, + AZURE_CUSTOM_NAME_CHAT_MODEL, + AZURE_GPT4O_CHAT_MODEL, + unique_marker, +) from e2e_http import unwrap from models import ChatBody, ChatMessage, ChatStreamChunk from passthrough_client import PassthroughClient @@ -134,3 +149,89 @@ def test_azure_openai_stream_returns_real_completion( f"{model}: stream chunks reassembled to an empty " f"completion (#28991): {result.events[:5]}" ) + + @pytest.mark.covers("llm.chat_completions.azure_openai.thinking.nonstream.works") + def test_azure_custom_deployment_name_reasoning_effort_none( + self, client: PassthroughClient, scoped_key: str + ) -> None: + response = unwrap( + client.gateway.chat( + scoped_key, + ChatBody( + model=AZURE_CUSTOM_NAME_CHAT_MODEL, + messages=[ + ChatMessage( + role="user", + content=( + "A farmer has 17 sheep, all but 9 run away, then " + "he buys twice as many as remain minus 3. How many " + "sheep? Reply with just the number. " + f"(session {unique_marker()})" + ), + ) + ], + max_completion_tokens=2000, + reasoning_effort="none", + ), + ) + ) + + assert response.choices, ( + f"{AZURE_CUSTOM_NAME_CHAT_MODEL}: reasoning_effort='none' on a " + f"custom-named deployment must resolve capabilities via base_model " + f"(GH #31243): {response}" + ) + message = response.choices[0].message + assert message is not None and message.content and message.content.strip(), ( + f"{AZURE_CUSTOM_NAME_CHAT_MODEL}: empty completion for " + f"reasoning_effort='none' (GH #31243): {response}" + ) + reasoning_tokens = ( + response.usage.completion_tokens_details.reasoning_tokens + if response.usage and response.usage.completion_tokens_details + else None + ) + assert not reasoning_tokens, ( + f"{AZURE_CUSTOM_NAME_CHAT_MODEL}: reasoning_effort='none' must " + f"disable reasoning, but the model spent {reasoning_tokens} " + f"reasoning tokens (GH #31243): {response.usage}" + ) + + @pytest.mark.covers( + "llm.chat_completions.azure_openai.basic.nonstream.token_param_dedup" + ) + @pytest.mark.xfail( + strict=True, + reason=( + "GH #31614: a config-level max_tokens default plus a client " + "max_completion_tokens forwards both to Azure, which rejects the pair" + ), + ) + def test_azure_config_token_cap_with_client_max_completion_tokens( + self, client: PassthroughClient, scoped_key: str + ) -> None: + response = unwrap( + client.gateway.chat( + scoped_key, + ChatBody( + model=AZURE_GPT4O_CHAT_MODEL, + messages=[ + ChatMessage( + role="user", + content=f"reply with one word {unique_marker()}", + ) + ], + max_completion_tokens=256, + ), + ) + ) + + assert response.choices, ( + f"{AZURE_GPT4O_CHAT_MODEL}: client max_completion_tokens on a " + f"deployment with a config max_tokens default must still complete " + f"(GH #31614): {response}" + ) + message = response.choices[0].message + assert message is not None and message.content and message.content.strip(), ( + f"{AZURE_GPT4O_CHAT_MODEL}: empty completion (GH #31614): {response}" + ) diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 6e3bd8ad5547..93a78ea38732 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -140,6 +140,7 @@ class ChatBody(BaseModel): messages: list[ChatMessage] stream: bool = False max_tokens: int | None = None + max_completion_tokens: int | None = None user: str | None = None metadata: ChatMetadata | None = None reasoning_effort: str | None = None @@ -174,6 +175,10 @@ class PromptTokensDetails(BaseModel): cached_tokens: int | None = None +class CompletionTokensDetails(BaseModel): + reasoning_tokens: int | None = None + + class Usage(BaseModel): prompt_tokens: int | None = None completion_tokens: int | None = None @@ -181,6 +186,7 @@ class Usage(BaseModel): cache_read_input_tokens: int | None = None cache_creation_input_tokens: int | None = None prompt_tokens_details: PromptTokensDetails | None = None + completion_tokens_details: CompletionTokensDetails | None = None class ChatResponse(BaseModel):