diff --git a/tests/e2e/claude_code/_gpt_cells.py b/tests/e2e/claude_code/_gpt_cells.py new file mode 100644 index 000000000000..870e9cea9180 --- /dev/null +++ b/tests/e2e/claude_code/_gpt_cells.py @@ -0,0 +1,61 @@ +"""Shared plumbing for the GPT-5.6 (Sol / Terra / Luna) provider columns. + +OpenAI shipped GPT-5.6 as a three-tier family on 2026-07-09 — Sol +(flagship), Terra (balanced), Luna (fast) — and Claude Code can drive +all three through a LiteLLM proxy that translates the Anthropic +Messages API to each provider's native shape. Four provider columns +cover "OpenAI plus the big three clouds": + + openai OpenAI API (openai/gpt-5.6-*) + azure_openai Azure OpenAI (azure/gpt-5.6-*) + bedrock_mantle AWS Bedrock, Mantle (bedrock_mantle/openai.gpt-5.6-*, + Responses API) + vertex_ai_gpt GCP Vertex AI not_applicable — Vertex does + not offer the closed-weight + GPT-5.6 family; Model Garden + carries only the open-weight + gpt-oss MaaS models + +The openai and azure_openai columns run unconditionally, like every +other live column: the environments that run the suite carry +`OPENAI_API_KEY` and `AZURE_API_BASE` + `AZURE_API_KEY` pointing at a +resource with gpt-5.6 deployments. The bedrock_mantle column is +opt-in via `COMPAT_MANTLE_CELLS=1` because the AWS account is still +waiting on the Bedrock Mantle allowlist for the `openai.gpt-5.6-*` +models; until the flag is set each Mantle cell skips and its matrix +cell publishes as `not_tested` instead of a credential-shaped red. +The `vertex_ai_gpt` column needs no flag either way: its cells report +a static `not_applicable` and never touch the network. +""" + +from __future__ import annotations + +import os + +import pytest + +MANTLE_CELLS_ENV = "COMPAT_MANTLE_CELLS" + +VERTEX_AI_GPT_NOT_APPLICABLE_REASON = ( + "GCP Vertex AI does not offer OpenAI's closed-weight GPT-5.6 family " + "(Sol / Terra / Luna); Model Garden carries only the open-weight " + "gpt-oss MaaS models. Convert this column's cells to live tests if " + "Google adds the GPT-5.6 models." +) + + +def skip_unless_mantle_cells_enabled() -> None: + """Skip the calling test unless `COMPAT_MANTLE_CELLS` opts the + Bedrock Mantle cells in. + + A skipped cell is recorded as `not_tested` in the published matrix + (see the skip handling in `tests/e2e/claude_code/conftest.py`), + which is the honest state while the AWS account has no Mantle + access to the GPT-5.6 models yet. + """ + if os.environ.get(MANTLE_CELLS_ENV, "").strip().lower() in {"1", "true", "yes"}: + return + pytest.skip( + f"Bedrock Mantle GPT-5.6 cells are opt-in; set {MANTLE_CELLS_ENV}=1 " + "once the AWS account is allowlisted for the openai.gpt-5.6-* models" + ) diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_azure_openai.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_azure_openai.py new file mode 100644 index 000000000000..77876c8f7eef --- /dev/null +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_azure_openai.py @@ -0,0 +1,44 @@ +"""basic_messaging_non_streaming x Azure OpenAI (GPT-5.6). + +Drive the real `claude` CLI in headless mode against a running LiteLLM +proxy that routes Anthropic Messages requests to Azure OpenAI +deployments of the GPT-5.6 family (Sol, Terra, Luna), and report the +outcome via `compat_result`. + +Azure OpenAI serves the same chat-completions wire shape as +openai.com behind per-resource deployments; LiteLLM's `azure/gpt-*` +route handles the deployment addressing while reusing the OpenAI +translation, so this cell catches Azure-specific regressions +(auth headers, api-version pinning, deployment routing) that the +`openai` column cannot. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/basic_messaging_non_streaming/test_azure_openai.py + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^ + feature_id provider + +Every GPT cell exercises the three GPT-5.6 tiers; the cell only goes +green if all three pass. +""" + +from __future__ import annotations + +from claude_code._basic_messaging import run_basic_messaging_cell + +AZURE_OPENAI_MODELS = [ + "gpt-5-6-sol-azure-openai", + "gpt-5-6-terra-azure-openai", + "gpt-5-6-luna-azure-openai", +] + + +def test_basic_messaging_non_streaming_azure_openai(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert a + non-empty reply from each GPT-5.6 tier.""" + run_basic_messaging_cell( + compat_result=compat_result, + models=AZURE_OPENAI_MODELS, + prompt="Reply with the single word 'pong' and nothing else.", + ) diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_mantle.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_mantle.py new file mode 100644 index 000000000000..8a64547a732a --- /dev/null +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_mantle.py @@ -0,0 +1,46 @@ +"""basic_messaging_non_streaming x AWS Bedrock Mantle (GPT-5.6). + +Drive the real `claude` CLI in headless mode against a running LiteLLM +proxy that routes Anthropic Messages requests to OpenAI's GPT-5.6 +family (Sol, Terra, Luna) hosted on AWS Bedrock, and report the +outcome via `compat_result`. + +Bedrock exposes the GPT-5.6 models through the Mantle endpoint, which +speaks the OpenAI Responses API rather than Converse/Invoke; LiteLLM's +`bedrock_mantle/openai.gpt-*` route signs the request with SigV4 and +translates Anthropic Messages to Responses, so this cell exercises a +translation path no other column covers. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_mantle.py + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^ + feature_id provider + +Every GPT cell exercises the three GPT-5.6 tiers; the cell only goes +green if all three pass. Mantle cells are opt-in via +COMPAT_MANTLE_CELLS=1 (see `claude_code._gpt_cells`). +""" + +from __future__ import annotations + +from claude_code._basic_messaging import run_basic_messaging_cell +from claude_code._gpt_cells import skip_unless_mantle_cells_enabled + +BEDROCK_MANTLE_MODELS = [ + "gpt-5-6-sol-bedrock-mantle", + "gpt-5-6-terra-bedrock-mantle", + "gpt-5-6-luna-bedrock-mantle", +] + + +def test_basic_messaging_non_streaming_bedrock_mantle(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert a + non-empty reply from each GPT-5.6 tier.""" + skip_unless_mantle_cells_enabled() + run_basic_messaging_cell( + compat_result=compat_result, + models=BEDROCK_MANTLE_MODELS, + prompt="Reply with the single word 'pong' and nothing else.", + ) diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_openai.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_openai.py new file mode 100644 index 000000000000..b0d143fa5e05 --- /dev/null +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_openai.py @@ -0,0 +1,41 @@ +"""basic_messaging_non_streaming x OpenAI (GPT-5.6). + +Drive the real `claude` CLI in headless mode against a running LiteLLM +proxy that routes Anthropic Messages requests to OpenAI's GPT-5.6 +family (Sol, Terra, Luna), and report the outcome via `compat_result`. + +Claude Code only speaks the Anthropic Messages API; LiteLLM's +`openai/gpt-*` route translates the request to OpenAI chat completions +and maps the response back, so this cell exercises the full +cross-provider translation layer in both directions. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/basic_messaging_non_streaming/test_openai.py + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^ + feature_id provider + +Every GPT cell exercises the three GPT-5.6 tiers; the cell only goes +green if all three pass. +""" + +from __future__ import annotations + +from claude_code._basic_messaging import run_basic_messaging_cell + +OPENAI_MODELS = [ + "gpt-5-6-sol-openai", + "gpt-5-6-terra-openai", + "gpt-5-6-luna-openai", +] + + +def test_basic_messaging_non_streaming_openai(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert a + non-empty reply from each GPT-5.6 tier.""" + run_basic_messaging_cell( + compat_result=compat_result, + models=OPENAI_MODELS, + prompt="Reply with the single word 'pong' and nothing else.", + ) diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_vertex_ai_gpt.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_vertex_ai_gpt.py new file mode 100644 index 000000000000..3b155b6ac9dd --- /dev/null +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_vertex_ai_gpt.py @@ -0,0 +1,29 @@ +"""basic_messaging_non_streaming x Vertex AI (GPT-5.6) — not applicable. + +GCP is the only one of the big-three clouds without OpenAI's +closed-weight GPT-5.6 family (Sol / Terra / Luna); Vertex AI Model +Garden carries only the open-weight gpt-oss MaaS models. The cell +reports `not_applicable` so the published matrix documents the gap +explicitly instead of leaving a `not_tested` hole. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/basic_messaging_non_streaming/test_vertex_ai_gpt.py + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +from claude_code._gpt_cells import VERTEX_AI_GPT_NOT_APPLICABLE_REASON + + +def test_basic_messaging_non_streaming_vertex_ai_gpt(compat_result): + """Record the static not_applicable outcome for this cell.""" + compat_result.set( + { + "status": "not_applicable", + "reason": VERTEX_AI_GPT_NOT_APPLICABLE_REASON, + } + ) diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_azure_openai.py b/tests/e2e/claude_code/basic_messaging_streaming/test_azure_openai.py new file mode 100644 index 000000000000..357596590c74 --- /dev/null +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_azure_openai.py @@ -0,0 +1,44 @@ +"""basic_messaging_streaming x Azure OpenAI (GPT-5.6). + +Drive the real `claude` CLI in headless `--output-format stream-json` +mode against a running LiteLLM proxy that routes Anthropic Messages +requests to Azure OpenAI deployments of the GPT-5.6 family (Sol, +Terra, Luna), and report the outcome via `compat_result`. + +Azure OpenAI streams the same chat-completions SSE shape as +openai.com; LiteLLM re-emits it as Anthropic stream events, and the +`verify_streaming=True` assertion (via `--include-partial-messages`) +proves the events arrived incrementally rather than as one buffered +response. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/basic_messaging_streaming/test_azure_openai.py + ^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^ + feature_id provider + +Every GPT cell exercises the three GPT-5.6 tiers; the cell only goes +green if all three pass. +""" + +from __future__ import annotations + +from claude_code._basic_messaging import run_basic_messaging_cell + +AZURE_OPENAI_MODELS = [ + "gpt-5-6-sol-azure-openai", + "gpt-5-6-terra-azure-openai", + "gpt-5-6-luna-azure-openai", +] + + +def test_basic_messaging_streaming_azure_openai(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert a + non-empty streamed reply from each GPT-5.6 tier.""" + run_basic_messaging_cell( + compat_result=compat_result, + models=AZURE_OPENAI_MODELS, + prompt="Count from 1 to 5, one number per line.", + verify_streaming=True, + ) diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_mantle.py b/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_mantle.py new file mode 100644 index 000000000000..38297e6a3e58 --- /dev/null +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_mantle.py @@ -0,0 +1,47 @@ +"""basic_messaging_streaming x AWS Bedrock Mantle (GPT-5.6). + +Drive the real `claude` CLI in headless `--output-format stream-json` +mode against a running LiteLLM proxy that routes Anthropic Messages +requests to OpenAI's GPT-5.6 family (Sol, Terra, Luna) on AWS +Bedrock's Mantle endpoint, and report the outcome via `compat_result`. + +Mantle streams OpenAI Responses API events over SigV4-signed SSE; +LiteLLM re-emits them as Anthropic stream events, and the +`verify_streaming=True` assertion (via `--include-partial-messages`) +proves the events arrived incrementally rather than as one buffered +response. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_mantle.py + ^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^ + feature_id provider + +Every GPT cell exercises the three GPT-5.6 tiers; the cell only goes +green if all three pass. Mantle cells are opt-in via +COMPAT_MANTLE_CELLS=1 (see `claude_code._gpt_cells`). +""" + +from __future__ import annotations + +from claude_code._basic_messaging import run_basic_messaging_cell +from claude_code._gpt_cells import skip_unless_mantle_cells_enabled + +BEDROCK_MANTLE_MODELS = [ + "gpt-5-6-sol-bedrock-mantle", + "gpt-5-6-terra-bedrock-mantle", + "gpt-5-6-luna-bedrock-mantle", +] + + +def test_basic_messaging_streaming_bedrock_mantle(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert a + non-empty streamed reply from each GPT-5.6 tier.""" + skip_unless_mantle_cells_enabled() + run_basic_messaging_cell( + compat_result=compat_result, + models=BEDROCK_MANTLE_MODELS, + prompt="Count from 1 to 5, one number per line.", + verify_streaming=True, + ) diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_openai.py b/tests/e2e/claude_code/basic_messaging_streaming/test_openai.py new file mode 100644 index 000000000000..402c763496bd --- /dev/null +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_openai.py @@ -0,0 +1,44 @@ +"""basic_messaging_streaming x OpenAI (GPT-5.6). + +Drive the real `claude` CLI in headless `--output-format stream-json` +mode against a running LiteLLM proxy that routes Anthropic Messages +requests to OpenAI's GPT-5.6 family (Sol, Terra, Luna), and report the +outcome via `compat_result`. + +LiteLLM translates OpenAI's chat-completions SSE chunks into Anthropic +`message_start` / `content_block_delta` / `message_stop` events on the +fly; the `verify_streaming=True` assertion (via +`--include-partial-messages`) proves the proxy re-emitted incremental +events instead of buffering the upstream stream into one response. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/basic_messaging_streaming/test_openai.py + ^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^ + feature_id provider + +Every GPT cell exercises the three GPT-5.6 tiers; the cell only goes +green if all three pass. +""" + +from __future__ import annotations + +from claude_code._basic_messaging import run_basic_messaging_cell + +OPENAI_MODELS = [ + "gpt-5-6-sol-openai", + "gpt-5-6-terra-openai", + "gpt-5-6-luna-openai", +] + + +def test_basic_messaging_streaming_openai(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert a + non-empty streamed reply from each GPT-5.6 tier.""" + run_basic_messaging_cell( + compat_result=compat_result, + models=OPENAI_MODELS, + prompt="Count from 1 to 5, one number per line.", + verify_streaming=True, + ) diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_vertex_ai_gpt.py b/tests/e2e/claude_code/basic_messaging_streaming/test_vertex_ai_gpt.py new file mode 100644 index 000000000000..f6aa01de5216 --- /dev/null +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_vertex_ai_gpt.py @@ -0,0 +1,29 @@ +"""basic_messaging_streaming x Vertex AI (GPT-5.6) — not applicable. + +GCP is the only one of the big-three clouds without OpenAI's +closed-weight GPT-5.6 family (Sol / Terra / Luna); Vertex AI Model +Garden carries only the open-weight gpt-oss MaaS models. The cell +reports `not_applicable` so the published matrix documents the gap +explicitly instead of leaving a `not_tested` hole. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/basic_messaging_streaming/test_vertex_ai_gpt.py + ^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +from claude_code._gpt_cells import VERTEX_AI_GPT_NOT_APPLICABLE_REASON + + +def test_basic_messaging_streaming_vertex_ai_gpt(compat_result): + """Record the static not_applicable outcome for this cell.""" + compat_result.set( + { + "status": "not_applicable", + "reason": VERTEX_AI_GPT_NOT_APPLICABLE_REASON, + } + ) diff --git a/tests/e2e/claude_code/manifest.yaml b/tests/e2e/claude_code/manifest.yaml index e5a956991cb1..ac6335d70d1d 100644 --- a/tests/e2e/claude_code/manifest.yaml +++ b/tests/e2e/claude_code/manifest.yaml @@ -12,13 +12,24 @@ schema_version: "1" -# Provider column order in the rendered matrix. +# Provider column order in the rendered matrix. The first five are +# the v0 Claude columns; the GPT-5.6 (Sol / Terra / Luna) columns +# added 2026-07 follow them. `vertex_ai_gpt` is a static +# not_applicable column: GCP does not offer the closed-weight GPT-5.6 +# family (Model Garden carries only the open-weight gpt-oss MaaS +# models), and the column documents that gap explicitly. GPT columns +# currently back the two basic_messaging rows plus tool_use and +# tool_use_streaming; other rows render not_tested for them. providers: - anthropic - bedrock_invoke - bedrock_converse - vertex_ai - azure + - openai + - azure_openai + - bedrock_mantle + - vertex_ai_gpt # Feature row order. features: diff --git a/tests/e2e/claude_code/rate_limiter.py b/tests/e2e/claude_code/rate_limiter.py index 06d21b838321..5818338ff7f8 100644 --- a/tests/e2e/claude_code/rate_limiter.py +++ b/tests/e2e/claude_code/rate_limiter.py @@ -24,6 +24,9 @@ LITELLM_COMPAT_RATE_VERTEX_AI (req/s, default 5.0) LITELLM_COMPAT_RATE_BEDROCK_CONVERSE (req/s, default 5.0) LITELLM_COMPAT_RATE_BEDROCK_INVOKE (req/s, default 5.0) + LITELLM_COMPAT_RATE_OPENAI (req/s, default 5.0) + LITELLM_COMPAT_RATE_AZURE_OPENAI (req/s, default 5.0) + LITELLM_COMPAT_RATE_BEDROCK_MANTLE (req/s, default 5.0) LITELLM_COMPAT_RATE_BURST (per-bucket burst override; default = rate) LITELLM_COMPAT_RATE_STATE_DIR (state file directory; @@ -36,7 +39,10 @@ The provider id is inferred from the model id by `infer_provider`, mirroring the matrix's column layout (`anthropic`, `azure`, -`vertex_ai`, `bedrock_converse`, `bedrock_invoke`). +`vertex_ai`, `bedrock_converse`, `bedrock_invoke`, `openai`, +`azure_openai`, `bedrock_mantle`). The `vertex_ai_gpt` matrix column +has no bucket: its cells are static not_applicable stubs that never +reach the network. """ from __future__ import annotations @@ -62,6 +68,9 @@ PROVIDER_VERTEX_AI = "vertex_ai" PROVIDER_BEDROCK_CONVERSE = "bedrock_converse" PROVIDER_BEDROCK_INVOKE = "bedrock_invoke" +PROVIDER_OPENAI = "openai" +PROVIDER_AZURE_OPENAI = "azure_openai" +PROVIDER_BEDROCK_MANTLE = "bedrock_mantle" ALL_PROVIDERS = ( PROVIDER_ANTHROPIC, @@ -69,6 +78,9 @@ PROVIDER_VERTEX_AI, PROVIDER_BEDROCK_CONVERSE, PROVIDER_BEDROCK_INVOKE, + PROVIDER_OPENAI, + PROVIDER_AZURE_OPENAI, + PROVIDER_BEDROCK_MANTLE, ) DEFAULT_RATE = 5.0 # req/s per provider, conservative starting point @@ -83,13 +95,21 @@ def infer_provider(model: str) -> str: The matrix column layout is fixed; aliases registered in the proxy encode the provider via a suffix (`-bedrock-converse`, - `-bedrock-invoke`, `-azure`, `-vertex`) or its absence (Anthropic). - Order matters: the bedrock suffixes both contain `bedrock`, so we - test the more-specific ones first. + `-bedrock-invoke`, `-azure`, `-vertex`, `-openai`, `-azure-openai`, + `-bedrock-mantle`) or its absence (Anthropic). Order matters: + `-azure-openai` also ends with `-openai`, and the bedrock suffixes + all contain `bedrock`, so the more-specific suffixes are tested + first. """ if not model: raise ValueError("model must be a non-empty string") lower = model.lower() + if lower.endswith("-azure-openai"): + return PROVIDER_AZURE_OPENAI + if lower.endswith("-openai"): + return PROVIDER_OPENAI + if lower.endswith("-bedrock-mantle"): + return PROVIDER_BEDROCK_MANTLE if lower.endswith("-bedrock-converse"): return PROVIDER_BEDROCK_CONVERSE if lower.endswith("-bedrock-invoke"): diff --git a/tests/e2e/claude_code/run_compat.sh b/tests/e2e/claude_code/run_compat.sh index e383792f45eb..b881cf1d31e5 100755 --- a/tests/e2e/claude_code/run_compat.sh +++ b/tests/e2e/claude_code/run_compat.sh @@ -21,8 +21,16 @@ # LITELLM_COMPAT_RATE_VERTEX_AI # LITELLM_COMPAT_RATE_BEDROCK_CONVERSE # LITELLM_COMPAT_RATE_BEDROCK_INVOKE +# LITELLM_COMPAT_RATE_OPENAI +# LITELLM_COMPAT_RATE_AZURE_OPENAI +# LITELLM_COMPAT_RATE_BEDROCK_MANTLE # LITELLM_COMPAT_RATE_BURST override per-bucket burst # +# Optional env (GPT-5.6 columns): +# COMPAT_MANTLE_CELLS=1 opt the Bedrock Mantle GPT-5.6 +# cells in; without it they skip +# and publish as not_tested +# # Optional env (parallelism): # COMPAT_XDIST_WORKERS passed to `pytest -n` (default: auto) # @@ -57,7 +65,7 @@ results_path="${COMPAT_RESULTS_PATH:-compat-results.json}" summary_path="${COMPAT_RATE_LIMIT_SUMMARY_PATH:-compat-rate-limit-summary.json}" echo "[run_compat] rates:" -for provider in ANTHROPIC AZURE VERTEX_AI BEDROCK_CONVERSE BEDROCK_INVOKE; do +for provider in ANTHROPIC AZURE VERTEX_AI BEDROCK_CONVERSE BEDROCK_INVOKE OPENAI AZURE_OPENAI BEDROCK_MANTLE; do var="LITELLM_COMPAT_RATE_${provider}" echo " ${provider}=${!var:-default(5/s)}" done diff --git a/tests/e2e/claude_code/test_config.yaml b/tests/e2e/claude_code/test_config.yaml index ba57ca4ccb77..eab913be7fe2 100644 --- a/tests/e2e/claude_code/test_config.yaml +++ b/tests/e2e/claude_code/test_config.yaml @@ -14,6 +14,14 @@ # - claude-{tier}-bedrock-converse → Bedrock Converse API # - claude-{tier}-vertex → GCP Vertex AI # - claude-{tier}-azure → Microsoft Foundry (Anthropic deployments) +# - gpt-5-6-{tier}-openai → OpenAI API +# - gpt-5-6-{tier}-azure-openai → Azure OpenAI deployments +# - gpt-5-6-{tier}-bedrock-mantle → Bedrock Mantle (Responses API) +# +# GPT-5.6 tiers are sol / terra / luna. There are no GPT aliases for +# GCP: Vertex AI does not offer the closed-weight GPT-5.6 family, so +# the matrix's `vertex_ai_gpt` column reports not_applicable without +# ever reaching the proxy. model_list: # ---- Anthropic ---- @@ -122,6 +130,54 @@ model_list: extra_headers: anthropic-beta: "context-1m-2025-08-07" + # ---- OpenAI (GPT-5.6) ---- + - model_name: gpt-5-6-sol-openai + litellm_params: + model: openai/gpt-5.6-sol + api_key: os.environ/OPENAI_API_KEY + - model_name: gpt-5-6-terra-openai + litellm_params: + model: openai/gpt-5.6-terra + api_key: os.environ/OPENAI_API_KEY + - model_name: gpt-5-6-luna-openai + litellm_params: + model: openai/gpt-5.6-luna + api_key: os.environ/OPENAI_API_KEY + + # ---- Azure OpenAI (GPT-5.6) ---- + - model_name: gpt-5-6-sol-azure-openai + litellm_params: + model: azure/gpt-5.6-sol + api_base: os.environ/AZURE_API_BASE + api_key: os.environ/AZURE_API_KEY + - model_name: gpt-5-6-terra-azure-openai + litellm_params: + model: azure/gpt-5.6-terra + api_base: os.environ/AZURE_API_BASE + api_key: os.environ/AZURE_API_KEY + - model_name: gpt-5-6-luna-azure-openai + litellm_params: + model: azure/gpt-5.6-luna + api_base: os.environ/AZURE_API_BASE + api_key: os.environ/AZURE_API_KEY + + # ---- Bedrock Mantle (GPT-5.6, Responses API) ---- + # Sol is only served from us-east-1 / us-east-2 as of 2026-07; + # Terra and Luna additionally have us-west-2. One region keeps the + # column comparable across tiers. + - model_name: gpt-5-6-sol-bedrock-mantle + litellm_params: + model: bedrock_mantle/openai.gpt-5.6-sol + aws_region_name: us-east-1 + - model_name: gpt-5-6-terra-bedrock-mantle + litellm_params: + model: bedrock_mantle/openai.gpt-5.6-terra + aws_region_name: us-east-1 + - model_name: gpt-5-6-luna-bedrock-mantle + litellm_params: + model: bedrock_mantle/openai.gpt-5.6-luna + aws_region_name: us-east-1 + general_settings: # Claude Code sends provider-specific headers (e.g. anthropic-beta) we # want to forward verbatim to the upstream so the wire-shape under diff --git a/tests/e2e/claude_code/tool_use/test_azure_openai.py b/tests/e2e/claude_code/tool_use/test_azure_openai.py new file mode 100644 index 000000000000..7e1eecdbc03a --- /dev/null +++ b/tests/e2e/claude_code/tool_use/test_azure_openai.py @@ -0,0 +1,109 @@ +"""tool_use x Azure OpenAI (GPT-5.6). + +Drive the real `claude` CLI against a running LiteLLM proxy that +routes Anthropic Messages requests to Azure OpenAI deployments of the +GPT-5.6 family (Sol, Terra, Luna), ask the model to invoke a built-in +tool (`Bash`), and assert that a `tool_use` content block came back +over the wire. + +Azure OpenAI serves the same function-calling wire shape as +openai.com behind per-resource deployments; LiteLLM's `azure/gpt-*` +route reuses the OpenAI tool translation on top of Azure's deployment +addressing and auth. + +Bash is restricted to the exact command `echo pong` plus +`--permission-mode dontAsk`; see `tool_use/test_anthropic.py` for the +security rationale. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/tool_use/test_azure_openai.py + ^^^^^^^^ ^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +from typing import Any, Mapping, Sequence + +import pytest + +from claude_code._env import require_proxy +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +AZURE_OPENAI_MODELS = [ + "gpt-5-6-sol-azure-openai", + "gpt-5-6-terra-azure-openai", + "gpt-5-6-luna-azure-openai", +] + +TOOL_USE_PROMPT = ( + "Use the Bash tool to run the command `echo pong` and report what it printed." +) +TOOL_USE_ARGS = [ + "--allowed-tools", + "Bash(echo pong)", + "--permission-mode", + "dontAsk", +] + + +def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + return True + return False + + +def test_tool_use_azure_openai(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert a + tool call was emitted on the wire by each GPT-5.6 tier.""" + proxy = require_proxy(compat_result) + + outcomes = run_claude_models_parallel( + models=AZURE_OPENAI_MODELS, + prompt=TOOL_USE_PROMPT, + base_url=proxy.base_url, + api_key=proxy.api_key, + extra_args=TOOL_USE_ARGS, + ) + + failures = [] + for model in AZURE_OPENAI_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_tool_use_event(outcome.events): + error = ( + f"[{model}] no tool_use content block observed in stream-json events" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/tool_use/test_bedrock_mantle.py b/tests/e2e/claude_code/tool_use/test_bedrock_mantle.py new file mode 100644 index 000000000000..e9cb70e74e97 --- /dev/null +++ b/tests/e2e/claude_code/tool_use/test_bedrock_mantle.py @@ -0,0 +1,115 @@ +"""tool_use x AWS Bedrock Mantle (GPT-5.6). + +Drive the real `claude` CLI against a running LiteLLM proxy that +routes Anthropic Messages requests to OpenAI's GPT-5.6 family (Sol, +Terra, Luna) on AWS Bedrock's Mantle endpoint, ask the model to invoke +a built-in tool (`Bash`), and assert that a `tool_use` content block +came back over the wire. + +Mantle speaks the OpenAI Responses API, whose tool declarations and +`function_call` outputs differ from both Anthropic Messages and +chat completions; LiteLLM's `bedrock_mantle/openai.gpt-*` route +translates Anthropic `tools` into Responses tool declarations and maps +the emitted function calls back to `tool_use` blocks. + +Bash is restricted to the exact command `echo pong` plus +`--permission-mode dontAsk`; see `tool_use/test_anthropic.py` for the +security rationale. + +Mantle cells are opt-in via COMPAT_MANTLE_CELLS=1 (see +`claude_code._gpt_cells`). + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/tool_use/test_bedrock_mantle.py + ^^^^^^^^ ^^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +from typing import Any, Mapping, Sequence + +import pytest + +from claude_code._env import require_proxy +from claude_code._gpt_cells import skip_unless_mantle_cells_enabled +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +BEDROCK_MANTLE_MODELS = [ + "gpt-5-6-sol-bedrock-mantle", + "gpt-5-6-terra-bedrock-mantle", + "gpt-5-6-luna-bedrock-mantle", +] + +TOOL_USE_PROMPT = ( + "Use the Bash tool to run the command `echo pong` and report what it printed." +) +TOOL_USE_ARGS = [ + "--allowed-tools", + "Bash(echo pong)", + "--permission-mode", + "dontAsk", +] + + +def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + return True + return False + + +def test_tool_use_bedrock_mantle(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert a + tool call was emitted on the wire by each GPT-5.6 tier.""" + skip_unless_mantle_cells_enabled() + proxy = require_proxy(compat_result) + + outcomes = run_claude_models_parallel( + models=BEDROCK_MANTLE_MODELS, + prompt=TOOL_USE_PROMPT, + base_url=proxy.base_url, + api_key=proxy.api_key, + extra_args=TOOL_USE_ARGS, + ) + + failures = [] + for model in BEDROCK_MANTLE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_tool_use_event(outcome.events): + error = ( + f"[{model}] no tool_use content block observed in stream-json events" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/tool_use/test_openai.py b/tests/e2e/claude_code/tool_use/test_openai.py new file mode 100644 index 000000000000..dbe60a65281c --- /dev/null +++ b/tests/e2e/claude_code/tool_use/test_openai.py @@ -0,0 +1,108 @@ +"""tool_use x OpenAI (GPT-5.6). + +Drive the real `claude` CLI against a running LiteLLM proxy that +routes Anthropic Messages requests to OpenAI's GPT-5.6 family (Sol, +Terra, Luna), ask the model to invoke a built-in tool (`Bash`), and +assert that a `tool_use` content block came back over the wire. + +Claude Code declares its tools in Anthropic `tools` format; LiteLLM's +`openai/gpt-*` route translates them to OpenAI function calling and +maps the returned `tool_calls` back to Anthropic `tool_use` blocks, so +this cell exercises the tool-schema translation in both directions. + +Bash is restricted to the exact command `echo pong` plus +`--permission-mode dontAsk`; see `tool_use/test_anthropic.py` for the +security rationale. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/tool_use/test_openai.py + ^^^^^^^^ ^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +from typing import Any, Mapping, Sequence + +import pytest + +from claude_code._env import require_proxy +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +OPENAI_MODELS = [ + "gpt-5-6-sol-openai", + "gpt-5-6-terra-openai", + "gpt-5-6-luna-openai", +] + +TOOL_USE_PROMPT = ( + "Use the Bash tool to run the command `echo pong` and report what it printed." +) +TOOL_USE_ARGS = [ + "--allowed-tools", + "Bash(echo pong)", + "--permission-mode", + "dontAsk", +] + + +def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + return True + return False + + +def test_tool_use_openai(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert a + tool call was emitted on the wire by each GPT-5.6 tier.""" + proxy = require_proxy(compat_result) + + outcomes = run_claude_models_parallel( + models=OPENAI_MODELS, + prompt=TOOL_USE_PROMPT, + base_url=proxy.base_url, + api_key=proxy.api_key, + extra_args=TOOL_USE_ARGS, + ) + + failures = [] + for model in OPENAI_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_tool_use_event(outcome.events): + error = ( + f"[{model}] no tool_use content block observed in stream-json events" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/tool_use/test_vertex_ai_gpt.py b/tests/e2e/claude_code/tool_use/test_vertex_ai_gpt.py new file mode 100644 index 000000000000..d1ebbced9dc5 --- /dev/null +++ b/tests/e2e/claude_code/tool_use/test_vertex_ai_gpt.py @@ -0,0 +1,33 @@ +"""tool_use x Vertex AI (GPT-5.6) — not applicable. + +GCP is the only one of the big-three clouds without OpenAI's +closed-weight GPT-5.6 family (Sol / Terra / Luna); Vertex AI Model +Garden carries only the open-weight gpt-oss MaaS models. The cell +reports `not_applicable` so the published matrix documents the gap +explicitly instead of leaving a `not_tested` hole. + +This stub never drives the `claude` CLI, so it grants no tools and is +exempt from the Bash allow-rule pin enforced by +`_pr_gate_unit_tests/test_bash_tool_restrictions.py`. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/tool_use/test_vertex_ai_gpt.py + ^^^^^^^^ ^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +from claude_code._gpt_cells import VERTEX_AI_GPT_NOT_APPLICABLE_REASON + + +def test_tool_use_vertex_ai_gpt(compat_result): + """Record the static not_applicable outcome for this cell.""" + compat_result.set( + { + "status": "not_applicable", + "reason": VERTEX_AI_GPT_NOT_APPLICABLE_REASON, + } + ) diff --git a/tests/e2e/claude_code/tool_use_streaming/test_azure_openai.py b/tests/e2e/claude_code/tool_use_streaming/test_azure_openai.py new file mode 100644 index 000000000000..ad5d4e0f613f --- /dev/null +++ b/tests/e2e/claude_code/tool_use_streaming/test_azure_openai.py @@ -0,0 +1,138 @@ +"""tool_use_streaming x Azure OpenAI (GPT-5.6). + +Drive the real `claude` CLI in headless `--output-format stream-json` +mode against a running LiteLLM proxy that routes Anthropic Messages +requests to Azure OpenAI deployments of the GPT-5.6 family (Sol, +Terra, Luna), ask the model to invoke a built-in tool (`Bash`), and +assert that the upstream (a) emitted a `tool_use` content block and +(b) streamed the tool input incrementally as `input_json_delta` +events. + +Azure OpenAI streams tool arguments in the same chat-completions +fragment shape as openai.com; LiteLLM must re-emit them as Anthropic +`input_json_delta` deltas rather than buffering the full input into +one complete block. + +Bash is restricted to the exact command `echo pong` plus +`--permission-mode dontAsk`; see `tool_use/test_anthropic.py` for the +security rationale. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/tool_use_streaming/test_azure_openai.py + ^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +from typing import Any, Mapping, Sequence + +import pytest + +from claude_code._env import require_proxy +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +AZURE_OPENAI_MODELS = [ + "gpt-5-6-sol-azure-openai", + "gpt-5-6-terra-azure-openai", + "gpt-5-6-luna-azure-openai", +] + +TOOL_USE_PROMPT = ( + "Use the Bash tool to run the command `echo pong` and report what it printed." +) +TOOL_USE_ARGS = [ + "--allowed-tools", + "Bash(echo pong)", + "--permission-mode", + "dontAsk", + "--include-partial-messages", +] + + +def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + return True + return False + + +def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: + """Count `input_json_delta` records among the `stream_event` + entries. Zero means the proxy collapsed the streamed tool input + into a single complete block instead of forwarding the incremental + deltas the upstream emitted.""" + inner_events = ( + event.get("event") for event in events if event.get("type") == "stream_event" + ) + return sum( + 1 + for inner in inner_events + if isinstance(inner, Mapping) + and inner.get("type") == "content_block_delta" + and isinstance(inner.get("delta"), Mapping) + and inner["delta"].get("type") == "input_json_delta" + ) + + +def test_tool_use_streaming_azure_openai(compat_result): + proxy = require_proxy(compat_result) + + outcomes = run_claude_models_parallel( + models=AZURE_OPENAI_MODELS, + prompt=TOOL_USE_PROMPT, + base_url=proxy.base_url, + api_key=proxy.api_key, + extra_args=TOOL_USE_ARGS, + ) + + failures = [] + for model in AZURE_OPENAI_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_tool_use_event(outcome.events): + error = ( + f"[{model}] no tool_use content block observed in stream-json events" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if _count_input_json_deltas(outcome.events) == 0: + error = ( + f"[{model}] no input_json_delta stream events observed; proxy " + f"likely buffered the tool input into a complete block or " + f"stripped fine-grained tool streaming" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/tool_use_streaming/test_bedrock_mantle.py b/tests/e2e/claude_code/tool_use_streaming/test_bedrock_mantle.py new file mode 100644 index 000000000000..20fae5d48dba --- /dev/null +++ b/tests/e2e/claude_code/tool_use_streaming/test_bedrock_mantle.py @@ -0,0 +1,143 @@ +"""tool_use_streaming x AWS Bedrock Mantle (GPT-5.6). + +Drive the real `claude` CLI in headless `--output-format stream-json` +mode against a running LiteLLM proxy that routes Anthropic Messages +requests to OpenAI's GPT-5.6 family (Sol, Terra, Luna) on AWS +Bedrock's Mantle endpoint, ask the model to invoke a built-in tool +(`Bash`), and assert that the upstream (a) emitted a `tool_use` +content block and (b) streamed the tool input incrementally as +`input_json_delta` events. + +Mantle streams OpenAI Responses API `function_call_arguments.delta` +events over SigV4-signed SSE; LiteLLM must re-emit them as Anthropic +`input_json_delta` deltas rather than buffering the full input into +one complete block. + +Bash is restricted to the exact command `echo pong` plus +`--permission-mode dontAsk`; see `tool_use/test_anthropic.py` for the +security rationale. + +Mantle cells are opt-in via COMPAT_MANTLE_CELLS=1 (see +`claude_code._gpt_cells`). + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/tool_use_streaming/test_bedrock_mantle.py + ^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +from typing import Any, Mapping, Sequence + +import pytest + +from claude_code._env import require_proxy +from claude_code._gpt_cells import skip_unless_mantle_cells_enabled +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +BEDROCK_MANTLE_MODELS = [ + "gpt-5-6-sol-bedrock-mantle", + "gpt-5-6-terra-bedrock-mantle", + "gpt-5-6-luna-bedrock-mantle", +] + +TOOL_USE_PROMPT = ( + "Use the Bash tool to run the command `echo pong` and report what it printed." +) +TOOL_USE_ARGS = [ + "--allowed-tools", + "Bash(echo pong)", + "--permission-mode", + "dontAsk", + "--include-partial-messages", +] + + +def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + return True + return False + + +def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: + """Count `input_json_delta` records among the `stream_event` + entries. Zero means the proxy collapsed the streamed tool input + into a single complete block instead of forwarding the incremental + deltas the upstream emitted.""" + inner_events = ( + event.get("event") for event in events if event.get("type") == "stream_event" + ) + return sum( + 1 + for inner in inner_events + if isinstance(inner, Mapping) + and inner.get("type") == "content_block_delta" + and isinstance(inner.get("delta"), Mapping) + and inner["delta"].get("type") == "input_json_delta" + ) + + +def test_tool_use_streaming_bedrock_mantle(compat_result): + skip_unless_mantle_cells_enabled() + proxy = require_proxy(compat_result) + + outcomes = run_claude_models_parallel( + models=BEDROCK_MANTLE_MODELS, + prompt=TOOL_USE_PROMPT, + base_url=proxy.base_url, + api_key=proxy.api_key, + extra_args=TOOL_USE_ARGS, + ) + + failures = [] + for model in BEDROCK_MANTLE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_tool_use_event(outcome.events): + error = ( + f"[{model}] no tool_use content block observed in stream-json events" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if _count_input_json_deltas(outcome.events) == 0: + error = ( + f"[{model}] no input_json_delta stream events observed; proxy " + f"likely buffered the tool input into a complete block or " + f"stripped fine-grained tool streaming" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/tool_use_streaming/test_openai.py b/tests/e2e/claude_code/tool_use_streaming/test_openai.py new file mode 100644 index 000000000000..895f88d994b8 --- /dev/null +++ b/tests/e2e/claude_code/tool_use_streaming/test_openai.py @@ -0,0 +1,136 @@ +"""tool_use_streaming x OpenAI (GPT-5.6). + +Drive the real `claude` CLI in headless `--output-format stream-json` +mode against a running LiteLLM proxy that routes Anthropic Messages +requests to OpenAI's GPT-5.6 family (Sol, Terra, Luna), ask the model +to invoke a built-in tool (`Bash`), and assert that the upstream (a) +emitted a `tool_use` content block and (b) streamed the tool input +incrementally as `input_json_delta` events. + +OpenAI streams tool arguments as incremental `tool_calls` argument +fragments; LiteLLM must re-emit them as Anthropic `input_json_delta` +deltas rather than buffering the full input into one complete block. + +Bash is restricted to the exact command `echo pong` plus +`--permission-mode dontAsk`; see `tool_use/test_anthropic.py` for the +security rationale. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/tool_use_streaming/test_openai.py + ^^^^^^^^^^^^^^^^^^ ^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +from typing import Any, Mapping, Sequence + +import pytest + +from claude_code._env import require_proxy +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +OPENAI_MODELS = [ + "gpt-5-6-sol-openai", + "gpt-5-6-terra-openai", + "gpt-5-6-luna-openai", +] + +TOOL_USE_PROMPT = ( + "Use the Bash tool to run the command `echo pong` and report what it printed." +) +TOOL_USE_ARGS = [ + "--allowed-tools", + "Bash(echo pong)", + "--permission-mode", + "dontAsk", + "--include-partial-messages", +] + + +def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + return True + return False + + +def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: + """Count `input_json_delta` records among the `stream_event` + entries. Zero means the proxy collapsed the streamed tool input + into a single complete block instead of forwarding the incremental + deltas the upstream emitted.""" + inner_events = ( + event.get("event") for event in events if event.get("type") == "stream_event" + ) + return sum( + 1 + for inner in inner_events + if isinstance(inner, Mapping) + and inner.get("type") == "content_block_delta" + and isinstance(inner.get("delta"), Mapping) + and inner["delta"].get("type") == "input_json_delta" + ) + + +def test_tool_use_streaming_openai(compat_result): + proxy = require_proxy(compat_result) + + outcomes = run_claude_models_parallel( + models=OPENAI_MODELS, + prompt=TOOL_USE_PROMPT, + base_url=proxy.base_url, + api_key=proxy.api_key, + extra_args=TOOL_USE_ARGS, + ) + + failures = [] + for model in OPENAI_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_tool_use_event(outcome.events): + error = ( + f"[{model}] no tool_use content block observed in stream-json events" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if _count_input_json_deltas(outcome.events) == 0: + error = ( + f"[{model}] no input_json_delta stream events observed; proxy " + f"likely buffered the tool input into a complete block or " + f"stripped fine-grained tool streaming" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/tool_use_streaming/test_vertex_ai_gpt.py b/tests/e2e/claude_code/tool_use_streaming/test_vertex_ai_gpt.py new file mode 100644 index 000000000000..7037e91fee02 --- /dev/null +++ b/tests/e2e/claude_code/tool_use_streaming/test_vertex_ai_gpt.py @@ -0,0 +1,33 @@ +"""tool_use_streaming x Vertex AI (GPT-5.6) — not applicable. + +GCP is the only one of the big-three clouds without OpenAI's +closed-weight GPT-5.6 family (Sol / Terra / Luna); Vertex AI Model +Garden carries only the open-weight gpt-oss MaaS models. The cell +reports `not_applicable` so the published matrix documents the gap +explicitly instead of leaving a `not_tested` hole. + +This stub never drives the `claude` CLI, so it grants no tools and is +exempt from the Bash allow-rule pin enforced by +`_pr_gate_unit_tests/test_bash_tool_restrictions.py`. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/tool_use_streaming/test_vertex_ai_gpt.py + ^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +from claude_code._gpt_cells import VERTEX_AI_GPT_NOT_APPLICABLE_REASON + + +def test_tool_use_streaming_vertex_ai_gpt(compat_result): + """Record the static not_applicable outcome for this cell.""" + compat_result.set( + { + "status": "not_applicable", + "reason": VERTEX_AI_GPT_NOT_APPLICABLE_REASON, + } + )