From 710a88eba70b99f519adef143af5c1ab8c7f1e06 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 15 Jul 2026 16:25:32 -0700 Subject: [PATCH 1/6] test(e2e/claude_code): add GPT-5.6 Sol/Terra/Luna provider columns for OpenAI, Azure OpenAI, and Bedrock Mantle --- tests/e2e/CLAUDE.md | 1 + .../test_matrix_builder.py | 18 +- .../_builder_unit_tests/test_v0_layout.py | 84 ++++++++- .../_driver_unit_tests/test_rate_limiter.py | 37 ++++ tests/e2e/claude_code/_gpt_cells.py | 60 +++++++ .../test_bash_tool_restrictions.py | 44 +++++ .../test_azure_openai.py | 47 +++++ .../test_bedrock_mantle.py | 46 +++++ .../test_openai.py | 44 +++++ .../test_vertex_ai_gpt.py | 29 ++++ .../test_azure_openai.py | 47 +++++ .../test_bedrock_mantle.py | 47 +++++ .../basic_messaging_streaming/test_openai.py | 47 +++++ .../test_vertex_ai_gpt.py | 29 ++++ tests/e2e/claude_code/manifest.yaml | 13 +- tests/e2e/claude_code/rate_limiter.py | 28 ++- tests/e2e/claude_code/run_compat.sh | 10 +- tests/e2e/claude_code/test_config.yaml | 56 ++++++ .../claude_code/tool_use/test_azure_openai.py | 131 ++++++++++++++ .../tool_use/test_bedrock_mantle.py | 132 +++++++++++++++ tests/e2e/claude_code/tool_use/test_openai.py | 130 ++++++++++++++ .../tool_use/test_vertex_ai_gpt.py | 33 ++++ .../tool_use_streaming/test_azure_openai.py | 160 ++++++++++++++++++ .../tool_use_streaming/test_bedrock_mantle.py | 160 ++++++++++++++++++ .../tool_use_streaming/test_openai.py | 158 +++++++++++++++++ .../tool_use_streaming/test_vertex_ai_gpt.py | 33 ++++ 26 files changed, 1616 insertions(+), 8 deletions(-) create mode 100644 tests/e2e/claude_code/_gpt_cells.py create mode 100644 tests/e2e/claude_code/basic_messaging_non_streaming/test_azure_openai.py create mode 100644 tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_mantle.py create mode 100644 tests/e2e/claude_code/basic_messaging_non_streaming/test_openai.py create mode 100644 tests/e2e/claude_code/basic_messaging_non_streaming/test_vertex_ai_gpt.py create mode 100644 tests/e2e/claude_code/basic_messaging_streaming/test_azure_openai.py create mode 100644 tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_mantle.py create mode 100644 tests/e2e/claude_code/basic_messaging_streaming/test_openai.py create mode 100644 tests/e2e/claude_code/basic_messaging_streaming/test_vertex_ai_gpt.py create mode 100644 tests/e2e/claude_code/tool_use/test_azure_openai.py create mode 100644 tests/e2e/claude_code/tool_use/test_bedrock_mantle.py create mode 100644 tests/e2e/claude_code/tool_use/test_openai.py create mode 100644 tests/e2e/claude_code/tool_use/test_vertex_ai_gpt.py create mode 100644 tests/e2e/claude_code/tool_use_streaming/test_azure_openai.py create mode 100644 tests/e2e/claude_code/tool_use_streaming/test_bedrock_mantle.py create mode 100644 tests/e2e/claude_code/tool_use_streaming/test_openai.py create mode 100644 tests/e2e/claude_code/tool_use_streaming/test_vertex_ai_gpt.py diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index f0d283629b06..d94d073c9e4b 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -6,6 +6,7 @@ Code-style rules for writing tests under `tests/e2e/`. The harness already encod Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family or behavior area. If you add a new folder, you must add a line here describing what kind of tests belong in it, so the layout stays self-describing. `gateway/` is the exception: it holds proxy configuration only and never tests +- `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI against a live proxy, one directory per feature row and one `test_.py` per column (see `claude_code/manifest.yaml`); results publish as `compat-results.json`, not the coverage registry - `llm_translation/` - LLM endpoint and provider-translation behavior: passthrough, custom pricing, OCR, and the non-chat inference endpoints (`/v1/responses`, `/v1/messages`, `/embeddings`, `/v1/rerank`, `/v1/audio/speech`, `/v1/images/generations`), each against a deployment the test creates via `/model/new` and deletes on teardown - `access_control/` - the gateway's authorization and error-shape contract: per-key model allow-lists, route-group permissions (`allowed_routes`), and unknown-model validation - `embeddings/` - the `/embeddings` endpoint across providers diff --git a/tests/e2e/claude_code/_builder_unit_tests/test_matrix_builder.py b/tests/e2e/claude_code/_builder_unit_tests/test_matrix_builder.py index 9ddbdd298465..5f1817d3075f 100644 --- a/tests/e2e/claude_code/_builder_unit_tests/test_matrix_builder.py +++ b/tests/e2e/claude_code/_builder_unit_tests/test_matrix_builder.py @@ -389,7 +389,23 @@ def test_build_matrix_6x5_grid_matches_published_sample(): for feature in full_manifest["features"] if feature["id"] in v0_feature_ids ] - manifest = {**full_manifest, "features": v0_features} + # The provider list is sliced to the five v0 columns for the same + # reason as the rows: the sample is a frozen 6x5 baseline, and the + # GPT-5.6 columns added 2026-07 (whose vertex_ai_gpt cells are + # not_applicable by design) are exercised by their own layout tests + # in `test_v0_layout.py` rather than by this golden file. + v0_provider_ids = [ + "anthropic", + "bedrock_invoke", + "bedrock_converse", + "vertex_ai", + "azure", + ] + manifest = { + **full_manifest, + "features": v0_features, + "providers": v0_provider_ids, + } feature_ids = [feature["id"] for feature in manifest["features"]] providers = manifest["providers"] diff --git a/tests/e2e/claude_code/_builder_unit_tests/test_v0_layout.py b/tests/e2e/claude_code/_builder_unit_tests/test_v0_layout.py index b1745008facd..2ea60dea5863 100644 --- a/tests/e2e/claude_code/_builder_unit_tests/test_v0_layout.py +++ b/tests/e2e/claude_code/_builder_unit_tests/test_v0_layout.py @@ -45,6 +45,37 @@ "azure", ] +# The GPT-5.6 (Sol / Terra / Luna) columns added 2026-07, in manifest +# order after the v0 Claude columns. Unlike the v0 columns they only +# back GPT_FEATURE_IDS below; other rows render not_tested for them. +GPT_PROVIDERS = [ + "openai", + "azure_openai", + "bedrock_mantle", + "vertex_ai_gpt", +] + +# GPT columns that drive the claude CLI against a live route. +# `vertex_ai_gpt` is excluded: GCP does not offer the closed-weight +# GPT-5.6 family, so its cells are static not_applicable stubs. +GPT_LIVE_PROVIDERS = [ + "openai", + "azure_openai", + "bedrock_mantle", +] + +# Feature rows backed by GPT cells. +GPT_FEATURE_IDS = [ + "basic_messaging_non_streaming", + "basic_messaging_streaming", + "tool_use", + "tool_use_streaming", +] + +# Every live GPT cell must exercise the three GPT-5.6 tiers, mirroring +# the three-Claude-tier rule for the v0 columns. +GPT_TIER_SUBSTRINGS = ("5-6-sol", "5-6-terra", "5-6-luna") + def _all_manifest_feature_ids() -> list[str]: """Every feature_id currently declared in `manifest.yaml`. @@ -80,7 +111,18 @@ def test_manifest_lists_all_six_v0_features_in_order(manifest): def test_manifest_lists_all_five_v0_providers_in_order(manifest): - assert manifest["providers"] == EXPECTED_PROVIDERS + """The v0 column set stays pinned at positions [0:5] for the + lifetime of the schema, mirroring the v0 feature-row pin above; + columns added later (the GPT-5.6 set) may only extend the list. + """ + assert manifest["providers"][: len(EXPECTED_PROVIDERS)] == EXPECTED_PROVIDERS + + +def test_manifest_lists_gpt_provider_columns_after_v0(manifest): + """The GPT-5.6 columns follow the v0 columns in a fixed order so + the rendered matrix keeps Claude and GPT column groups contiguous. + """ + assert manifest["providers"][len(EXPECTED_PROVIDERS) :] == GPT_PROVIDERS def test_manifest_every_feature_has_human_readable_name(manifest): @@ -158,6 +200,46 @@ def test_per_provider_test_file_imports_and_parametrizes_three_models( ), f"{feature_id}/test_{provider}.py does not reference {tier}" +@pytest.mark.parametrize("feature_id", GPT_FEATURE_IDS) +@pytest.mark.parametrize("provider", GPT_PROVIDERS) +def test_gpt_cell_test_file_exists(feature_id, provider): + """Every (GPT feature, GPT provider) cell must be backed by a test + file; a missing file silently becomes a `not_tested` cell in the + published matrix rather than a CI failure surfacing the drift.""" + test_file = REPO_ROOT / feature_id / f"test_{provider}.py" + assert test_file.is_file(), f"missing per-provider test file: {test_file}" + + +@pytest.mark.parametrize("feature_id", GPT_FEATURE_IDS) +@pytest.mark.parametrize("provider", GPT_LIVE_PROVIDERS) +def test_gpt_cell_references_three_gpt_tiers(feature_id, provider): + """Every live GPT cell must exercise Sol, Terra, and Luna — the + same all-tiers-or-red rule the v0 columns apply to the three + Claude tiers.""" + text = (REPO_ROOT / feature_id / f"test_{provider}.py").read_text() + for tier in GPT_TIER_SUBSTRINGS: + assert ( + tier in text + ), f"{feature_id}/test_{provider}.py does not reference {tier}" + + +@pytest.mark.parametrize("feature_id", GPT_FEATURE_IDS) +def test_vertex_ai_gpt_cell_is_a_static_not_applicable_stub(feature_id): + """GCP does not offer the closed-weight GPT-5.6 family, so the + `vertex_ai_gpt` cells must report `not_applicable` and must not + drive the claude CLI. If Google adds the models, flip the stubs to + live cells and update this pin alongside GPT_LIVE_PROVIDERS.""" + text = (REPO_ROOT / feature_id / "test_vertex_ai_gpt.py").read_text() + assert '"status": "not_applicable"' in text, ( + f"{feature_id}/test_vertex_ai_gpt.py must report not_applicable while " + "GCP Vertex AI does not offer the GPT-5.6 family." + ) + assert "run_claude" not in text, ( + f"{feature_id}/test_vertex_ai_gpt.py must not drive the claude CLI; " + "there is no GPT-5.6 route on Vertex AI to exercise." + ) + + @pytest.mark.parametrize("feature_id", EXPECTED_FEATURE_IDS) def test_azure_test_file_drives_the_proxy(feature_id): """Azure (Microsoft Foundry) hosts Anthropic Claude as of 2025-11-18, diff --git a/tests/e2e/claude_code/_driver_unit_tests/test_rate_limiter.py b/tests/e2e/claude_code/_driver_unit_tests/test_rate_limiter.py index 92907eda3c47..b4ecacac0347 100644 --- a/tests/e2e/claude_code/_driver_unit_tests/test_rate_limiter.py +++ b/tests/e2e/claude_code/_driver_unit_tests/test_rate_limiter.py @@ -38,8 +38,11 @@ DEFAULT_RATE, PROVIDER_ANTHROPIC, PROVIDER_AZURE, + PROVIDER_AZURE_OPENAI, PROVIDER_BEDROCK_CONVERSE, PROVIDER_BEDROCK_INVOKE, + PROVIDER_BEDROCK_MANTLE, + PROVIDER_OPENAI, PROVIDER_VERTEX_AI, ProviderConfig, RateLimiter, @@ -67,6 +70,9 @@ ("claude-opus-4-7-vertex", PROVIDER_VERTEX_AI), ("claude-haiku-4-5-bedrock-converse", PROVIDER_BEDROCK_CONVERSE), ("claude-haiku-4-5-bedrock-invoke", PROVIDER_BEDROCK_INVOKE), + ("gpt-5-6-sol-openai", PROVIDER_OPENAI), + ("gpt-5-6-terra-azure-openai", PROVIDER_AZURE_OPENAI), + ("gpt-5-6-luna-bedrock-mantle", PROVIDER_BEDROCK_MANTLE), ], ) def test_infer_provider_maps_alias_suffix_to_column(model, expected): @@ -79,6 +85,23 @@ def test_infer_provider_bedrock_converse_beats_bedrock_invoke_lookup_order(): assert infer_provider("claude-foo-bedrock-invoke") == PROVIDER_BEDROCK_INVOKE +def test_infer_provider_azure_openai_beats_openai_and_azure_lookup_order(): + """`-azure-openai` also ends with `-openai`; the more-specific + suffix must win so Azure OpenAI traffic doesn't drain the OpenAI + bucket (and never falls through to the Claude `-azure` column).""" + assert infer_provider("gpt-5-6-sol-azure-openai") == PROVIDER_AZURE_OPENAI + assert infer_provider("gpt-5-6-sol-openai") == PROVIDER_OPENAI + assert infer_provider("claude-opus-4-7-azure") == PROVIDER_AZURE + + +def test_infer_provider_bedrock_mantle_beats_other_bedrock_suffixes(): + """All three bedrock suffixes contain `bedrock`; each alias must + land in its own bucket.""" + assert infer_provider("gpt-5-6-terra-bedrock-mantle") == PROVIDER_BEDROCK_MANTLE + assert infer_provider("claude-foo-bedrock-converse") == PROVIDER_BEDROCK_CONVERSE + assert infer_provider("claude-foo-bedrock-invoke") == PROVIDER_BEDROCK_INVOKE + + def test_infer_provider_rejects_empty_string(): with pytest.raises(ValueError, match="non-empty"): infer_provider("") @@ -114,6 +137,20 @@ def test_load_config_reads_per_provider_rate(): assert cfg[PROVIDER_VERTEX_AI].rate_per_sec == DEFAULT_RATE +def test_load_config_reads_gpt_provider_rates(): + cfg = load_config( + env={ + "LITELLM_COMPAT_RATE_OPENAI": "2", + "LITELLM_COMPAT_RATE_AZURE_OPENAI": "3", + "LITELLM_COMPAT_RATE_BEDROCK_MANTLE": "4", + } + ) + assert cfg[PROVIDER_OPENAI].rate_per_sec == 2.0 + assert cfg[PROVIDER_AZURE_OPENAI].rate_per_sec == 3.0 + assert cfg[PROVIDER_BEDROCK_MANTLE].rate_per_sec == 4.0 + assert cfg[PROVIDER_ANTHROPIC].rate_per_sec == DEFAULT_RATE + + def test_load_config_zero_rate_disables_provider(): cfg = load_config(env={"LITELLM_COMPAT_RATE_BEDROCK_INVOKE": "0"}) assert cfg[PROVIDER_BEDROCK_INVOKE].enabled is False diff --git a/tests/e2e/claude_code/_gpt_cells.py b/tests/e2e/claude_code/_gpt_cells.py new file mode 100644 index 000000000000..15b35da7e694 --- /dev/null +++ b/tests/e2e/claude_code/_gpt_cells.py @@ -0,0 +1,60 @@ +"""Shared plumbing for the GPT-5.6 (Sol / Terra / Luna) provider columns. + +OpenAI shipped GPT-5.6 as a three-tier family on 2026-07-09 — Sol +(flagship), Terra (balanced), Luna (fast) — and Claude Code can drive +all three through a LiteLLM proxy that translates the Anthropic +Messages API to each provider's native shape. Four provider columns +cover "OpenAI plus the big three clouds": + + openai OpenAI API (openai/gpt-5.6-*) + azure_openai Azure OpenAI (azure/gpt-5.6-*) + bedrock_mantle AWS Bedrock, Mantle (bedrock_mantle/openai.gpt-5.6-*, + Responses API) + vertex_ai_gpt GCP Vertex AI not_applicable — Vertex does + not offer the closed-weight + GPT-5.6 family; Model Garden + carries only the open-weight + gpt-oss MaaS models + +Live GPT cells are opt-in via `COMPAT_GPT_CELLS=1`. The external PR +gate and the daily cron VM must be provisioned with the GPT-route +credentials (`OPENAI_API_KEY`, `AZURE_OPENAI_API_BASE` + +`AZURE_OPENAI_API_KEY`, and Bedrock Mantle model access) before these +cells can pass, so until the flag is set each live cell skips and its +matrix cell stays `not_tested` — landing this suite change cannot flip +the existing gate red. The `vertex_ai_gpt` column ignores the flag: +its cells report a static `not_applicable` and never touch the +network. +""" + +from __future__ import annotations + +import os + +import pytest + +GPT_CELLS_ENV = "COMPAT_GPT_CELLS" + +VERTEX_AI_GPT_NOT_APPLICABLE_REASON = ( + "GCP Vertex AI does not offer OpenAI's closed-weight GPT-5.6 family " + "(Sol / Terra / Luna); Model Garden carries only the open-weight " + "gpt-oss MaaS models. Convert this column's cells to live tests if " + "Google adds the GPT-5.6 models." +) + + +def skip_unless_gpt_cells_enabled() -> None: + """Skip the calling test unless `COMPAT_GPT_CELLS` opts GPT cells in. + + A skipped cell is recorded as `not_tested` in the published matrix + (see the skip handling in `tests/e2e/claude_code/conftest.py`), + which is the honest state for an environment that has no GPT-route + credentials yet. + """ + if os.environ.get(GPT_CELLS_ENV, "").strip().lower() in {"1", "true", "yes"}: + return + pytest.skip( + f"GPT-5.6 cells are opt-in; set {GPT_CELLS_ENV}=1 once the proxy has " + "OpenAI / Azure OpenAI / Bedrock Mantle credentials for the " + "gpt-5-6-* aliases" + ) diff --git a/tests/e2e/claude_code/_pr_gate_unit_tests/test_bash_tool_restrictions.py b/tests/e2e/claude_code/_pr_gate_unit_tests/test_bash_tool_restrictions.py index d698131670a8..a0402e15752e 100644 --- a/tests/e2e/claude_code/_pr_gate_unit_tests/test_bash_tool_restrictions.py +++ b/tests/e2e/claude_code/_pr_gate_unit_tests/test_bash_tool_restrictions.py @@ -60,6 +60,21 @@ def _bash_cells() -> Iterable[Path]: yield path +def _is_exempt_stub(text: str) -> bool: + """Return True for cells that never drive the `claude` CLI and + never pass `--allowed-tools`. + + Such a cell (e.g. the static `not_applicable` stubs in the + `vertex_ai_gpt` column) cannot grant Bash — or any tool — to a + model-controlled response, so the allow-rule pins below don't + apply to it. Both conditions are required: a file that references + `--allowed-tools` without a visible `run_claude` entrypoint is NOT + exempt and must still carry the pinned shape, so a cell can't dodge + the scan by hiding its driver behind an indirection. + """ + return "run_claude" not in text and "--allowed-tools" not in text + + def _has_bare_bash_token(text: str) -> bool: """Return True if `text` contains a `"Bash"` token outside the `"Bash(echo pong)"` allow rule. @@ -80,6 +95,8 @@ def test_bash_allow_rule_is_pinned_to_exact_echo_pong(cell: Path) -> None: """The cell must pass `Bash(echo pong)` as the allow rule, not the unrestricted `Bash` value that was originally flagged.""" text = cell.read_text() + if _is_exempt_stub(text): + return assert '"Bash(echo pong)"' in text, ( f"{cell.relative_to(REPO_ROOT)} must restrict `--allowed-tools` to " f'`Bash(echo pong)` (exact-match pattern). Unrestricted `"Bash"` ' @@ -138,6 +155,8 @@ def test_bash_cell_uses_dontask_permission_mode(cell: Path) -> None: opposed to defaulting to "ask", which in headless mode would succeed without ever surfacing the security issue).""" text = cell.read_text() + if _is_exempt_stub(text): + return assert '"--permission-mode"' in text and '"dontAsk"' in text, ( f"{cell.relative_to(REPO_ROOT)} must pass `--permission-mode dontAsk` " f"alongside the `Bash(echo pong)` allow rule. Without dontAsk, " @@ -145,3 +164,28 @@ def test_bash_cell_uses_dontask_permission_mode(cell: Path) -> None: f"mode behavior, which in `--print` (headless) mode is non-" f"interactive — defeating the explicit-allow contract." ) + + +def test_is_exempt_stub_accepts_not_applicable_stub(): + """A static not_applicable stub (no CLI driver, no tool grants) is + outside the Bash pin's threat model and must be exempt — this is + the shape of the `vertex_ai_gpt` cells.""" + text = 'compat_result.set({"status": "not_applicable", "reason": REASON})' + assert _is_exempt_stub(text) + + +def test_is_exempt_stub_rejects_cli_driving_cell(): + """Any cell that drives the CLI stays subject to the pins, whether + or not it currently grants tools.""" + text = ( + "run_claude_models_parallel(models=MODELS, " + 'extra_args=["--allowed-tools", "Bash(echo pong)"])' + ) + assert not _is_exempt_stub(text) + + +def test_is_exempt_stub_rejects_allowed_tools_without_visible_driver(): + """A cell that passes `--allowed-tools` while hiding its driver + behind an indirection must not slip out of the pinned shape.""" + text = 'helper(extra_args=["--allowed-tools", "Bash"])' + assert not _is_exempt_stub(text) diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_azure_openai.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_azure_openai.py new file mode 100644 index 000000000000..fb0b5e9aa770 --- /dev/null +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_azure_openai.py @@ -0,0 +1,47 @@ +"""basic_messaging_non_streaming x Azure OpenAI (GPT-5.6). + +Drive the real `claude` CLI in headless mode against a running LiteLLM +proxy that routes Anthropic Messages requests to Azure OpenAI +deployments of the GPT-5.6 family (Sol, Terra, Luna), and report the +outcome via `compat_result`. + +Azure OpenAI serves the same chat-completions wire shape as +openai.com behind per-resource deployments; LiteLLM's `azure/gpt-*` +route handles the deployment addressing while reusing the OpenAI +translation, so this cell catches Azure-specific regressions +(auth headers, api-version pinning, deployment routing) that the +`openai` column cannot. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/basic_messaging_non_streaming/test_azure_openai.py + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^ + feature_id provider + +Every GPT cell exercises the three GPT-5.6 tiers; the cell only goes +green if all three pass. Cells are opt-in via COMPAT_GPT_CELLS=1 (see +`claude_code._gpt_cells`). +""" + +from __future__ import annotations + +from claude_code._basic_messaging import run_basic_messaging_cell +from claude_code._gpt_cells import skip_unless_gpt_cells_enabled + +AZURE_OPENAI_MODELS = [ + "gpt-5-6-sol-azure-openai", + "gpt-5-6-terra-azure-openai", + "gpt-5-6-luna-azure-openai", +] + + +def test_basic_messaging_non_streaming_azure_openai(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert a + non-empty reply from each GPT-5.6 tier.""" + skip_unless_gpt_cells_enabled() + run_basic_messaging_cell( + compat_result=compat_result, + models=AZURE_OPENAI_MODELS, + prompt="Reply with the single word 'pong' and nothing else.", + ) diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_mantle.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_mantle.py new file mode 100644 index 000000000000..51614570fc01 --- /dev/null +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_mantle.py @@ -0,0 +1,46 @@ +"""basic_messaging_non_streaming x AWS Bedrock Mantle (GPT-5.6). + +Drive the real `claude` CLI in headless mode against a running LiteLLM +proxy that routes Anthropic Messages requests to OpenAI's GPT-5.6 +family (Sol, Terra, Luna) hosted on AWS Bedrock, and report the +outcome via `compat_result`. + +Bedrock exposes the GPT-5.6 models through the Mantle endpoint, which +speaks the OpenAI Responses API rather than Converse/Invoke; LiteLLM's +`bedrock_mantle/openai.gpt-*` route signs the request with SigV4 and +translates Anthropic Messages to Responses, so this cell exercises a +translation path no other column covers. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_mantle.py + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^ + feature_id provider + +Every GPT cell exercises the three GPT-5.6 tiers; the cell only goes +green if all three pass. Cells are opt-in via COMPAT_GPT_CELLS=1 (see +`claude_code._gpt_cells`). +""" + +from __future__ import annotations + +from claude_code._basic_messaging import run_basic_messaging_cell +from claude_code._gpt_cells import skip_unless_gpt_cells_enabled + +BEDROCK_MANTLE_MODELS = [ + "gpt-5-6-sol-bedrock-mantle", + "gpt-5-6-terra-bedrock-mantle", + "gpt-5-6-luna-bedrock-mantle", +] + + +def test_basic_messaging_non_streaming_bedrock_mantle(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert a + non-empty reply from each GPT-5.6 tier.""" + skip_unless_gpt_cells_enabled() + run_basic_messaging_cell( + compat_result=compat_result, + models=BEDROCK_MANTLE_MODELS, + prompt="Reply with the single word 'pong' and nothing else.", + ) diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_openai.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_openai.py new file mode 100644 index 000000000000..57270158328f --- /dev/null +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_openai.py @@ -0,0 +1,44 @@ +"""basic_messaging_non_streaming x OpenAI (GPT-5.6). + +Drive the real `claude` CLI in headless mode against a running LiteLLM +proxy that routes Anthropic Messages requests to OpenAI's GPT-5.6 +family (Sol, Terra, Luna), and report the outcome via `compat_result`. + +Claude Code only speaks the Anthropic Messages API; LiteLLM's +`openai/gpt-*` route translates the request to OpenAI chat completions +and maps the response back, so this cell exercises the full +cross-provider translation layer in both directions. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/basic_messaging_non_streaming/test_openai.py + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^ + feature_id provider + +Every GPT cell exercises the three GPT-5.6 tiers; the cell only goes +green if all three pass. Cells are opt-in via COMPAT_GPT_CELLS=1 (see +`claude_code._gpt_cells`). +""" + +from __future__ import annotations + +from claude_code._basic_messaging import run_basic_messaging_cell +from claude_code._gpt_cells import skip_unless_gpt_cells_enabled + +OPENAI_MODELS = [ + "gpt-5-6-sol-openai", + "gpt-5-6-terra-openai", + "gpt-5-6-luna-openai", +] + + +def test_basic_messaging_non_streaming_openai(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert a + non-empty reply from each GPT-5.6 tier.""" + skip_unless_gpt_cells_enabled() + run_basic_messaging_cell( + compat_result=compat_result, + models=OPENAI_MODELS, + prompt="Reply with the single word 'pong' and nothing else.", + ) diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_vertex_ai_gpt.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_vertex_ai_gpt.py new file mode 100644 index 000000000000..3b155b6ac9dd --- /dev/null +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_vertex_ai_gpt.py @@ -0,0 +1,29 @@ +"""basic_messaging_non_streaming x Vertex AI (GPT-5.6) — not applicable. + +GCP is the only one of the big-three clouds without OpenAI's +closed-weight GPT-5.6 family (Sol / Terra / Luna); Vertex AI Model +Garden carries only the open-weight gpt-oss MaaS models. The cell +reports `not_applicable` so the published matrix documents the gap +explicitly instead of leaving a `not_tested` hole. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/basic_messaging_non_streaming/test_vertex_ai_gpt.py + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +from claude_code._gpt_cells import VERTEX_AI_GPT_NOT_APPLICABLE_REASON + + +def test_basic_messaging_non_streaming_vertex_ai_gpt(compat_result): + """Record the static not_applicable outcome for this cell.""" + compat_result.set( + { + "status": "not_applicable", + "reason": VERTEX_AI_GPT_NOT_APPLICABLE_REASON, + } + ) diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_azure_openai.py b/tests/e2e/claude_code/basic_messaging_streaming/test_azure_openai.py new file mode 100644 index 000000000000..603a575d7513 --- /dev/null +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_azure_openai.py @@ -0,0 +1,47 @@ +"""basic_messaging_streaming x Azure OpenAI (GPT-5.6). + +Drive the real `claude` CLI in headless `--output-format stream-json` +mode against a running LiteLLM proxy that routes Anthropic Messages +requests to Azure OpenAI deployments of the GPT-5.6 family (Sol, +Terra, Luna), and report the outcome via `compat_result`. + +Azure OpenAI streams the same chat-completions SSE shape as +openai.com; LiteLLM re-emits it as Anthropic stream events, and the +`verify_streaming=True` assertion (via `--include-partial-messages`) +proves the events arrived incrementally rather than as one buffered +response. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/basic_messaging_streaming/test_azure_openai.py + ^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^ + feature_id provider + +Every GPT cell exercises the three GPT-5.6 tiers; the cell only goes +green if all three pass. Cells are opt-in via COMPAT_GPT_CELLS=1 (see +`claude_code._gpt_cells`). +""" + +from __future__ import annotations + +from claude_code._basic_messaging import run_basic_messaging_cell +from claude_code._gpt_cells import skip_unless_gpt_cells_enabled + +AZURE_OPENAI_MODELS = [ + "gpt-5-6-sol-azure-openai", + "gpt-5-6-terra-azure-openai", + "gpt-5-6-luna-azure-openai", +] + + +def test_basic_messaging_streaming_azure_openai(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert a + non-empty streamed reply from each GPT-5.6 tier.""" + skip_unless_gpt_cells_enabled() + run_basic_messaging_cell( + compat_result=compat_result, + models=AZURE_OPENAI_MODELS, + prompt="Count from 1 to 5, one number per line.", + verify_streaming=True, + ) diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_mantle.py b/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_mantle.py new file mode 100644 index 000000000000..59303edc5150 --- /dev/null +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_mantle.py @@ -0,0 +1,47 @@ +"""basic_messaging_streaming x AWS Bedrock Mantle (GPT-5.6). + +Drive the real `claude` CLI in headless `--output-format stream-json` +mode against a running LiteLLM proxy that routes Anthropic Messages +requests to OpenAI's GPT-5.6 family (Sol, Terra, Luna) on AWS +Bedrock's Mantle endpoint, and report the outcome via `compat_result`. + +Mantle streams OpenAI Responses API events over SigV4-signed SSE; +LiteLLM re-emits them as Anthropic stream events, and the +`verify_streaming=True` assertion (via `--include-partial-messages`) +proves the events arrived incrementally rather than as one buffered +response. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_mantle.py + ^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^ + feature_id provider + +Every GPT cell exercises the three GPT-5.6 tiers; the cell only goes +green if all three pass. Cells are opt-in via COMPAT_GPT_CELLS=1 (see +`claude_code._gpt_cells`). +""" + +from __future__ import annotations + +from claude_code._basic_messaging import run_basic_messaging_cell +from claude_code._gpt_cells import skip_unless_gpt_cells_enabled + +BEDROCK_MANTLE_MODELS = [ + "gpt-5-6-sol-bedrock-mantle", + "gpt-5-6-terra-bedrock-mantle", + "gpt-5-6-luna-bedrock-mantle", +] + + +def test_basic_messaging_streaming_bedrock_mantle(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert a + non-empty streamed reply from each GPT-5.6 tier.""" + skip_unless_gpt_cells_enabled() + run_basic_messaging_cell( + compat_result=compat_result, + models=BEDROCK_MANTLE_MODELS, + prompt="Count from 1 to 5, one number per line.", + verify_streaming=True, + ) diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_openai.py b/tests/e2e/claude_code/basic_messaging_streaming/test_openai.py new file mode 100644 index 000000000000..58767b2fd104 --- /dev/null +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_openai.py @@ -0,0 +1,47 @@ +"""basic_messaging_streaming x OpenAI (GPT-5.6). + +Drive the real `claude` CLI in headless `--output-format stream-json` +mode against a running LiteLLM proxy that routes Anthropic Messages +requests to OpenAI's GPT-5.6 family (Sol, Terra, Luna), and report the +outcome via `compat_result`. + +LiteLLM translates OpenAI's chat-completions SSE chunks into Anthropic +`message_start` / `content_block_delta` / `message_stop` events on the +fly; the `verify_streaming=True` assertion (via +`--include-partial-messages`) proves the proxy re-emitted incremental +events instead of buffering the upstream stream into one response. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/basic_messaging_streaming/test_openai.py + ^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^ + feature_id provider + +Every GPT cell exercises the three GPT-5.6 tiers; the cell only goes +green if all three pass. Cells are opt-in via COMPAT_GPT_CELLS=1 (see +`claude_code._gpt_cells`). +""" + +from __future__ import annotations + +from claude_code._basic_messaging import run_basic_messaging_cell +from claude_code._gpt_cells import skip_unless_gpt_cells_enabled + +OPENAI_MODELS = [ + "gpt-5-6-sol-openai", + "gpt-5-6-terra-openai", + "gpt-5-6-luna-openai", +] + + +def test_basic_messaging_streaming_openai(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert a + non-empty streamed reply from each GPT-5.6 tier.""" + skip_unless_gpt_cells_enabled() + run_basic_messaging_cell( + compat_result=compat_result, + models=OPENAI_MODELS, + prompt="Count from 1 to 5, one number per line.", + verify_streaming=True, + ) diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_vertex_ai_gpt.py b/tests/e2e/claude_code/basic_messaging_streaming/test_vertex_ai_gpt.py new file mode 100644 index 000000000000..f6aa01de5216 --- /dev/null +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_vertex_ai_gpt.py @@ -0,0 +1,29 @@ +"""basic_messaging_streaming x Vertex AI (GPT-5.6) — not applicable. + +GCP is the only one of the big-three clouds without OpenAI's +closed-weight GPT-5.6 family (Sol / Terra / Luna); Vertex AI Model +Garden carries only the open-weight gpt-oss MaaS models. The cell +reports `not_applicable` so the published matrix documents the gap +explicitly instead of leaving a `not_tested` hole. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/basic_messaging_streaming/test_vertex_ai_gpt.py + ^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +from claude_code._gpt_cells import VERTEX_AI_GPT_NOT_APPLICABLE_REASON + + +def test_basic_messaging_streaming_vertex_ai_gpt(compat_result): + """Record the static not_applicable outcome for this cell.""" + compat_result.set( + { + "status": "not_applicable", + "reason": VERTEX_AI_GPT_NOT_APPLICABLE_REASON, + } + ) diff --git a/tests/e2e/claude_code/manifest.yaml b/tests/e2e/claude_code/manifest.yaml index f7cccf0cef2f..41ca4e0fce72 100644 --- a/tests/e2e/claude_code/manifest.yaml +++ b/tests/e2e/claude_code/manifest.yaml @@ -12,13 +12,24 @@ schema_version: "1" -# Provider column order in the rendered matrix. +# Provider column order in the rendered matrix. The first five are +# the v0 Claude columns; the GPT-5.6 (Sol / Terra / Luna) columns +# added 2026-07 follow them. `vertex_ai_gpt` is a static +# not_applicable column: GCP does not offer the closed-weight GPT-5.6 +# family (Model Garden carries only the open-weight gpt-oss MaaS +# models), and the column documents that gap explicitly. GPT columns +# currently back the two basic_messaging rows plus tool_use and +# tool_use_streaming; other rows render not_tested for them. providers: - anthropic - bedrock_invoke - bedrock_converse - vertex_ai - azure + - openai + - azure_openai + - bedrock_mantle + - vertex_ai_gpt # Feature row order. features: diff --git a/tests/e2e/claude_code/rate_limiter.py b/tests/e2e/claude_code/rate_limiter.py index 06d21b838321..5818338ff7f8 100644 --- a/tests/e2e/claude_code/rate_limiter.py +++ b/tests/e2e/claude_code/rate_limiter.py @@ -24,6 +24,9 @@ LITELLM_COMPAT_RATE_VERTEX_AI (req/s, default 5.0) LITELLM_COMPAT_RATE_BEDROCK_CONVERSE (req/s, default 5.0) LITELLM_COMPAT_RATE_BEDROCK_INVOKE (req/s, default 5.0) + LITELLM_COMPAT_RATE_OPENAI (req/s, default 5.0) + LITELLM_COMPAT_RATE_AZURE_OPENAI (req/s, default 5.0) + LITELLM_COMPAT_RATE_BEDROCK_MANTLE (req/s, default 5.0) LITELLM_COMPAT_RATE_BURST (per-bucket burst override; default = rate) LITELLM_COMPAT_RATE_STATE_DIR (state file directory; @@ -36,7 +39,10 @@ The provider id is inferred from the model id by `infer_provider`, mirroring the matrix's column layout (`anthropic`, `azure`, -`vertex_ai`, `bedrock_converse`, `bedrock_invoke`). +`vertex_ai`, `bedrock_converse`, `bedrock_invoke`, `openai`, +`azure_openai`, `bedrock_mantle`). The `vertex_ai_gpt` matrix column +has no bucket: its cells are static not_applicable stubs that never +reach the network. """ from __future__ import annotations @@ -62,6 +68,9 @@ PROVIDER_VERTEX_AI = "vertex_ai" PROVIDER_BEDROCK_CONVERSE = "bedrock_converse" PROVIDER_BEDROCK_INVOKE = "bedrock_invoke" +PROVIDER_OPENAI = "openai" +PROVIDER_AZURE_OPENAI = "azure_openai" +PROVIDER_BEDROCK_MANTLE = "bedrock_mantle" ALL_PROVIDERS = ( PROVIDER_ANTHROPIC, @@ -69,6 +78,9 @@ PROVIDER_VERTEX_AI, PROVIDER_BEDROCK_CONVERSE, PROVIDER_BEDROCK_INVOKE, + PROVIDER_OPENAI, + PROVIDER_AZURE_OPENAI, + PROVIDER_BEDROCK_MANTLE, ) DEFAULT_RATE = 5.0 # req/s per provider, conservative starting point @@ -83,13 +95,21 @@ def infer_provider(model: str) -> str: The matrix column layout is fixed; aliases registered in the proxy encode the provider via a suffix (`-bedrock-converse`, - `-bedrock-invoke`, `-azure`, `-vertex`) or its absence (Anthropic). - Order matters: the bedrock suffixes both contain `bedrock`, so we - test the more-specific ones first. + `-bedrock-invoke`, `-azure`, `-vertex`, `-openai`, `-azure-openai`, + `-bedrock-mantle`) or its absence (Anthropic). Order matters: + `-azure-openai` also ends with `-openai`, and the bedrock suffixes + all contain `bedrock`, so the more-specific suffixes are tested + first. """ if not model: raise ValueError("model must be a non-empty string") lower = model.lower() + if lower.endswith("-azure-openai"): + return PROVIDER_AZURE_OPENAI + if lower.endswith("-openai"): + return PROVIDER_OPENAI + if lower.endswith("-bedrock-mantle"): + return PROVIDER_BEDROCK_MANTLE if lower.endswith("-bedrock-converse"): return PROVIDER_BEDROCK_CONVERSE if lower.endswith("-bedrock-invoke"): diff --git a/tests/e2e/claude_code/run_compat.sh b/tests/e2e/claude_code/run_compat.sh index 4d8d0b6d7b26..8aafa6449945 100755 --- a/tests/e2e/claude_code/run_compat.sh +++ b/tests/e2e/claude_code/run_compat.sh @@ -21,8 +21,16 @@ # LITELLM_COMPAT_RATE_VERTEX_AI # LITELLM_COMPAT_RATE_BEDROCK_CONVERSE # LITELLM_COMPAT_RATE_BEDROCK_INVOKE +# LITELLM_COMPAT_RATE_OPENAI +# LITELLM_COMPAT_RATE_AZURE_OPENAI +# LITELLM_COMPAT_RATE_BEDROCK_MANTLE # LITELLM_COMPAT_RATE_BURST override per-bucket burst # +# Optional env (GPT-5.6 columns): +# COMPAT_GPT_CELLS=1 opt the GPT-5.6 (Sol/Terra/Luna) +# cells in; without it they skip +# and publish as not_tested +# # Optional env (parallelism): # COMPAT_XDIST_WORKERS passed to `pytest -n` (default: auto) # @@ -57,7 +65,7 @@ results_path="${COMPAT_RESULTS_PATH:-compat-results.json}" summary_path="${COMPAT_RATE_LIMIT_SUMMARY_PATH:-compat-rate-limit-summary.json}" echo "[run_compat] rates:" -for provider in ANTHROPIC AZURE VERTEX_AI BEDROCK_CONVERSE BEDROCK_INVOKE; do +for provider in ANTHROPIC AZURE VERTEX_AI BEDROCK_CONVERSE BEDROCK_INVOKE OPENAI AZURE_OPENAI BEDROCK_MANTLE; do var="LITELLM_COMPAT_RATE_${provider}" echo " ${provider}=${!var:-default(5/s)}" done diff --git a/tests/e2e/claude_code/test_config.yaml b/tests/e2e/claude_code/test_config.yaml index eec68d11dcfc..9de26e26b9a3 100644 --- a/tests/e2e/claude_code/test_config.yaml +++ b/tests/e2e/claude_code/test_config.yaml @@ -14,6 +14,14 @@ # - claude-{tier}-bedrock-converse → Bedrock Converse API # - claude-{tier}-vertex → GCP Vertex AI # - claude-{tier}-azure → Microsoft Foundry (Anthropic deployments) +# - gpt-5-6-{tier}-openai → OpenAI API +# - gpt-5-6-{tier}-azure-openai → Azure OpenAI deployments +# - gpt-5-6-{tier}-bedrock-mantle → Bedrock Mantle (Responses API) +# +# GPT-5.6 tiers are sol / terra / luna. There are no GPT aliases for +# GCP: Vertex AI does not offer the closed-weight GPT-5.6 family, so +# the matrix's `vertex_ai_gpt` column reports not_applicable without +# ever reaching the proxy. model_list: # ---- Anthropic ---- @@ -92,6 +100,54 @@ model_list: api_base: os.environ/AZURE_FOUNDRY_API_BASE api_key: os.environ/AZURE_FOUNDRY_API_KEY + # ---- OpenAI (GPT-5.6) ---- + - model_name: gpt-5-6-sol-openai + litellm_params: + model: openai/gpt-5.6-sol + api_key: os.environ/OPENAI_API_KEY + - model_name: gpt-5-6-terra-openai + litellm_params: + model: openai/gpt-5.6-terra + api_key: os.environ/OPENAI_API_KEY + - model_name: gpt-5-6-luna-openai + litellm_params: + model: openai/gpt-5.6-luna + api_key: os.environ/OPENAI_API_KEY + + # ---- Azure OpenAI (GPT-5.6) ---- + - model_name: gpt-5-6-sol-azure-openai + litellm_params: + model: azure/gpt-5.6-sol + api_base: os.environ/AZURE_OPENAI_API_BASE + api_key: os.environ/AZURE_OPENAI_API_KEY + - model_name: gpt-5-6-terra-azure-openai + litellm_params: + model: azure/gpt-5.6-terra + api_base: os.environ/AZURE_OPENAI_API_BASE + api_key: os.environ/AZURE_OPENAI_API_KEY + - model_name: gpt-5-6-luna-azure-openai + litellm_params: + model: azure/gpt-5.6-luna + api_base: os.environ/AZURE_OPENAI_API_BASE + api_key: os.environ/AZURE_OPENAI_API_KEY + + # ---- Bedrock Mantle (GPT-5.6, Responses API) ---- + # Sol is only served from us-east-1 / us-east-2 as of 2026-07; + # Terra and Luna additionally have us-west-2. One region keeps the + # column comparable across tiers. + - model_name: gpt-5-6-sol-bedrock-mantle + litellm_params: + model: bedrock_mantle/openai.gpt-5.6-sol + aws_region_name: us-east-1 + - model_name: gpt-5-6-terra-bedrock-mantle + litellm_params: + model: bedrock_mantle/openai.gpt-5.6-terra + aws_region_name: us-east-1 + - model_name: gpt-5-6-luna-bedrock-mantle + litellm_params: + model: bedrock_mantle/openai.gpt-5.6-luna + aws_region_name: us-east-1 + general_settings: # Claude Code sends provider-specific headers (e.g. anthropic-beta) we # want to forward verbatim to the upstream so the wire-shape under diff --git a/tests/e2e/claude_code/tool_use/test_azure_openai.py b/tests/e2e/claude_code/tool_use/test_azure_openai.py new file mode 100644 index 000000000000..cf7809d13a5c --- /dev/null +++ b/tests/e2e/claude_code/tool_use/test_azure_openai.py @@ -0,0 +1,131 @@ +"""tool_use x Azure OpenAI (GPT-5.6). + +Drive the real `claude` CLI against a running LiteLLM proxy that +routes Anthropic Messages requests to Azure OpenAI deployments of the +GPT-5.6 family (Sol, Terra, Luna), ask the model to invoke a built-in +tool (`Bash`), and assert that a `tool_use` content block came back +over the wire. + +Azure OpenAI serves the same function-calling wire shape as +openai.com behind per-resource deployments; LiteLLM's `azure/gpt-*` +route reuses the OpenAI tool translation on top of Azure's deployment +addressing and auth. + +Bash is restricted to the exact command `echo pong` plus +`--permission-mode dontAsk`; see `tool_use/test_anthropic.py` for the +security rationale. + +GPT cells are opt-in via COMPAT_GPT_CELLS=1 (see +`claude_code._gpt_cells`). + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/tool_use/test_azure_openai.py + ^^^^^^^^ ^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from claude_code._gpt_cells import skip_unless_gpt_cells_enabled +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +AZURE_OPENAI_MODELS = [ + "gpt-5-6-sol-azure-openai", + "gpt-5-6-terra-azure-openai", + "gpt-5-6-luna-azure-openai", +] + +TOOL_USE_PROMPT = ( + "Use the Bash tool to run the command `echo pong` and report what it printed." +) +TOOL_USE_ARGS = [ + "--allowed-tools", + "Bash(echo pong)", + "--permission-mode", + "dontAsk", +] + + +def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + return True + return False + + +def test_tool_use_azure_openai(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert a + tool call was emitted on the wire by each GPT-5.6 tier.""" + skip_unless_gpt_cells_enabled() + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=AZURE_OPENAI_MODELS, + prompt=TOOL_USE_PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=TOOL_USE_ARGS, + ) + + failures = [] + for model in AZURE_OPENAI_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_tool_use_event(outcome.events): + error = ( + f"[{model}] no tool_use content block observed in stream-json events" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/tool_use/test_bedrock_mantle.py b/tests/e2e/claude_code/tool_use/test_bedrock_mantle.py new file mode 100644 index 000000000000..cf630b3be193 --- /dev/null +++ b/tests/e2e/claude_code/tool_use/test_bedrock_mantle.py @@ -0,0 +1,132 @@ +"""tool_use x AWS Bedrock Mantle (GPT-5.6). + +Drive the real `claude` CLI against a running LiteLLM proxy that +routes Anthropic Messages requests to OpenAI's GPT-5.6 family (Sol, +Terra, Luna) on AWS Bedrock's Mantle endpoint, ask the model to invoke +a built-in tool (`Bash`), and assert that a `tool_use` content block +came back over the wire. + +Mantle speaks the OpenAI Responses API, whose tool declarations and +`function_call` outputs differ from both Anthropic Messages and +chat completions; LiteLLM's `bedrock_mantle/openai.gpt-*` route +translates Anthropic `tools` into Responses tool declarations and maps +the emitted function calls back to `tool_use` blocks. + +Bash is restricted to the exact command `echo pong` plus +`--permission-mode dontAsk`; see `tool_use/test_anthropic.py` for the +security rationale. + +GPT cells are opt-in via COMPAT_GPT_CELLS=1 (see +`claude_code._gpt_cells`). + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/tool_use/test_bedrock_mantle.py + ^^^^^^^^ ^^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from claude_code._gpt_cells import skip_unless_gpt_cells_enabled +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_MANTLE_MODELS = [ + "gpt-5-6-sol-bedrock-mantle", + "gpt-5-6-terra-bedrock-mantle", + "gpt-5-6-luna-bedrock-mantle", +] + +TOOL_USE_PROMPT = ( + "Use the Bash tool to run the command `echo pong` and report what it printed." +) +TOOL_USE_ARGS = [ + "--allowed-tools", + "Bash(echo pong)", + "--permission-mode", + "dontAsk", +] + + +def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + return True + return False + + +def test_tool_use_bedrock_mantle(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert a + tool call was emitted on the wire by each GPT-5.6 tier.""" + skip_unless_gpt_cells_enabled() + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=BEDROCK_MANTLE_MODELS, + prompt=TOOL_USE_PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=TOOL_USE_ARGS, + ) + + failures = [] + for model in BEDROCK_MANTLE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_tool_use_event(outcome.events): + error = ( + f"[{model}] no tool_use content block observed in stream-json events" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/tool_use/test_openai.py b/tests/e2e/claude_code/tool_use/test_openai.py new file mode 100644 index 000000000000..7fa671f8e6e7 --- /dev/null +++ b/tests/e2e/claude_code/tool_use/test_openai.py @@ -0,0 +1,130 @@ +"""tool_use x OpenAI (GPT-5.6). + +Drive the real `claude` CLI against a running LiteLLM proxy that +routes Anthropic Messages requests to OpenAI's GPT-5.6 family (Sol, +Terra, Luna), ask the model to invoke a built-in tool (`Bash`), and +assert that a `tool_use` content block came back over the wire. + +Claude Code declares its tools in Anthropic `tools` format; LiteLLM's +`openai/gpt-*` route translates them to OpenAI function calling and +maps the returned `tool_calls` back to Anthropic `tool_use` blocks, so +this cell exercises the tool-schema translation in both directions. + +Bash is restricted to the exact command `echo pong` plus +`--permission-mode dontAsk`; see `tool_use/test_anthropic.py` for the +security rationale. + +GPT cells are opt-in via COMPAT_GPT_CELLS=1 (see +`claude_code._gpt_cells`). + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/tool_use/test_openai.py + ^^^^^^^^ ^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from claude_code._gpt_cells import skip_unless_gpt_cells_enabled +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +OPENAI_MODELS = [ + "gpt-5-6-sol-openai", + "gpt-5-6-terra-openai", + "gpt-5-6-luna-openai", +] + +TOOL_USE_PROMPT = ( + "Use the Bash tool to run the command `echo pong` and report what it printed." +) +TOOL_USE_ARGS = [ + "--allowed-tools", + "Bash(echo pong)", + "--permission-mode", + "dontAsk", +] + + +def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + return True + return False + + +def test_tool_use_openai(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert a + tool call was emitted on the wire by each GPT-5.6 tier.""" + skip_unless_gpt_cells_enabled() + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=OPENAI_MODELS, + prompt=TOOL_USE_PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=TOOL_USE_ARGS, + ) + + failures = [] + for model in OPENAI_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_tool_use_event(outcome.events): + error = ( + f"[{model}] no tool_use content block observed in stream-json events" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/tool_use/test_vertex_ai_gpt.py b/tests/e2e/claude_code/tool_use/test_vertex_ai_gpt.py new file mode 100644 index 000000000000..d1ebbced9dc5 --- /dev/null +++ b/tests/e2e/claude_code/tool_use/test_vertex_ai_gpt.py @@ -0,0 +1,33 @@ +"""tool_use x Vertex AI (GPT-5.6) — not applicable. + +GCP is the only one of the big-three clouds without OpenAI's +closed-weight GPT-5.6 family (Sol / Terra / Luna); Vertex AI Model +Garden carries only the open-weight gpt-oss MaaS models. The cell +reports `not_applicable` so the published matrix documents the gap +explicitly instead of leaving a `not_tested` hole. + +This stub never drives the `claude` CLI, so it grants no tools and is +exempt from the Bash allow-rule pin enforced by +`_pr_gate_unit_tests/test_bash_tool_restrictions.py`. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/tool_use/test_vertex_ai_gpt.py + ^^^^^^^^ ^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +from claude_code._gpt_cells import VERTEX_AI_GPT_NOT_APPLICABLE_REASON + + +def test_tool_use_vertex_ai_gpt(compat_result): + """Record the static not_applicable outcome for this cell.""" + compat_result.set( + { + "status": "not_applicable", + "reason": VERTEX_AI_GPT_NOT_APPLICABLE_REASON, + } + ) diff --git a/tests/e2e/claude_code/tool_use_streaming/test_azure_openai.py b/tests/e2e/claude_code/tool_use_streaming/test_azure_openai.py new file mode 100644 index 000000000000..f85ffa9c4b4b --- /dev/null +++ b/tests/e2e/claude_code/tool_use_streaming/test_azure_openai.py @@ -0,0 +1,160 @@ +"""tool_use_streaming x Azure OpenAI (GPT-5.6). + +Drive the real `claude` CLI in headless `--output-format stream-json` +mode against a running LiteLLM proxy that routes Anthropic Messages +requests to Azure OpenAI deployments of the GPT-5.6 family (Sol, +Terra, Luna), ask the model to invoke a built-in tool (`Bash`), and +assert that the upstream (a) emitted a `tool_use` content block and +(b) streamed the tool input incrementally as `input_json_delta` +events. + +Azure OpenAI streams tool arguments in the same chat-completions +fragment shape as openai.com; LiteLLM must re-emit them as Anthropic +`input_json_delta` deltas rather than buffering the full input into +one complete block. + +Bash is restricted to the exact command `echo pong` plus +`--permission-mode dontAsk`; see `tool_use/test_anthropic.py` for the +security rationale. + +GPT cells are opt-in via COMPAT_GPT_CELLS=1 (see +`claude_code._gpt_cells`). + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/tool_use_streaming/test_azure_openai.py + ^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from claude_code._gpt_cells import skip_unless_gpt_cells_enabled +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +AZURE_OPENAI_MODELS = [ + "gpt-5-6-sol-azure-openai", + "gpt-5-6-terra-azure-openai", + "gpt-5-6-luna-azure-openai", +] + +TOOL_USE_PROMPT = ( + "Use the Bash tool to run the command `echo pong` and report what it printed." +) +TOOL_USE_ARGS = [ + "--allowed-tools", + "Bash(echo pong)", + "--permission-mode", + "dontAsk", + "--include-partial-messages", +] + + +def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + return True + return False + + +def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: + """Count `input_json_delta` records among the `stream_event` + entries. Zero means the proxy collapsed the streamed tool input + into a single complete block instead of forwarding the incremental + deltas the upstream emitted.""" + inner_events = ( + event.get("event") for event in events if event.get("type") == "stream_event" + ) + return sum( + 1 + for inner in inner_events + if isinstance(inner, Mapping) + and inner.get("type") == "content_block_delta" + and isinstance(inner.get("delta"), Mapping) + and inner["delta"].get("type") == "input_json_delta" + ) + + +def test_tool_use_streaming_azure_openai(compat_result): + skip_unless_gpt_cells_enabled() + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=AZURE_OPENAI_MODELS, + prompt=TOOL_USE_PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=TOOL_USE_ARGS, + ) + + failures = [] + for model in AZURE_OPENAI_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_tool_use_event(outcome.events): + error = ( + f"[{model}] no tool_use content block observed in stream-json events" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if _count_input_json_deltas(outcome.events) == 0: + error = ( + f"[{model}] no input_json_delta stream events observed; proxy " + f"likely buffered the tool input into a complete block or " + f"stripped fine-grained tool streaming" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/tool_use_streaming/test_bedrock_mantle.py b/tests/e2e/claude_code/tool_use_streaming/test_bedrock_mantle.py new file mode 100644 index 000000000000..0e80f2319dec --- /dev/null +++ b/tests/e2e/claude_code/tool_use_streaming/test_bedrock_mantle.py @@ -0,0 +1,160 @@ +"""tool_use_streaming x AWS Bedrock Mantle (GPT-5.6). + +Drive the real `claude` CLI in headless `--output-format stream-json` +mode against a running LiteLLM proxy that routes Anthropic Messages +requests to OpenAI's GPT-5.6 family (Sol, Terra, Luna) on AWS +Bedrock's Mantle endpoint, ask the model to invoke a built-in tool +(`Bash`), and assert that the upstream (a) emitted a `tool_use` +content block and (b) streamed the tool input incrementally as +`input_json_delta` events. + +Mantle streams OpenAI Responses API `function_call_arguments.delta` +events over SigV4-signed SSE; LiteLLM must re-emit them as Anthropic +`input_json_delta` deltas rather than buffering the full input into +one complete block. + +Bash is restricted to the exact command `echo pong` plus +`--permission-mode dontAsk`; see `tool_use/test_anthropic.py` for the +security rationale. + +GPT cells are opt-in via COMPAT_GPT_CELLS=1 (see +`claude_code._gpt_cells`). + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/tool_use_streaming/test_bedrock_mantle.py + ^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from claude_code._gpt_cells import skip_unless_gpt_cells_enabled +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_MANTLE_MODELS = [ + "gpt-5-6-sol-bedrock-mantle", + "gpt-5-6-terra-bedrock-mantle", + "gpt-5-6-luna-bedrock-mantle", +] + +TOOL_USE_PROMPT = ( + "Use the Bash tool to run the command `echo pong` and report what it printed." +) +TOOL_USE_ARGS = [ + "--allowed-tools", + "Bash(echo pong)", + "--permission-mode", + "dontAsk", + "--include-partial-messages", +] + + +def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + return True + return False + + +def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: + """Count `input_json_delta` records among the `stream_event` + entries. Zero means the proxy collapsed the streamed tool input + into a single complete block instead of forwarding the incremental + deltas the upstream emitted.""" + inner_events = ( + event.get("event") for event in events if event.get("type") == "stream_event" + ) + return sum( + 1 + for inner in inner_events + if isinstance(inner, Mapping) + and inner.get("type") == "content_block_delta" + and isinstance(inner.get("delta"), Mapping) + and inner["delta"].get("type") == "input_json_delta" + ) + + +def test_tool_use_streaming_bedrock_mantle(compat_result): + skip_unless_gpt_cells_enabled() + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=BEDROCK_MANTLE_MODELS, + prompt=TOOL_USE_PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=TOOL_USE_ARGS, + ) + + failures = [] + for model in BEDROCK_MANTLE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_tool_use_event(outcome.events): + error = ( + f"[{model}] no tool_use content block observed in stream-json events" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if _count_input_json_deltas(outcome.events) == 0: + error = ( + f"[{model}] no input_json_delta stream events observed; proxy " + f"likely buffered the tool input into a complete block or " + f"stripped fine-grained tool streaming" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/tool_use_streaming/test_openai.py b/tests/e2e/claude_code/tool_use_streaming/test_openai.py new file mode 100644 index 000000000000..6ed05c732132 --- /dev/null +++ b/tests/e2e/claude_code/tool_use_streaming/test_openai.py @@ -0,0 +1,158 @@ +"""tool_use_streaming x OpenAI (GPT-5.6). + +Drive the real `claude` CLI in headless `--output-format stream-json` +mode against a running LiteLLM proxy that routes Anthropic Messages +requests to OpenAI's GPT-5.6 family (Sol, Terra, Luna), ask the model +to invoke a built-in tool (`Bash`), and assert that the upstream (a) +emitted a `tool_use` content block and (b) streamed the tool input +incrementally as `input_json_delta` events. + +OpenAI streams tool arguments as incremental `tool_calls` argument +fragments; LiteLLM must re-emit them as Anthropic `input_json_delta` +deltas rather than buffering the full input into one complete block. + +Bash is restricted to the exact command `echo pong` plus +`--permission-mode dontAsk`; see `tool_use/test_anthropic.py` for the +security rationale. + +GPT cells are opt-in via COMPAT_GPT_CELLS=1 (see +`claude_code._gpt_cells`). + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/tool_use_streaming/test_openai.py + ^^^^^^^^^^^^^^^^^^ ^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from claude_code._gpt_cells import skip_unless_gpt_cells_enabled +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +OPENAI_MODELS = [ + "gpt-5-6-sol-openai", + "gpt-5-6-terra-openai", + "gpt-5-6-luna-openai", +] + +TOOL_USE_PROMPT = ( + "Use the Bash tool to run the command `echo pong` and report what it printed." +) +TOOL_USE_ARGS = [ + "--allowed-tools", + "Bash(echo pong)", + "--permission-mode", + "dontAsk", + "--include-partial-messages", +] + + +def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + return True + return False + + +def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: + """Count `input_json_delta` records among the `stream_event` + entries. Zero means the proxy collapsed the streamed tool input + into a single complete block instead of forwarding the incremental + deltas the upstream emitted.""" + inner_events = ( + event.get("event") for event in events if event.get("type") == "stream_event" + ) + return sum( + 1 + for inner in inner_events + if isinstance(inner, Mapping) + and inner.get("type") == "content_block_delta" + and isinstance(inner.get("delta"), Mapping) + and inner["delta"].get("type") == "input_json_delta" + ) + + +def test_tool_use_streaming_openai(compat_result): + skip_unless_gpt_cells_enabled() + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=OPENAI_MODELS, + prompt=TOOL_USE_PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=TOOL_USE_ARGS, + ) + + failures = [] + for model in OPENAI_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_tool_use_event(outcome.events): + error = ( + f"[{model}] no tool_use content block observed in stream-json events" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if _count_input_json_deltas(outcome.events) == 0: + error = ( + f"[{model}] no input_json_delta stream events observed; proxy " + f"likely buffered the tool input into a complete block or " + f"stripped fine-grained tool streaming" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/tool_use_streaming/test_vertex_ai_gpt.py b/tests/e2e/claude_code/tool_use_streaming/test_vertex_ai_gpt.py new file mode 100644 index 000000000000..7037e91fee02 --- /dev/null +++ b/tests/e2e/claude_code/tool_use_streaming/test_vertex_ai_gpt.py @@ -0,0 +1,33 @@ +"""tool_use_streaming x Vertex AI (GPT-5.6) — not applicable. + +GCP is the only one of the big-three clouds without OpenAI's +closed-weight GPT-5.6 family (Sol / Terra / Luna); Vertex AI Model +Garden carries only the open-weight gpt-oss MaaS models. The cell +reports `not_applicable` so the published matrix documents the gap +explicitly instead of leaving a `not_tested` hole. + +This stub never drives the `claude` CLI, so it grants no tools and is +exempt from the Bash allow-rule pin enforced by +`_pr_gate_unit_tests/test_bash_tool_restrictions.py`. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/tool_use_streaming/test_vertex_ai_gpt.py + ^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +from claude_code._gpt_cells import VERTEX_AI_GPT_NOT_APPLICABLE_REASON + + +def test_tool_use_streaming_vertex_ai_gpt(compat_result): + """Record the static not_applicable outcome for this cell.""" + compat_result.set( + { + "status": "not_applicable", + "reason": VERTEX_AI_GPT_NOT_APPLICABLE_REASON, + } + ) From 90f7807830846ae539b2876da1f245eca456fc3b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 15 Jul 2026 16:27:58 -0700 Subject: [PATCH 2/6] test(e2e/claude_code): scope the GPT cell opt-in rationale to the cron VM --- tests/e2e/claude_code/_gpt_cells.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/e2e/claude_code/_gpt_cells.py b/tests/e2e/claude_code/_gpt_cells.py index 15b35da7e694..8a15f384d30b 100644 --- a/tests/e2e/claude_code/_gpt_cells.py +++ b/tests/e2e/claude_code/_gpt_cells.py @@ -16,15 +16,15 @@ carries only the open-weight gpt-oss MaaS models -Live GPT cells are opt-in via `COMPAT_GPT_CELLS=1`. The external PR -gate and the daily cron VM must be provisioned with the GPT-route -credentials (`OPENAI_API_KEY`, `AZURE_OPENAI_API_BASE` + -`AZURE_OPENAI_API_KEY`, and Bedrock Mantle model access) before these -cells can pass, so until the flag is set each live cell skips and its -matrix cell stays `not_tested` — landing this suite change cannot flip -the existing gate red. The `vertex_ai_gpt` column ignores the flag: -its cells report a static `not_applicable` and never touch the -network. +Live GPT cells are opt-in via `COMPAT_GPT_CELLS=1`. The cron VM that +runs the scheduled suite and publishes the matrix must be provisioned +with the GPT-route credentials (`OPENAI_API_KEY` with available +quota, `AZURE_OPENAI_API_BASE` + `AZURE_OPENAI_API_KEY` with gpt-5.6 +deployments, and Bedrock Mantle model access) before these cells can +pass, so until the flag is set each live cell skips and its matrix +cell publishes as `not_tested` instead of a credential-shaped red. +The `vertex_ai_gpt` column ignores the flag: its cells report a +static `not_applicable` and never touch the network. """ from __future__ import annotations From c462c51e2557ad28c51994149ce8f5e7f402bea1 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 16 Jul 2026 15:20:24 -0700 Subject: [PATCH 3/6] docs(e2e): drop duplicate claude_code suite entry left by the base merge --- tests/e2e/CLAUDE.md | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 202d7173bc94..5d16761ac444 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -6,7 +6,6 @@ Code-style rules for writing tests under `tests/e2e/`. The harness already encod Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family or behavior area. If you add a new folder, you must add a line here describing what kind of tests belong in it, so the layout stays self-describing. `gateway/` is the exception: it holds proxy configuration only and never tests -- `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI against a live proxy, one directory per feature row and one `test_.py` per column (see `claude_code/manifest.yaml`); results publish as `compat-results.json`, not the coverage registry - `llm_translation/` - LLM endpoint and provider-translation behavior: passthrough, custom pricing, OCR, and the non-chat inference endpoints (`/v1/responses`, `/v1/messages`, `/embeddings`, `/v1/rerank`, `/v1/audio/speech`, `/v1/images/generations`), each against a deployment the test creates via `/model/new` and deletes on teardown - `access_control/` - the gateway's authorization and error-shape contract: per-key model allow-lists, route-group permissions (`allowed_routes`), and unknown-model validation - `embeddings/` - the `/embeddings` endpoint across providers From f93a84b01e608281f993b51d6e0d4b134a02e81b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 16 Jul 2026 16:45:18 -0700 Subject: [PATCH 4/6] test(e2e/claude_code): reuse AZURE_API_BASE/KEY for the azure_openai GPT column --- tests/e2e/claude_code/_gpt_cells.py | 2 +- tests/e2e/claude_code/test_config.yaml | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/e2e/claude_code/_gpt_cells.py b/tests/e2e/claude_code/_gpt_cells.py index 8a15f384d30b..7f0e085b9b8a 100644 --- a/tests/e2e/claude_code/_gpt_cells.py +++ b/tests/e2e/claude_code/_gpt_cells.py @@ -19,7 +19,7 @@ Live GPT cells are opt-in via `COMPAT_GPT_CELLS=1`. The cron VM that runs the scheduled suite and publishes the matrix must be provisioned with the GPT-route credentials (`OPENAI_API_KEY` with available -quota, `AZURE_OPENAI_API_BASE` + `AZURE_OPENAI_API_KEY` with gpt-5.6 +quota, `AZURE_API_BASE` + `AZURE_API_KEY` with gpt-5.6 deployments, and Bedrock Mantle model access) before these cells can pass, so until the flag is set each live cell skips and its matrix cell publishes as `not_tested` instead of a credential-shaped red. diff --git a/tests/e2e/claude_code/test_config.yaml b/tests/e2e/claude_code/test_config.yaml index cde463640111..eab913be7fe2 100644 --- a/tests/e2e/claude_code/test_config.yaml +++ b/tests/e2e/claude_code/test_config.yaml @@ -148,18 +148,18 @@ model_list: - model_name: gpt-5-6-sol-azure-openai litellm_params: model: azure/gpt-5.6-sol - api_base: os.environ/AZURE_OPENAI_API_BASE - api_key: os.environ/AZURE_OPENAI_API_KEY + api_base: os.environ/AZURE_API_BASE + api_key: os.environ/AZURE_API_KEY - model_name: gpt-5-6-terra-azure-openai litellm_params: model: azure/gpt-5.6-terra - api_base: os.environ/AZURE_OPENAI_API_BASE - api_key: os.environ/AZURE_OPENAI_API_KEY + api_base: os.environ/AZURE_API_BASE + api_key: os.environ/AZURE_API_KEY - model_name: gpt-5-6-luna-azure-openai litellm_params: model: azure/gpt-5.6-luna - api_base: os.environ/AZURE_OPENAI_API_BASE - api_key: os.environ/AZURE_OPENAI_API_KEY + api_base: os.environ/AZURE_API_BASE + api_key: os.environ/AZURE_API_KEY # ---- Bedrock Mantle (GPT-5.6, Responses API) ---- # Sol is only served from us-east-1 / us-east-2 as of 2026-07; From bb04a1ed1599eadde9dc8c175fc4a8e750093a18 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 16 Jul 2026 17:25:39 -0700 Subject: [PATCH 5/6] test(e2e/claude_code): run openai and azure GPT cells unconditionally, gate only bedrock_mantle The openai and azure_openai columns have working credentials in every suite runner, so only the bedrock_mantle column still needs an opt-in flag while the AWS account waits on the Mantle allowlist; COMPAT_GPT_CELLS becomes COMPAT_MANTLE_CELLS. The six tool_use cells now resolve the proxy through claude_code._env like the basic_messaging cells instead of hardcoding LITELLM_PROXY_BASE_URL/LITELLM_PROXY_API_KEY. --- tests/e2e/claude_code/_gpt_cells.py | 35 ++++++++++--------- .../test_azure_openai.py | 5 +-- .../test_bedrock_mantle.py | 8 ++--- .../test_openai.py | 5 +-- .../test_azure_openai.py | 5 +-- .../test_bedrock_mantle.py | 8 ++--- .../basic_messaging_streaming/test_openai.py | 5 +-- .../claude_code/tool_use/test_azure_openai.py | 30 +++------------- .../tool_use/test_bedrock_mantle.py | 31 ++++------------ tests/e2e/claude_code/tool_use/test_openai.py | 30 +++------------- .../tool_use_streaming/test_azure_openai.py | 30 +++------------- .../tool_use_streaming/test_bedrock_mantle.py | 31 ++++------------ .../tool_use_streaming/test_openai.py | 30 +++------------- 13 files changed, 60 insertions(+), 193 deletions(-) diff --git a/tests/e2e/claude_code/_gpt_cells.py b/tests/e2e/claude_code/_gpt_cells.py index 7f0e085b9b8a..870e9cea9180 100644 --- a/tests/e2e/claude_code/_gpt_cells.py +++ b/tests/e2e/claude_code/_gpt_cells.py @@ -16,15 +16,16 @@ carries only the open-weight gpt-oss MaaS models -Live GPT cells are opt-in via `COMPAT_GPT_CELLS=1`. The cron VM that -runs the scheduled suite and publishes the matrix must be provisioned -with the GPT-route credentials (`OPENAI_API_KEY` with available -quota, `AZURE_API_BASE` + `AZURE_API_KEY` with gpt-5.6 -deployments, and Bedrock Mantle model access) before these cells can -pass, so until the flag is set each live cell skips and its matrix +The openai and azure_openai columns run unconditionally, like every +other live column: the environments that run the suite carry +`OPENAI_API_KEY` and `AZURE_API_BASE` + `AZURE_API_KEY` pointing at a +resource with gpt-5.6 deployments. The bedrock_mantle column is +opt-in via `COMPAT_MANTLE_CELLS=1` because the AWS account is still +waiting on the Bedrock Mantle allowlist for the `openai.gpt-5.6-*` +models; until the flag is set each Mantle cell skips and its matrix cell publishes as `not_tested` instead of a credential-shaped red. -The `vertex_ai_gpt` column ignores the flag: its cells report a -static `not_applicable` and never touch the network. +The `vertex_ai_gpt` column needs no flag either way: its cells report +a static `not_applicable` and never touch the network. """ from __future__ import annotations @@ -33,7 +34,7 @@ import pytest -GPT_CELLS_ENV = "COMPAT_GPT_CELLS" +MANTLE_CELLS_ENV = "COMPAT_MANTLE_CELLS" VERTEX_AI_GPT_NOT_APPLICABLE_REASON = ( "GCP Vertex AI does not offer OpenAI's closed-weight GPT-5.6 family " @@ -43,18 +44,18 @@ ) -def skip_unless_gpt_cells_enabled() -> None: - """Skip the calling test unless `COMPAT_GPT_CELLS` opts GPT cells in. +def skip_unless_mantle_cells_enabled() -> None: + """Skip the calling test unless `COMPAT_MANTLE_CELLS` opts the + Bedrock Mantle cells in. A skipped cell is recorded as `not_tested` in the published matrix (see the skip handling in `tests/e2e/claude_code/conftest.py`), - which is the honest state for an environment that has no GPT-route - credentials yet. + which is the honest state while the AWS account has no Mantle + access to the GPT-5.6 models yet. """ - if os.environ.get(GPT_CELLS_ENV, "").strip().lower() in {"1", "true", "yes"}: + if os.environ.get(MANTLE_CELLS_ENV, "").strip().lower() in {"1", "true", "yes"}: return pytest.skip( - f"GPT-5.6 cells are opt-in; set {GPT_CELLS_ENV}=1 once the proxy has " - "OpenAI / Azure OpenAI / Bedrock Mantle credentials for the " - "gpt-5-6-* aliases" + f"Bedrock Mantle GPT-5.6 cells are opt-in; set {MANTLE_CELLS_ENV}=1 " + "once the AWS account is allowlisted for the openai.gpt-5.6-* models" ) diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_azure_openai.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_azure_openai.py index fb0b5e9aa770..77876c8f7eef 100644 --- a/tests/e2e/claude_code/basic_messaging_non_streaming/test_azure_openai.py +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_azure_openai.py @@ -20,14 +20,12 @@ feature_id provider Every GPT cell exercises the three GPT-5.6 tiers; the cell only goes -green if all three pass. Cells are opt-in via COMPAT_GPT_CELLS=1 (see -`claude_code._gpt_cells`). +green if all three pass. """ from __future__ import annotations from claude_code._basic_messaging import run_basic_messaging_cell -from claude_code._gpt_cells import skip_unless_gpt_cells_enabled AZURE_OPENAI_MODELS = [ "gpt-5-6-sol-azure-openai", @@ -39,7 +37,6 @@ def test_basic_messaging_non_streaming_azure_openai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty reply from each GPT-5.6 tier.""" - skip_unless_gpt_cells_enabled() run_basic_messaging_cell( compat_result=compat_result, models=AZURE_OPENAI_MODELS, diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_mantle.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_mantle.py index 51614570fc01..8a64547a732a 100644 --- a/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_mantle.py +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_mantle.py @@ -19,14 +19,14 @@ feature_id provider Every GPT cell exercises the three GPT-5.6 tiers; the cell only goes -green if all three pass. Cells are opt-in via COMPAT_GPT_CELLS=1 (see -`claude_code._gpt_cells`). +green if all three pass. Mantle cells are opt-in via +COMPAT_MANTLE_CELLS=1 (see `claude_code._gpt_cells`). """ from __future__ import annotations from claude_code._basic_messaging import run_basic_messaging_cell -from claude_code._gpt_cells import skip_unless_gpt_cells_enabled +from claude_code._gpt_cells import skip_unless_mantle_cells_enabled BEDROCK_MANTLE_MODELS = [ "gpt-5-6-sol-bedrock-mantle", @@ -38,7 +38,7 @@ def test_basic_messaging_non_streaming_bedrock_mantle(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty reply from each GPT-5.6 tier.""" - skip_unless_gpt_cells_enabled() + skip_unless_mantle_cells_enabled() run_basic_messaging_cell( compat_result=compat_result, models=BEDROCK_MANTLE_MODELS, diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_openai.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_openai.py index 57270158328f..b0d143fa5e05 100644 --- a/tests/e2e/claude_code/basic_messaging_non_streaming/test_openai.py +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_openai.py @@ -17,14 +17,12 @@ feature_id provider Every GPT cell exercises the three GPT-5.6 tiers; the cell only goes -green if all three pass. Cells are opt-in via COMPAT_GPT_CELLS=1 (see -`claude_code._gpt_cells`). +green if all three pass. """ from __future__ import annotations from claude_code._basic_messaging import run_basic_messaging_cell -from claude_code._gpt_cells import skip_unless_gpt_cells_enabled OPENAI_MODELS = [ "gpt-5-6-sol-openai", @@ -36,7 +34,6 @@ def test_basic_messaging_non_streaming_openai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty reply from each GPT-5.6 tier.""" - skip_unless_gpt_cells_enabled() run_basic_messaging_cell( compat_result=compat_result, models=OPENAI_MODELS, diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_azure_openai.py b/tests/e2e/claude_code/basic_messaging_streaming/test_azure_openai.py index 603a575d7513..357596590c74 100644 --- a/tests/e2e/claude_code/basic_messaging_streaming/test_azure_openai.py +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_azure_openai.py @@ -19,14 +19,12 @@ feature_id provider Every GPT cell exercises the three GPT-5.6 tiers; the cell only goes -green if all three pass. Cells are opt-in via COMPAT_GPT_CELLS=1 (see -`claude_code._gpt_cells`). +green if all three pass. """ from __future__ import annotations from claude_code._basic_messaging import run_basic_messaging_cell -from claude_code._gpt_cells import skip_unless_gpt_cells_enabled AZURE_OPENAI_MODELS = [ "gpt-5-6-sol-azure-openai", @@ -38,7 +36,6 @@ def test_basic_messaging_streaming_azure_openai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty streamed reply from each GPT-5.6 tier.""" - skip_unless_gpt_cells_enabled() run_basic_messaging_cell( compat_result=compat_result, models=AZURE_OPENAI_MODELS, diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_mantle.py b/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_mantle.py index 59303edc5150..38297e6a3e58 100644 --- a/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_mantle.py +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_mantle.py @@ -19,14 +19,14 @@ feature_id provider Every GPT cell exercises the three GPT-5.6 tiers; the cell only goes -green if all three pass. Cells are opt-in via COMPAT_GPT_CELLS=1 (see -`claude_code._gpt_cells`). +green if all three pass. Mantle cells are opt-in via +COMPAT_MANTLE_CELLS=1 (see `claude_code._gpt_cells`). """ from __future__ import annotations from claude_code._basic_messaging import run_basic_messaging_cell -from claude_code._gpt_cells import skip_unless_gpt_cells_enabled +from claude_code._gpt_cells import skip_unless_mantle_cells_enabled BEDROCK_MANTLE_MODELS = [ "gpt-5-6-sol-bedrock-mantle", @@ -38,7 +38,7 @@ def test_basic_messaging_streaming_bedrock_mantle(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty streamed reply from each GPT-5.6 tier.""" - skip_unless_gpt_cells_enabled() + skip_unless_mantle_cells_enabled() run_basic_messaging_cell( compat_result=compat_result, models=BEDROCK_MANTLE_MODELS, diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_openai.py b/tests/e2e/claude_code/basic_messaging_streaming/test_openai.py index 58767b2fd104..402c763496bd 100644 --- a/tests/e2e/claude_code/basic_messaging_streaming/test_openai.py +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_openai.py @@ -19,14 +19,12 @@ feature_id provider Every GPT cell exercises the three GPT-5.6 tiers; the cell only goes -green if all three pass. Cells are opt-in via COMPAT_GPT_CELLS=1 (see -`claude_code._gpt_cells`). +green if all three pass. """ from __future__ import annotations from claude_code._basic_messaging import run_basic_messaging_cell -from claude_code._gpt_cells import skip_unless_gpt_cells_enabled OPENAI_MODELS = [ "gpt-5-6-sol-openai", @@ -38,7 +36,6 @@ def test_basic_messaging_streaming_openai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty streamed reply from each GPT-5.6 tier.""" - skip_unless_gpt_cells_enabled() run_basic_messaging_cell( compat_result=compat_result, models=OPENAI_MODELS, diff --git a/tests/e2e/claude_code/tool_use/test_azure_openai.py b/tests/e2e/claude_code/tool_use/test_azure_openai.py index cf7809d13a5c..7e1eecdbc03a 100644 --- a/tests/e2e/claude_code/tool_use/test_azure_openai.py +++ b/tests/e2e/claude_code/tool_use/test_azure_openai.py @@ -15,9 +15,6 @@ `--permission-mode dontAsk`; see `tool_use/test_anthropic.py` for the security rationale. -GPT cells are opt-in via COMPAT_GPT_CELLS=1 (see -`claude_code._gpt_cells`). - The (feature, provider) for this cell is inferred from the file path by `tests/e2e/claude_code/conftest.py`: @@ -28,21 +25,17 @@ from __future__ import annotations -import os from typing import Any, Mapping, Sequence import pytest -from claude_code._gpt_cells import skip_unless_gpt_cells_enabled +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" - AZURE_OPENAI_MODELS = [ "gpt-5-6-sol-azure-openai", "gpt-5-6-terra-azure-openai", @@ -77,28 +70,13 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: def test_tool_use_azure_openai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a tool call was emitted on the wire by each GPT-5.6 tier.""" - skip_unless_gpt_cells_enabled() - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + proxy = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=AZURE_OPENAI_MODELS, prompt=TOOL_USE_PROMPT, - base_url=base_url, - api_key=api_key, + base_url=proxy.base_url, + api_key=proxy.api_key, extra_args=TOOL_USE_ARGS, ) diff --git a/tests/e2e/claude_code/tool_use/test_bedrock_mantle.py b/tests/e2e/claude_code/tool_use/test_bedrock_mantle.py index cf630b3be193..e9cb70e74e97 100644 --- a/tests/e2e/claude_code/tool_use/test_bedrock_mantle.py +++ b/tests/e2e/claude_code/tool_use/test_bedrock_mantle.py @@ -16,7 +16,7 @@ `--permission-mode dontAsk`; see `tool_use/test_anthropic.py` for the security rationale. -GPT cells are opt-in via COMPAT_GPT_CELLS=1 (see +Mantle cells are opt-in via COMPAT_MANTLE_CELLS=1 (see `claude_code._gpt_cells`). The (feature, provider) for this cell is inferred from the file path by @@ -29,21 +29,18 @@ from __future__ import annotations -import os from typing import Any, Mapping, Sequence import pytest -from claude_code._gpt_cells import skip_unless_gpt_cells_enabled +from claude_code._env import require_proxy +from claude_code._gpt_cells import skip_unless_mantle_cells_enabled from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" - BEDROCK_MANTLE_MODELS = [ "gpt-5-6-sol-bedrock-mantle", "gpt-5-6-terra-bedrock-mantle", @@ -78,28 +75,14 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: def test_tool_use_bedrock_mantle(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a tool call was emitted on the wire by each GPT-5.6 tier.""" - skip_unless_gpt_cells_enabled() - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + skip_unless_mantle_cells_enabled() + proxy = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=BEDROCK_MANTLE_MODELS, prompt=TOOL_USE_PROMPT, - base_url=base_url, - api_key=api_key, + base_url=proxy.base_url, + api_key=proxy.api_key, extra_args=TOOL_USE_ARGS, ) diff --git a/tests/e2e/claude_code/tool_use/test_openai.py b/tests/e2e/claude_code/tool_use/test_openai.py index 7fa671f8e6e7..dbe60a65281c 100644 --- a/tests/e2e/claude_code/tool_use/test_openai.py +++ b/tests/e2e/claude_code/tool_use/test_openai.py @@ -14,9 +14,6 @@ `--permission-mode dontAsk`; see `tool_use/test_anthropic.py` for the security rationale. -GPT cells are opt-in via COMPAT_GPT_CELLS=1 (see -`claude_code._gpt_cells`). - The (feature, provider) for this cell is inferred from the file path by `tests/e2e/claude_code/conftest.py`: @@ -27,21 +24,17 @@ from __future__ import annotations -import os from typing import Any, Mapping, Sequence import pytest -from claude_code._gpt_cells import skip_unless_gpt_cells_enabled +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" - OPENAI_MODELS = [ "gpt-5-6-sol-openai", "gpt-5-6-terra-openai", @@ -76,28 +69,13 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: def test_tool_use_openai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a tool call was emitted on the wire by each GPT-5.6 tier.""" - skip_unless_gpt_cells_enabled() - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + proxy = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=OPENAI_MODELS, prompt=TOOL_USE_PROMPT, - base_url=base_url, - api_key=api_key, + base_url=proxy.base_url, + api_key=proxy.api_key, extra_args=TOOL_USE_ARGS, ) diff --git a/tests/e2e/claude_code/tool_use_streaming/test_azure_openai.py b/tests/e2e/claude_code/tool_use_streaming/test_azure_openai.py index f85ffa9c4b4b..ad5d4e0f613f 100644 --- a/tests/e2e/claude_code/tool_use_streaming/test_azure_openai.py +++ b/tests/e2e/claude_code/tool_use_streaming/test_azure_openai.py @@ -17,9 +17,6 @@ `--permission-mode dontAsk`; see `tool_use/test_anthropic.py` for the security rationale. -GPT cells are opt-in via COMPAT_GPT_CELLS=1 (see -`claude_code._gpt_cells`). - The (feature, provider) for this cell is inferred from the file path by `tests/e2e/claude_code/conftest.py`: @@ -30,21 +27,17 @@ from __future__ import annotations -import os from typing import Any, Mapping, Sequence import pytest -from claude_code._gpt_cells import skip_unless_gpt_cells_enabled +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" - AZURE_OPENAI_MODELS = [ "gpt-5-6-sol-azure-openai", "gpt-5-6-terra-azure-openai", @@ -96,28 +89,13 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: def test_tool_use_streaming_azure_openai(compat_result): - skip_unless_gpt_cells_enabled() - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + proxy = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=AZURE_OPENAI_MODELS, prompt=TOOL_USE_PROMPT, - base_url=base_url, - api_key=api_key, + base_url=proxy.base_url, + api_key=proxy.api_key, extra_args=TOOL_USE_ARGS, ) diff --git a/tests/e2e/claude_code/tool_use_streaming/test_bedrock_mantle.py b/tests/e2e/claude_code/tool_use_streaming/test_bedrock_mantle.py index 0e80f2319dec..20fae5d48dba 100644 --- a/tests/e2e/claude_code/tool_use_streaming/test_bedrock_mantle.py +++ b/tests/e2e/claude_code/tool_use_streaming/test_bedrock_mantle.py @@ -17,7 +17,7 @@ `--permission-mode dontAsk`; see `tool_use/test_anthropic.py` for the security rationale. -GPT cells are opt-in via COMPAT_GPT_CELLS=1 (see +Mantle cells are opt-in via COMPAT_MANTLE_CELLS=1 (see `claude_code._gpt_cells`). The (feature, provider) for this cell is inferred from the file path by @@ -30,21 +30,18 @@ from __future__ import annotations -import os from typing import Any, Mapping, Sequence import pytest -from claude_code._gpt_cells import skip_unless_gpt_cells_enabled +from claude_code._env import require_proxy +from claude_code._gpt_cells import skip_unless_mantle_cells_enabled from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" - BEDROCK_MANTLE_MODELS = [ "gpt-5-6-sol-bedrock-mantle", "gpt-5-6-terra-bedrock-mantle", @@ -96,28 +93,14 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: def test_tool_use_streaming_bedrock_mantle(compat_result): - skip_unless_gpt_cells_enabled() - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + skip_unless_mantle_cells_enabled() + proxy = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=BEDROCK_MANTLE_MODELS, prompt=TOOL_USE_PROMPT, - base_url=base_url, - api_key=api_key, + base_url=proxy.base_url, + api_key=proxy.api_key, extra_args=TOOL_USE_ARGS, ) diff --git a/tests/e2e/claude_code/tool_use_streaming/test_openai.py b/tests/e2e/claude_code/tool_use_streaming/test_openai.py index 6ed05c732132..895f88d994b8 100644 --- a/tests/e2e/claude_code/tool_use_streaming/test_openai.py +++ b/tests/e2e/claude_code/tool_use_streaming/test_openai.py @@ -15,9 +15,6 @@ `--permission-mode dontAsk`; see `tool_use/test_anthropic.py` for the security rationale. -GPT cells are opt-in via COMPAT_GPT_CELLS=1 (see -`claude_code._gpt_cells`). - The (feature, provider) for this cell is inferred from the file path by `tests/e2e/claude_code/conftest.py`: @@ -28,21 +25,17 @@ from __future__ import annotations -import os from typing import Any, Mapping, Sequence import pytest -from claude_code._gpt_cells import skip_unless_gpt_cells_enabled +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" - OPENAI_MODELS = [ "gpt-5-6-sol-openai", "gpt-5-6-terra-openai", @@ -94,28 +87,13 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: def test_tool_use_streaming_openai(compat_result): - skip_unless_gpt_cells_enabled() - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + proxy = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=OPENAI_MODELS, prompt=TOOL_USE_PROMPT, - base_url=base_url, - api_key=api_key, + base_url=proxy.base_url, + api_key=proxy.api_key, extra_args=TOOL_USE_ARGS, ) From a017b95e2ffc6d5d904f64bc60abb54e6737174b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 16 Jul 2026 17:26:46 -0700 Subject: [PATCH 6/6] test(e2e/claude_code): update run_compat.sh flag docs to COMPAT_MANTLE_CELLS --- tests/e2e/claude_code/run_compat.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/claude_code/run_compat.sh b/tests/e2e/claude_code/run_compat.sh index 0b30fb51f1c4..b881cf1d31e5 100755 --- a/tests/e2e/claude_code/run_compat.sh +++ b/tests/e2e/claude_code/run_compat.sh @@ -27,7 +27,7 @@ # LITELLM_COMPAT_RATE_BURST override per-bucket burst # # Optional env (GPT-5.6 columns): -# COMPAT_GPT_CELLS=1 opt the GPT-5.6 (Sol/Terra/Luna) +# COMPAT_MANTLE_CELLS=1 opt the Bedrock Mantle GPT-5.6 # cells in; without it they skip # and publish as not_tested #