test(e2e/claude_code): add GPT-5.6 Sol/Terra/Luna columns for OpenAI, Azure OpenAI, and Bedrock Mantle

tests/e2e/claude_code/_gpt_cells.py

@@ -0,0 +1,61 @@
+"""Shared plumbing for the GPT-5.6 (Sol / Terra / Luna) provider columns.
+
+OpenAI shipped GPT-5.6 as a three-tier family on 2026-07-09 — Sol
+(flagship), Terra (balanced), Luna (fast) — and Claude Code can drive
+all three through a LiteLLM proxy that translates the Anthropic
+Messages API to each provider's native shape. Four provider columns
+cover "OpenAI plus the big three clouds":
+
+    openai          OpenAI API             (openai/gpt-5.6-*)
+    azure_openai    Azure OpenAI           (azure/gpt-5.6-*)
+    bedrock_mantle  AWS Bedrock, Mantle    (bedrock_mantle/openai.gpt-5.6-*,
+                                            Responses API)
+    vertex_ai_gpt   GCP Vertex AI          not_applicable — Vertex does
+                                           not offer the closed-weight
+                                           GPT-5.6 family; Model Garden
+                                           carries only the open-weight
+                                           gpt-oss MaaS models
+
+The openai and azure_openai columns run unconditionally, like every
+other live column: the environments that run the suite carry
+`OPENAI_API_KEY` and `AZURE_API_BASE` + `AZURE_API_KEY` pointing at a
+resource with gpt-5.6 deployments. The bedrock_mantle column is
+opt-in via `COMPAT_MANTLE_CELLS=1` because the AWS account is still
+waiting on the Bedrock Mantle allowlist for the `openai.gpt-5.6-*`
+models; until the flag is set each Mantle cell skips and its matrix
+cell publishes as `not_tested` instead of a credential-shaped red.
+The `vertex_ai_gpt` column needs no flag either way: its cells report
+a static `not_applicable` and never touch the network.
+"""
+
+from __future__ import annotations
+
+import os
+
+import pytest
+
+MANTLE_CELLS_ENV = "COMPAT_MANTLE_CELLS"
+
+VERTEX_AI_GPT_NOT_APPLICABLE_REASON = (
+    "GCP Vertex AI does not offer OpenAI's closed-weight GPT-5.6 family "
+    "(Sol / Terra / Luna); Model Garden carries only the open-weight "
+    "gpt-oss MaaS models. Convert this column's cells to live tests if "
+    "Google adds the GPT-5.6 models."
+)
+
+
+def skip_unless_mantle_cells_enabled() -> None:
+    """Skip the calling test unless `COMPAT_MANTLE_CELLS` opts the
+    Bedrock Mantle cells in.
+
+    A skipped cell is recorded as `not_tested` in the published matrix
+    (see the skip handling in `tests/e2e/claude_code/conftest.py`),
+    which is the honest state while the AWS account has no Mantle
+    access to the GPT-5.6 models yet.
+    """
+    if os.environ.get(MANTLE_CELLS_ENV, "").strip().lower() in {"1", "true", "yes"}:
+        return
+    pytest.skip(
+        f"Bedrock Mantle GPT-5.6 cells are opt-in; set {MANTLE_CELLS_ENV}=1 "
+        "once the AWS account is allowlisted for the openai.gpt-5.6-* models"
+    )

tests/e2e/claude_code/basic_messaging_non_streaming/test_azure_openai.py

@@ -0,0 +1,44 @@
+"""basic_messaging_non_streaming x Azure OpenAI (GPT-5.6).
+
+Drive the real `claude` CLI in headless mode against a running LiteLLM
+proxy that routes Anthropic Messages requests to Azure OpenAI
+deployments of the GPT-5.6 family (Sol, Terra, Luna), and report the
+outcome via `compat_result`.
+
+Azure OpenAI serves the same chat-completions wire shape as
+openai.com behind per-resource deployments; LiteLLM's `azure/gpt-*`
+route handles the deployment addressing while reusing the OpenAI
+translation, so this cell catches Azure-specific regressions
+(auth headers, api-version pinning, deployment routing) that the
+`openai` column cannot.
+
+The (feature, provider) for this cell is inferred from the file path by
+`tests/e2e/claude_code/conftest.py`:
+
+    tests/e2e/claude_code/basic_messaging_non_streaming/test_azure_openai.py
+                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^      ^^^^^^^^^^^^
+                       feature_id                         provider
+
+Every GPT cell exercises the three GPT-5.6 tiers; the cell only goes
+green if all three pass.
+"""
+
+from __future__ import annotations
+
+from claude_code._basic_messaging import run_basic_messaging_cell
+
+AZURE_OPENAI_MODELS = [
+    "gpt-5-6-sol-azure-openai",
+    "gpt-5-6-terra-azure-openai",
+    "gpt-5-6-luna-azure-openai",
+]
+
+
+def test_basic_messaging_non_streaming_azure_openai(compat_result):
+    """Drive the `claude` CLI against the LiteLLM proxy and assert a
+    non-empty reply from each GPT-5.6 tier."""
+    run_basic_messaging_cell(
+        compat_result=compat_result,
+        models=AZURE_OPENAI_MODELS,
+        prompt="Reply with the single word 'pong' and nothing else.",
+    )

tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_mantle.py

@@ -0,0 +1,46 @@
+"""basic_messaging_non_streaming x AWS Bedrock Mantle (GPT-5.6).
+
+Drive the real `claude` CLI in headless mode against a running LiteLLM
+proxy that routes Anthropic Messages requests to OpenAI's GPT-5.6
+family (Sol, Terra, Luna) hosted on AWS Bedrock, and report the
+outcome via `compat_result`.
+
+Bedrock exposes the GPT-5.6 models through the Mantle endpoint, which
+speaks the OpenAI Responses API rather than Converse/Invoke; LiteLLM's
+`bedrock_mantle/openai.gpt-*` route signs the request with SigV4 and
+translates Anthropic Messages to Responses, so this cell exercises a
+translation path no other column covers.
+
+The (feature, provider) for this cell is inferred from the file path by
+`tests/e2e/claude_code/conftest.py`:
+
+    tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_mantle.py
+                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^      ^^^^^^^^^^^^^^
+                       feature_id                         provider
+
+Every GPT cell exercises the three GPT-5.6 tiers; the cell only goes
+green if all three pass. Mantle cells are opt-in via
+COMPAT_MANTLE_CELLS=1 (see `claude_code._gpt_cells`).
+"""
+
+from __future__ import annotations
+
+from claude_code._basic_messaging import run_basic_messaging_cell
+from claude_code._gpt_cells import skip_unless_mantle_cells_enabled
+
+BEDROCK_MANTLE_MODELS = [
+    "gpt-5-6-sol-bedrock-mantle",
+    "gpt-5-6-terra-bedrock-mantle",
+    "gpt-5-6-luna-bedrock-mantle",
+]
+
+
+def test_basic_messaging_non_streaming_bedrock_mantle(compat_result):
+    """Drive the `claude` CLI against the LiteLLM proxy and assert a
+    non-empty reply from each GPT-5.6 tier."""
+    skip_unless_mantle_cells_enabled()
+    run_basic_messaging_cell(
+        compat_result=compat_result,
+        models=BEDROCK_MANTLE_MODELS,
+        prompt="Reply with the single word 'pong' and nothing else.",
+    )

tests/e2e/claude_code/basic_messaging_non_streaming/test_openai.py

@@ -0,0 +1,41 @@
+"""basic_messaging_non_streaming x OpenAI (GPT-5.6).
+
+Drive the real `claude` CLI in headless mode against a running LiteLLM
+proxy that routes Anthropic Messages requests to OpenAI's GPT-5.6
+family (Sol, Terra, Luna), and report the outcome via `compat_result`.
+
+Claude Code only speaks the Anthropic Messages API; LiteLLM's
+`openai/gpt-*` route translates the request to OpenAI chat completions
+and maps the response back, so this cell exercises the full
+cross-provider translation layer in both directions.
+
+The (feature, provider) for this cell is inferred from the file path by
+`tests/e2e/claude_code/conftest.py`:
+
+    tests/e2e/claude_code/basic_messaging_non_streaming/test_openai.py
+                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^      ^^^^^^
+                       feature_id                         provider
+
+Every GPT cell exercises the three GPT-5.6 tiers; the cell only goes
+green if all three pass.
+"""
+
+from __future__ import annotations
+
+from claude_code._basic_messaging import run_basic_messaging_cell
+
+OPENAI_MODELS = [
+    "gpt-5-6-sol-openai",
+    "gpt-5-6-terra-openai",
+    "gpt-5-6-luna-openai",
+]
+
+
+def test_basic_messaging_non_streaming_openai(compat_result):
+    """Drive the `claude` CLI against the LiteLLM proxy and assert a
+    non-empty reply from each GPT-5.6 tier."""
+    run_basic_messaging_cell(
+        compat_result=compat_result,
+        models=OPENAI_MODELS,
+        prompt="Reply with the single word 'pong' and nothing else.",
+    )

tests/e2e/claude_code/basic_messaging_non_streaming/test_vertex_ai_gpt.py

@@ -0,0 +1,29 @@
+"""basic_messaging_non_streaming x Vertex AI (GPT-5.6) — not applicable.
+
+GCP is the only one of the big-three clouds without OpenAI's
+closed-weight GPT-5.6 family (Sol / Terra / Luna); Vertex AI Model
+Garden carries only the open-weight gpt-oss MaaS models. The cell
+reports `not_applicable` so the published matrix documents the gap
+explicitly instead of leaving a `not_tested` hole.
+
+The (feature, provider) for this cell is inferred from the file path by
+`tests/e2e/claude_code/conftest.py`:
+
+    tests/e2e/claude_code/basic_messaging_non_streaming/test_vertex_ai_gpt.py
+                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^      ^^^^^^^^^^^^^
+                       feature_id                         provider
+"""
+
+from __future__ import annotations
+
+from claude_code._gpt_cells import VERTEX_AI_GPT_NOT_APPLICABLE_REASON
+
+
+def test_basic_messaging_non_streaming_vertex_ai_gpt(compat_result):
+    """Record the static not_applicable outcome for this cell."""
+    compat_result.set(
+        {
+            "status": "not_applicable",
+            "reason": VERTEX_AI_GPT_NOT_APPLICABLE_REASON,
+        }
+    )

tests/e2e/claude_code/basic_messaging_streaming/test_azure_openai.py

@@ -0,0 +1,44 @@
+"""basic_messaging_streaming x Azure OpenAI (GPT-5.6).
+
+Drive the real `claude` CLI in headless `--output-format stream-json`
+mode against a running LiteLLM proxy that routes Anthropic Messages
+requests to Azure OpenAI deployments of the GPT-5.6 family (Sol,
+Terra, Luna), and report the outcome via `compat_result`.
+
+Azure OpenAI streams the same chat-completions SSE shape as
+openai.com; LiteLLM re-emits it as Anthropic stream events, and the
+`verify_streaming=True` assertion (via `--include-partial-messages`)
+proves the events arrived incrementally rather than as one buffered
+response.
+
+The (feature, provider) for this cell is inferred from the file path by
+`tests/e2e/claude_code/conftest.py`:
+
+    tests/e2e/claude_code/basic_messaging_streaming/test_azure_openai.py
+                       ^^^^^^^^^^^^^^^^^^^^^^^^^      ^^^^^^^^^^^^
+                       feature_id                     provider
+
+Every GPT cell exercises the three GPT-5.6 tiers; the cell only goes
+green if all three pass.
+"""
+
+from __future__ import annotations
+
+from claude_code._basic_messaging import run_basic_messaging_cell
+
+AZURE_OPENAI_MODELS = [
+    "gpt-5-6-sol-azure-openai",
+    "gpt-5-6-terra-azure-openai",
+    "gpt-5-6-luna-azure-openai",
+]
+
+
+def test_basic_messaging_streaming_azure_openai(compat_result):
+    """Drive the `claude` CLI against the LiteLLM proxy and assert a
+    non-empty streamed reply from each GPT-5.6 tier."""
+    run_basic_messaging_cell(
+        compat_result=compat_result,
+        models=AZURE_OPENAI_MODELS,
+        prompt="Count from 1 to 5, one number per line.",
+        verify_streaming=True,
+    )

tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_mantle.py

@@ -0,0 +1,47 @@
+"""basic_messaging_streaming x AWS Bedrock Mantle (GPT-5.6).
+
+Drive the real `claude` CLI in headless `--output-format stream-json`
+mode against a running LiteLLM proxy that routes Anthropic Messages
+requests to OpenAI's GPT-5.6 family (Sol, Terra, Luna) on AWS
+Bedrock's Mantle endpoint, and report the outcome via `compat_result`.
+
+Mantle streams OpenAI Responses API events over SigV4-signed SSE;
+LiteLLM re-emits them as Anthropic stream events, and the
+`verify_streaming=True` assertion (via `--include-partial-messages`)
+proves the events arrived incrementally rather than as one buffered
+response.
+
+The (feature, provider) for this cell is inferred from the file path by
+`tests/e2e/claude_code/conftest.py`:
+
+    tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_mantle.py
+                       ^^^^^^^^^^^^^^^^^^^^^^^^^      ^^^^^^^^^^^^^^
+                       feature_id                     provider
+
+Every GPT cell exercises the three GPT-5.6 tiers; the cell only goes
+green if all three pass. Mantle cells are opt-in via
+COMPAT_MANTLE_CELLS=1 (see `claude_code._gpt_cells`).
+"""
+
+from __future__ import annotations
+
+from claude_code._basic_messaging import run_basic_messaging_cell
+from claude_code._gpt_cells import skip_unless_mantle_cells_enabled
+
+BEDROCK_MANTLE_MODELS = [
+    "gpt-5-6-sol-bedrock-mantle",
+    "gpt-5-6-terra-bedrock-mantle",
+    "gpt-5-6-luna-bedrock-mantle",
+]
+
+
+def test_basic_messaging_streaming_bedrock_mantle(compat_result):
+    """Drive the `claude` CLI against the LiteLLM proxy and assert a
+    non-empty streamed reply from each GPT-5.6 tier."""
+    skip_unless_mantle_cells_enabled()
+    run_basic_messaging_cell(
+        compat_result=compat_result,
+        models=BEDROCK_MANTLE_MODELS,
+        prompt="Count from 1 to 5, one number per line.",
+        verify_streaming=True,
+    )

tests/e2e/claude_code/basic_messaging_streaming/test_openai.py

@@ -0,0 +1,44 @@
+"""basic_messaging_streaming x OpenAI (GPT-5.6).
+
+Drive the real `claude` CLI in headless `--output-format stream-json`
+mode against a running LiteLLM proxy that routes Anthropic Messages
+requests to OpenAI's GPT-5.6 family (Sol, Terra, Luna), and report the
+outcome via `compat_result`.
+
+LiteLLM translates OpenAI's chat-completions SSE chunks into Anthropic
+`message_start` / `content_block_delta` / `message_stop` events on the
+fly; the `verify_streaming=True` assertion (via
+`--include-partial-messages`) proves the proxy re-emitted incremental
+events instead of buffering the upstream stream into one response.
+
+The (feature, provider) for this cell is inferred from the file path by
+`tests/e2e/claude_code/conftest.py`:
+
+    tests/e2e/claude_code/basic_messaging_streaming/test_openai.py
+                       ^^^^^^^^^^^^^^^^^^^^^^^^^      ^^^^^^
+                       feature_id                     provider
+
+Every GPT cell exercises the three GPT-5.6 tiers; the cell only goes
+green if all three pass.
+"""
+
+from __future__ import annotations
+
+from claude_code._basic_messaging import run_basic_messaging_cell
+
+OPENAI_MODELS = [
+    "gpt-5-6-sol-openai",
+    "gpt-5-6-terra-openai",
+    "gpt-5-6-luna-openai",
+]
+
+
+def test_basic_messaging_streaming_openai(compat_result):
+    """Drive the `claude` CLI against the LiteLLM proxy and assert a
+    non-empty streamed reply from each GPT-5.6 tier."""
+    run_basic_messaging_cell(
+        compat_result=compat_result,
+        models=OPENAI_MODELS,
+        prompt="Count from 1 to 5, one number per line.",
+        verify_streaming=True,
+    )

tests/e2e/claude_code/basic_messaging_streaming/test_vertex_ai_gpt.py

@@ -0,0 +1,29 @@
+"""basic_messaging_streaming x Vertex AI (GPT-5.6) — not applicable.
+
+GCP is the only one of the big-three clouds without OpenAI's
+closed-weight GPT-5.6 family (Sol / Terra / Luna); Vertex AI Model
+Garden carries only the open-weight gpt-oss MaaS models. The cell
+reports `not_applicable` so the published matrix documents the gap
+explicitly instead of leaving a `not_tested` hole.
+
+The (feature, provider) for this cell is inferred from the file path by
+`tests/e2e/claude_code/conftest.py`:
+
+    tests/e2e/claude_code/basic_messaging_streaming/test_vertex_ai_gpt.py
+                       ^^^^^^^^^^^^^^^^^^^^^^^^^      ^^^^^^^^^^^^^
+                       feature_id                     provider
+"""
+
+from __future__ import annotations
+
+from claude_code._gpt_cells import VERTEX_AI_GPT_NOT_APPLICABLE_REASON
+
+
+def test_basic_messaging_streaming_vertex_ai_gpt(compat_result):
+    """Record the static not_applicable outcome for this cell."""
+    compat_result.set(
+        {
+            "status": "not_applicable",
+            "reason": VERTEX_AI_GPT_NOT_APPLICABLE_REASON,
+        }
+    )