From 3befa3afc304cce3593c386d2a35ce45119ea5ba Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 15 Jul 2026 16:47:31 -0700 Subject: [PATCH] test(e2e): bump claude code compat matrix tiers to sonnet-5 and opus-4-8 Anthropic and both Bedrock surfaces move their mid and large tiers from claude-sonnet-4-6 / claude-opus-4-7 to claude-sonnet-5 / claude-opus-4-8. The haiku-4-5 small tier is unchanged everywhere. Vertex AI stays fully on the old tiers and Azure keeps sonnet-4-6 in its sonnet slot (opus does bump to 4-8): live validation showed both new base models 429 with 0 TPM quota in the suite's Vertex project/region, and the Foundry subscription has a hard 0 TPM quota limit for Claude Sonnet 5, so a sonnet-5 deployment cannot be created yet. test_v0_layout now encodes the per-provider tier expectations so the pins are enforced and documented. --- .../fixtures/expected_matrix.json | 2 +- .../_builder_unit_tests/fixtures/results.json | 8 ++--- .../test_matrix_builder.py | 14 ++++----- .../_builder_unit_tests/test_v0_layout.py | 31 +++++++++++++++---- .../test_basic_messaging.py | 4 +-- .../_driver_unit_tests/test_cli_driver.py | 18 +++++------ .../_driver_unit_tests/test_rate_limiter.py | 10 +++--- .../test_anthropic.py | 8 ++--- .../test_azure.py | 4 +-- .../test_bedrock_converse.py | 8 ++--- .../test_bedrock_invoke.py | 8 ++--- .../test_anthropic.py | 4 +-- .../basic_messaging_streaming/test_azure.py | 2 +- .../test_bedrock_converse.py | 4 +-- .../test_bedrock_invoke.py | 4 +-- .../count_tokens/test_anthropic.py | 4 +-- .../claude_code/count_tokens/test_azure.py | 2 +- .../count_tokens/test_bedrock_converse.py | 4 +-- .../count_tokens/test_bedrock_invoke.py | 4 +-- .../long_context_1m/test_anthropic.py | 6 ++-- .../claude_code/long_context_1m/test_azure.py | 4 +-- .../long_context_1m/test_bedrock_converse.py | 6 ++-- .../long_context_1m/test_bedrock_invoke.py | 6 ++-- tests/e2e/claude_code/manifest.yaml | 6 ++-- .../claude_code/pdf_input/test_anthropic.py | 4 +-- tests/e2e/claude_code/pdf_input/test_azure.py | 2 +- .../pdf_input/test_bedrock_converse.py | 4 +-- .../pdf_input/test_bedrock_invoke.py | 4 +-- .../prompt_caching_1h/test_anthropic.py | 4 +-- .../prompt_caching_1h/test_azure.py | 2 +- .../test_bedrock_converse.py | 4 +-- .../prompt_caching_1h/test_bedrock_invoke.py | 4 +-- .../prompt_caching_5m/test_anthropic.py | 4 +-- .../prompt_caching_5m/test_azure.py | 2 +- .../test_bedrock_converse.py | 4 +-- .../prompt_caching_5m/test_bedrock_invoke.py | 4 +-- .../structured_outputs/test_anthropic.py | 4 +-- .../structured_outputs/test_azure.py | 2 +- .../test_bedrock_converse.py | 4 +-- .../structured_outputs/test_bedrock_invoke.py | 4 +-- tests/e2e/claude_code/test_config.yaml | 28 ++++++++--------- .../claude_code/thinking/test_anthropic.py | 10 +++--- tests/e2e/claude_code/thinking/test_azure.py | 7 ++--- .../thinking/test_bedrock_converse.py | 4 +-- .../thinking/test_bedrock_invoke.py | 4 +-- .../thinking_with_tool_use/test_anthropic.py | 4 +-- .../thinking_with_tool_use/test_azure.py | 2 +- .../test_bedrock_converse.py | 4 +-- .../test_bedrock_invoke.py | 4 +-- .../claude_code/tool_search/test_anthropic.py | 4 +-- .../e2e/claude_code/tool_search/test_azure.py | 2 +- .../tool_search/test_bedrock_converse.py | 4 +-- .../tool_search/test_bedrock_invoke.py | 4 +-- .../claude_code/tool_use/test_anthropic.py | 4 +-- tests/e2e/claude_code/tool_use/test_azure.py | 2 +- .../tool_use/test_bedrock_converse.py | 4 +-- .../tool_use/test_bedrock_invoke.py | 4 +-- .../tool_use_streaming/test_anthropic.py | 4 +-- .../tool_use_streaming/test_azure.py | 2 +- .../test_bedrock_converse.py | 4 +-- .../tool_use_streaming/test_bedrock_invoke.py | 4 +-- .../e2e/claude_code/vision/test_anthropic.py | 4 +-- tests/e2e/claude_code/vision/test_azure.py | 2 +- .../vision/test_bedrock_converse.py | 4 +-- .../claude_code/vision/test_bedrock_invoke.py | 4 +-- .../claude_code/web_search/test_anthropic.py | 4 +-- .../e2e/claude_code/web_search/test_azure.py | 2 +- .../web_search/test_bedrock_converse.py | 4 +-- .../web_search/test_bedrock_invoke.py | 4 +-- 69 files changed, 190 insertions(+), 174 deletions(-) diff --git a/tests/e2e/claude_code/_builder_unit_tests/fixtures/expected_matrix.json b/tests/e2e/claude_code/_builder_unit_tests/fixtures/expected_matrix.json index d3aca0142dc4..4b45390c86d2 100644 --- a/tests/e2e/claude_code/_builder_unit_tests/fixtures/expected_matrix.json +++ b/tests/e2e/claude_code/_builder_unit_tests/fixtures/expected_matrix.json @@ -26,7 +26,7 @@ "providers": { "anthropic": { "status": "fail", - "error": "[claude-sonnet-4-6] tool call dropped" + "error": "[claude-sonnet-5] tool call dropped" }, "bedrock_invoke": { "status": "not_applicable", diff --git a/tests/e2e/claude_code/_builder_unit_tests/fixtures/results.json b/tests/e2e/claude_code/_builder_unit_tests/fixtures/results.json index a01540c394f3..40d18f3ef975 100644 --- a/tests/e2e/claude_code/_builder_unit_tests/fixtures/results.json +++ b/tests/e2e/claude_code/_builder_unit_tests/fixtures/results.json @@ -10,13 +10,13 @@ { "feature_id": "basic_messaging_non_streaming", "provider": "anthropic", - "nodeid": "tests/e2e/claude_code/basic_messaging_non_streaming/test_anthropic.py::test_basic_messaging_non_streaming_anthropic[claude-sonnet-4-6]", + "nodeid": "tests/e2e/claude_code/basic_messaging_non_streaming/test_anthropic.py::test_basic_messaging_non_streaming_anthropic[claude-sonnet-5]", "result": {"status": "pass"} }, { "feature_id": "basic_messaging_non_streaming", "provider": "anthropic", - "nodeid": "tests/e2e/claude_code/basic_messaging_non_streaming/test_anthropic.py::test_basic_messaging_non_streaming_anthropic[claude-opus-4-7]", + "nodeid": "tests/e2e/claude_code/basic_messaging_non_streaming/test_anthropic.py::test_basic_messaging_non_streaming_anthropic[claude-opus-4-8]", "result": {"status": "pass"} }, { @@ -28,8 +28,8 @@ { "feature_id": "tool_use", "provider": "anthropic", - "nodeid": "tests/e2e/claude_code/tool_use/test_anthropic.py::test_x[claude-sonnet-4-6]", - "result": {"status": "fail", "error": "[claude-sonnet-4-6] tool call dropped"} + "nodeid": "tests/e2e/claude_code/tool_use/test_anthropic.py::test_x[claude-sonnet-5]", + "result": {"status": "fail", "error": "[claude-sonnet-5] tool call dropped"} }, { "feature_id": "tool_use", diff --git a/tests/e2e/claude_code/_builder_unit_tests/test_matrix_builder.py b/tests/e2e/claude_code/_builder_unit_tests/test_matrix_builder.py index 9ddbdd298465..7a5da133b908 100644 --- a/tests/e2e/claude_code/_builder_unit_tests/test_matrix_builder.py +++ b/tests/e2e/claude_code/_builder_unit_tests/test_matrix_builder.py @@ -74,7 +74,7 @@ def test_build_matrix_any_fail_makes_cell_fail(): { "feature_id": "f", "provider": "anthropic", - "result": {"status": "fail", "error": "[claude-opus-4-7] timeout"}, + "result": {"status": "fail", "error": "[claude-opus-4-8] timeout"}, }, {"feature_id": "f", "provider": "anthropic", "result": {"status": "pass"}}, ] @@ -87,7 +87,7 @@ def test_build_matrix_any_fail_makes_cell_fail(): ) cell = matrix["features"][0]["providers"]["anthropic"] assert cell["status"] == "fail" - assert cell["error"] == "[claude-opus-4-7] timeout" + assert cell["error"] == "[claude-opus-4-8] timeout" def test_build_matrix_joins_all_failure_errors_in_one_cell(): @@ -110,7 +110,7 @@ def test_build_matrix_joins_all_failure_errors_in_one_cell(): { "feature_id": "f", "provider": "anthropic", - "result": {"status": "fail", "error": "[claude-opus-4-7] timeout"}, + "result": {"status": "fail", "error": "[claude-opus-4-8] timeout"}, }, ] matrix = build_matrix( @@ -123,7 +123,7 @@ def test_build_matrix_joins_all_failure_errors_in_one_cell(): cell = matrix["features"][0]["providers"]["anthropic"] assert cell["status"] == "fail" assert "[claude-haiku-4-5] 429" in cell["error"] - assert "[claude-opus-4-7] timeout" in cell["error"] + assert "[claude-opus-4-8] timeout" in cell["error"] def test_build_matrix_mixed_pass_and_not_tested_surfaces_pass(): @@ -393,7 +393,7 @@ def test_build_matrix_6x5_grid_matches_published_sample(): feature_ids = [feature["id"] for feature in manifest["features"]] providers = manifest["providers"] - models = ["claude-haiku-4-5", "claude-sonnet-4-6", "claude-opus-4-7"] + models = ["claude-haiku-4-5", "claude-sonnet-5", "claude-opus-4-8"] results = [] for feature_id in feature_ids: @@ -440,7 +440,7 @@ def test_build_matrix_1x5_grid_one_failing_model_breaks_cell(): "provider": "bedrock_invoke", "result": { "status": "fail", - "error": "[claude-opus-4-7-bedrock-invoke] claude CLI exited 1: throttled", + "error": "[claude-opus-4-8-bedrock-invoke] claude CLI exited 1: throttled", }, }, { @@ -459,7 +459,7 @@ def test_build_matrix_1x5_grid_one_failing_model_breaks_cell(): ) cell = matrix["features"][0]["providers"]["bedrock_invoke"] assert cell["status"] == "fail" - assert "claude-opus-4-7-bedrock-invoke" in cell["error"] + assert "claude-opus-4-8-bedrock-invoke" in cell["error"] def test_build_from_paths_writes_output(tmp_path): diff --git a/tests/e2e/claude_code/_builder_unit_tests/test_v0_layout.py b/tests/e2e/claude_code/_builder_unit_tests/test_v0_layout.py index b1745008facd..d06e475619c3 100644 --- a/tests/e2e/claude_code/_builder_unit_tests/test_v0_layout.py +++ b/tests/e2e/claude_code/_builder_unit_tests/test_v0_layout.py @@ -141,18 +141,37 @@ def test_every_manifest_feature_has_per_provider_test_file(feature_id, provider) assert test_file.is_file(), f"missing per-provider test file: {test_file}" +DEFAULT_TIERS = ("haiku-4-5", "sonnet-5", "opus-4-8") + +PROVIDER_TIER_OVERRIDES = { + "vertex_ai": ("haiku-4-5", "sonnet-4-6", "opus-4-7"), + "azure": ("haiku-4-5", "sonnet-4-6", "opus-4-8"), +} + + @pytest.mark.parametrize("feature_id", EXPECTED_FEATURE_IDS) @pytest.mark.parametrize("provider", EXPECTED_PROVIDERS) def test_per_provider_test_file_imports_and_parametrizes_three_models( feature_id, provider ): """Every test file must reference the three Claude tiers required - by the PRD: Haiku 4.5, Sonnet 4.6, Opus 4.7. Implementations may - use plain aliases or per-provider-suffixed aliases (e.g. - `claude-opus-4-7-bedrock-invoke`), so we check for the tier - substrings rather than exact alias names.""" + by the PRD: small, mid, and large; Haiku 4.5, Sonnet 5, and Opus + 4.8 as of the 2026-07 tier bump. Implementations may use plain + aliases or per-provider-suffixed aliases (e.g. + `claude-opus-4-8-bedrock-invoke`), so we check for the tier + substrings rather than exact alias names. + + Two provider columns are pinned to older tiers until quota is + granted: Vertex AI has 0 TPM for the sonnet-5 / opus-4-8 base + models in the suite's project+region (deterministic 429), and + Microsoft Foundry has a hard 0 TPM quota limit for Claude Sonnet 5 + in the suite's subscription, so its sonnet slot stays on + sonnet-4-6 while its opus slot runs the deployed opus-4-8. Once + quota requests land, flip the overrides here and the matching + aliases in `test_config.yaml` and the per-provider test files.""" + tiers = PROVIDER_TIER_OVERRIDES.get(provider, DEFAULT_TIERS) text = (REPO_ROOT / feature_id / f"test_{provider}.py").read_text() - for tier in ("haiku-4-5", "sonnet-4-6", "opus-4-7"): + for tier in tiers: assert ( tier in text ), f"{feature_id}/test_{provider}.py does not reference {tier}" @@ -179,5 +198,5 @@ def test_azure_test_file_drives_the_proxy(feature_id): ) assert '"status": "not_applicable"' not in text, ( f"{feature_id}/test_azure.py still reports not_applicable; Microsoft Foundry " - "now hosts Claude (Haiku 4.5, Sonnet 4.6, Opus 4.7), so this row must run." + "now hosts Claude (Haiku 4.5, Sonnet 4.6, Opus 4.8), so this row must run." ) diff --git a/tests/e2e/claude_code/_driver_unit_tests/test_basic_messaging.py b/tests/e2e/claude_code/_driver_unit_tests/test_basic_messaging.py index 018121a8e5c9..9279c549aa64 100644 --- a/tests/e2e/claude_code/_driver_unit_tests/test_basic_messaging.py +++ b/tests/e2e/claude_code/_driver_unit_tests/test_basic_messaging.py @@ -184,8 +184,8 @@ def test_verify_streaming_requires_all_models_to_stream(monkeypatch): fake_result = _FakeResult() outcomes = { "claude-haiku-4-5": DriverResult(text="ok", events=_streamed_events(5)), - "claude-sonnet-4-6": DriverResult(text="ok", events=_buffered_events()), - "claude-opus-4-7": DriverResult(text="ok", events=_streamed_events(5)), + "claude-sonnet-5": DriverResult(text="ok", events=_buffered_events()), + "claude-opus-4-8": DriverResult(text="ok", events=_streamed_events(5)), } _install_fake_runner(monkeypatch, outcomes_by_model=outcomes) diff --git a/tests/e2e/claude_code/_driver_unit_tests/test_cli_driver.py b/tests/e2e/claude_code/_driver_unit_tests/test_cli_driver.py index 786f60299931..1e73c03068f2 100644 --- a/tests/e2e/claude_code/_driver_unit_tests/test_cli_driver.py +++ b/tests/e2e/claude_code/_driver_unit_tests/test_cli_driver.py @@ -97,7 +97,7 @@ def test_run_claude_overlays_proxy_env(): runner, captured = _make_runner(stdout="") run_claude( prompt="hi", - model="claude-opus-4-7", + model="claude-opus-4-8", base_url="http://proxy.example:4000", api_key="sk-abc", runner=runner, @@ -112,7 +112,7 @@ def test_run_claude_extra_env_is_added_to_subprocess_env(): runner, captured = _make_runner(stdout="") run_claude( prompt="hi", - model="claude-opus-4-7", + model="claude-opus-4-8", base_url="http://localhost", api_key="sk-abc", extra_env={"MAX_THINKING_TOKENS": "4096"}, @@ -146,7 +146,7 @@ def test_run_claude_inherits_only_allowlisted_os_environ(monkeypatch): runner, captured = _make_runner(stdout="") run_claude( prompt="hi", - model="claude-opus-4-7", + model="claude-opus-4-8", base_url="http://localhost", api_key="sk-abc", runner=runner, @@ -178,7 +178,7 @@ def test_run_claude_uses_isolated_per_invocation_home(monkeypatch, tmp_path): runner, captured = _make_runner(stdout="") run_claude( prompt="hi", - model="claude-opus-4-7", + model="claude-opus-4-8", base_url="http://localhost", api_key="sk-abc", runner=runner, @@ -209,7 +209,7 @@ def test_run_claude_isolated_home_is_distinct_per_invocation(monkeypatch): runner, captured = _make_runner(stdout="") run_claude( prompt="hi", - model="claude-opus-4-7", + model="claude-opus-4-8", base_url="http://localhost", api_key="sk-abc", runner=runner, @@ -219,7 +219,7 @@ def test_run_claude_isolated_home_is_distinct_per_invocation(monkeypatch): runner, captured = _make_runner(stdout="") run_claude( prompt="hi", - model="claude-opus-4-7", + model="claude-opus-4-8", base_url="http://localhost", api_key="sk-abc", runner=runner, @@ -243,7 +243,7 @@ def test_run_claude_isolated_home_cleaned_up_after_run(monkeypatch): runner, captured = _make_runner(stdout="") run_claude( prompt="hi", - model="claude-opus-4-7", + model="claude-opus-4-8", base_url="http://localhost", api_key="sk-abc", runner=runner, @@ -275,7 +275,7 @@ def runner(cmd, env, capture_output, text, timeout, check, input=None): with pytest.raises(ClaudeCLIError): run_claude( prompt="hi", - model="claude-opus-4-7", + model="claude-opus-4-8", base_url="http://localhost", api_key="sk-abc", runner=runner, @@ -294,7 +294,7 @@ def test_run_claude_extra_env_can_pass_through_otherwise_blocked_var(monkeypatch runner, captured = _make_runner(stdout="") run_claude( prompt="hi", - model="claude-opus-4-7", + model="claude-opus-4-8", base_url="http://localhost", api_key="sk-abc", extra_env={"ANTHROPIC_API_KEY": "from-arg"}, diff --git a/tests/e2e/claude_code/_driver_unit_tests/test_rate_limiter.py b/tests/e2e/claude_code/_driver_unit_tests/test_rate_limiter.py index 92907eda3c47..71ea6bb53962 100644 --- a/tests/e2e/claude_code/_driver_unit_tests/test_rate_limiter.py +++ b/tests/e2e/claude_code/_driver_unit_tests/test_rate_limiter.py @@ -60,11 +60,11 @@ "model, expected", [ ("claude-haiku-4-5", PROVIDER_ANTHROPIC), - ("claude-sonnet-4-6", PROVIDER_ANTHROPIC), - ("claude-opus-4-7", PROVIDER_ANTHROPIC), + ("claude-sonnet-5", PROVIDER_ANTHROPIC), + ("claude-opus-4-8", PROVIDER_ANTHROPIC), ("claude-haiku-4-5-azure", PROVIDER_AZURE), - ("claude-sonnet-4-6-azure", PROVIDER_AZURE), - ("claude-opus-4-7-vertex", PROVIDER_VERTEX_AI), + ("claude-sonnet-5-azure", PROVIDER_AZURE), + ("claude-opus-4-8-vertex", PROVIDER_VERTEX_AI), ("claude-haiku-4-5-bedrock-converse", PROVIDER_BEDROCK_CONVERSE), ("claude-haiku-4-5-bedrock-invoke", PROVIDER_BEDROCK_INVOKE), ], @@ -87,7 +87,7 @@ def test_infer_provider_rejects_empty_string(): def test_infer_provider_is_case_insensitive(): """Aliases in the proxy config sometimes drift between cases; we should still route them to the right column.""" - assert infer_provider("CLAUDE-OPUS-4-7-AZURE") == PROVIDER_AZURE + assert infer_provider("CLAUDE-OPUS-4-8-AZURE") == PROVIDER_AZURE # --------------------------------------------------------------------------- diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_anthropic.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_anthropic.py index c06fff28d2d5..7665f9cd9ea2 100644 --- a/tests/e2e/claude_code/basic_messaging_non_streaming/test_anthropic.py +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_anthropic.py @@ -11,8 +11,8 @@ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^ feature_id provider -Per the PRD, every cell exercises Claude Haiku 4.5, Sonnet 4.6, and Opus -4.7; the cell only goes green if all three pass. The shared +Per the PRD, every cell exercises Claude Haiku 4.5, Sonnet 5, and Opus +4.8; the cell only goes green if all three pass. The shared `run_basic_messaging_cell` helper fans the three model runs out in parallel and reports one `compat_result.add(...)` entry per model so the matrix builder still sees three rows for this (feature, provider). @@ -27,8 +27,8 @@ # routing config; the driver only sends the alias. ANTHROPIC_MODELS = [ "claude-haiku-4-5", - "claude-sonnet-4-6", - "claude-opus-4-7", + "claude-sonnet-5", + "claude-opus-4-8", ] diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_azure.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_azure.py index 2a962b244a86..dad5b89eb11e 100644 --- a/tests/e2e/claude_code/basic_messaging_non_streaming/test_azure.py +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_azure.py @@ -17,7 +17,7 @@ feature_id provider Per the PRD, every cell exercises Claude Haiku 4.5, Sonnet 4.6, and Opus -4.7; the cell only goes green if all three pass. The shared +4.8; the cell only goes green if all three pass. The shared `run_basic_messaging_cell` helper fans the three model runs out in parallel and reports one `compat_result.add(...)` entry per model so the matrix builder still sees three rows for this (feature, provider). @@ -34,7 +34,7 @@ AZURE_MODELS = [ "claude-haiku-4-5-azure", "claude-sonnet-4-6-azure", - "claude-opus-4-7-azure", + "claude-opus-4-8-azure", ] diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_converse.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_converse.py index 2245ed7417a6..1fcbf03ac421 100644 --- a/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_converse.py +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_converse.py @@ -11,8 +11,8 @@ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^ feature_id provider -Per the PRD, every cell exercises Claude Haiku 4.5, Sonnet 4.6, and Opus -4.7; the cell only goes green if all three pass. The shared +Per the PRD, every cell exercises Claude Haiku 4.5, Sonnet 5, and Opus +4.8; the cell only goes green if all three pass. The shared `run_basic_messaging_cell` helper fans the three model runs out in parallel and reports one `compat_result.add(...)` entry per model so the matrix builder still sees three rows for this (feature, provider). @@ -28,8 +28,8 @@ # strategy. BEDROCK_CONVERSE_MODELS = [ "claude-haiku-4-5-bedrock-converse", - "claude-sonnet-4-6-bedrock-converse", - "claude-opus-4-7-bedrock-converse", + "claude-sonnet-5-bedrock-converse", + "claude-opus-4-8-bedrock-converse", ] diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_invoke.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_invoke.py index e0a6e77f3c12..aad21e5a2edc 100644 --- a/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_invoke.py @@ -11,8 +11,8 @@ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^ feature_id provider -Per the PRD, every cell exercises Claude Haiku 4.5, Sonnet 4.6, and Opus -4.7; the cell only goes green if all three pass. The shared +Per the PRD, every cell exercises Claude Haiku 4.5, Sonnet 5, and Opus +4.8; the cell only goes green if all three pass. The shared `run_basic_messaging_cell` helper fans the three model runs out in parallel and reports one `compat_result.add(...)` entry per model so the matrix builder still sees three rows for this (feature, provider). @@ -28,8 +28,8 @@ # routing strategy. BEDROCK_INVOKE_MODELS = [ "claude-haiku-4-5-bedrock-invoke", - "claude-sonnet-4-6-bedrock-invoke", - "claude-opus-4-7-bedrock-invoke", + "claude-sonnet-5-bedrock-invoke", + "claude-opus-4-8-bedrock-invoke", ] diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_anthropic.py b/tests/e2e/claude_code/basic_messaging_streaming/test_anthropic.py index 56e3fb6c1818..c41a93eb645d 100644 --- a/tests/e2e/claude_code/basic_messaging_streaming/test_anthropic.py +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_anthropic.py @@ -29,8 +29,8 @@ ANTHROPIC_MODELS = [ "claude-haiku-4-5", - "claude-sonnet-4-6", - "claude-opus-4-7", + "claude-sonnet-5", + "claude-opus-4-8", ] diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_azure.py b/tests/e2e/claude_code/basic_messaging_streaming/test_azure.py index b6c002d0b279..9c65e9548c3a 100644 --- a/tests/e2e/claude_code/basic_messaging_streaming/test_azure.py +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_azure.py @@ -24,7 +24,7 @@ AZURE_MODELS = [ "claude-haiku-4-5-azure", "claude-sonnet-4-6-azure", - "claude-opus-4-7-azure", + "claude-opus-4-8-azure", ] diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_converse.py b/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_converse.py index 44ac54515f01..7f01ba221368 100644 --- a/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_converse.py +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_converse.py @@ -19,8 +19,8 @@ BEDROCK_CONVERSE_MODELS = [ "claude-haiku-4-5-bedrock-converse", - "claude-sonnet-4-6-bedrock-converse", - "claude-opus-4-7-bedrock-converse", + "claude-sonnet-5-bedrock-converse", + "claude-opus-4-8-bedrock-converse", ] diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_invoke.py b/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_invoke.py index 1d59d16cdc1c..392406271d86 100644 --- a/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_invoke.py @@ -19,8 +19,8 @@ BEDROCK_INVOKE_MODELS = [ "claude-haiku-4-5-bedrock-invoke", - "claude-sonnet-4-6-bedrock-invoke", - "claude-opus-4-7-bedrock-invoke", + "claude-sonnet-5-bedrock-invoke", + "claude-opus-4-8-bedrock-invoke", ] diff --git a/tests/e2e/claude_code/count_tokens/test_anthropic.py b/tests/e2e/claude_code/count_tokens/test_anthropic.py index 3508063459cb..bc77f0079db4 100644 --- a/tests/e2e/claude_code/count_tokens/test_anthropic.py +++ b/tests/e2e/claude_code/count_tokens/test_anthropic.py @@ -51,8 +51,8 @@ ANTHROPIC_MODELS = [ "claude-haiku-4-5", - "claude-sonnet-4-6", - "claude-opus-4-7", + "claude-sonnet-5", + "claude-opus-4-8", ] diff --git a/tests/e2e/claude_code/count_tokens/test_azure.py b/tests/e2e/claude_code/count_tokens/test_azure.py index 2b8707b50b09..514d9701a7bf 100644 --- a/tests/e2e/claude_code/count_tokens/test_azure.py +++ b/tests/e2e/claude_code/count_tokens/test_azure.py @@ -52,7 +52,7 @@ AZURE_MODELS = [ "claude-haiku-4-5-azure", "claude-sonnet-4-6-azure", - "claude-opus-4-7-azure", + "claude-opus-4-8-azure", ] diff --git a/tests/e2e/claude_code/count_tokens/test_bedrock_converse.py b/tests/e2e/claude_code/count_tokens/test_bedrock_converse.py index 4221773ead2c..c352c0e38660 100644 --- a/tests/e2e/claude_code/count_tokens/test_bedrock_converse.py +++ b/tests/e2e/claude_code/count_tokens/test_bedrock_converse.py @@ -51,8 +51,8 @@ BEDROCK_CONVERSE_MODELS = [ "claude-haiku-4-5-bedrock-converse", - "claude-sonnet-4-6-bedrock-converse", - "claude-opus-4-7-bedrock-converse", + "claude-sonnet-5-bedrock-converse", + "claude-opus-4-8-bedrock-converse", ] diff --git a/tests/e2e/claude_code/count_tokens/test_bedrock_invoke.py b/tests/e2e/claude_code/count_tokens/test_bedrock_invoke.py index cc70bf123928..ad629f68c4a0 100644 --- a/tests/e2e/claude_code/count_tokens/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/count_tokens/test_bedrock_invoke.py @@ -51,8 +51,8 @@ BEDROCK_INVOKE_MODELS = [ "claude-haiku-4-5-bedrock-invoke", - "claude-sonnet-4-6-bedrock-invoke", - "claude-opus-4-7-bedrock-invoke", + "claude-sonnet-5-bedrock-invoke", + "claude-opus-4-8-bedrock-invoke", ] diff --git a/tests/e2e/claude_code/long_context_1m/test_anthropic.py b/tests/e2e/claude_code/long_context_1m/test_anthropic.py index fb74d5fd40d8..332de6fe1440 100644 --- a/tests/e2e/claude_code/long_context_1m/test_anthropic.py +++ b/tests/e2e/claude_code/long_context_1m/test_anthropic.py @@ -68,12 +68,12 @@ PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" -# Haiku 4.5 is excluded -- only Sonnet 4.6 and Opus 4.7 support the +# Haiku 4.5 is excluded -- only Sonnet 5 and Opus 4.8 support the # 1M-context beta. See module docstring for the per-cell-aggregator # rationale. ANTHROPIC_MODELS: Sequence[str] = ( - "claude-sonnet-4-6", - "claude-opus-4-7", + "claude-sonnet-5", + "claude-opus-4-8", ) # Beta header that opts an Anthropic-shape model into the 1M context diff --git a/tests/e2e/claude_code/long_context_1m/test_azure.py b/tests/e2e/claude_code/long_context_1m/test_azure.py index 5800fdadbfce..a5da2653681f 100644 --- a/tests/e2e/claude_code/long_context_1m/test_azure.py +++ b/tests/e2e/claude_code/long_context_1m/test_azure.py @@ -68,12 +68,12 @@ PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" -# Haiku 4.5 is excluded -- only Sonnet 4.6 and Opus 4.7 support the +# Haiku 4.5 is excluded -- only Sonnet 4.6 and Opus 4.8 support the # 1M-context beta. See module docstring for the per-cell-aggregator # rationale. AZURE_MODELS: Sequence[str] = ( "claude-sonnet-4-6-azure", - "claude-opus-4-7-azure", + "claude-opus-4-8-azure", ) # Beta header that opts an Anthropic-shape model into the 1M context diff --git a/tests/e2e/claude_code/long_context_1m/test_bedrock_converse.py b/tests/e2e/claude_code/long_context_1m/test_bedrock_converse.py index 18587f7c2d63..9bec724d0ed9 100644 --- a/tests/e2e/claude_code/long_context_1m/test_bedrock_converse.py +++ b/tests/e2e/claude_code/long_context_1m/test_bedrock_converse.py @@ -68,12 +68,12 @@ PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" -# Haiku 4.5 is excluded -- only Sonnet 4.6 and Opus 4.7 support the +# Haiku 4.5 is excluded -- only Sonnet 5 and Opus 4.8 support the # 1M-context beta. See module docstring for the per-cell-aggregator # rationale. BEDROCK_CONVERSE_MODELS: Sequence[str] = ( - "claude-sonnet-4-6-bedrock-converse", - "claude-opus-4-7-bedrock-converse", + "claude-sonnet-5-bedrock-converse", + "claude-opus-4-8-bedrock-converse", ) # Beta header that opts an Anthropic-shape model into the 1M context diff --git a/tests/e2e/claude_code/long_context_1m/test_bedrock_invoke.py b/tests/e2e/claude_code/long_context_1m/test_bedrock_invoke.py index 0270197ce2ac..b13b679e118a 100644 --- a/tests/e2e/claude_code/long_context_1m/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/long_context_1m/test_bedrock_invoke.py @@ -68,12 +68,12 @@ PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" -# Haiku 4.5 is excluded -- only Sonnet 4.6 and Opus 4.7 support the +# Haiku 4.5 is excluded -- only Sonnet 5 and Opus 4.8 support the # 1M-context beta. See module docstring for the per-cell-aggregator # rationale. BEDROCK_INVOKE_MODELS: Sequence[str] = ( - "claude-sonnet-4-6-bedrock-invoke", - "claude-opus-4-7-bedrock-invoke", + "claude-sonnet-5-bedrock-invoke", + "claude-opus-4-8-bedrock-invoke", ) # Beta header that opts an Anthropic-shape model into the 1M context diff --git a/tests/e2e/claude_code/manifest.yaml b/tests/e2e/claude_code/manifest.yaml index f7cccf0cef2f..3f5b97ff4652 100644 --- a/tests/e2e/claude_code/manifest.yaml +++ b/tests/e2e/claude_code/manifest.yaml @@ -36,7 +36,7 @@ features: name: Thinking # The single row covers both API shapes Anthropic exposes — manual # `thinking: {type: "enabled", budget_tokens: N}` (Haiku 4.5) and - # `thinking: {type: "adaptive"}` (Opus 4.7); Sonnet 4.6 supports + # `thinking: {type: "adaptive"}` (Opus 4.8); Sonnet 5 supports # either and Claude Code picks per model. A break in either # transformer surfaces as a red cell because all three tiers must # pass for the cell to go green. The row was named @@ -98,6 +98,6 @@ features: # 200k context window so the request can only succeed when the # beta header makes it all the way through the proxy to the # upstream. Haiku 4.5 is intentionally omitted from this row's - # model list (its window is 200k); Sonnet 4.6 and Opus 4.7 are - # the only tiers exercised. Costs roughly $4/cell/run -- + # model list (its window is 200k); the Sonnet and Opus tiers are + # the only ones exercised. Costs roughly $4/cell/run -- # tighten the prompt-token target if pricing changes meaningfully. diff --git a/tests/e2e/claude_code/pdf_input/test_anthropic.py b/tests/e2e/claude_code/pdf_input/test_anthropic.py index 36fb69a1db6e..95b18f4b29f9 100644 --- a/tests/e2e/claude_code/pdf_input/test_anthropic.py +++ b/tests/e2e/claude_code/pdf_input/test_anthropic.py @@ -37,8 +37,8 @@ ANTHROPIC_MODELS = [ "claude-haiku-4-5", - "claude-sonnet-4-6", - "claude-opus-4-7", + "claude-sonnet-5", + "claude-opus-4-8", ] # Smallest valid PDF that renders a single visible word ("PONG"). Built diff --git a/tests/e2e/claude_code/pdf_input/test_azure.py b/tests/e2e/claude_code/pdf_input/test_azure.py index 810c857e407e..a2cd18afd3df 100644 --- a/tests/e2e/claude_code/pdf_input/test_azure.py +++ b/tests/e2e/claude_code/pdf_input/test_azure.py @@ -31,7 +31,7 @@ AZURE_MODELS = [ "claude-haiku-4-5-azure", "claude-sonnet-4-6-azure", - "claude-opus-4-7-azure", + "claude-opus-4-8-azure", ] PDF_MARKER = "PONG" diff --git a/tests/e2e/claude_code/pdf_input/test_bedrock_converse.py b/tests/e2e/claude_code/pdf_input/test_bedrock_converse.py index 191a27c6d463..cbfd907829f5 100644 --- a/tests/e2e/claude_code/pdf_input/test_bedrock_converse.py +++ b/tests/e2e/claude_code/pdf_input/test_bedrock_converse.py @@ -36,8 +36,8 @@ BEDROCK_CONVERSE_MODELS = [ "claude-haiku-4-5-bedrock-converse", - "claude-sonnet-4-6-bedrock-converse", - "claude-opus-4-7-bedrock-converse", + "claude-sonnet-5-bedrock-converse", + "claude-opus-4-8-bedrock-converse", ] PDF_MARKER = "PONG" diff --git a/tests/e2e/claude_code/pdf_input/test_bedrock_invoke.py b/tests/e2e/claude_code/pdf_input/test_bedrock_invoke.py index 163cabb45a00..1274852e3fb2 100644 --- a/tests/e2e/claude_code/pdf_input/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/pdf_input/test_bedrock_invoke.py @@ -35,8 +35,8 @@ BEDROCK_INVOKE_MODELS = [ "claude-haiku-4-5-bedrock-invoke", - "claude-sonnet-4-6-bedrock-invoke", - "claude-opus-4-7-bedrock-invoke", + "claude-sonnet-5-bedrock-invoke", + "claude-opus-4-8-bedrock-invoke", ] PDF_MARKER = "PONG" diff --git a/tests/e2e/claude_code/prompt_caching_1h/test_anthropic.py b/tests/e2e/claude_code/prompt_caching_1h/test_anthropic.py index d81887231d87..b3de00d7e53b 100644 --- a/tests/e2e/claude_code/prompt_caching_1h/test_anthropic.py +++ b/tests/e2e/claude_code/prompt_caching_1h/test_anthropic.py @@ -40,8 +40,8 @@ ANTHROPIC_MODELS = [ "claude-haiku-4-5", - "claude-sonnet-4-6", - "claude-opus-4-7", + "claude-sonnet-5", + "claude-opus-4-8", ] # Per the changelog (2.1.108): `ENABLE_PROMPT_CACHING_1H` flips Claude diff --git a/tests/e2e/claude_code/prompt_caching_1h/test_azure.py b/tests/e2e/claude_code/prompt_caching_1h/test_azure.py index 416757f86916..79dbefccfadd 100644 --- a/tests/e2e/claude_code/prompt_caching_1h/test_azure.py +++ b/tests/e2e/claude_code/prompt_caching_1h/test_azure.py @@ -32,7 +32,7 @@ AZURE_MODELS = [ "claude-haiku-4-5-azure", "claude-sonnet-4-6-azure", - "claude-opus-4-7-azure", + "claude-opus-4-8-azure", ] CACHE_1H_ENV = {"ENABLE_PROMPT_CACHING_1H": "1"} diff --git a/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_converse.py b/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_converse.py index 5bc632c6f1b5..53c1a8a6a78f 100644 --- a/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_converse.py +++ b/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_converse.py @@ -36,8 +36,8 @@ BEDROCK_CONVERSE_MODELS = [ "claude-haiku-4-5-bedrock-converse", - "claude-sonnet-4-6-bedrock-converse", - "claude-opus-4-7-bedrock-converse", + "claude-sonnet-5-bedrock-converse", + "claude-opus-4-8-bedrock-converse", ] CACHE_1H_ENV = { diff --git a/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_invoke.py b/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_invoke.py index 4501834956bb..60c7a323f409 100644 --- a/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_invoke.py @@ -38,8 +38,8 @@ BEDROCK_INVOKE_MODELS = [ "claude-haiku-4-5-bedrock-invoke", - "claude-sonnet-4-6-bedrock-invoke", - "claude-opus-4-7-bedrock-invoke", + "claude-sonnet-5-bedrock-invoke", + "claude-opus-4-8-bedrock-invoke", ] # Set both the modern and the deprecated-but-honored Bedrock var so we diff --git a/tests/e2e/claude_code/prompt_caching_5m/test_anthropic.py b/tests/e2e/claude_code/prompt_caching_5m/test_anthropic.py index 4b20a65f31b8..62b6c6173764 100644 --- a/tests/e2e/claude_code/prompt_caching_5m/test_anthropic.py +++ b/tests/e2e/claude_code/prompt_caching_5m/test_anthropic.py @@ -38,8 +38,8 @@ ANTHROPIC_MODELS = [ "claude-haiku-4-5", - "claude-sonnet-4-6", - "claude-opus-4-7", + "claude-sonnet-5", + "claude-opus-4-8", ] diff --git a/tests/e2e/claude_code/prompt_caching_5m/test_azure.py b/tests/e2e/claude_code/prompt_caching_5m/test_azure.py index 22bd5aa70483..f1df273efcba 100644 --- a/tests/e2e/claude_code/prompt_caching_5m/test_azure.py +++ b/tests/e2e/claude_code/prompt_caching_5m/test_azure.py @@ -39,7 +39,7 @@ AZURE_MODELS = [ "claude-haiku-4-5-azure", "claude-sonnet-4-6-azure", - "claude-opus-4-7-azure", + "claude-opus-4-8-azure", ] diff --git a/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_converse.py b/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_converse.py index 681a6ecce10b..e6c9e956cca9 100644 --- a/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_converse.py +++ b/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_converse.py @@ -31,8 +31,8 @@ BEDROCK_CONVERSE_MODELS = [ "claude-haiku-4-5-bedrock-converse", - "claude-sonnet-4-6-bedrock-converse", - "claude-opus-4-7-bedrock-converse", + "claude-sonnet-5-bedrock-converse", + "claude-opus-4-8-bedrock-converse", ] diff --git a/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_invoke.py b/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_invoke.py index f1a3109b3a1e..c0b7a951574b 100644 --- a/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_invoke.py @@ -31,8 +31,8 @@ BEDROCK_INVOKE_MODELS = [ "claude-haiku-4-5-bedrock-invoke", - "claude-sonnet-4-6-bedrock-invoke", - "claude-opus-4-7-bedrock-invoke", + "claude-sonnet-5-bedrock-invoke", + "claude-opus-4-8-bedrock-invoke", ] diff --git a/tests/e2e/claude_code/structured_outputs/test_anthropic.py b/tests/e2e/claude_code/structured_outputs/test_anthropic.py index 610d8433b729..ae0bb77e031f 100644 --- a/tests/e2e/claude_code/structured_outputs/test_anthropic.py +++ b/tests/e2e/claude_code/structured_outputs/test_anthropic.py @@ -65,8 +65,8 @@ ANTHROPIC_MODELS = [ "claude-haiku-4-5", - "claude-sonnet-4-6", - "claude-opus-4-7", + "claude-sonnet-5", + "claude-opus-4-8", ] # Minimal schema with one required integer field. Kept intentionally diff --git a/tests/e2e/claude_code/structured_outputs/test_azure.py b/tests/e2e/claude_code/structured_outputs/test_azure.py index 290f91569101..6cf4da9184a7 100644 --- a/tests/e2e/claude_code/structured_outputs/test_azure.py +++ b/tests/e2e/claude_code/structured_outputs/test_azure.py @@ -66,7 +66,7 @@ AZURE_MODELS = [ "claude-haiku-4-5-azure", "claude-sonnet-4-6-azure", - "claude-opus-4-7-azure", + "claude-opus-4-8-azure", ] # Minimal schema with one required integer field. Kept intentionally diff --git a/tests/e2e/claude_code/structured_outputs/test_bedrock_converse.py b/tests/e2e/claude_code/structured_outputs/test_bedrock_converse.py index 5179014773c6..db1687f41006 100644 --- a/tests/e2e/claude_code/structured_outputs/test_bedrock_converse.py +++ b/tests/e2e/claude_code/structured_outputs/test_bedrock_converse.py @@ -65,8 +65,8 @@ BEDROCK_CONVERSE_MODELS = [ "claude-haiku-4-5-bedrock-converse", - "claude-sonnet-4-6-bedrock-converse", - "claude-opus-4-7-bedrock-converse", + "claude-sonnet-5-bedrock-converse", + "claude-opus-4-8-bedrock-converse", ] # Minimal schema with one required integer field. Kept intentionally diff --git a/tests/e2e/claude_code/structured_outputs/test_bedrock_invoke.py b/tests/e2e/claude_code/structured_outputs/test_bedrock_invoke.py index 313a714be34e..24e54ed22ba5 100644 --- a/tests/e2e/claude_code/structured_outputs/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/structured_outputs/test_bedrock_invoke.py @@ -65,8 +65,8 @@ BEDROCK_INVOKE_MODELS = [ "claude-haiku-4-5-bedrock-invoke", - "claude-sonnet-4-6-bedrock-invoke", - "claude-opus-4-7-bedrock-invoke", + "claude-sonnet-5-bedrock-invoke", + "claude-opus-4-8-bedrock-invoke", ] # Minimal schema with one required integer field. Kept intentionally diff --git a/tests/e2e/claude_code/test_config.yaml b/tests/e2e/claude_code/test_config.yaml index eec68d11dcfc..9f1b43f8f3f1 100644 --- a/tests/e2e/claude_code/test_config.yaml +++ b/tests/e2e/claude_code/test_config.yaml @@ -21,13 +21,13 @@ model_list: litellm_params: model: anthropic/claude-haiku-4-5 api_key: os.environ/ANTHROPIC_API_KEY - - model_name: claude-sonnet-4-6 + - model_name: claude-sonnet-5 litellm_params: - model: anthropic/claude-sonnet-4-6 + model: anthropic/claude-sonnet-5 api_key: os.environ/ANTHROPIC_API_KEY - - model_name: claude-opus-4-7 + - model_name: claude-opus-4-8 litellm_params: - model: anthropic/claude-opus-4-7 + model: anthropic/claude-opus-4-8 api_key: os.environ/ANTHROPIC_API_KEY # ---- Bedrock (InvokeModel) ---- @@ -35,13 +35,13 @@ model_list: litellm_params: model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 aws_region_name: us-east-1 - - model_name: claude-sonnet-4-6-bedrock-invoke + - model_name: claude-sonnet-5-bedrock-invoke litellm_params: - model: bedrock/us.anthropic.claude-sonnet-4-6 + model: bedrock/us.anthropic.claude-sonnet-5 aws_region_name: us-east-1 - - model_name: claude-opus-4-7-bedrock-invoke + - model_name: claude-opus-4-8-bedrock-invoke litellm_params: - model: bedrock/us.anthropic.claude-opus-4-7 + model: bedrock/us.anthropic.claude-opus-4-8 aws_region_name: us-east-1 # ---- Bedrock (Converse) ---- @@ -49,13 +49,13 @@ model_list: litellm_params: model: bedrock/converse/us.anthropic.claude-haiku-4-5-20251001-v1:0 aws_region_name: us-east-1 - - model_name: claude-sonnet-4-6-bedrock-converse + - model_name: claude-sonnet-5-bedrock-converse litellm_params: - model: bedrock/converse/us.anthropic.claude-sonnet-4-6 + model: bedrock/converse/us.anthropic.claude-sonnet-5 aws_region_name: us-east-1 - - model_name: claude-opus-4-7-bedrock-converse + - model_name: claude-opus-4-8-bedrock-converse litellm_params: - model: bedrock/converse/us.anthropic.claude-opus-4-7 + model: bedrock/converse/us.anthropic.claude-opus-4-8 aws_region_name: us-east-1 # ---- Vertex AI ---- @@ -86,9 +86,9 @@ model_list: model: azure_ai/claude-sonnet-4-6 api_base: os.environ/AZURE_FOUNDRY_API_BASE api_key: os.environ/AZURE_FOUNDRY_API_KEY - - model_name: claude-opus-4-7-azure + - model_name: claude-opus-4-8-azure litellm_params: - model: azure_ai/claude-opus-4-7 + model: azure_ai/claude-opus-4-8 api_base: os.environ/AZURE_FOUNDRY_API_BASE api_key: os.environ/AZURE_FOUNDRY_API_KEY diff --git a/tests/e2e/claude_code/thinking/test_anthropic.py b/tests/e2e/claude_code/thinking/test_anthropic.py index 1090d1b384e3..1cd5e082b65a 100644 --- a/tests/e2e/claude_code/thinking/test_anthropic.py +++ b/tests/e2e/claude_code/thinking/test_anthropic.py @@ -36,8 +36,8 @@ ANTHROPIC_MODELS = [ "claude-haiku-4-5", - "claude-sonnet-4-6", - "claude-opus-4-7", + "claude-sonnet-5", + "claude-opus-4-8", ] # --effort max maps to the largest thinking budget on every supported @@ -45,15 +45,15 @@ # use a CLI flag (rather than the legacy MAX_THINKING_TOKENS env var) # because Claude Code 2.x reads thinking config from --effort, not from # the env, and silently no-ops the env var. We use `max` rather than -# `high` because Sonnet 4.6 / Opus 4.7 only emit thinking blocks when +# `high` because Sonnet 5 / Opus 4.8 only emit thinking blocks when # the budget is generous and the prompt is non-trivial. THINKING_ARGS = ["--effort", "max"] # A puzzle non-trivial enough that Sonnet/Opus actually engage thinking # rather than answer from memory. Trivial arithmetic ("3-2=?") is # optimized away on the modern tiers and arrives without a thinking # block, which would make this test silently false-fail under -# `--effort max`. Haiku 4.5 thinks even for trivial prompts; Sonnet 4.6 -# and Opus 4.7 only emit thinking when the upstream judges it useful. +# `--effort max`. Haiku 4.5 thinks even for trivial prompts; Sonnet 5 +# and Opus 4.8 only emit thinking when the upstream judges it useful. THINKING_PROMPT = ( "I have a 3-gallon jug and a 5-gallon jug. How can I measure " "exactly 4 gallons of water? Think through the steps carefully." diff --git a/tests/e2e/claude_code/thinking/test_azure.py b/tests/e2e/claude_code/thinking/test_azure.py index 1fd5138d574c..f121645af797 100644 --- a/tests/e2e/claude_code/thinking/test_azure.py +++ b/tests/e2e/claude_code/thinking/test_azure.py @@ -8,10 +8,7 @@ Foundry's Claude deployments advertise `supports_reasoning: true` in LiteLLM's pricing metadata; the `thinking={"type": "enabled", ...}` parameter passes through `azure_ai/claude-*` to Foundry's -`/anthropic/v1/messages` endpoint unchanged. Note that -`claude-opus-4-7-preview` documents thinking as not supported on -Foundry; if that lands, this row may flip to a partial pass and we'll -re-evaluate. +`/anthropic/v1/messages` endpoint unchanged. The (feature, provider) for this cell is inferred from the file path by `tests/e2e/claude_code/conftest.py`: @@ -40,7 +37,7 @@ AZURE_MODELS = [ "claude-haiku-4-5-azure", "claude-sonnet-4-6-azure", - "claude-opus-4-7-azure", + "claude-opus-4-8-azure", ] THINKING_ARGS = ["--effort", "max"] diff --git a/tests/e2e/claude_code/thinking/test_bedrock_converse.py b/tests/e2e/claude_code/thinking/test_bedrock_converse.py index 793ce8542daf..485d7a17443c 100644 --- a/tests/e2e/claude_code/thinking/test_bedrock_converse.py +++ b/tests/e2e/claude_code/thinking/test_bedrock_converse.py @@ -31,8 +31,8 @@ BEDROCK_CONVERSE_MODELS = [ "claude-haiku-4-5-bedrock-converse", - "claude-sonnet-4-6-bedrock-converse", - "claude-opus-4-7-bedrock-converse", + "claude-sonnet-5-bedrock-converse", + "claude-opus-4-8-bedrock-converse", ] THINKING_ARGS = ["--effort", "max"] diff --git a/tests/e2e/claude_code/thinking/test_bedrock_invoke.py b/tests/e2e/claude_code/thinking/test_bedrock_invoke.py index e31b60eb004f..b81fa3c1b5d7 100644 --- a/tests/e2e/claude_code/thinking/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/thinking/test_bedrock_invoke.py @@ -31,8 +31,8 @@ BEDROCK_INVOKE_MODELS = [ "claude-haiku-4-5-bedrock-invoke", - "claude-sonnet-4-6-bedrock-invoke", - "claude-opus-4-7-bedrock-invoke", + "claude-sonnet-5-bedrock-invoke", + "claude-opus-4-8-bedrock-invoke", ] THINKING_ARGS = ["--effort", "max"] diff --git a/tests/e2e/claude_code/thinking_with_tool_use/test_anthropic.py b/tests/e2e/claude_code/thinking_with_tool_use/test_anthropic.py index 2c573ea039e5..9ae2e7e3f535 100644 --- a/tests/e2e/claude_code/thinking_with_tool_use/test_anthropic.py +++ b/tests/e2e/claude_code/thinking_with_tool_use/test_anthropic.py @@ -40,8 +40,8 @@ ANTHROPIC_MODELS = [ "claude-haiku-4-5", - "claude-sonnet-4-6", - "claude-opus-4-7", + "claude-sonnet-5", + "claude-opus-4-8", ] # Extended thinking on, with a small budget — enough to surface a diff --git a/tests/e2e/claude_code/thinking_with_tool_use/test_azure.py b/tests/e2e/claude_code/thinking_with_tool_use/test_azure.py index 3d65e82cdeca..6676b5b51a40 100644 --- a/tests/e2e/claude_code/thinking_with_tool_use/test_azure.py +++ b/tests/e2e/claude_code/thinking_with_tool_use/test_azure.py @@ -35,7 +35,7 @@ AZURE_MODELS = [ "claude-haiku-4-5-azure", "claude-sonnet-4-6-azure", - "claude-opus-4-7-azure", + "claude-opus-4-8-azure", ] THINKING_ARGS = ["--effort", "max"] diff --git a/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_converse.py b/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_converse.py index eb9163235465..24f53c1979b8 100644 --- a/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_converse.py +++ b/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_converse.py @@ -39,8 +39,8 @@ BEDROCK_CONVERSE_MODELS = [ "claude-haiku-4-5-bedrock-converse", - "claude-sonnet-4-6-bedrock-converse", - "claude-opus-4-7-bedrock-converse", + "claude-sonnet-5-bedrock-converse", + "claude-opus-4-8-bedrock-converse", ] THINKING_ARGS = ["--effort", "max"] diff --git a/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_invoke.py b/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_invoke.py index d1a61a597724..a57d114dd8d7 100644 --- a/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_invoke.py @@ -41,8 +41,8 @@ BEDROCK_INVOKE_MODELS = [ "claude-haiku-4-5-bedrock-invoke", - "claude-sonnet-4-6-bedrock-invoke", - "claude-opus-4-7-bedrock-invoke", + "claude-sonnet-5-bedrock-invoke", + "claude-opus-4-8-bedrock-invoke", ] THINKING_ARGS = ["--effort", "max"] diff --git a/tests/e2e/claude_code/tool_search/test_anthropic.py b/tests/e2e/claude_code/tool_search/test_anthropic.py index 3495c882e063..bdd6693f095a 100644 --- a/tests/e2e/claude_code/tool_search/test_anthropic.py +++ b/tests/e2e/claude_code/tool_search/test_anthropic.py @@ -57,8 +57,8 @@ ANTHROPIC_MODELS = [ "claude-haiku-4-5", - "claude-sonnet-4-6", - "claude-opus-4-7", + "claude-sonnet-5", + "claude-opus-4-8", ] diff --git a/tests/e2e/claude_code/tool_search/test_azure.py b/tests/e2e/claude_code/tool_search/test_azure.py index 1d9cb5673c50..f36f9ab76d77 100644 --- a/tests/e2e/claude_code/tool_search/test_azure.py +++ b/tests/e2e/claude_code/tool_search/test_azure.py @@ -58,7 +58,7 @@ AZURE_MODELS = [ "claude-haiku-4-5-azure", "claude-sonnet-4-6-azure", - "claude-opus-4-7-azure", + "claude-opus-4-8-azure", ] diff --git a/tests/e2e/claude_code/tool_search/test_bedrock_converse.py b/tests/e2e/claude_code/tool_search/test_bedrock_converse.py index 5ca0792529aa..b4f07550db19 100644 --- a/tests/e2e/claude_code/tool_search/test_bedrock_converse.py +++ b/tests/e2e/claude_code/tool_search/test_bedrock_converse.py @@ -57,8 +57,8 @@ BEDROCK_CONVERSE_MODELS = [ "claude-haiku-4-5-bedrock-converse", - "claude-sonnet-4-6-bedrock-converse", - "claude-opus-4-7-bedrock-converse", + "claude-sonnet-5-bedrock-converse", + "claude-opus-4-8-bedrock-converse", ] diff --git a/tests/e2e/claude_code/tool_search/test_bedrock_invoke.py b/tests/e2e/claude_code/tool_search/test_bedrock_invoke.py index 21bb33e34bd1..b3c06baf09a6 100644 --- a/tests/e2e/claude_code/tool_search/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/tool_search/test_bedrock_invoke.py @@ -57,8 +57,8 @@ BEDROCK_INVOKE_MODELS = [ "claude-haiku-4-5-bedrock-invoke", - "claude-sonnet-4-6-bedrock-invoke", - "claude-opus-4-7-bedrock-invoke", + "claude-sonnet-5-bedrock-invoke", + "claude-opus-4-8-bedrock-invoke", ] diff --git a/tests/e2e/claude_code/tool_use/test_anthropic.py b/tests/e2e/claude_code/tool_use/test_anthropic.py index 7d2aa4be683b..66e9e7b2bf1a 100644 --- a/tests/e2e/claude_code/tool_use/test_anthropic.py +++ b/tests/e2e/claude_code/tool_use/test_anthropic.py @@ -31,8 +31,8 @@ ANTHROPIC_MODELS = [ "claude-haiku-4-5", - "claude-sonnet-4-6", - "claude-opus-4-7", + "claude-sonnet-5", + "claude-opus-4-8", ] # Built-in tool-use prompt: ask Claude to use the `Bash` tool. The CLI diff --git a/tests/e2e/claude_code/tool_use/test_azure.py b/tests/e2e/claude_code/tool_use/test_azure.py index 484f50a55080..d0d455a5803c 100644 --- a/tests/e2e/claude_code/tool_use/test_azure.py +++ b/tests/e2e/claude_code/tool_use/test_azure.py @@ -36,7 +36,7 @@ AZURE_MODELS = [ "claude-haiku-4-5-azure", "claude-sonnet-4-6-azure", - "claude-opus-4-7-azure", + "claude-opus-4-8-azure", ] TOOL_USE_PROMPT = ( diff --git a/tests/e2e/claude_code/tool_use/test_bedrock_converse.py b/tests/e2e/claude_code/tool_use/test_bedrock_converse.py index 7d1b58fce90b..28c6533f31f4 100644 --- a/tests/e2e/claude_code/tool_use/test_bedrock_converse.py +++ b/tests/e2e/claude_code/tool_use/test_bedrock_converse.py @@ -31,8 +31,8 @@ BEDROCK_CONVERSE_MODELS = [ "claude-haiku-4-5-bedrock-converse", - "claude-sonnet-4-6-bedrock-converse", - "claude-opus-4-7-bedrock-converse", + "claude-sonnet-5-bedrock-converse", + "claude-opus-4-8-bedrock-converse", ] TOOL_USE_PROMPT = ( diff --git a/tests/e2e/claude_code/tool_use/test_bedrock_invoke.py b/tests/e2e/claude_code/tool_use/test_bedrock_invoke.py index 7d2b72b951d1..9cc5c7f2cb48 100644 --- a/tests/e2e/claude_code/tool_use/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/tool_use/test_bedrock_invoke.py @@ -31,8 +31,8 @@ BEDROCK_INVOKE_MODELS = [ "claude-haiku-4-5-bedrock-invoke", - "claude-sonnet-4-6-bedrock-invoke", - "claude-opus-4-7-bedrock-invoke", + "claude-sonnet-5-bedrock-invoke", + "claude-opus-4-8-bedrock-invoke", ] TOOL_USE_PROMPT = ( diff --git a/tests/e2e/claude_code/tool_use_streaming/test_anthropic.py b/tests/e2e/claude_code/tool_use_streaming/test_anthropic.py index 9aa94c892412..fbabbd2c63e8 100644 --- a/tests/e2e/claude_code/tool_use_streaming/test_anthropic.py +++ b/tests/e2e/claude_code/tool_use_streaming/test_anthropic.py @@ -41,8 +41,8 @@ ANTHROPIC_MODELS = [ "claude-haiku-4-5", - "claude-sonnet-4-6", - "claude-opus-4-7", + "claude-sonnet-5", + "claude-opus-4-8", ] # Same shape as the non-streaming `tool_use` cell: ask Claude to call diff --git a/tests/e2e/claude_code/tool_use_streaming/test_azure.py b/tests/e2e/claude_code/tool_use_streaming/test_azure.py index c73062b72cd5..6abf15ba049c 100644 --- a/tests/e2e/claude_code/tool_use_streaming/test_azure.py +++ b/tests/e2e/claude_code/tool_use_streaming/test_azure.py @@ -34,7 +34,7 @@ AZURE_MODELS = [ "claude-haiku-4-5-azure", "claude-sonnet-4-6-azure", - "claude-opus-4-7-azure", + "claude-opus-4-8-azure", ] TOOL_USE_PROMPT = ( diff --git a/tests/e2e/claude_code/tool_use_streaming/test_bedrock_converse.py b/tests/e2e/claude_code/tool_use_streaming/test_bedrock_converse.py index 3642551c7c3e..ca0e743f7bc0 100644 --- a/tests/e2e/claude_code/tool_use_streaming/test_bedrock_converse.py +++ b/tests/e2e/claude_code/tool_use_streaming/test_bedrock_converse.py @@ -39,8 +39,8 @@ BEDROCK_CONVERSE_MODELS = [ "claude-haiku-4-5-bedrock-converse", - "claude-sonnet-4-6-bedrock-converse", - "claude-opus-4-7-bedrock-converse", + "claude-sonnet-5-bedrock-converse", + "claude-opus-4-8-bedrock-converse", ] TOOL_USE_PROMPT = ( diff --git a/tests/e2e/claude_code/tool_use_streaming/test_bedrock_invoke.py b/tests/e2e/claude_code/tool_use_streaming/test_bedrock_invoke.py index af4689b2847a..c34ae2bc712f 100644 --- a/tests/e2e/claude_code/tool_use_streaming/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/tool_use_streaming/test_bedrock_invoke.py @@ -37,8 +37,8 @@ BEDROCK_INVOKE_MODELS = [ "claude-haiku-4-5-bedrock-invoke", - "claude-sonnet-4-6-bedrock-invoke", - "claude-opus-4-7-bedrock-invoke", + "claude-sonnet-5-bedrock-invoke", + "claude-opus-4-8-bedrock-invoke", ] TOOL_USE_PROMPT = ( diff --git a/tests/e2e/claude_code/vision/test_anthropic.py b/tests/e2e/claude_code/vision/test_anthropic.py index 650940248ead..01ee6fbaedb0 100644 --- a/tests/e2e/claude_code/vision/test_anthropic.py +++ b/tests/e2e/claude_code/vision/test_anthropic.py @@ -40,8 +40,8 @@ ANTHROPIC_MODELS = [ "claude-haiku-4-5", - "claude-sonnet-4-6", - "claude-opus-4-7", + "claude-sonnet-5", + "claude-opus-4-8", ] # Minimal 1x1 red PNG, base64-encoded. We embed it directly as the diff --git a/tests/e2e/claude_code/vision/test_azure.py b/tests/e2e/claude_code/vision/test_azure.py index 3b03c0f2b355..4835bbfcc720 100644 --- a/tests/e2e/claude_code/vision/test_azure.py +++ b/tests/e2e/claude_code/vision/test_azure.py @@ -41,7 +41,7 @@ AZURE_MODELS = [ "claude-haiku-4-5-azure", "claude-sonnet-4-6-azure", - "claude-opus-4-7-azure", + "claude-opus-4-8-azure", ] # Minimal 1x1 red PNG, base64-encoded. We embed it directly as the diff --git a/tests/e2e/claude_code/vision/test_bedrock_converse.py b/tests/e2e/claude_code/vision/test_bedrock_converse.py index 4201f9e64fcd..9f54bd8989f1 100644 --- a/tests/e2e/claude_code/vision/test_bedrock_converse.py +++ b/tests/e2e/claude_code/vision/test_bedrock_converse.py @@ -40,8 +40,8 @@ BEDROCK_CONVERSE_MODELS = [ "claude-haiku-4-5-bedrock-converse", - "claude-sonnet-4-6-bedrock-converse", - "claude-opus-4-7-bedrock-converse", + "claude-sonnet-5-bedrock-converse", + "claude-opus-4-8-bedrock-converse", ] # Minimal 1x1 red PNG, base64-encoded. We embed it directly as the diff --git a/tests/e2e/claude_code/vision/test_bedrock_invoke.py b/tests/e2e/claude_code/vision/test_bedrock_invoke.py index d2e641f14622..5bee52e06e41 100644 --- a/tests/e2e/claude_code/vision/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/vision/test_bedrock_invoke.py @@ -40,8 +40,8 @@ BEDROCK_INVOKE_MODELS = [ "claude-haiku-4-5-bedrock-invoke", - "claude-sonnet-4-6-bedrock-invoke", - "claude-opus-4-7-bedrock-invoke", + "claude-sonnet-5-bedrock-invoke", + "claude-opus-4-8-bedrock-invoke", ] # Minimal 1x1 red PNG, base64-encoded. We embed it directly as the diff --git a/tests/e2e/claude_code/web_search/test_anthropic.py b/tests/e2e/claude_code/web_search/test_anthropic.py index b8fa806f9238..ed2f6467bbb0 100644 --- a/tests/e2e/claude_code/web_search/test_anthropic.py +++ b/tests/e2e/claude_code/web_search/test_anthropic.py @@ -43,8 +43,8 @@ ANTHROPIC_MODELS = [ "claude-haiku-4-5", - "claude-sonnet-4-6", - "claude-opus-4-7", + "claude-sonnet-5", + "claude-opus-4-8", ] # A prompt the model cannot answer from training data alone — it forces diff --git a/tests/e2e/claude_code/web_search/test_azure.py b/tests/e2e/claude_code/web_search/test_azure.py index e70dc848dcf4..402964446fe8 100644 --- a/tests/e2e/claude_code/web_search/test_azure.py +++ b/tests/e2e/claude_code/web_search/test_azure.py @@ -44,7 +44,7 @@ AZURE_MODELS = [ "claude-haiku-4-5-azure", "claude-sonnet-4-6-azure", - "claude-opus-4-7-azure", + "claude-opus-4-8-azure", ] # A prompt the model cannot answer from training data alone — it forces diff --git a/tests/e2e/claude_code/web_search/test_bedrock_converse.py b/tests/e2e/claude_code/web_search/test_bedrock_converse.py index cbeea03df40d..146a9816b906 100644 --- a/tests/e2e/claude_code/web_search/test_bedrock_converse.py +++ b/tests/e2e/claude_code/web_search/test_bedrock_converse.py @@ -43,8 +43,8 @@ BEDROCK_CONVERSE_MODELS = [ "claude-haiku-4-5-bedrock-converse", - "claude-sonnet-4-6-bedrock-converse", - "claude-opus-4-7-bedrock-converse", + "claude-sonnet-5-bedrock-converse", + "claude-opus-4-8-bedrock-converse", ] # A prompt the model cannot answer from training data alone — it forces diff --git a/tests/e2e/claude_code/web_search/test_bedrock_invoke.py b/tests/e2e/claude_code/web_search/test_bedrock_invoke.py index 86068e1e22bd..e5bfa26f1947 100644 --- a/tests/e2e/claude_code/web_search/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/web_search/test_bedrock_invoke.py @@ -43,8 +43,8 @@ BEDROCK_INVOKE_MODELS = [ "claude-haiku-4-5-bedrock-invoke", - "claude-sonnet-4-6-bedrock-invoke", - "claude-opus-4-7-bedrock-invoke", + "claude-sonnet-5-bedrock-invoke", + "claude-opus-4-8-bedrock-invoke", ] # A prompt the model cannot answer from training data alone — it forces