From 95fe8687c8a20782216b331189d463e97e3a77cc Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 15 Jul 2026 16:22:02 -0700 Subject: [PATCH 1/3] test(e2e/claude_code): add passthrough matrix row for the big-3 clouds and Anthropic API Adds a 'passthrough' feature row to the Claude Code compat matrix that drives the real claude CLI in each cloud's native mode against LiteLLM's passthrough routes (the LLM-gateway setup from code.claude.com/docs/en/gateway) instead of the /v1/messages translation layer: - anthropic: ANTHROPIC_BASE_URL={proxy}/anthropic, forwarded verbatim to api.anthropic.com - bedrock_invoke: CLAUDE_CODE_USE_BEDROCK=1 against {proxy}/bedrock; the router resolves the alias in /model/{alias}/invoke-with-response-stream - vertex_ai: CLAUDE_CODE_USE_VERTEX=1 against {proxy}/vertex_ai/v1; alias, project, location and credentials resolve from the deployment, which now sets use_in_pass_through: true (and the canonical vertex_project/vertex_location param names) in test_config.yaml - azure: CLAUDE_CODE_USE_FOUNDRY=1 against {proxy}/azure via the AZURE_API_BASE/AZURE_API_KEY fallback (documented in the cron env example) - bedrock_converse: not_applicable; Claude Code has no Converse-wire client The shared cell body lives in _passthrough.py with injectable runner and env (no monkeypatching), unit-covered in _driver_unit_tests/test_passthrough.py including pins on the per-mode CLI env contracts captured from a real claude CLI (2.1.210) run against a request-logging sink. --- .../_driver_unit_tests/test_passthrough.py | 195 +++++++++++++++++ tests/e2e/claude_code/_passthrough.py | 196 ++++++++++++++++++ .../cron_vm/litellm-compat-matrix.env.example | 9 + tests/e2e/claude_code/manifest.yaml | 17 ++ tests/e2e/claude_code/passthrough/__init__.py | 0 .../claude_code/passthrough/test_anthropic.py | 44 ++++ .../e2e/claude_code/passthrough/test_azure.py | 50 +++++ .../passthrough/test_bedrock_converse.py | 34 +++ .../passthrough/test_bedrock_invoke.py | 42 ++++ .../claude_code/passthrough/test_vertex_ai.py | 45 ++++ tests/e2e/claude_code/test_config.yaml | 22 +- 11 files changed, 648 insertions(+), 6 deletions(-) create mode 100644 tests/e2e/claude_code/_driver_unit_tests/test_passthrough.py create mode 100644 tests/e2e/claude_code/_passthrough.py create mode 100644 tests/e2e/claude_code/passthrough/__init__.py create mode 100644 tests/e2e/claude_code/passthrough/test_anthropic.py create mode 100644 tests/e2e/claude_code/passthrough/test_azure.py create mode 100644 tests/e2e/claude_code/passthrough/test_bedrock_converse.py create mode 100644 tests/e2e/claude_code/passthrough/test_bedrock_invoke.py create mode 100644 tests/e2e/claude_code/passthrough/test_vertex_ai.py diff --git a/tests/e2e/claude_code/_driver_unit_tests/test_passthrough.py b/tests/e2e/claude_code/_driver_unit_tests/test_passthrough.py new file mode 100644 index 000000000000..2d17a84d418b --- /dev/null +++ b/tests/e2e/claude_code/_driver_unit_tests/test_passthrough.py @@ -0,0 +1,195 @@ +"""Unit tests for the shared `run_passthrough_cell` helper. + +These tests inject a fake `run_models` callable and an explicit `env` +mapping (both are first-class parameters, no monkeypatching), so they +exercise the helper's branching -- env-missing guard, base-URL +assembly, extra-env forwarding, per-model pass/fail -- without +spawning the real CLI. + +The env-builder tests pin the provider-mode contract itself: the +CLAUDE_CODE_USE_* / CLAUDE_CODE_SKIP_*_AUTH flags and the passthrough +route each mode must target. Those values are the feature -- e.g. +dropping the `/v1` from the vertex base URL produces a request Google +404s on -- so a mutation to any of them must fail here before it burns +a live matrix run. +""" + +from __future__ import annotations + +from typing import Any, Dict, List, Mapping, Optional + +import pytest + +from claude_code._passthrough import ( + ANTHROPIC_PASSTHROUGH_BASE_PATH, + CLIENT_SIDE_AWS_REGION, + VERTEX_PLACEHOLDER_PROJECT, + VERTEX_PLACEHOLDER_REGION, + bedrock_extra_env, + foundry_extra_env, + run_passthrough_cell, + vertex_extra_env, +) +from claude_code.cli_driver import ClaudeCLIError, DriverResult + +PROXY_ENV = { + "LITELLM_PROXY_BASE_URL": "http://localhost:4000", + "LITELLM_PROXY_API_KEY": "sk-test", +} + + +class _FakeResult: + def __init__(self) -> None: + self.rows: List[Dict[str, Any]] = [] + self.single: Optional[Dict[str, Any]] = None + + def set(self, payload: Mapping[str, Any]) -> None: + self.single = dict(payload) + + def add(self, payload: Mapping[str, Any]) -> None: + self.rows.append(dict(payload)) + + +def _fake_run_models(outcomes_by_model, captured: Dict[str, Any]): + def fake(*, models, prompt, base_url, api_key, extra_env=None, **_kwargs): + captured["models"] = list(models) + captured["prompt"] = prompt + captured["base_url"] = base_url + captured["api_key"] = api_key + captured["extra_env"] = dict(extra_env) if extra_env is not None else None + return {model: outcomes_by_model[model] for model in models} + + return fake + + +def test_env_missing_guard_reports_fail_and_aborts(): + fake_result = _FakeResult() + with pytest.raises(pytest.fail.Exception): + run_passthrough_cell( + compat_result=fake_result, + models=["claude-haiku-4-5"], + prompt="ping", + env={}, + ) + assert fake_result.single is not None + assert fake_result.single["status"] == "fail" + assert "LITELLM_PROXY_BASE_URL" in fake_result.single["error"] + + +def test_anthropic_base_path_appended_to_normalized_proxy_url(): + fake_result = _FakeResult() + captured: Dict[str, Any] = {} + outcome = DriverResult(text="pong") + + run_passthrough_cell( + compat_result=fake_result, + models=["claude-haiku-4-5"], + prompt="ping", + passthrough_base_path=ANTHROPIC_PASSTHROUGH_BASE_PATH, + run_models=_fake_run_models({"claude-haiku-4-5": outcome}, captured), + env={**PROXY_ENV, "LITELLM_PROXY_BASE_URL": "http://localhost:4000/"}, + ) + + assert captured["base_url"] == "http://localhost:4000/anthropic" + assert captured["extra_env"] is None + assert fake_result.rows == [{"status": "pass"}] + + +def test_extra_env_builder_receives_normalized_base_and_is_forwarded(): + fake_result = _FakeResult() + captured: Dict[str, Any] = {} + outcome = DriverResult(text="pong") + seen_bases: List[str] = [] + + def build(proxy_base: str) -> Dict[str, str]: + seen_bases.append(proxy_base) + return {"SOME_FLAG": "1"} + + run_passthrough_cell( + compat_result=fake_result, + models=["claude-haiku-4-5"], + prompt="ping", + build_extra_env=build, + run_models=_fake_run_models({"claude-haiku-4-5": outcome}, captured), + env={**PROXY_ENV, "LITELLM_PROXY_BASE_URL": "http://localhost:4000/"}, + ) + + assert seen_bases == ["http://localhost:4000"] + assert captured["extra_env"] == {"SOME_FLAG": "1"} + assert captured["base_url"] == "http://localhost:4000" + + +def test_per_model_failures_reported_individually(): + fake_result = _FakeResult() + captured: Dict[str, Any] = {} + outcomes = { + "claude-haiku-4-5": DriverResult(text="pong"), + "claude-sonnet-4-6": ClaudeCLIError("claude CLI timed out after 120s"), + "claude-opus-4-7": DriverResult(text="", exit_code=1), + } + + with pytest.raises(pytest.fail.Exception): + run_passthrough_cell( + compat_result=fake_result, + models=list(outcomes.keys()), + prompt="ping", + run_models=_fake_run_models(outcomes, captured), + env=PROXY_ENV, + ) + + statuses = [row["status"] for row in fake_result.rows] + assert statuses == ["pass", "fail", "fail"] + assert "timed out" in fake_result.rows[1]["error"] + assert "claude CLI failed" in fake_result.rows[2]["error"] + + +def test_empty_assistant_text_is_a_fail(): + fake_result = _FakeResult() + captured: Dict[str, Any] = {} + outcomes = {"claude-haiku-4-5": DriverResult(text=" ")} + + with pytest.raises(pytest.fail.Exception): + run_passthrough_cell( + compat_result=fake_result, + models=["claude-haiku-4-5"], + prompt="ping", + run_models=_fake_run_models(outcomes, captured), + env=PROXY_ENV, + ) + + assert fake_result.rows == [ + { + "status": "fail", + "error": "[claude-haiku-4-5] claude returned empty assistant text", + } + ] + + +def test_bedrock_extra_env_targets_proxy_bedrock_route(): + env = bedrock_extra_env("http://localhost:4000") + assert env == { + "CLAUDE_CODE_USE_BEDROCK": "1", + "CLAUDE_CODE_SKIP_BEDROCK_AUTH": "1", + "ANTHROPIC_BEDROCK_BASE_URL": "http://localhost:4000/bedrock", + "AWS_REGION": CLIENT_SIDE_AWS_REGION, + } + + +def test_vertex_extra_env_keeps_the_api_version_in_the_base_url(): + env = vertex_extra_env("http://localhost:4000") + assert env == { + "CLAUDE_CODE_USE_VERTEX": "1", + "CLAUDE_CODE_SKIP_VERTEX_AUTH": "1", + "ANTHROPIC_VERTEX_BASE_URL": "http://localhost:4000/vertex_ai/v1", + "ANTHROPIC_VERTEX_PROJECT_ID": VERTEX_PLACEHOLDER_PROJECT, + "CLOUD_ML_REGION": VERTEX_PLACEHOLDER_REGION, + } + + +def test_foundry_extra_env_targets_proxy_azure_route(): + env = foundry_extra_env("http://localhost:4000") + assert env == { + "CLAUDE_CODE_USE_FOUNDRY": "1", + "CLAUDE_CODE_SKIP_FOUNDRY_AUTH": "1", + "ANTHROPIC_FOUNDRY_BASE_URL": "http://localhost:4000/azure", + } diff --git a/tests/e2e/claude_code/_passthrough.py b/tests/e2e/claude_code/_passthrough.py new file mode 100644 index 000000000000..24b6d694d3fa --- /dev/null +++ b/tests/e2e/claude_code/_passthrough.py @@ -0,0 +1,196 @@ +"""Shared body for the `passthrough` × compat cells. + +Every other matrix row drives the proxy's `/v1/messages` translation +layer: Claude Code speaks the first-party Anthropic wire and LiteLLM +transforms the request per provider. This row instead exercises +LiteLLM's *native passthrough* routes -- the "LLM gateway" +configuration documented at https://code.claude.com/docs/en/gateway -- +where Claude Code speaks each cloud's own wire format and the proxy +forwards it, attaching provider credentials on the way out: + + anthropic ANTHROPIC_BASE_URL={proxy}/anthropic. The CLI's + first-party wire, forwarded verbatim to + api.anthropic.com, so the model ids are real + Anthropic ids rather than proxy aliases. + bedrock_invoke CLAUDE_CODE_USE_BEDROCK=1 + + ANTHROPIC_BEDROCK_BASE_URL={proxy}/bedrock. The + CLI POSTs /model/{model}/invoke-with-response-stream; + the proxy recognizes a router alias in the model + segment, rewrites it to the deployment's upstream + model id, and SigV4-signs with its own AWS creds. + vertex_ai CLAUDE_CODE_USE_VERTEX=1 + + ANTHROPIC_VERTEX_BASE_URL={proxy}/vertex_ai/v1. + The CLI POSTs + .../models/{model}:streamRawPredict; the proxy + resolves a router alias in the model segment and + takes project, location, and credentials from the + deployment (which is why the deployment must set + `use_in_pass_through: true` -- see + test_config.yaml). + azure CLAUDE_CODE_USE_FOUNDRY=1 + + ANTHROPIC_FOUNDRY_BASE_URL={proxy}/azure. Foundry + mode sends the model in the JSON body, not the + URL, so the proxy's /azure route cannot resolve a + router alias and falls back to the env-configured + AZURE_API_BASE / AZURE_API_KEY target. + bedrock_converse not applicable -- Claude Code's bedrock mode is + InvokeModel-only; no Converse-wire client exists. + +Auth is the same in every mode: the CLI's provider-native signing is +disabled via CLAUDE_CODE_SKIP__AUTH, and the LiteLLM virtual +key travels as `Authorization: Bearer` (ANTHROPIC_AUTH_TOKEN), exactly +like the translation rows. The proxy holds the real provider +credentials. + +The per-mode env vars and URL shapes above were captured from a real +`claude` CLI (2.1.210) run against a request-logging sink, not from +docs; if a CLI release changes them, the cells fail with the CLI's own +diagnostic rather than silently testing the wrong wire. + +`run_models` and `env` are injection seams for +`_driver_unit_tests/test_passthrough.py`; production callers leave +them unset. +""" + +from __future__ import annotations + +import os +from typing import Any, Callable, Dict, Mapping, Optional, Sequence + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +ANTHROPIC_PASSTHROUGH_BASE_PATH = "/anthropic" + +CLIENT_SIDE_AWS_REGION = "us-east-1" +"""Satisfies the CLI's embedded AWS SDK, which refuses to construct a +client without a region. The value never influences routing: the proxy +signs the upstream request with its own credentials and region.""" + +VERTEX_PLACEHOLDER_PROJECT = "proxy-resolved-project" +VERTEX_PLACEHOLDER_REGION = "us-east5" +"""The CLI refuses to build a Vertex URL without a project id and +region, but the proxy replaces both path segments with the resolved +deployment's `vertex_project` / `vertex_location` before forwarding, +so deliberately-fake values prove the resolution actually happened.""" + + +def bedrock_extra_env(proxy_base_url: str) -> Dict[str, str]: + return { + "CLAUDE_CODE_USE_BEDROCK": "1", + "CLAUDE_CODE_SKIP_BEDROCK_AUTH": "1", + "ANTHROPIC_BEDROCK_BASE_URL": f"{proxy_base_url}/bedrock", + "AWS_REGION": CLIENT_SIDE_AWS_REGION, + } + + +def vertex_extra_env(proxy_base_url: str) -> Dict[str, str]: + """Vertex-mode CLI env pointed at the proxy's /vertex_ai route. + + The `/v1` suffix on ANTHROPIC_VERTEX_BASE_URL is load-bearing: the + CLI's Vertex SDK ships its API version inside its *default* base + URL (`https://{region}-aiplatform.googleapis.com/v1`), so + overriding the base drops the version from the request path unless + the override carries it. LiteLLM's /vertex_ai route reuses the + incoming path verbatim when it contains `/projects/.../locations/...`, + so a version-less path would reach Google as + `aiplatform.googleapis.com/projects/...` and 404. + """ + return { + "CLAUDE_CODE_USE_VERTEX": "1", + "CLAUDE_CODE_SKIP_VERTEX_AUTH": "1", + "ANTHROPIC_VERTEX_BASE_URL": f"{proxy_base_url}/vertex_ai/v1", + "ANTHROPIC_VERTEX_PROJECT_ID": VERTEX_PLACEHOLDER_PROJECT, + "CLOUD_ML_REGION": VERTEX_PLACEHOLDER_REGION, + } + + +def foundry_extra_env(proxy_base_url: str) -> Dict[str, str]: + return { + "CLAUDE_CODE_USE_FOUNDRY": "1", + "CLAUDE_CODE_SKIP_FOUNDRY_AUTH": "1", + "ANTHROPIC_FOUNDRY_BASE_URL": f"{proxy_base_url}/azure", + } + + +def run_passthrough_cell( + *, + compat_result, + models: Sequence[str], + prompt: str, + passthrough_base_path: str = "", + build_extra_env: Optional[Callable[[str], Mapping[str, str]]] = None, + run_models: Callable[..., Mapping[str, Any]] = run_claude_models_parallel, + env: Optional[Mapping[str, str]] = None, +) -> None: + """Run the shared `passthrough` × cell body. + + `passthrough_base_path` is appended to the proxy base URL and + becomes the CLI's ANTHROPIC_BASE_URL (only the anthropic column + uses it; the cloud columns ignore ANTHROPIC_BASE_URL entirely once + their CLAUDE_CODE_USE_* flag is set). `build_extra_env` receives + the trailing-slash-normalized proxy base URL and returns the + provider-mode env for the CLI subprocess. + """ + environ = env if env is not None else os.environ + base_url = environ.get(PROXY_BASE_URL_ENV) + api_key = environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", + pytrace=False, + ) + + proxy_base = base_url.rstrip("/") + extra_env = dict(build_extra_env(proxy_base)) if build_extra_env else None + + outcomes = run_models( + models=models, + prompt=prompt, + base_url=proxy_base + passthrough_base_path, + api_key=api_key, + extra_env=extra_env, + ) + + failures = [] + for model in models: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not outcome.text.strip(): + error = f"[{model}] claude returned empty assistant text" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.env.example b/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.env.example index 11633810533d..5ca7937a4261 100644 --- a/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.env.example +++ b/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.env.example @@ -27,6 +27,15 @@ VERTEXAI_LOCATION=global AZURE_FOUNDRY_API_KEY= AZURE_FOUNDRY_API_BASE= +# Azure cell of the `passthrough` row. Foundry-mode Claude Code sends +# the model in the request body, so the proxy's /azure passthrough +# cannot resolve a router alias and falls back to these env vars. +# AZURE_API_BASE is the Foundry resource's Anthropic surface, i.e. +# https://.services.ai.azure.com/anthropic ; AZURE_API_KEY +# is the same key as AZURE_FOUNDRY_API_KEY. +AZURE_API_BASE= +AZURE_API_KEY= + # REQUIRED for publishing: PAT for the `agent-shin` user, used to push # the daily compat-matrix branch to its fork (agent-shin/litellm-docs) # and open the cross-repo PR against BerriAI/litellm-docs. Scopes: diff --git a/tests/e2e/claude_code/manifest.yaml b/tests/e2e/claude_code/manifest.yaml index f7cccf0cef2f..e5a956991cb1 100644 --- a/tests/e2e/claude_code/manifest.yaml +++ b/tests/e2e/claude_code/manifest.yaml @@ -91,6 +91,23 @@ features: # Code releases. The HTTP probe hits the bug surface LiteLLM # has actually shipped fixes for (2.1.117, 2.1.72, 2.1.70 per # the Claude Code release notes). + - id: passthrough + name: Native API passthrough + # Drives the CLI in each cloud's native mode against LiteLLM's + # passthrough routes instead of the /v1/messages translation + # layer -- the "LLM gateway" setup from + # https://code.claude.com/docs/en/gateway. anthropic uses + # ANTHROPIC_BASE_URL={proxy}/anthropic; bedrock_invoke uses + # CLAUDE_CODE_USE_BEDROCK=1 against {proxy}/bedrock (InvokeModel + # wire, alias resolved from the URL by the router); vertex_ai + # uses CLAUDE_CODE_USE_VERTEX=1 against {proxy}/vertex_ai/v1 + # (rawPredict wire, alias + project + location resolved from the + # deployment, which therefore needs `use_in_pass_through: true`); + # azure uses CLAUDE_CODE_USE_FOUNDRY=1 against {proxy}/azure and + # needs AZURE_API_BASE/AZURE_API_KEY on the proxy (see + # passthrough/test_azure.py and the cron env example). + # bedrock_converse is structurally not_applicable: Claude Code + # has no Converse-wire client. - id: long_context_1m name: Long context (1M) # Sends a ~210k-token padded prompt with the diff --git a/tests/e2e/claude_code/passthrough/__init__.py b/tests/e2e/claude_code/passthrough/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/e2e/claude_code/passthrough/test_anthropic.py b/tests/e2e/claude_code/passthrough/test_anthropic.py new file mode 100644 index 000000000000..aa0443e06258 --- /dev/null +++ b/tests/e2e/claude_code/passthrough/test_anthropic.py @@ -0,0 +1,44 @@ +"""passthrough x Anthropic. + +Drive the real `claude` CLI in its default first-party mode, but with +ANTHROPIC_BASE_URL aimed at the proxy's `/anthropic` passthrough route +instead of the `/v1/messages` translation endpoint. The proxy forwards +the request verbatim to api.anthropic.com, swapping the virtual-key +bearer for its own ANTHROPIC_API_KEY. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/passthrough/test_anthropic.py + ^^^^^^^^^^^ ^^^^^^^^^ + feature_id provider + +Because nothing is translated, the model ids are the real Anthropic API +ids (which happen to equal the proxy aliases for this column). A red +cell here means the passthrough route broke forwarding itself -- auth +header swap, streaming SSE relay, or beta-header propagation -- since +no per-provider transformation is involved. +""" + +from __future__ import annotations + +from claude_code._passthrough import ( + ANTHROPIC_PASSTHROUGH_BASE_PATH, + run_passthrough_cell, +) + +ANTHROPIC_MODELS = [ + "claude-haiku-4-5", + "claude-sonnet-4-6", + "claude-opus-4-7", +] + + +def test_passthrough_anthropic(compat_result): + """Drive the `claude` CLI through `{proxy}/anthropic` and assert a reply.""" + run_passthrough_cell( + compat_result=compat_result, + models=ANTHROPIC_MODELS, + prompt="Reply with the single word 'pong' and nothing else.", + passthrough_base_path=ANTHROPIC_PASSTHROUGH_BASE_PATH, + ) diff --git a/tests/e2e/claude_code/passthrough/test_azure.py b/tests/e2e/claude_code/passthrough/test_azure.py new file mode 100644 index 000000000000..32aaf566e923 --- /dev/null +++ b/tests/e2e/claude_code/passthrough/test_azure.py @@ -0,0 +1,50 @@ +"""passthrough x Azure (Microsoft Foundry). + +Drive the real `claude` CLI in foundry mode (CLAUDE_CODE_USE_FOUNDRY=1) +with ANTHROPIC_FOUNDRY_BASE_URL aimed at the proxy's `/azure` +passthrough route. The CLI POSTs `/v1/messages` with the model in the +JSON body -- unlike the bedrock/vertex modes there is no model segment +in the URL, so the proxy's router-alias resolution cannot engage and +the `/azure` route falls back to its env-configured target: the proxy +must set AZURE_API_BASE to the Foundry resource's Anthropic surface +(`https://.services.ai.azure.com/anthropic`) and +AZURE_API_KEY to the Foundry key (see +cron_vm/litellm-compat-matrix.env.example). The model ids are the +Foundry deployment names, which this matrix provisions to match the +Anthropic ids. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/passthrough/test_azure.py + ^^^^^^^^^^^ ^^^^^ + feature_id provider + +Wiring verified live at authoring time: through `{proxy}/azure` the +Foundry Anthropic surface accepted the `api-key` / `Authorization: +Bearer` headers the fallback sends (a bogus key 401s, the real key +proceeds to deployment lookup), so a red cell here means missing +AZURE_API_BASE/AZURE_API_KEY on the proxy, missing Foundry deployments +for the three tiers, or a genuine forwarding regression -- not an +auth-scheme mismatch. +""" + +from __future__ import annotations + +from claude_code._passthrough import foundry_extra_env, run_passthrough_cell + +AZURE_MODELS = [ + "claude-haiku-4-5", + "claude-sonnet-4-6", + "claude-opus-4-7", +] + + +def test_passthrough_azure(compat_result): + """Drive the `claude` CLI through `{proxy}/azure` and assert a reply.""" + run_passthrough_cell( + compat_result=compat_result, + models=AZURE_MODELS, + prompt="Reply with the single word 'pong' and nothing else.", + build_extra_env=foundry_extra_env, + ) diff --git a/tests/e2e/claude_code/passthrough/test_bedrock_converse.py b/tests/e2e/claude_code/passthrough/test_bedrock_converse.py new file mode 100644 index 000000000000..d1093a7a9580 --- /dev/null +++ b/tests/e2e/claude_code/passthrough/test_bedrock_converse.py @@ -0,0 +1,34 @@ +"""passthrough x Bedrock (Converse). + +Structurally not applicable. In bedrock mode the `claude` CLI speaks +only the InvokeModel wire (`/model/{id}/invoke-with-response-stream`); +it has no Converse-wire client, so there is no Claude Code traffic a +Converse passthrough could serve. LiteLLM's `/bedrock` route does +accept `/model/{id}/converse-stream`, but exercising it would test a +wire no Claude Code user can produce, which is out of scope for this +matrix. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/passthrough/test_bedrock_converse.py + ^^^^^^^^^^^ ^^^^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + + +def test_passthrough_bedrock_converse(compat_result): + """Report not_applicable: Claude Code has no Converse-wire mode.""" + compat_result.set( + { + "status": "not_applicable", + "reason": ( + "Claude Code's bedrock mode speaks only the InvokeModel wire " + "(/model/{id}/invoke-with-response-stream); it has no " + "Converse-wire client, so there is no Claude Code surface " + "for Converse passthrough." + ), + } + ) diff --git a/tests/e2e/claude_code/passthrough/test_bedrock_invoke.py b/tests/e2e/claude_code/passthrough/test_bedrock_invoke.py new file mode 100644 index 000000000000..6e84dea67798 --- /dev/null +++ b/tests/e2e/claude_code/passthrough/test_bedrock_invoke.py @@ -0,0 +1,42 @@ +"""passthrough x Bedrock (Invoke). + +Drive the real `claude` CLI in bedrock mode (CLAUDE_CODE_USE_BEDROCK=1) +with ANTHROPIC_BEDROCK_BASE_URL aimed at the proxy's `/bedrock` +passthrough route. The CLI speaks the native InvokeModel wire -- +`POST /model/{model}/invoke-with-response-stream` -- with the proxy +alias in the model segment; the proxy resolves the alias through its +router, rewrites the path to the deployment's upstream model id, and +SigV4-signs the forwarded request with its own AWS credentials +(CLAUDE_CODE_SKIP_BEDROCK_AUTH=1 keeps the CLI from signing). + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/passthrough/test_bedrock_invoke.py + ^^^^^^^^^^^ ^^^^^^^^^^^^^^ + feature_id provider + +The CLI also fires a best-effort `GET /bedrock/inference-profiles` +listing at startup; its failure is non-fatal and does not gate this +cell. +""" + +from __future__ import annotations + +from claude_code._passthrough import bedrock_extra_env, run_passthrough_cell + +BEDROCK_INVOKE_MODELS = [ + "claude-haiku-4-5-bedrock-invoke", + "claude-sonnet-4-6-bedrock-invoke", + "claude-opus-4-7-bedrock-invoke", +] + + +def test_passthrough_bedrock_invoke(compat_result): + """Drive the `claude` CLI through `{proxy}/bedrock` and assert a reply.""" + run_passthrough_cell( + compat_result=compat_result, + models=BEDROCK_INVOKE_MODELS, + prompt="Reply with the single word 'pong' and nothing else.", + build_extra_env=bedrock_extra_env, + ) diff --git a/tests/e2e/claude_code/passthrough/test_vertex_ai.py b/tests/e2e/claude_code/passthrough/test_vertex_ai.py new file mode 100644 index 000000000000..5e3c6bce419b --- /dev/null +++ b/tests/e2e/claude_code/passthrough/test_vertex_ai.py @@ -0,0 +1,45 @@ +"""passthrough x Vertex AI. + +Drive the real `claude` CLI in vertex mode (CLAUDE_CODE_USE_VERTEX=1) +with ANTHROPIC_VERTEX_BASE_URL aimed at the proxy's `/vertex_ai` +passthrough route. The CLI speaks the native rawPredict wire -- +`POST .../projects/{p}/locations/{l}/publishers/anthropic/models/{model}:streamRawPredict` +-- with the proxy alias in the model segment; the proxy resolves the +alias through its router, replaces the placeholder project/location +path segments with the deployment's `vertex_project` / +`vertex_location`, and attaches its own Google credentials +(CLAUDE_CODE_SKIP_VERTEX_AUTH=1 keeps the CLI from minting a token). + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/passthrough/test_vertex_ai.py + ^^^^^^^^^^^ ^^^^^^^^^ + feature_id provider + +This cell requires the vertex deployments in the proxy config to carry +`use_in_pass_through: true` (see test_config.yaml) -- that is what +registers their credentials with the passthrough router. Without it +the proxy forwards the CLI's own headers (the virtual-key bearer) to +Google and every tier fails with a 401. +""" + +from __future__ import annotations + +from claude_code._passthrough import run_passthrough_cell, vertex_extra_env + +VERTEX_MODELS = [ + "claude-haiku-4-5-vertex", + "claude-sonnet-4-6-vertex", + "claude-opus-4-7-vertex", +] + + +def test_passthrough_vertex_ai(compat_result): + """Drive the `claude` CLI through `{proxy}/vertex_ai` and assert a reply.""" + run_passthrough_cell( + compat_result=compat_result, + models=VERTEX_MODELS, + prompt="Reply with the single word 'pong' and nothing else.", + build_extra_env=vertex_extra_env, + ) diff --git a/tests/e2e/claude_code/test_config.yaml b/tests/e2e/claude_code/test_config.yaml index eec68d11dcfc..e9253da2b3c1 100644 --- a/tests/e2e/claude_code/test_config.yaml +++ b/tests/e2e/claude_code/test_config.yaml @@ -59,21 +59,31 @@ model_list: aws_region_name: us-east-1 # ---- Vertex AI ---- + # `use_in_pass_through: true` registers each deployment's + # project/location/credentials with the /vertex_ai passthrough + # router, which the `passthrough` row needs to resolve + # .../models/{alias}:streamRawPredict URLs. That registration only + # reads the canonical `vertex_project`/`vertex_location` param names + # (not the `vertex_ai_*` aliases); the chat translation path accepts + # both. - model_name: claude-haiku-4-5-vertex litellm_params: model: vertex_ai/claude-haiku-4-5 - vertex_ai_project: os.environ/VERTEXAI_PROJECT - vertex_ai_location: os.environ/VERTEXAI_LOCATION + vertex_project: os.environ/VERTEXAI_PROJECT + vertex_location: os.environ/VERTEXAI_LOCATION + use_in_pass_through: true - model_name: claude-sonnet-4-6-vertex litellm_params: model: vertex_ai/claude-sonnet-4-6 - vertex_ai_project: os.environ/VERTEXAI_PROJECT - vertex_ai_location: os.environ/VERTEXAI_LOCATION + vertex_project: os.environ/VERTEXAI_PROJECT + vertex_location: os.environ/VERTEXAI_LOCATION + use_in_pass_through: true - model_name: claude-opus-4-7-vertex litellm_params: model: vertex_ai/claude-opus-4-7 - vertex_ai_project: os.environ/VERTEXAI_PROJECT - vertex_ai_location: os.environ/VERTEXAI_LOCATION + vertex_project: os.environ/VERTEXAI_PROJECT + vertex_location: os.environ/VERTEXAI_LOCATION + use_in_pass_through: true # ---- Microsoft Foundry (Anthropic deployments on Azure) ---- - model_name: claude-haiku-4-5-azure From 79bfe2e9969a82667cec2c184823c19545dc5463 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 15 Jul 2026 17:25:56 -0700 Subject: [PATCH 2/3] docs(e2e/claude_code): document the known-red azure passthrough cell (anthropic-version dropped by the /azure fallback) --- tests/e2e/claude_code/passthrough/test_azure.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/e2e/claude_code/passthrough/test_azure.py b/tests/e2e/claude_code/passthrough/test_azure.py index 32aaf566e923..09b0824047a9 100644 --- a/tests/e2e/claude_code/passthrough/test_azure.py +++ b/tests/e2e/claude_code/passthrough/test_azure.py @@ -25,8 +25,17 @@ Bearer` headers the fallback sends (a bogus key 401s, the real key proceeds to deployment lookup), so a red cell here means missing AZURE_API_BASE/AZURE_API_KEY on the proxy, missing Foundry deployments -for the three tiers, or a genuine forwarding regression -- not an -auth-scheme mismatch. +for the three tiers, or a genuine forwarding gap -- not an auth-scheme +mismatch. + +Known-red at authoring time against a healthy Foundry resource: the +`/azure` fallback assembles only its own auth headers and drops the +rest of the client's headers, including the `anthropic-version` header +the CLI sends, and Foundry's Anthropic surface rejects the request +with 400 "anthropic-version: header is required" (the same request +sent directly to Foundry with that header succeeds). This cell stays +red until that forwarding gap is fixed, which is precisely the class +of bug the row exists to surface. """ from __future__ import annotations From c4fee0eafe665abf0513864453ef7f8b773a6a90 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 15 Jul 2026 18:06:28 -0700 Subject: [PATCH 3/3] test(e2e/claude_code): retry rate-limit-shaped CLI failures with backoff --- .../_driver_unit_tests/test_cli_driver.py | 197 ++++++++++++++++++ tests/e2e/claude_code/cli_driver.py | 108 ++++++++-- tests/e2e/claude_code/conftest.py | 12 +- 3 files changed, 295 insertions(+), 22 deletions(-) diff --git a/tests/e2e/claude_code/_driver_unit_tests/test_cli_driver.py b/tests/e2e/claude_code/_driver_unit_tests/test_cli_driver.py index 786f60299931..f1a0534906e6 100644 --- a/tests/e2e/claude_code/_driver_unit_tests/test_cli_driver.py +++ b/tests/e2e/claude_code/_driver_unit_tests/test_cli_driver.py @@ -19,6 +19,7 @@ ClaudeCLIError, DriverResult, failure_diagnostic, + is_rate_limit_shaped, run_claude, run_claude_models_parallel, ) @@ -793,3 +794,199 @@ def test_failure_diagnostic_uses_last_result_event_status(): diag = failure_diagnostic(result) assert "api_status=429" in diag assert "500" not in diag + + +_RATE_LIMITED_STDOUT = ( + json.dumps( + { + "type": "assistant", + "message": { + "content": [ + {"type": "text", "text": "API Error: 429 Too Many Requests"} + ] + }, + } + ) + + "\n" + + json.dumps({"type": "result", "api_error_status": 429}) + + "\n" +) + +_OK_STDOUT = ( + json.dumps( + { + "type": "assistant", + "message": {"content": [{"type": "text", "text": "pong"}]}, + } + ) + + "\n" +) + + +class _FlakyRunner: + """Fake runner that rate-limits each model N times before succeeding. + + Keeps a per-model call count so tests can assert exactly how many + attempts the retry loop made — the load-bearing detail a canned + single-response runner can't express. + """ + + def __init__(self, failures_before_success: dict): + self.failures_before_success = dict(failures_before_success) + self.calls: dict = {} + + def __call__(self, cmd, env, capture_output, text, timeout, check, input=None): + model = cmd[cmd.index("--model") + 1] + self.calls[model] = self.calls.get(model, 0) + 1 + if self.calls[model] <= self.failures_before_success.get(model, 0): + return _Completed(returncode=1, stdout=_RATE_LIMITED_STDOUT) + return _Completed(returncode=0, stdout=_OK_STDOUT) + + +@pytest.mark.parametrize( + "outcome,expected", + [ + (ClaudeCLIError("claude CLI timed out after 120.0s"), True), + (ClaudeCLIError("claude CLI not found at 'claude'"), False), + ( + DriverResult( + text="", + events=[{"type": "result", "api_error_status": 429}], + exit_code=1, + ), + True, + ), + (DriverResult(text="Too Many Requests", exit_code=1), True), + (DriverResult(text="", stderr="throttled by upstream", exit_code=1), True), + (DriverResult(text="rate limit exceeded", exit_code=0), False), + (DriverResult(text="", stderr="auth failed", exit_code=2), False), + ], +) +def test_is_rate_limit_shaped_classification(outcome, expected): + """The retry trigger must match 429/throttle/timeout markers on + failures only — a passing result mentioning '429' in its reply text + must never be classified as retryable.""" + assert is_rate_limit_shaped(outcome) is expected + + +def test_run_claude_models_parallel_retries_rate_limited_model_until_success(): + """A model that 429s once must be retried after the backoff sleep and + end up green, while an untroubled sibling model runs exactly once.""" + runner = _FlakyRunner({"flaky": 1}) + sleeps: List[float] = [] + + outcomes = run_claude_models_parallel( + models=["flaky", "steady"], + prompt="hi", + base_url="http://x", + api_key="k", + runner=runner, + rate_limit_retries=2, + rate_limit_backoff_seconds=0.5, + sleep=sleeps.append, + ) + + assert isinstance(outcomes["flaky"], DriverResult) + assert outcomes["flaky"].exit_code == 0 + assert outcomes["flaky"].text == "pong" + assert runner.calls == {"flaky": 2, "steady": 1} + assert sleeps == [0.5] + + +def test_run_claude_models_parallel_does_not_retry_non_rate_limit_failures(): + """A deterministic failure (bad auth) must fail fast: no sleeps, one + attempt — retrying it would just triple the matrix wall time.""" + + def runner(cmd, env, capture_output, text, timeout, check, input=None): + return _Completed(returncode=2, stdout="", stderr="auth failed") + + sleeps: List[float] = [] + outcomes = run_claude_models_parallel( + models=["a"], + prompt="hi", + base_url="http://x", + api_key="k", + runner=runner, + rate_limit_retries=2, + rate_limit_backoff_seconds=0.5, + sleep=sleeps.append, + ) + + assert outcomes["a"].exit_code == 2 + assert sleeps == [] + + +def test_run_claude_models_parallel_returns_last_failure_when_retries_exhausted(): + """A persistently rate-limited model exhausts its budget (initial + attempt + N retries, each preceded by one backoff sleep) and still + surfaces the 429 diagnostic instead of masking it.""" + runner = _FlakyRunner({"stuck": 99}) + sleeps: List[float] = [] + + outcomes = run_claude_models_parallel( + models=["stuck"], + prompt="hi", + base_url="http://x", + api_key="k", + runner=runner, + rate_limit_retries=2, + rate_limit_backoff_seconds=0.25, + sleep=sleeps.append, + ) + + assert runner.calls == {"stuck": 3} + assert sleeps == [0.25, 0.25] + assert outcomes["stuck"].exit_code == 1 + assert "429" in failure_diagnostic(outcomes["stuck"]) + + +def test_run_claude_models_parallel_retries_timeout_shaped_cli_errors(): + """CLI timeouts are how saturated upstreams usually present (the CLI + retries 429s internally until the harness kills it), so a timeout + must be retried like an explicit 429.""" + calls: List[int] = [] + + def runner(cmd, env, capture_output, text, timeout, check, input=None): + calls.append(1) + if len(calls) == 1: + raise subprocess.TimeoutExpired(cmd="claude", timeout=1) + return _Completed(returncode=0, stdout=_OK_STDOUT) + + sleeps: List[float] = [] + outcomes = run_claude_models_parallel( + models=["a"], + prompt="hi", + base_url="http://x", + api_key="k", + runner=runner, + rate_limit_retries=1, + rate_limit_backoff_seconds=0.5, + sleep=sleeps.append, + ) + + assert isinstance(outcomes["a"], DriverResult) + assert outcomes["a"].text == "pong" + assert len(calls) == 2 + assert sleeps == [0.5] + + +def test_run_claude_models_parallel_zero_retries_disables_backoff(): + """`rate_limit_retries=0` must restore the old single-attempt + behavior exactly: one call, no sleeps, failure returned as-is.""" + runner = _FlakyRunner({"stuck": 99}) + sleeps: List[float] = [] + + outcomes = run_claude_models_parallel( + models=["stuck"], + prompt="hi", + base_url="http://x", + api_key="k", + runner=runner, + rate_limit_retries=0, + rate_limit_backoff_seconds=0.5, + sleep=sleeps.append, + ) + + assert runner.calls == {"stuck": 1} + assert sleeps == [] + assert outcomes["stuck"].exit_code == 1 diff --git a/tests/e2e/claude_code/cli_driver.py b/tests/e2e/claude_code/cli_driver.py index 97eaa0e68476..5b18c1c291a4 100644 --- a/tests/e2e/claude_code/cli_driver.py +++ b/tests/e2e/claude_code/cli_driver.py @@ -15,6 +15,7 @@ import json import os +import re import shutil import subprocess import sys @@ -40,6 +41,29 @@ os.environ.get("LITELLM_COMPAT_CLI_TIMEOUT_SECONDS") or 120 ) +RATE_LIMIT_SHAPED_RE = re.compile( + r"(?:\b429\b|rate[\s_-]?limit|too\s+many\s+requests|throttl(?:ed|ing)|" + r"claude\s+CLI\s+timed\s+out)", + re.IGNORECASE, +) +"""Heuristic shared with the conftest rate-limit summary: 429s and +throttle markers anywhere in the failure text, plus CLI timeouts -- +the CLI retries 429s internally until the harness timeout kills it, +so a saturated upstream usually surfaces as a timeout rather than a +clean 429.""" + +DEFAULT_RATE_LIMIT_RETRIES = int( + os.environ.get("LITELLM_COMPAT_RATE_LIMIT_RETRIES") or 2 +) +DEFAULT_RATE_LIMIT_BACKOFF_SECONDS = float( + os.environ.get("LITELLM_COMPAT_RATE_LIMIT_BACKOFF_SECONDS") or 65 +) +"""Rate-limit-shaped failures are retried after a backoff long enough +for a per-minute quota window (the dominant 429 source across +Anthropic / Bedrock / Vertex) to reset. Both knobs are env-tunable so +a matrix run can trade wall time for resilience without code edits; +retries=0 disables the behavior entirely.""" + # Env vars the `claude` Node CLI legitimately needs to function: # locating its own binary + node, basic locale/terminal plumbing. # Deliberately excludes every credential-bearing var that the @@ -265,6 +289,22 @@ def run_claude( ModelResult = Union[DriverResult, ClaudeCLIError] +def is_rate_limit_shaped(outcome: ModelResult) -> bool: + """Classify an outcome as a retryable rate-limit-shaped failure. + + A `ClaudeCLIError` matches on its message (which is where the + driver's own timeout diagnostic lands); a failing `DriverResult` + matches on its full `failure_diagnostic` so 429s buried in the + CLI's stdout text or `api_error_status` are both caught. Passing + results are never rate-limit-shaped. + """ + if isinstance(outcome, ClaudeCLIError): + return bool(RATE_LIMIT_SHAPED_RE.search(str(outcome))) + if outcome.exit_code == 0: + return False + return bool(RATE_LIMIT_SHAPED_RE.search(failure_diagnostic(outcome))) + + def run_claude_models_parallel( *, models: Sequence[str], @@ -277,6 +317,9 @@ def run_claude_models_parallel( cli_path: str = CLAUDE_CLI_DEFAULT, timeout: float = DEFAULT_TIMEOUT_SECONDS, runner: Optional[Callable[..., Any]] = None, + rate_limit_retries: Optional[int] = None, + rate_limit_backoff_seconds: Optional[float] = None, + sleep: Callable[[float], None] = time.sleep, ) -> Dict[str, ModelResult]: """Invoke `run_claude` for every `models[i]` concurrently and collect outcomes. @@ -290,6 +333,14 @@ def run_claude_models_parallel( keep the synchronous CLI driver unchanged so unit tests can keep injecting a fake `runner`. + Rate-limit-shaped failures (see `is_rate_limit_shaped`) are retried + per model up to `rate_limit_retries` times, sleeping + `rate_limit_backoff_seconds` before each retry so per-minute quota + windows can reset; both default to the `LITELLM_COMPAT_RATE_LIMIT_*` + env knobs. Each retry goes back through `run_claude`, so it + re-acquires a token from the provider rate limiter like any other + invocation. `sleep` is an injection seam for unit tests. + Returns a dict keyed by model id. Each value is either the `DriverResult` produced by `run_claude` or the `ClaudeCLIError` that aborted that model's run — callers decide how to map either @@ -300,14 +351,20 @@ def run_claude_models_parallel( if not models: raise ValueError("models must be a non-empty sequence") - def _one(model: str) -> Tuple[str, ModelResult, float]: - # Per-model wall clock: this is what the matrix run actually pays for. - # We record it whether the run succeeded or raised so the breakdown - # log below covers both code paths and surfaces "which model is the - # long pole?" without requiring per-test instrumentation. - started = time.monotonic() + retries = ( + DEFAULT_RATE_LIMIT_RETRIES + if rate_limit_retries is None + else max(0, rate_limit_retries) + ) + backoff = ( + DEFAULT_RATE_LIMIT_BACKOFF_SECONDS + if rate_limit_backoff_seconds is None + else max(0.0, rate_limit_backoff_seconds) + ) + + def _run_once(model: str) -> ModelResult: try: - result = run_claude( + return run_claude( prompt=prompt, model=model, base_url=base_url, @@ -319,14 +376,8 @@ def _one(model: str) -> Tuple[str, ModelResult, float]: timeout=timeout, runner=runner, ) - elapsed = time.monotonic() - started - # Stamp the duration onto the DriverResult so callers (tests, - # diagnostics) can attribute slow cells without re-timing. - result.duration_ms = int(elapsed * 1000) - return model, result, elapsed except ClaudeCLIError as exc: - elapsed = time.monotonic() - started - return model, exc, elapsed + return exc except Exception as exc: # Honor the documented "errors as values" contract for any # exception type — not just ClaudeCLIError. The rate @@ -334,13 +385,38 @@ def _one(model: str) -> Tuple[str, ModelResult, float]: # raise ValueError on edge-case model strings, and a future # bug elsewhere in the call stack must not abort the entire # parallel batch and lose the other models' outcomes. - elapsed = time.monotonic() - started wrapped = ClaudeCLIError( f"unexpected error running model {model!r}: " f"{type(exc).__name__}: {exc}" ) wrapped.__cause__ = exc - return model, wrapped, elapsed + return wrapped + + def _one(model: str) -> Tuple[str, ModelResult, float]: + # Per-model wall clock: this is what the matrix run actually pays + # for, retries and backoff sleeps included. We record it whether + # the run succeeded or raised so the breakdown log below covers + # both code paths and surfaces "which model is the long pole?" + # without requiring per-test instrumentation. + started = time.monotonic() + outcome = _run_once(model) + for attempt in range(retries): + if not is_rate_limit_shaped(outcome): + break + print( + f"[retry] {model}: rate-limit-shaped failure; sleeping " + f"{backoff:.0f}s before attempt {attempt + 2}/{retries + 1}", + file=sys.stderr, + flush=True, + ) + sleep(backoff) + outcome = _run_once(model) + elapsed = time.monotonic() - started + if isinstance(outcome, DriverResult): + # Stamp the duration onto the DriverResult so callers (tests, + # diagnostics) can attribute slow cells without re-timing. + outcome.duration_ms = int(elapsed * 1000) + return model, outcome, elapsed outcomes: Dict[str, ModelResult] = {} durations: Dict[str, float] = {} diff --git a/tests/e2e/claude_code/conftest.py b/tests/e2e/claude_code/conftest.py index d2bfa1a54bff..6ee8b940648a 100644 --- a/tests/e2e/claude_code/conftest.py +++ b/tests/e2e/claude_code/conftest.py @@ -33,7 +33,6 @@ import functools import json import os -import re import sys from collections import Counter, defaultdict from dataclasses import dataclass, field @@ -43,6 +42,8 @@ import pytest import yaml +from claude_code.cli_driver import RATE_LIMIT_SHAPED_RE + VALID_STATUSES = {"pass", "fail", "not_applicable", "not_tested"} RESULTS_ARTIFACT_ENV = "COMPAT_RESULTS_PATH" DEFAULT_ARTIFACT_PATH = "compat-results.json" @@ -62,11 +63,10 @@ # the rate limiter is supposed to back off from. False positives on a # genuinely slow upstream are tolerable here because the worst case is # the binary search runs at a slightly lower rate than necessary. -_RATE_LIMIT_RE = re.compile( - r"(?:\b429\b|rate[\s_-]?limit|too\s+many\s+requests|throttl(?:ed|ing)|" - r"claude\s+CLI\s+timed\s+out)", - re.IGNORECASE, -) +# +# The pattern lives in `cli_driver` so the driver's retry-on-rate-limit +# logic and this summary classify failures identically. +_RATE_LIMIT_RE = RATE_LIMIT_SHAPED_RE @dataclass