From 12a48178caf8e37ac2862f7d53522d893fe1123b Mon Sep 17 00:00:00 2001 From: kerry Date: Sat, 3 Oct 2026 21:14:49 +0000 Subject: [PATCH 1/8] test(integration): exact four-part translation cases on a shared fake provider and shared YAML deployment Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/README.md | 6 ++ tests/integration/_support/manifest.py | 1 + tests/integration/_support/provider.py | 44 +++++++++++ tests/integration/conftest.py | 26 +++++++ tests/integration/proxy_config.yaml | 7 +- tests/integration/run.py | 2 +- tests/integration/translation/__init__.py | 0 tests/integration/translation/case.py | 25 +++++++ tests/integration/translation/conftest.py | 3 + .../translation/messages/__init__.py | 0 .../translation/messages/bases/__init__.py | 0 .../translation/messages/bases/anthropic.py | 70 ++++++++++++++++++ .../translation/messages/basic/__init__.py | 0 .../basic/test_messages_basic_anthropic.py | 11 +++ .../messages/reasoning/__init__.py | 0 .../test_messages_reasoning_anthropic.py | 73 +++++++++++++++++++ tests/integration/translation/runner.py | 19 +++++ 17 files changed, 285 insertions(+), 2 deletions(-) create mode 100644 tests/integration/_support/provider.py create mode 100644 tests/integration/translation/__init__.py create mode 100644 tests/integration/translation/case.py create mode 100644 tests/integration/translation/conftest.py create mode 100644 tests/integration/translation/messages/__init__.py create mode 100644 tests/integration/translation/messages/bases/__init__.py create mode 100644 tests/integration/translation/messages/bases/anthropic.py create mode 100644 tests/integration/translation/messages/basic/__init__.py create mode 100644 tests/integration/translation/messages/basic/test_messages_basic_anthropic.py create mode 100644 tests/integration/translation/messages/reasoning/__init__.py create mode 100644 tests/integration/translation/messages/reasoning/test_messages_reasoning_anthropic.py create mode 100644 tests/integration/translation/runner.py diff --git a/tests/integration/README.md b/tests/integration/README.md index 79a5d9c7c033..852fd434cafe 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -18,6 +18,12 @@ There is no per-node manifest. A positional argument is a file of the group or a Provider sentinels currently use the controlled server, not live recordings. The provider shard also runs the existing strict replay controls for changed requests, exhausted interactions, leftover interactions and no provider connection. Future recorded scenarios must use that replay-only implementation; missing recordings cannot fall back to a real provider. The observation endpoint is destructive and the current selection runs serially against one owned upstream +Translation tests in `translation/` check one request through the proxy as four literals on a `TranslationTestCase`: the client request, the exact provider request, the provider response and the exact client response. The runner compares the provider request body and the client response body in full, so an added, removed, renamed or moved field fails. Folders follow the client endpoint and then the feature, for example `translation/messages/reasoning/`. Each endpoint keeps one complete base case per model in `/bases/.py`, which `/basic/` runs on its own, and a feature case is `dataclasses.replace(BASE, ...)` that lists only the fields it changes + +Deployments used by translation tests are shared by the whole suite and declared in `proxy_config.yaml` under `model_list`, with `model_name` equal to the litellm model string, `api_base: http://127.0.0.1:8191` and a synthetic key. A case names the deployment literally in its client request. The fake provider behind them is the `provider` fixture: one `wire_server` on port 8191 inside the pytest process, started the first time a test asks for it. A test queues its replies with `provider.expect(...)` and reads what the proxy sent with `provider.received()`. Tests that use it run one at a time. After each test the fixture fails if a queued reply was never requested or a received request was never read, and before each test it fails if a request arrived in between, naming the previous test. It refuses to start under pytest-xdist, so the `mcp` and `cost` groups cannot use it. Client requests carry `"cache": {"no-cache": True}` because the proxy caches responses in Redis + +Provider responses in translation cases are captured once from the real provider and stored verbatim. First run the new case against the fake provider; the body the proxy sends is the case's provider request. Send that exact body to the real provider endpoint with a key from the 1Password `Shared` vault (`/qa-keys`), and only store the case when the provider answers 2xx. Keep only the response body and drop every response header, since headers carry account identifiers such as the organization id and rate limits. Never print or save the request headers you sent. Before committing, check the body contains no key and no account identifier, such as an organization id, an AWS account id inside an ARN, a GCP project id or an Azure resource name, and that the prompt is synthetic. Paste the body as the case's provider response without shortening ids, token counts or signatures, and move long opaque values such as thinking signatures into module-level constants used by both the provider response and the client response. Do not commit the script used for the capture + Fixtures must contain synthetic data only. Keep private incident records and source documents out of code, fixtures, logs and PR descriptions Database cases own their temporary schemas, roles, constraints and proxy processes. They prove reader-versus-writer execution with PostgreSQL lock observations, exercise real transaction wait limits and verify rollback after a reached database failure diff --git a/tests/integration/_support/manifest.py b/tests/integration/_support/manifest.py index a4a86a215684..fb17ebb38640 100644 --- a/tests/integration/_support/manifest.py +++ b/tests/integration/_support/manifest.py @@ -11,6 +11,7 @@ "providers", "streaming", "messages_endpoint", + "translation", "configuration", "mcp", "observability", diff --git a/tests/integration/_support/provider.py b/tests/integration/_support/provider.py new file mode 100644 index 000000000000..278cd054c614 --- /dev/null +++ b/tests/integration/_support/provider.py @@ -0,0 +1,44 @@ +"""The fake provider shared by every integration test that takes the `provider` fixture. + +Deployments in `proxy_config.yaml` point at `PROVIDER_URL`, so one server answers for all of them. A test +queues the replies it expects with `expect` and reads what the proxy sent with `received`. Tests run one at a +time against it; the `provider` fixture checks nothing is left over between tests. +""" + +from __future__ import annotations + +from collections import deque +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass, field +from typing import Final + +from tests.integration._support.wire import Reply, Request, Wire, wire_server + +PROVIDER_PORT: Final = 8191 +PROVIDER_URL: Final = f"http://127.0.0.1:{PROVIDER_PORT}" +_UNQUEUED: Final = Reply(status=500, body=b'{"error": "the shared fake provider has no reply queued for this request"}') + + +@dataclass(slots=True) +class SharedProvider: + wire: Wire + replies: deque[Reply] + last_test: str | None = field(default=None) + + def expect(self, *replies: Reply) -> None: + self.replies.extend(replies) + + def received(self) -> tuple[Request, ...]: + return self.wire.drain() + + +@contextmanager +def shared_provider() -> Iterator[SharedProvider]: + replies: Final[deque[Reply]] = deque() + + def respond(request: Request) -> Reply: + return replies.popleft() if replies else _UNQUEUED + + with wire_server(respond, port=PROVIDER_PORT) as wire: + yield SharedProvider(wire, replies) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 1b39eb81b018..7040f784f71c 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -15,6 +15,7 @@ from tests.integration._support.client import Gateway, eventually, gateway_from_environment from tests.integration._support.generation import LIFECYCLE_SETTINGS from tests.integration._support.manifest import OWNED_DIRECTORIES +from tests.integration._support.provider import SharedProvider, shared_provider from tests.integration._support.routing import RoutingPlugin COLLECTED: Final = pytest.StashKey[tuple[str, ...]]() @@ -126,6 +127,31 @@ def gateway() -> Iterator[Gateway]: yield value +@pytest.fixture(scope="session") +def shared_provider_server() -> Iterator[SharedProvider]: + if os.environ.get("PYTEST_XDIST_WORKER"): + pytest.fail("the shared fake provider needs tests to run one at a time; this group runs under pytest-xdist") + with shared_provider() as server: + yield server + + +@pytest.fixture +def provider(shared_provider_server: SharedProvider, request: pytest.FixtureRequest) -> Iterator[SharedProvider]: + stray: Final = shared_provider_server.received() + shared_provider_server.replies.clear() + assert stray == (), ( + f"the shared fake provider got {[item.target for item in stray]} " + f"after {shared_provider_server.last_test} finished" + ) + yield shared_provider_server + shared_provider_server.last_test = request.node.nodeid + unused: Final = len(shared_provider_server.replies) + unread: Final = shared_provider_server.received() + shared_provider_server.replies.clear() + assert unused == 0, f"{unused} queued provider replies were never requested" + assert unread == (), f"the test never read the provider requests {[item.target for item in unread]}" + + @pytest.fixture def peer(gateway: Gateway) -> Iterator[Gateway]: url: Final = os.environ["INTEGRATION_PEER_URL"] diff --git a/tests/integration/proxy_config.yaml b/tests/integration/proxy_config.yaml index 085adea81ac1..50fd6c9d32dc 100644 --- a/tests/integration/proxy_config.yaml +++ b/tests/integration/proxy_config.yaml @@ -1,4 +1,9 @@ -model_list: [] +model_list: + - model_name: anthropic/claude-sonnet-4-6 + litellm_params: + model: anthropic/claude-sonnet-4-6 + api_base: http://127.0.0.1:8191 + api_key: synthetic-anthropic-key general_settings: master_key: os.environ/LITELLM_MASTER_KEY database_url: os.environ/DATABASE_URL diff --git a/tests/integration/run.py b/tests/integration/run.py index c5facec0bd2b..0e3264b770bf 100644 --- a/tests/integration/run.py +++ b/tests/integration/run.py @@ -15,7 +15,7 @@ "management": ("management", "authorization", "configuration"), "accounting": ("pricing", "spend"), "database": ("database",), - "providers": ("providers", "routing", "streaming", "messages_endpoint"), + "providers": ("providers", "routing", "streaming", "messages_endpoint", "translation"), "extensions": ("observability", "compatibility"), "mcp": ("mcp",), "sdk": ("sdk",), diff --git a/tests/integration/translation/__init__.py b/tests/integration/translation/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/translation/case.py b/tests/integration/translation/case.py new file mode 100644 index 000000000000..fc03ed00e32a --- /dev/null +++ b/tests/integration/translation/case.py @@ -0,0 +1,25 @@ +from collections.abc import Mapping +from dataclasses import dataclass + +from pydantic import JsonValue + + +@dataclass(frozen=True, slots=True, kw_only=True) +class TranslationTestCase: + """One client request through the proxy to a deployment in `proxy_config.yaml`: what the client sends, the + exact request the provider must receive, the provider's reply, and the exact response the client must get.""" + + scenario: str + regressions: tuple[str, ...] = () + client_path: str + client_request: Mapping[str, JsonValue] + provider_path: str + provider_headers: Mapping[str, str] + provider_request: Mapping[str, JsonValue] + provider_response: Mapping[str, JsonValue] + client_status: int = 200 + client_response: Mapping[str, JsonValue] + + @property + def id(self) -> str: + return f"{self.client_request['model']}-{self.scenario}" diff --git a/tests/integration/translation/conftest.py b/tests/integration/translation/conftest.py new file mode 100644 index 000000000000..c0dc4462caec --- /dev/null +++ b/tests/integration/translation/conftest.py @@ -0,0 +1,3 @@ +import pytest + +pytest.register_assert_rewrite("integration.translation.runner") diff --git a/tests/integration/translation/messages/__init__.py b/tests/integration/translation/messages/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/translation/messages/bases/__init__.py b/tests/integration/translation/messages/bases/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/translation/messages/bases/anthropic.py b/tests/integration/translation/messages/bases/anthropic.py new file mode 100644 index 000000000000..4b3ea7fa50e7 --- /dev/null +++ b/tests/integration/translation/messages/bases/anthropic.py @@ -0,0 +1,70 @@ +from typing import Final + +from integration.translation.case import TranslationTestCase + +CLAUDE_SONNET_4_6: Final = TranslationTestCase( + scenario="basic", + client_path="/v1/messages", + client_request={ + "model": "anthropic/claude-sonnet-4-6", + "max_tokens": 64, + "system": "You are a terse assistant.", + "messages": [{"role": "user", "content": "Say hello."}], + "cache": {"no-cache": True}, + }, + provider_path="/v1/messages", + provider_headers={ + "x-api-key": "synthetic-anthropic-key", + "anthropic-version": "2023-06-01", + "content-type": "application/json", + }, + provider_request={ + "model": "claude-sonnet-4-6", + "max_tokens": 64, + "stream": False, + "system": "You are a terse assistant.", + "messages": [{"role": "user", "content": "Say hello."}], + }, + provider_response={ + "model": "claude-sonnet-4-6", + "id": "msg_011CffzUNHaEfzVxCh5hskBG", + "type": "message", + "role": "assistant", + "content": [{"type": "text", "text": "Hello!"}], + "container": None, + "stop_reason": "end_turn", + "stop_sequence": None, + "stop_details": None, + "usage": { + "input_tokens": 18, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 5, + "service_tier": "standard", + "inference_geo": "global", + }, + "diagnostics": None, + }, + client_response={ + "model": "anthropic/claude-sonnet-4-6", + "id": "msg_011CffzUNHaEfzVxCh5hskBG", + "type": "message", + "role": "assistant", + "content": [{"type": "text", "text": "Hello!"}], + "container": None, + "stop_reason": "end_turn", + "stop_sequence": None, + "stop_details": None, + "usage": { + "input_tokens": 18, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 5, + "service_tier": "standard", + "inference_geo": "global", + }, + "diagnostics": None, + }, +) diff --git a/tests/integration/translation/messages/basic/__init__.py b/tests/integration/translation/messages/basic/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/translation/messages/basic/test_messages_basic_anthropic.py b/tests/integration/translation/messages/basic/test_messages_basic_anthropic.py new file mode 100644 index 000000000000..afd0e6a0362c --- /dev/null +++ b/tests/integration/translation/messages/basic/test_messages_basic_anthropic.py @@ -0,0 +1,11 @@ +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.messages.bases.anthropic import CLAUDE_SONNET_4_6 +from integration.translation.runner import run + + +@pytest.mark.parametrize("case", [CLAUDE_SONNET_4_6], ids=lambda case: case.id) +def test_messages_basic_anthropic(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + run(case, gateway, provider) diff --git a/tests/integration/translation/messages/reasoning/__init__.py b/tests/integration/translation/messages/reasoning/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/translation/messages/reasoning/test_messages_reasoning_anthropic.py b/tests/integration/translation/messages/reasoning/test_messages_reasoning_anthropic.py new file mode 100644 index 000000000000..e7ece3841937 --- /dev/null +++ b/tests/integration/translation/messages/reasoning/test_messages_reasoning_anthropic.py @@ -0,0 +1,73 @@ +from dataclasses import replace +from typing import Final + +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.messages.bases.anthropic import CLAUDE_SONNET_4_6 as BASE +from integration.translation.runner import run + +SIGNATURE_1: Final = ( + "EpECCqgBCBIYAipAivUPApu85FYYe3+cXal8EiJOza7QGqKyekC8vDSn4oyeqGa2CrarO4abiuG7dzBXjmYR8+daw4h50ZjKmak7czIRY2xh" + "dWRlLXNvbm5ldC00LTY4AEIIdGhpbmtpbmdaJGQwMDgxZjJiLWQ5NjEtNGFhYi05ZTRjLTcxYmU3ZTA0ZTY3MJoBEwoRY2xhdWRlLXNvbm5l" + "dC00LTaoAY3fhdYGEgwJHsjNkCTlV9k1jWQaDOmeP/z67YtLTojSqCIwhRXGrNzSuGfMD1HqA72lctQCy83Wkr0u8W5lBXXn+MD6WfJGTJqM" + "1FW7qRmOMOKJKhbheeMpsTs7XvdvsiDQqgM4PAJt4cwgGAE=" +) + +THINKING_BUDGET: Final = replace( + BASE, + scenario="thinking_budget", + regressions=("LIT-4957",), + client_request={ + **BASE.client_request, + "max_tokens": 2048, + "thinking": {"type": "enabled", "budget_tokens": 1024}, + }, + provider_request={ + **BASE.provider_request, + "max_tokens": 2048, + "thinking": {"type": "enabled", "budget_tokens": 1024}, + }, + provider_response={ + **BASE.provider_response, + "id": "msg_011CffzUREgTzMm1dXRqP2LR", + "content": [ + {"type": "thinking", "thinking": "Hello!", "signature": SIGNATURE_1}, + {"type": "text", "text": "Hello!"}, + ], + "usage": { + "input_tokens": 47, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 15, + "output_tokens_details": {"thinking_tokens": 7}, + "service_tier": "standard", + "inference_geo": "global", + }, + }, + client_response={ + **BASE.client_response, + "id": "msg_011CffzUREgTzMm1dXRqP2LR", + "content": [ + {"type": "thinking", "thinking": "Hello!", "signature": SIGNATURE_1}, + {"type": "text", "text": "Hello!"}, + ], + "usage": { + "input_tokens": 47, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 15, + "output_tokens_details": {"thinking_tokens": 7}, + "service_tier": "standard", + "inference_geo": "global", + }, + }, +) + + +@pytest.mark.parametrize("case", [THINKING_BUDGET], ids=lambda case: case.id) +def test_messages_reasoning_anthropic(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + run(case, gateway, provider) diff --git a/tests/integration/translation/runner.py b/tests/integration/translation/runner.py new file mode 100644 index 000000000000..ea61cb591b5a --- /dev/null +++ b/tests/integration/translation/runner.py @@ -0,0 +1,19 @@ +import json +from typing import Final + +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration._support.wire import Reply +from integration.translation.case import TranslationTestCase + + +def run(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + provider.expect(Reply(body=json.dumps(case.provider_response).encode())) + response: Final = gateway.request("POST", case.client_path, case.client_request) + received: Final = provider.received() + assert [(request.method, request.target) for request in received] == [("POST", case.provider_path)] + sent: Final = received[0] + assert {name: sent.headers.get(name) for name in case.provider_headers} == dict(case.provider_headers) + assert json.loads(sent.body) == case.provider_request + assert response.status_code == case.client_status, response.text + assert response.json() == case.client_response From ae6ed760bb3d8227a87394a3bb8557255ea9390a Mon Sep 17 00:00:00 2001 From: kerry Date: Sat, 3 Oct 2026 23:26:14 +0000 Subject: [PATCH 2/8] test(integration): compare every non-transport provider header and check for late provider requests at session end Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/conftest.py | 2 ++ tests/integration/translation/runner.py | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 7040f784f71c..c4d6dd9c6231 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -133,6 +133,8 @@ def shared_provider_server() -> Iterator[SharedProvider]: pytest.fail("the shared fake provider needs tests to run one at a time; this group runs under pytest-xdist") with shared_provider() as server: yield server + late: Final = server.received() + assert late == (), f"the shared fake provider got {[item.target for item in late]} after {server.last_test} finished" @pytest.fixture diff --git a/tests/integration/translation/runner.py b/tests/integration/translation/runner.py index ea61cb591b5a..2f793d7f73ce 100644 --- a/tests/integration/translation/runner.py +++ b/tests/integration/translation/runner.py @@ -6,6 +6,8 @@ from integration._support.wire import Reply from integration.translation.case import TranslationTestCase +TRANSPORT_HEADERS: Final = frozenset({"host", "accept", "accept-encoding", "connection", "content-length", "user-agent"}) + def run(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: provider.expect(Reply(body=json.dumps(case.provider_response).encode())) @@ -13,7 +15,7 @@ def run(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) - received: Final = provider.received() assert [(request.method, request.target) for request in received] == [("POST", case.provider_path)] sent: Final = received[0] - assert {name: sent.headers.get(name) for name in case.provider_headers} == dict(case.provider_headers) + assert {name: value for name, value in sent.headers.items() if name not in TRANSPORT_HEADERS} == dict(case.provider_headers) assert json.loads(sent.body) == case.provider_request assert response.status_code == case.client_status, response.text assert response.json() == case.client_response From b1c46c2c413a275c10629a5239a9b6e4ba7a643a Mon Sep 17 00:00:00 2001 From: kerry Date: Sat, 3 Oct 2026 23:34:16 +0000 Subject: [PATCH 3/8] test(integration): name TranslationTestCase fields after litellm and provider sides and drop regressions Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/translation/case.py | 13 ++++++------- .../translation/messages/bases/anthropic.py | 8 ++++---- .../reasoning/test_messages_reasoning_anthropic.py | 9 ++++----- tests/integration/translation/runner.py | 8 ++++---- 4 files changed, 18 insertions(+), 20 deletions(-) diff --git a/tests/integration/translation/case.py b/tests/integration/translation/case.py index fc03ed00e32a..64185566a6b6 100644 --- a/tests/integration/translation/case.py +++ b/tests/integration/translation/case.py @@ -10,16 +10,15 @@ class TranslationTestCase: exact request the provider must receive, the provider's reply, and the exact response the client must get.""" scenario: str - regressions: tuple[str, ...] = () - client_path: str - client_request: Mapping[str, JsonValue] - provider_path: str + litellm_endpoint: str + litellm_request: Mapping[str, JsonValue] + provider_endpoint: str provider_headers: Mapping[str, str] provider_request: Mapping[str, JsonValue] provider_response: Mapping[str, JsonValue] - client_status: int = 200 - client_response: Mapping[str, JsonValue] + litellm_status_code: int = 200 + litellm_response: Mapping[str, JsonValue] @property def id(self) -> str: - return f"{self.client_request['model']}-{self.scenario}" + return f"{self.litellm_request['model']}-{self.scenario}" diff --git a/tests/integration/translation/messages/bases/anthropic.py b/tests/integration/translation/messages/bases/anthropic.py index 4b3ea7fa50e7..cdf042f68808 100644 --- a/tests/integration/translation/messages/bases/anthropic.py +++ b/tests/integration/translation/messages/bases/anthropic.py @@ -4,15 +4,15 @@ CLAUDE_SONNET_4_6: Final = TranslationTestCase( scenario="basic", - client_path="/v1/messages", - client_request={ + litellm_endpoint="/v1/messages", + litellm_request={ "model": "anthropic/claude-sonnet-4-6", "max_tokens": 64, "system": "You are a terse assistant.", "messages": [{"role": "user", "content": "Say hello."}], "cache": {"no-cache": True}, }, - provider_path="/v1/messages", + provider_endpoint="/v1/messages", provider_headers={ "x-api-key": "synthetic-anthropic-key", "anthropic-version": "2023-06-01", @@ -46,7 +46,7 @@ }, "diagnostics": None, }, - client_response={ + litellm_response={ "model": "anthropic/claude-sonnet-4-6", "id": "msg_011CffzUNHaEfzVxCh5hskBG", "type": "message", diff --git a/tests/integration/translation/messages/reasoning/test_messages_reasoning_anthropic.py b/tests/integration/translation/messages/reasoning/test_messages_reasoning_anthropic.py index e7ece3841937..022f44f5ce8f 100644 --- a/tests/integration/translation/messages/reasoning/test_messages_reasoning_anthropic.py +++ b/tests/integration/translation/messages/reasoning/test_messages_reasoning_anthropic.py @@ -18,9 +18,8 @@ THINKING_BUDGET: Final = replace( BASE, scenario="thinking_budget", - regressions=("LIT-4957",), - client_request={ - **BASE.client_request, + litellm_request={ + **BASE.litellm_request, "max_tokens": 2048, "thinking": {"type": "enabled", "budget_tokens": 1024}, }, @@ -47,8 +46,8 @@ "inference_geo": "global", }, }, - client_response={ - **BASE.client_response, + litellm_response={ + **BASE.litellm_response, "id": "msg_011CffzUREgTzMm1dXRqP2LR", "content": [ {"type": "thinking", "thinking": "Hello!", "signature": SIGNATURE_1}, diff --git a/tests/integration/translation/runner.py b/tests/integration/translation/runner.py index 2f793d7f73ce..9d83f88aef9f 100644 --- a/tests/integration/translation/runner.py +++ b/tests/integration/translation/runner.py @@ -11,11 +11,11 @@ def run(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: provider.expect(Reply(body=json.dumps(case.provider_response).encode())) - response: Final = gateway.request("POST", case.client_path, case.client_request) + response: Final = gateway.request("POST", case.litellm_endpoint, case.litellm_request) received: Final = provider.received() - assert [(request.method, request.target) for request in received] == [("POST", case.provider_path)] + assert [(request.method, request.target) for request in received] == [("POST", case.provider_endpoint)] sent: Final = received[0] assert {name: value for name, value in sent.headers.items() if name not in TRANSPORT_HEADERS} == dict(case.provider_headers) assert json.loads(sent.body) == case.provider_request - assert response.status_code == case.client_status, response.text - assert response.json() == case.client_response + assert response.status_code == case.litellm_status_code, response.text + assert response.json() == case.litellm_response From 6e866cc75d63551ad100c1845b5f510d59e1ed68 Mon Sep 17 00:00:00 2001 From: kerry Date: Sat, 3 Oct 2026 23:36:09 +0000 Subject: [PATCH 4/8] test(integration): prefix checked TranslationTestCase fields with expected_ and name the fake reply mock_provider_response Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/translation/case.py | 12 ++++++------ .../translation/messages/bases/anthropic.py | 10 +++++----- .../reasoning/test_messages_reasoning_anthropic.py | 12 ++++++------ tests/integration/translation/runner.py | 12 ++++++------ 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/tests/integration/translation/case.py b/tests/integration/translation/case.py index 64185566a6b6..1b140abe1f7f 100644 --- a/tests/integration/translation/case.py +++ b/tests/integration/translation/case.py @@ -12,12 +12,12 @@ class TranslationTestCase: scenario: str litellm_endpoint: str litellm_request: Mapping[str, JsonValue] - provider_endpoint: str - provider_headers: Mapping[str, str] - provider_request: Mapping[str, JsonValue] - provider_response: Mapping[str, JsonValue] - litellm_status_code: int = 200 - litellm_response: Mapping[str, JsonValue] + expected_provider_endpoint: str + expected_provider_headers: Mapping[str, str] + expected_provider_request: Mapping[str, JsonValue] + mock_provider_response: Mapping[str, JsonValue] + expected_litellm_status_code: int = 200 + expected_litellm_response: Mapping[str, JsonValue] @property def id(self) -> str: diff --git a/tests/integration/translation/messages/bases/anthropic.py b/tests/integration/translation/messages/bases/anthropic.py index cdf042f68808..de00d4563df3 100644 --- a/tests/integration/translation/messages/bases/anthropic.py +++ b/tests/integration/translation/messages/bases/anthropic.py @@ -12,20 +12,20 @@ "messages": [{"role": "user", "content": "Say hello."}], "cache": {"no-cache": True}, }, - provider_endpoint="/v1/messages", - provider_headers={ + expected_provider_endpoint="/v1/messages", + expected_provider_headers={ "x-api-key": "synthetic-anthropic-key", "anthropic-version": "2023-06-01", "content-type": "application/json", }, - provider_request={ + expected_provider_request={ "model": "claude-sonnet-4-6", "max_tokens": 64, "stream": False, "system": "You are a terse assistant.", "messages": [{"role": "user", "content": "Say hello."}], }, - provider_response={ + mock_provider_response={ "model": "claude-sonnet-4-6", "id": "msg_011CffzUNHaEfzVxCh5hskBG", "type": "message", @@ -46,7 +46,7 @@ }, "diagnostics": None, }, - litellm_response={ + expected_litellm_response={ "model": "anthropic/claude-sonnet-4-6", "id": "msg_011CffzUNHaEfzVxCh5hskBG", "type": "message", diff --git a/tests/integration/translation/messages/reasoning/test_messages_reasoning_anthropic.py b/tests/integration/translation/messages/reasoning/test_messages_reasoning_anthropic.py index 022f44f5ce8f..b3009a9ce500 100644 --- a/tests/integration/translation/messages/reasoning/test_messages_reasoning_anthropic.py +++ b/tests/integration/translation/messages/reasoning/test_messages_reasoning_anthropic.py @@ -23,13 +23,13 @@ "max_tokens": 2048, "thinking": {"type": "enabled", "budget_tokens": 1024}, }, - provider_request={ - **BASE.provider_request, + expected_provider_request={ + **BASE.expected_provider_request, "max_tokens": 2048, "thinking": {"type": "enabled", "budget_tokens": 1024}, }, - provider_response={ - **BASE.provider_response, + mock_provider_response={ + **BASE.mock_provider_response, "id": "msg_011CffzUREgTzMm1dXRqP2LR", "content": [ {"type": "thinking", "thinking": "Hello!", "signature": SIGNATURE_1}, @@ -46,8 +46,8 @@ "inference_geo": "global", }, }, - litellm_response={ - **BASE.litellm_response, + expected_litellm_response={ + **BASE.expected_litellm_response, "id": "msg_011CffzUREgTzMm1dXRqP2LR", "content": [ {"type": "thinking", "thinking": "Hello!", "signature": SIGNATURE_1}, diff --git a/tests/integration/translation/runner.py b/tests/integration/translation/runner.py index 9d83f88aef9f..81c858ad303a 100644 --- a/tests/integration/translation/runner.py +++ b/tests/integration/translation/runner.py @@ -10,12 +10,12 @@ def run(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: - provider.expect(Reply(body=json.dumps(case.provider_response).encode())) + provider.expect(Reply(body=json.dumps(case.mock_provider_response).encode())) response: Final = gateway.request("POST", case.litellm_endpoint, case.litellm_request) received: Final = provider.received() - assert [(request.method, request.target) for request in received] == [("POST", case.provider_endpoint)] + assert [(request.method, request.target) for request in received] == [("POST", case.expected_provider_endpoint)] sent: Final = received[0] - assert {name: value for name, value in sent.headers.items() if name not in TRANSPORT_HEADERS} == dict(case.provider_headers) - assert json.loads(sent.body) == case.provider_request - assert response.status_code == case.litellm_status_code, response.text - assert response.json() == case.litellm_response + assert {name: value for name, value in sent.headers.items() if name not in TRANSPORT_HEADERS} == dict(case.expected_provider_headers) + assert json.loads(sent.body) == case.expected_provider_request + assert response.status_code == case.expected_litellm_status_code, response.text + assert response.json() == case.expected_litellm_response From e32373985eb198c648db4f128148ecf60deb5f71 Mon Sep 17 00:00:00 2001 From: kerry Date: Sat, 3 Oct 2026 23:36:54 +0000 Subject: [PATCH 5/8] docs(integration): name TranslationTestCase fields in the translation README Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/README.md | 4 ++-- tests/integration/translation/case.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/README.md b/tests/integration/README.md index 852fd434cafe..92fe3090177a 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -18,11 +18,11 @@ There is no per-node manifest. A positional argument is a file of the group or a Provider sentinels currently use the controlled server, not live recordings. The provider shard also runs the existing strict replay controls for changed requests, exhausted interactions, leftover interactions and no provider connection. Future recorded scenarios must use that replay-only implementation; missing recordings cannot fall back to a real provider. The observation endpoint is destructive and the current selection runs serially against one owned upstream -Translation tests in `translation/` check one request through the proxy as four literals on a `TranslationTestCase`: the client request, the exact provider request, the provider response and the exact client response. The runner compares the provider request body and the client response body in full, so an added, removed, renamed or moved field fails. Folders follow the client endpoint and then the feature, for example `translation/messages/reasoning/`. Each endpoint keeps one complete base case per model in `/bases/.py`, which `/basic/` runs on its own, and a feature case is `dataclasses.replace(BASE, ...)` that lists only the fields it changes +Translation tests in `translation/` check one request through the proxy as literals on a `TranslationTestCase`: the `litellm_endpoint` and `litellm_request` the test sends, the `expected_provider_endpoint`, `expected_provider_headers` and `expected_provider_request` the fake provider must receive, the `mock_provider_response` it answers with, and the `expected_litellm_status_code` and `expected_litellm_response` the test must get back. The runner compares the provider request body, every provider header other than transport headers, and the LiteLLM response body in full, so an added, removed, renamed or moved field fails. Folders follow the client endpoint and then the feature, for example `translation/messages/reasoning/`. Each endpoint keeps one complete base case per model in `/bases/.py`, which `/basic/` runs on its own, and a feature case is `dataclasses.replace(BASE, ...)` that lists only the fields it changes Deployments used by translation tests are shared by the whole suite and declared in `proxy_config.yaml` under `model_list`, with `model_name` equal to the litellm model string, `api_base: http://127.0.0.1:8191` and a synthetic key. A case names the deployment literally in its client request. The fake provider behind them is the `provider` fixture: one `wire_server` on port 8191 inside the pytest process, started the first time a test asks for it. A test queues its replies with `provider.expect(...)` and reads what the proxy sent with `provider.received()`. Tests that use it run one at a time. After each test the fixture fails if a queued reply was never requested or a received request was never read, and before each test it fails if a request arrived in between, naming the previous test. It refuses to start under pytest-xdist, so the `mcp` and `cost` groups cannot use it. Client requests carry `"cache": {"no-cache": True}` because the proxy caches responses in Redis -Provider responses in translation cases are captured once from the real provider and stored verbatim. First run the new case against the fake provider; the body the proxy sends is the case's provider request. Send that exact body to the real provider endpoint with a key from the 1Password `Shared` vault (`/qa-keys`), and only store the case when the provider answers 2xx. Keep only the response body and drop every response header, since headers carry account identifiers such as the organization id and rate limits. Never print or save the request headers you sent. Before committing, check the body contains no key and no account identifier, such as an organization id, an AWS account id inside an ARN, a GCP project id or an Azure resource name, and that the prompt is synthetic. Paste the body as the case's provider response without shortening ids, token counts or signatures, and move long opaque values such as thinking signatures into module-level constants used by both the provider response and the client response. Do not commit the script used for the capture +Provider responses in translation cases are captured once from the real provider and stored verbatim. First run the new case against the fake provider; the body the proxy sends is the case's `expected_provider_request`. Send that exact body to the real provider endpoint with a key from the 1Password `Shared` vault (`/qa-keys`), and only store the case when the provider answers 2xx. Keep only the response body and drop every response header, since headers carry account identifiers such as the organization id and rate limits. Never print or save the request headers you sent. Before committing, check the body contains no key and no account identifier, such as an organization id, an AWS account id inside an ARN, a GCP project id or an Azure resource name, and that the prompt is synthetic. Paste the body as the case's `mock_provider_response` without shortening ids, token counts or signatures, and move long opaque values such as thinking signatures into module-level constants used by both `mock_provider_response` and `expected_litellm_response`. Do not commit the script used for the capture Fixtures must contain synthetic data only. Keep private incident records and source documents out of code, fixtures, logs and PR descriptions diff --git a/tests/integration/translation/case.py b/tests/integration/translation/case.py index 1b140abe1f7f..ac8f43439cee 100644 --- a/tests/integration/translation/case.py +++ b/tests/integration/translation/case.py @@ -6,8 +6,8 @@ @dataclass(frozen=True, slots=True, kw_only=True) class TranslationTestCase: - """One client request through the proxy to a deployment in `proxy_config.yaml`: what the client sends, the - exact request the provider must receive, the provider's reply, and the exact response the client must get.""" + """One request through the proxy to a deployment in `proxy_config.yaml`: what the test sends to LiteLLM, the + exact request the provider must receive, the fake provider's reply, and the exact response LiteLLM must return.""" scenario: str litellm_endpoint: str From fffdcb1d127a6839c3e149131840c2e024eaeb52 Mon Sep 17 00:00:00 2001 From: kerry Date: Sat, 3 Oct 2026 23:45:05 +0000 Subject: [PATCH 6/8] test(integration): add a claude-opus-5-5 base case and deployment next to claude-sonnet-4-6 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/proxy_config.yaml | 5 ++ .../translation/messages/bases/anthropic.py | 69 +++++++++++++++++++ .../basic/test_messages_basic_anthropic.py | 4 +- 3 files changed, 76 insertions(+), 2 deletions(-) diff --git a/tests/integration/proxy_config.yaml b/tests/integration/proxy_config.yaml index 50fd6c9d32dc..4c9db3945655 100644 --- a/tests/integration/proxy_config.yaml +++ b/tests/integration/proxy_config.yaml @@ -1,4 +1,9 @@ model_list: + - model_name: anthropic/claude-opus-5-5 + litellm_params: + model: anthropic/claude-opus-5-5 + api_base: http://127.0.0.1:8191 + api_key: synthetic-anthropic-key - model_name: anthropic/claude-sonnet-4-6 litellm_params: model: anthropic/claude-sonnet-4-6 diff --git a/tests/integration/translation/messages/bases/anthropic.py b/tests/integration/translation/messages/bases/anthropic.py index de00d4563df3..1f9cd2419bf6 100644 --- a/tests/integration/translation/messages/bases/anthropic.py +++ b/tests/integration/translation/messages/bases/anthropic.py @@ -2,6 +2,75 @@ from integration.translation.case import TranslationTestCase +CLAUDE_OPUS_5_5: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/messages", + litellm_request={ + "model": "anthropic/claude-opus-5-5", + "max_tokens": 64, + "system": "You are a terse assistant.", + "messages": [{"role": "user", "content": "Say hello."}], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/v1/messages", + expected_provider_headers={ + "x-api-key": "synthetic-anthropic-key", + "anthropic-version": "2023-06-01", + "content-type": "application/json", + }, + expected_provider_request={ + "model": "claude-opus-5-5", + "max_tokens": 64, + "stream": False, + "system": "You are a terse assistant.", + "messages": [{"role": "user", "content": "Say hello."}], + }, + mock_provider_response={ + "model": "claude-opus-5-5", + "id": "msg_011CfgC5HKvyve78CTFAw97f", + "type": "message", + "role": "assistant", + "content": [{"type": "text", "text": "Hello."}], + "container": None, + "stop_reason": "end_turn", + "stop_sequence": None, + "stop_details": None, + "usage": { + "input_tokens": 23, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 6, + "output_tokens_details": {"thinking_tokens": 0}, + "service_tier": "standard", + "inference_geo": "global", + }, + "diagnostics": None, + }, + expected_litellm_response={ + "model": "anthropic/claude-opus-5-5", + "id": "msg_011CfgC5HKvyve78CTFAw97f", + "type": "message", + "role": "assistant", + "content": [{"type": "text", "text": "Hello."}], + "container": None, + "stop_reason": "end_turn", + "stop_sequence": None, + "stop_details": None, + "usage": { + "input_tokens": 23, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 6, + "output_tokens_details": {"thinking_tokens": 0}, + "service_tier": "standard", + "inference_geo": "global", + }, + "diagnostics": None, + }, +) + CLAUDE_SONNET_4_6: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/messages", diff --git a/tests/integration/translation/messages/basic/test_messages_basic_anthropic.py b/tests/integration/translation/messages/basic/test_messages_basic_anthropic.py index afd0e6a0362c..75c2344c1adb 100644 --- a/tests/integration/translation/messages/basic/test_messages_basic_anthropic.py +++ b/tests/integration/translation/messages/basic/test_messages_basic_anthropic.py @@ -2,10 +2,10 @@ from integration._support.client import Gateway from integration._support.provider import SharedProvider from integration.translation.case import TranslationTestCase -from integration.translation.messages.bases.anthropic import CLAUDE_SONNET_4_6 +from integration.translation.messages.bases.anthropic import CLAUDE_OPUS_5_5, CLAUDE_SONNET_4_6 from integration.translation.runner import run -@pytest.mark.parametrize("case", [CLAUDE_SONNET_4_6], ids=lambda case: case.id) +@pytest.mark.parametrize("case", [CLAUDE_OPUS_5_5, CLAUDE_SONNET_4_6], ids=lambda case: case.id) def test_messages_basic_anthropic(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: run(case, gateway, provider) From 980460636aa46237c7515f9c0ab856a41575a206 Mon Sep 17 00:00:00 2001 From: kerry Date: Sat, 3 Oct 2026 23:54:27 +0000 Subject: [PATCH 7/8] test(integration): name translation cases _TEST_CASE and document the naming rule Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/AGENTS.md | 18 ++++++++++++++++++ tests/integration/README.md | 2 +- .../translation/messages/bases/anthropic.py | 4 ++-- .../basic/test_messages_basic_anthropic.py | 4 ++-- .../test_messages_reasoning_anthropic.py | 16 ++++++++-------- 5 files changed, 31 insertions(+), 13 deletions(-) diff --git a/tests/integration/AGENTS.md b/tests/integration/AGENTS.md index 0499bfc97c83..7688f2ebee49 100644 --- a/tests/integration/AGENTS.md +++ b/tests/integration/AGENTS.md @@ -25,3 +25,21 @@ By the domain a user would name: `pricing`, `spend`, `routing`, `mcp`. A file on directory that a `GROUPS` entry in `run.py` selects; there is no manifest and no `covers` marker on new tests. A product bug the test exposes is `pytest.skip("BUG: ")` at the top of the body, not a fix in the test and not a deletion. Needs no proxy, DB or Redis: `tests/unit` + +## Translation cases + +Each model gets one complete base `TranslationTestCase` in `translation//bases/.py`, +named `_TEST_CASE` after its deployment (`anthropic/claude-sonnet-4-6` is +`CLAUDE_SONNET_4_6_TEST_CASE`). A feature case is `__TEST_CASE`. Import a base under its +own name, never aliased to `BASE`, so every case shows which model it derives from + +```python +from integration.translation.messages.bases.anthropic import CLAUDE_SONNET_4_6_TEST_CASE + +CLAUDE_SONNET_4_6_THINKING_BUDGET_TEST_CASE: Final = replace( + CLAUDE_SONNET_4_6_TEST_CASE, + scenario="thinking_budget", + litellm_request={**CLAUDE_SONNET_4_6_TEST_CASE.litellm_request, "max_tokens": 2048, "thinking": ...}, + ... +) +``` diff --git a/tests/integration/README.md b/tests/integration/README.md index 92fe3090177a..7d09f2a97486 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -18,7 +18,7 @@ There is no per-node manifest. A positional argument is a file of the group or a Provider sentinels currently use the controlled server, not live recordings. The provider shard also runs the existing strict replay controls for changed requests, exhausted interactions, leftover interactions and no provider connection. Future recorded scenarios must use that replay-only implementation; missing recordings cannot fall back to a real provider. The observation endpoint is destructive and the current selection runs serially against one owned upstream -Translation tests in `translation/` check one request through the proxy as literals on a `TranslationTestCase`: the `litellm_endpoint` and `litellm_request` the test sends, the `expected_provider_endpoint`, `expected_provider_headers` and `expected_provider_request` the fake provider must receive, the `mock_provider_response` it answers with, and the `expected_litellm_status_code` and `expected_litellm_response` the test must get back. The runner compares the provider request body, every provider header other than transport headers, and the LiteLLM response body in full, so an added, removed, renamed or moved field fails. Folders follow the client endpoint and then the feature, for example `translation/messages/reasoning/`. Each endpoint keeps one complete base case per model in `/bases/.py`, which `/basic/` runs on its own, and a feature case is `dataclasses.replace(BASE, ...)` that lists only the fields it changes +Translation tests in `translation/` check one request through the proxy as literals on a `TranslationTestCase`: the `litellm_endpoint` and `litellm_request` the test sends, the `expected_provider_endpoint`, `expected_provider_headers` and `expected_provider_request` the fake provider must receive, the `mock_provider_response` it answers with, and the `expected_litellm_status_code` and `expected_litellm_response` the test must get back. The runner compares the provider request body, every provider header other than transport headers, and the LiteLLM response body in full, so an added, removed, renamed or moved field fails. Folders follow the client endpoint and then the feature, for example `translation/messages/reasoning/`. Each endpoint keeps one complete base case per model in `/bases/.py`, named `_TEST_CASE` (for example `CLAUDE_SONNET_4_6_TEST_CASE`), which `/basic/` runs on its own. A feature case is `__TEST_CASE = dataclasses.replace(_TEST_CASE, ...)`, imports the base under its own name rather than as `BASE`, and lists only the fields it changes Deployments used by translation tests are shared by the whole suite and declared in `proxy_config.yaml` under `model_list`, with `model_name` equal to the litellm model string, `api_base: http://127.0.0.1:8191` and a synthetic key. A case names the deployment literally in its client request. The fake provider behind them is the `provider` fixture: one `wire_server` on port 8191 inside the pytest process, started the first time a test asks for it. A test queues its replies with `provider.expect(...)` and reads what the proxy sent with `provider.received()`. Tests that use it run one at a time. After each test the fixture fails if a queued reply was never requested or a received request was never read, and before each test it fails if a request arrived in between, naming the previous test. It refuses to start under pytest-xdist, so the `mcp` and `cost` groups cannot use it. Client requests carry `"cache": {"no-cache": True}` because the proxy caches responses in Redis diff --git a/tests/integration/translation/messages/bases/anthropic.py b/tests/integration/translation/messages/bases/anthropic.py index 1f9cd2419bf6..0642175a739c 100644 --- a/tests/integration/translation/messages/bases/anthropic.py +++ b/tests/integration/translation/messages/bases/anthropic.py @@ -2,7 +2,7 @@ from integration.translation.case import TranslationTestCase -CLAUDE_OPUS_5_5: Final = TranslationTestCase( +CLAUDE_OPUS_5_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/messages", litellm_request={ @@ -71,7 +71,7 @@ }, ) -CLAUDE_SONNET_4_6: Final = TranslationTestCase( +CLAUDE_SONNET_4_6_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/messages", litellm_request={ diff --git a/tests/integration/translation/messages/basic/test_messages_basic_anthropic.py b/tests/integration/translation/messages/basic/test_messages_basic_anthropic.py index 75c2344c1adb..3fbca2addd5f 100644 --- a/tests/integration/translation/messages/basic/test_messages_basic_anthropic.py +++ b/tests/integration/translation/messages/basic/test_messages_basic_anthropic.py @@ -2,10 +2,10 @@ from integration._support.client import Gateway from integration._support.provider import SharedProvider from integration.translation.case import TranslationTestCase -from integration.translation.messages.bases.anthropic import CLAUDE_OPUS_5_5, CLAUDE_SONNET_4_6 +from integration.translation.messages.bases.anthropic import CLAUDE_OPUS_5_5_TEST_CASE, CLAUDE_SONNET_4_6_TEST_CASE from integration.translation.runner import run -@pytest.mark.parametrize("case", [CLAUDE_OPUS_5_5, CLAUDE_SONNET_4_6], ids=lambda case: case.id) +@pytest.mark.parametrize("case", [CLAUDE_OPUS_5_5_TEST_CASE, CLAUDE_SONNET_4_6_TEST_CASE], ids=lambda case: case.id) def test_messages_basic_anthropic(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: run(case, gateway, provider) diff --git a/tests/integration/translation/messages/reasoning/test_messages_reasoning_anthropic.py b/tests/integration/translation/messages/reasoning/test_messages_reasoning_anthropic.py index b3009a9ce500..555e55cc4a58 100644 --- a/tests/integration/translation/messages/reasoning/test_messages_reasoning_anthropic.py +++ b/tests/integration/translation/messages/reasoning/test_messages_reasoning_anthropic.py @@ -5,7 +5,7 @@ from integration._support.client import Gateway from integration._support.provider import SharedProvider from integration.translation.case import TranslationTestCase -from integration.translation.messages.bases.anthropic import CLAUDE_SONNET_4_6 as BASE +from integration.translation.messages.bases.anthropic import CLAUDE_SONNET_4_6_TEST_CASE from integration.translation.runner import run SIGNATURE_1: Final = ( @@ -15,21 +15,21 @@ "1FW7qRmOMOKJKhbheeMpsTs7XvdvsiDQqgM4PAJt4cwgGAE=" ) -THINKING_BUDGET: Final = replace( - BASE, +CLAUDE_SONNET_4_6_THINKING_BUDGET_TEST_CASE: Final = replace( + CLAUDE_SONNET_4_6_TEST_CASE, scenario="thinking_budget", litellm_request={ - **BASE.litellm_request, + **CLAUDE_SONNET_4_6_TEST_CASE.litellm_request, "max_tokens": 2048, "thinking": {"type": "enabled", "budget_tokens": 1024}, }, expected_provider_request={ - **BASE.expected_provider_request, + **CLAUDE_SONNET_4_6_TEST_CASE.expected_provider_request, "max_tokens": 2048, "thinking": {"type": "enabled", "budget_tokens": 1024}, }, mock_provider_response={ - **BASE.mock_provider_response, + **CLAUDE_SONNET_4_6_TEST_CASE.mock_provider_response, "id": "msg_011CffzUREgTzMm1dXRqP2LR", "content": [ {"type": "thinking", "thinking": "Hello!", "signature": SIGNATURE_1}, @@ -47,7 +47,7 @@ }, }, expected_litellm_response={ - **BASE.expected_litellm_response, + **CLAUDE_SONNET_4_6_TEST_CASE.expected_litellm_response, "id": "msg_011CffzUREgTzMm1dXRqP2LR", "content": [ {"type": "thinking", "thinking": "Hello!", "signature": SIGNATURE_1}, @@ -67,6 +67,6 @@ ) -@pytest.mark.parametrize("case", [THINKING_BUDGET], ids=lambda case: case.id) +@pytest.mark.parametrize("case", [CLAUDE_SONNET_4_6_THINKING_BUDGET_TEST_CASE], ids=lambda case: case.id) def test_messages_reasoning_anthropic(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: run(case, gateway, provider) From 502f51a50ef2704ee2df11ff46d6c9513fc98d1d Mon Sep 17 00:00:00 2001 From: kerry Date: Sun, 4 Oct 2026 00:01:18 +0000 Subject: [PATCH 8/8] docs(integration): move translation test rules into tests/integration/translation Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/AGENTS.md | 18 ------------------ tests/integration/README.md | 6 +----- tests/integration/translation/AGENTS.md | 21 +++++++++++++++++++++ tests/integration/translation/README.md | 7 +++++++ 4 files changed, 29 insertions(+), 23 deletions(-) create mode 100644 tests/integration/translation/AGENTS.md create mode 100644 tests/integration/translation/README.md diff --git a/tests/integration/AGENTS.md b/tests/integration/AGENTS.md index 7688f2ebee49..0499bfc97c83 100644 --- a/tests/integration/AGENTS.md +++ b/tests/integration/AGENTS.md @@ -25,21 +25,3 @@ By the domain a user would name: `pricing`, `spend`, `routing`, `mcp`. A file on directory that a `GROUPS` entry in `run.py` selects; there is no manifest and no `covers` marker on new tests. A product bug the test exposes is `pytest.skip("BUG: ")` at the top of the body, not a fix in the test and not a deletion. Needs no proxy, DB or Redis: `tests/unit` - -## Translation cases - -Each model gets one complete base `TranslationTestCase` in `translation//bases/.py`, -named `_TEST_CASE` after its deployment (`anthropic/claude-sonnet-4-6` is -`CLAUDE_SONNET_4_6_TEST_CASE`). A feature case is `__TEST_CASE`. Import a base under its -own name, never aliased to `BASE`, so every case shows which model it derives from - -```python -from integration.translation.messages.bases.anthropic import CLAUDE_SONNET_4_6_TEST_CASE - -CLAUDE_SONNET_4_6_THINKING_BUDGET_TEST_CASE: Final = replace( - CLAUDE_SONNET_4_6_TEST_CASE, - scenario="thinking_budget", - litellm_request={**CLAUDE_SONNET_4_6_TEST_CASE.litellm_request, "max_tokens": 2048, "thinking": ...}, - ... -) -``` diff --git a/tests/integration/README.md b/tests/integration/README.md index 7d09f2a97486..6cadb536ee4e 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -18,11 +18,7 @@ There is no per-node manifest. A positional argument is a file of the group or a Provider sentinels currently use the controlled server, not live recordings. The provider shard also runs the existing strict replay controls for changed requests, exhausted interactions, leftover interactions and no provider connection. Future recorded scenarios must use that replay-only implementation; missing recordings cannot fall back to a real provider. The observation endpoint is destructive and the current selection runs serially against one owned upstream -Translation tests in `translation/` check one request through the proxy as literals on a `TranslationTestCase`: the `litellm_endpoint` and `litellm_request` the test sends, the `expected_provider_endpoint`, `expected_provider_headers` and `expected_provider_request` the fake provider must receive, the `mock_provider_response` it answers with, and the `expected_litellm_status_code` and `expected_litellm_response` the test must get back. The runner compares the provider request body, every provider header other than transport headers, and the LiteLLM response body in full, so an added, removed, renamed or moved field fails. Folders follow the client endpoint and then the feature, for example `translation/messages/reasoning/`. Each endpoint keeps one complete base case per model in `/bases/.py`, named `_TEST_CASE` (for example `CLAUDE_SONNET_4_6_TEST_CASE`), which `/basic/` runs on its own. A feature case is `__TEST_CASE = dataclasses.replace(_TEST_CASE, ...)`, imports the base under its own name rather than as `BASE`, and lists only the fields it changes - -Deployments used by translation tests are shared by the whole suite and declared in `proxy_config.yaml` under `model_list`, with `model_name` equal to the litellm model string, `api_base: http://127.0.0.1:8191` and a synthetic key. A case names the deployment literally in its client request. The fake provider behind them is the `provider` fixture: one `wire_server` on port 8191 inside the pytest process, started the first time a test asks for it. A test queues its replies with `provider.expect(...)` and reads what the proxy sent with `provider.received()`. Tests that use it run one at a time. After each test the fixture fails if a queued reply was never requested or a received request was never read, and before each test it fails if a request arrived in between, naming the previous test. It refuses to start under pytest-xdist, so the `mcp` and `cost` groups cannot use it. Client requests carry `"cache": {"no-cache": True}` because the proxy caches responses in Redis - -Provider responses in translation cases are captured once from the real provider and stored verbatim. First run the new case against the fake provider; the body the proxy sends is the case's `expected_provider_request`. Send that exact body to the real provider endpoint with a key from the 1Password `Shared` vault (`/qa-keys`), and only store the case when the provider answers 2xx. Keep only the response body and drop every response header, since headers carry account identifiers such as the organization id and rate limits. Never print or save the request headers you sent. Before committing, check the body contains no key and no account identifier, such as an organization id, an AWS account id inside an ARN, a GCP project id or an Azure resource name, and that the prompt is synthetic. Paste the body as the case's `mock_provider_response` without shortening ids, token counts or signatures, and move long opaque values such as thinking signatures into module-level constants used by both `mock_provider_response` and `expected_litellm_response`. Do not commit the script used for the capture +Translation tests in `translation/` compare exact provider requests and LiteLLM responses against a shared fake provider; `translation/README.md` has their rules Fixtures must contain synthetic data only. Keep private incident records and source documents out of code, fixtures, logs and PR descriptions diff --git a/tests/integration/translation/AGENTS.md b/tests/integration/translation/AGENTS.md new file mode 100644 index 000000000000..33d176256e3e --- /dev/null +++ b/tests/integration/translation/AGENTS.md @@ -0,0 +1,21 @@ +# tests/integration/translation + +Exact translation cases on the shared fake provider. `README.md` here has the case fields, deployments and capture steps + +## Naming + +Each model gets one complete base `TranslationTestCase` in `translation//bases/.py`, +named `_TEST_CASE` after its deployment (`anthropic/claude-sonnet-4-6` is +`CLAUDE_SONNET_4_6_TEST_CASE`). A feature case is `__TEST_CASE`. Import a base under its +own name, never aliased to `BASE`, so every case shows which model it derives from + +```python +from integration.translation.messages.bases.anthropic import CLAUDE_SONNET_4_6_TEST_CASE + +CLAUDE_SONNET_4_6_THINKING_BUDGET_TEST_CASE: Final = replace( + CLAUDE_SONNET_4_6_TEST_CASE, + scenario="thinking_budget", + litellm_request={**CLAUDE_SONNET_4_6_TEST_CASE.litellm_request, "max_tokens": 2048, "thinking": ...}, + ... +) +``` diff --git a/tests/integration/translation/README.md b/tests/integration/translation/README.md new file mode 100644 index 000000000000..65b8e0659260 --- /dev/null +++ b/tests/integration/translation/README.md @@ -0,0 +1,7 @@ +# Translation tests + +These tests check one request through the proxy as literals on a `TranslationTestCase`: the `litellm_endpoint` and `litellm_request` the test sends, the `expected_provider_endpoint`, `expected_provider_headers` and `expected_provider_request` the fake provider must receive, the `mock_provider_response` it answers with, and the `expected_litellm_status_code` and `expected_litellm_response` the test must get back. The runner compares the provider request body, every provider header other than transport headers, and the LiteLLM response body in full, so an added, removed, renamed or moved field fails. Folders follow the client endpoint and then the feature, for example `translation/messages/reasoning/`. Each endpoint keeps one complete base case per model in `/bases/.py`, named `_TEST_CASE` (for example `CLAUDE_SONNET_4_6_TEST_CASE`), which `/basic/` runs on its own. A feature case is `__TEST_CASE = dataclasses.replace(_TEST_CASE, ...)`, imports the base under its own name rather than as `BASE`, and lists only the fields it changes + +Deployments used by translation tests are shared by the whole suite and declared in `proxy_config.yaml` under `model_list`, with `model_name` equal to the litellm model string, `api_base: http://127.0.0.1:8191` and a synthetic key. A case names the deployment literally in its client request. The fake provider behind them is the `provider` fixture: one `wire_server` on port 8191 inside the pytest process, started the first time a test asks for it. A test queues its replies with `provider.expect(...)` and reads what the proxy sent with `provider.received()`. Tests that use it run one at a time. After each test the fixture fails if a queued reply was never requested or a received request was never read, and before each test it fails if a request arrived in between, naming the previous test. It refuses to start under pytest-xdist, so the `mcp` and `cost` groups cannot use it. Client requests carry `"cache": {"no-cache": True}` because the proxy caches responses in Redis + +Provider responses in translation cases are captured once from the real provider and stored verbatim. First run the new case against the fake provider; the body the proxy sends is the case's `expected_provider_request`. Send that exact body to the real provider endpoint with a key from the 1Password `Shared` vault (`/qa-keys`), and only store the case when the provider answers 2xx. Keep only the response body and drop every response header, since headers carry account identifiers such as the organization id and rate limits. Never print or save the request headers you sent. Before committing, check the body contains no key and no account identifier, such as an organization id, an AWS account id inside an ARN, a GCP project id or an Azure resource name, and that the prompt is synthetic. Paste the body as the case's `mock_provider_response` without shortening ids, token counts or signatures, and move long opaque values such as thinking signatures into module-level constants used by both `mock_provider_response` and `expected_litellm_response`. Do not commit the script used for the capture