ci: gate tests/e2e on zero basedpyright errors in pre-commit and lint CI

.github/workflows/test-linting.yml

@@ -48,7 +48,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          uv sync --frozen --group proxy-dev
+          uv sync --frozen --group proxy-dev --group e2e-dev
 
       # basedpyright resolves Prisma's generated client (litellm/proxy/schema.prisma)
       # only after `prisma generate` writes prisma/client.py et al. Without this the
@@ -107,6 +107,16 @@ jobs:
         run: |
           (uv run --no-sync basedpyright --outputjson || true) | uv run --no-sync python scripts/type_check_gate.py --base "$BASE_SHA"
 
+      - name: Check tests/e2e basedpyright (zero errors)
+        env:
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+        run: |
+          if git diff --name-only --diff-filter=ACMRD "$BASE_SHA"...HEAD -- 'tests/e2e/**/*.py' | grep -q .; then
+            uv run --no-sync basedpyright tests/e2e
+          else
+            echo "No changed tests/e2e Python files; skipping."
+          fi
+
       - name: Check for circular imports
         run: |
           cd litellm

Makefile

@@ -5,7 +5,7 @@
 	test-unit-integrations test-unit-core-utils test-unit-other test-unit-root \
 	test-proxy-unit-a test-proxy-unit-b test-integration test-unit-helm \
 	info lint lint-dev lint-checks format \
-	lint-basedpyright lint-basedpyright-budget-update lint-type-discipline lint-type-discipline-budget-update \
+	lint-basedpyright lint-e2e-basedpyright lint-basedpyright-budget-update lint-type-discipline lint-type-discipline-budget-update \
 	lint-ruff-budget lint-ruff-budget-update lint-budget-update lint-gate \
 	install-dev install-proxy-dev install-test-deps install-hooks \
 	install-helm-unittest check-circular-imports check-import-safety pre-commit \
@@ -27,6 +27,7 @@ help:
 	@echo "  make lint               - Run all linting (Ruff, basedpyright, format check, circular imports, import safety)"
 	@echo "  make lint-ruff          - Run Ruff linting only"
 	@echo "  make lint-basedpyright  - Run basedpyright strict, gated by per-rule error counts"
+	@echo "  make lint-e2e-basedpyright - Run basedpyright over tests/e2e (zero errors allowed)"
 	@echo "  make lint-basedpyright-budget-update - Ratchet basedpyright limits down by what this branch fixed"
 	@echo "  make lint-format        - Check ruff format formatting (matches CI)"
 	@echo "  make lint-ruff-budget - Gate the codebase total of each strict ruff rule against its limit"
@@ -54,6 +55,7 @@ UV := uv
 UV_RUN := $(UV) run --no-sync
 
 LINT_DEP_INSTALL ?= install-dev
+LINT_E2E_DEP_INSTALL ?= lint-install
 LINT_DEP_BASE ?= lint-fetch-base
 LINT_JOBS := $(shell sysctl -n hw.ncpu 2>/dev/null || nproc 2>/dev/null || echo 4)
 LINT_OUTPUT_SYNC := $(if $(filter output-sync,$(.FEATURES)),--output-sync=target,)
@@ -111,7 +113,7 @@ lint-fetch-base:
 # CI's). --inexact tops up the venv instead of pruning the proxy extras gen:api and the
 # running proxy need.
 lint-install:
-	$(UV) sync --inexact --frozen --group proxy-dev
+	$(UV) sync --inexact --frozen --group proxy-dev --group e2e-dev
 	$(UV_RUN) python scripts/prisma_generate_if_needed.py
 
 # Diff-scoped format check, identical to test-linting.yml's "Check ruff format" step:
@@ -164,6 +166,9 @@ lint-ruff-FULL-dev: install-dev
 lint-basedpyright: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
 	($(UV_RUN) basedpyright --outputjson || true) | $(UV_RUN) python scripts/type_check_gate.py --base origin/litellm_internal_staging
 
+lint-e2e-basedpyright: $(LINT_E2E_DEP_INSTALL)
+	$(UV_RUN) basedpyright tests/e2e
+
 # Type-discipline budget (mutable collections / casts / type guards / kwargs /
 # unexplained suppressions), the test-linting.yml step `make lint` used to omit.
 lint-type-discipline: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
@@ -208,9 +213,9 @@ check-import-safety: $(LINT_DEP_INSTALL)
 # base fetch) runs once up front; the checks themselves are independent, so a sub-make
 # fans them out with -j and the fast ones finish under basedpyright's shadow.
 lint: lint-install lint-fetch-base
-	$(MAKE) -j $(LINT_JOBS) $(LINT_OUTPUT_SYNC) LINT_DEP_INSTALL= LINT_DEP_BASE= lint-checks
+	$(MAKE) -j $(LINT_JOBS) $(LINT_OUTPUT_SYNC) LINT_DEP_INSTALL= LINT_E2E_DEP_INSTALL= LINT_DEP_BASE= lint-checks
 
-lint-checks: lint-format-check-changed lint-ruff lint-gate lint-type-discipline lint-basedpyright check-circular-imports check-import-safety
+lint-checks: lint-format-check-changed lint-ruff lint-gate lint-type-discipline lint-basedpyright lint-e2e-basedpyright check-circular-imports check-import-safety
 
 # Faster linting for local development (only checks changed code)
 lint-dev: lint-format-changed check-circular-imports check-import-safety

pyproject.toml

@@ -184,6 +184,10 @@ dev = [
     "vcrpy==8.2.1",
     "pytest-recording==0.13.4",
 ]
+e2e-dev = [
+    "playwright==1.61.0",
+    "websockets>=15.0.1,<16.0",
+]
 proxy-dev = [
     "prisma==0.11.0",
     "hypercorn==0.17.3",

scripts/pre_commit_lint.sh

@@ -4,6 +4,7 @@
 # before `git commit`; it inspects your staged files and runs only the matching
 # gating CI checks, so a clean run means a green CI lint:
 #   - litellm/ Python staged -> `make lint` (test-linting.yml's lint job)
+#   - tests/e2e Python staged -> `make lint-e2e-basedpyright` (test-linting.yml's e2e type-check step)
 #   - dashboard staged        -> prettier + eslint + lint budgets (test-litellm-ui-build.yml's frontend-lint)
 #   - proxy/types staged      -> regenerate dashboard API types and fail on drift (check-ui-api-types.yml)
 #
@@ -25,6 +26,7 @@ staged_match() { printf '%s\n' "$staged" | grep -E "$1" || true; }
 # scripts-only commit can't turn it red; scope the trigger there to skip the slow
 # make lint when it couldn't catch anything.
 litellm_py_files=$(staged_match '^litellm/.*\.py$')
+e2e_py_files=$(staged_match '^tests/e2e/.*\.py$')
 # ruff format (and CI's format step) skip enterprise; the rest of make lint covers it.
 fmt_files=$(printf '%s\n' "$litellm_py_files" | grep -v '^litellm/enterprise/' || true)
 # check-ui-api-types.yml triggers on any file under litellm/proxy or litellm/types
@@ -107,6 +109,11 @@ if [ -n "$litellm_py_files" ]; then
     fi
 fi
 
+if [ -n "$e2e_py_files" ] && [ -z "$litellm_py_files" ]; then
+    echo "pre-commit: type-checking tests/e2e (make lint-e2e-basedpyright)"
+    make lint-e2e-basedpyright || { echo "✗ tests/e2e basedpyright failed. Fix the errors above, then re-run make pre-commit." >&2; status=1; }
+fi
+
 if [ -n "$ui_prettier_files" ] || [ -n "$ui_eslint_files" ]; then
     echo "pre-commit: linting dashboard (prettier + eslint + lint budgets)"
     lint_dashboard || { echo "✗ Dashboard lint failed. See above; format with: (cd ui/litellm-dashboard && npm run format)." >&2; status=1; }

tests/e2e/CLAUDE.md

@@ -57,7 +57,7 @@ Mark live tests with `@pytest.mark.e2e` (on the class or the module). Pure cover
 
 ## Typing
 
-The harness is fully typed and new code must not add `Any` or widen the basedpyright budgets. When a response field is untyped, model it in `models.py` (just the fields you read) and let pydantic validate it, rather than threading a `dict` or `Any` through the test
+The harness is fully typed with no error budget: `make lint-e2e-basedpyright` must report zero basedpyright errors, and CI enforces that on any PR touching `tests/e2e/**/*.py`. When a response field is untyped, model it in `models.py` (just the fields you read) and let pydantic validate it, rather than threading a `dict` or `Any` through the test
 
 ## Coverage registry
 

tests/e2e/CONTRIBUTING.md

@@ -34,10 +34,10 @@ The suites run against a live proxy, so bring one up first. `docker-compose.yml`
    uv run pytest tests/e2e/llm_translation/ -v
    ```
 
-   The browser tests in the `management/` suite drive the dashboard the proxy serves at `/ui` through playwright, an optional dependency behind `importorskip` (the suite's API tests run without it). Install it once into your environment along with its browser:
+   The browser tests in the `management/` suite drive the dashboard the proxy serves at `/ui` through playwright, an optional dependency behind `importorskip` (the suite's API tests run without it). It lives in the `e2e-dev` dependency group; install it along with its browser:
 
    ```bash
-   uv pip install playwright
+   uv sync --inexact --group e2e-dev
    uv run playwright install chromium
    ```
 
@@ -140,7 +140,7 @@ Mark live tests with `@pytest.mark.e2e` (on the class or the module). Pure cover
 
 Before you push
 
-1. Run basedpyright over your changes; the harness is fully typed and new code must not add `Any` or widen the budgets
+1. Run `make lint-e2e-basedpyright` (or `make pre-commit` with your changes staged); the harness is fully typed and the gate allows zero basedpyright errors, enforced in CI on any PR touching `tests/e2e/**/*.py`
 
 2. Add the models your test needs to the inline config in `docker-compose.yml`
 

tests/e2e/budgets/budget_client.py

@@ -342,7 +342,7 @@ def _wait_for_team(self, team_id: str) -> None:
                 case _:
                     time.sleep(_TEAM_READY_SLEEP_SECONDS)
         assert last is not None
-        _ = unwrap(last)
+        raise AssertionError(last)
 
     def add_team_member(self, team_id: str, user_id: str, *, max_budget_in_team: float | None = None) -> None:
         last_body = ""

tests/e2e/budgets/test_budget_fallback_e2e.py

@@ -4,14 +4,14 @@
 budget_fallbacks feature in litellm/proxy/hooks/model_max_budget_limiter.py.
 """
 
-import json
 import time
 
 import pytest
 
 from budget_client import BudgetClient, model_budget
 from e2e_config import unique_marker
 from lifecycle import ResourceManager
+from models import AnthropicMessagesResponse
 
 pytestmark = pytest.mark.e2e
 
@@ -42,8 +42,8 @@ def test_budget_fallback_reroutes_anthropic_messages_to_openai(
                 "budget_fallbacks must reroute transparently, never surface a "
                 f"block; status={result.status_code} body={result.body[:300]}"
             )
-        served_by = json.loads(result.body)["model"]
-        if FALLBACK_MODEL in served_by:
+        served_by = AnthropicMessagesResponse.model_validate_json(result.body).model
+        if served_by is not None and FALLBACK_MODEL in served_by:
             break
         time.sleep(1)
     assert served_by is not None and FALLBACK_MODEL in served_by, (

tests/e2e/budgets/test_spend_counter_reseed_e2e.py

@@ -23,6 +23,7 @@
 from typing import TYPE_CHECKING
 
 import pytest
+from pydantic import TypeAdapter, ValidationError
 
 from budget_client import BudgetClient
 from e2e_config import unique_marker
@@ -42,6 +43,8 @@
 # expires the counter; this waits out both.
 COLD_WAIT_SECONDS = 80
 
+_JSON_FLOAT: TypeAdapter[float] = TypeAdapter(float)
+
 
 def _redis() -> "redis.Redis[str] | RedisCluster[str]":
     """The proxy's Redis. The deployed runner sets REDIS_HOST to the serverless
@@ -75,12 +78,11 @@ def _parse_counter(raw: object) -> float | None:
     try:
         return float(text)
     except ValueError:
-        try:
-            import json
-
-            return float(json.loads(text))
-        except Exception:
-            return None
+        pass
+    try:
+        return _JSON_FLOAT.validate_json(text)
+    except ValidationError:
+        return None
 
 
 def _spend_counter(rds: "redis.Redis[str] | RedisCluster[str]", key: str) -> float | None:

tests/e2e/coverage_registry/collector.py

@@ -17,8 +17,10 @@
 from argparse import ArgumentParser
 from dataclasses import dataclass
 from pathlib import Path
+from typing import Literal
 
 import pytest
+from pydantic import BaseModel
 
 from .registry import load_registry
 from .schema import MODULE_ORDER, Cell, Tier, dashboard_module, loki_module_label
@@ -35,12 +37,13 @@ def __init__(self) -> None:
         self.collection_errors: tuple[str, ...] = ()
 
     def pytest_collection_finish(self, session: pytest.Session) -> None:
-        self.covered_ids = frozenset(
-            arg
+        marker_args: tuple[tuple[object, ...], ...] = tuple(
+            marker.args
             for item in session.items
             for marker in item.iter_markers(name="covers")
-            for arg in marker.args
-            if isinstance(arg, str)
+        )
+        self.covered_ids = frozenset(
+            arg for args in marker_args for arg in args if isinstance(arg, str)
         )
 
     def pytest_collectreport(self, report: pytest.CollectReport) -> None:
@@ -257,6 +260,12 @@ def render_loki(report: CoverageReport) -> str:
     return "\n".join(lines)
 
 
+class _CliArgs(BaseModel):
+    format: Literal["text", "json", "prometheus", "loki"]
+    strict: bool
+    fail_on_collection_errors: bool
+
+
 def main() -> int:
     parser = ArgumentParser()
     parser.add_argument(
@@ -275,7 +284,7 @@ def main() -> int:
         action="store_true",
         help="Exit non-zero if pytest collection errors are found.",
     )
-    args = parser.parse_args()
+    args = _CliArgs.model_validate(vars(parser.parse_args()))
     cells = load_registry()
     covered, errors = collect_covered_ids()
     report = compute_coverage(cells, covered, errors)

tests/e2e/coverage_registry/registry.py

@@ -6,20 +6,23 @@
 from pathlib import Path
 
 import yaml
+from pydantic import TypeAdapter
 
-from .schema import CELL_ADAPTER, Cell
+from .schema import Cell
 
 REGISTRY_DIR = Path(__file__).resolve().parent
 
+_CELLS_ADAPTER: TypeAdapter[tuple[Cell, ...]] = TypeAdapter(tuple[Cell, ...])
+
+
+def _load_cells(path: Path) -> tuple[Cell, ...]:
+    return _CELLS_ADAPTER.validate_python(yaml.safe_load(path.read_text()) or ())
+
 
 def load_registry(registry_dir: Path = REGISTRY_DIR) -> tuple[Cell, ...]:
     """Every cell across every `*.yaml`, validated. Raises on a schema violation or
     a duplicate id, since either would corrupt the coverage denominator."""
-    cells = tuple(
-        CELL_ADAPTER.validate_python(row)
-        for path in sorted(registry_dir.glob("*.yaml"))
-        for row in (yaml.safe_load(path.read_text()) or ())
-    )
+    cells = tuple(cell for path in sorted(registry_dir.glob("*.yaml")) for cell in _load_cells(path))
     duplicates = sorted(cid for cid, n in Counter(c.id for c in cells).items() if n > 1)
     if duplicates:
         raise ValueError(f"duplicate cell ids in registry: {duplicates}")

tests/e2e/llm_translation/realtime/realtime_client.py

@@ -14,7 +14,7 @@
 from collections.abc import Generator, Mapping
 from contextlib import contextmanager
 from dataclasses import dataclass
-from typing import Any, TypeVar
+from typing import TypeVar
 from urllib.parse import urlencode
 
 from pydantic import BaseModel, ConfigDict
@@ -28,15 +28,15 @@
 _M = TypeVar("_M", bound=BaseModel)
 
 
-def _ws_base_url() -> str:
+def ws_base_url() -> str:
     for scheme, ws_scheme in (("https://", "wss://"), ("http://", "ws://")):
         if PROXY_BASE_URL.startswith(scheme):
             return ws_scheme + PROXY_BASE_URL[len(scheme) :]
     return PROXY_BASE_URL
 
 
 def realtime_ws_url(model: str) -> str:
-    return f"{_ws_base_url()}/v1/realtime?{urlencode({'model': model})}"
+    return f"{ws_base_url()}/v1/realtime?{urlencode({'model': model})}"
 
 
 @dataclass(frozen=True, slots=True)
@@ -217,7 +217,7 @@ class OutputItemDone(BaseModel):
 
 class ResponsePayload(BaseModel):
     model_config = ConfigDict(extra="allow")
-    usage: dict[str, Any] | None = None
+    usage: dict[str, object] | None = None
     output: list[OutputItem] | None = None
 
 

tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e.py

@@ -30,7 +30,7 @@
 from realtime_client import (
     PROVIDERS,
     RealtimeProvider,
-    _ws_base_url,
+    ws_base_url,
     realtime_model,
 )
 
@@ -154,7 +154,7 @@ async def get_weather(params: FunctionCallParams) -> None:
 
     llm = LiteLLMRealtimeLLMService(
         api_key=key,
-        base_url=f"{_ws_base_url()}/v1/realtime",
+        base_url=f"{ws_base_url()}/v1/realtime",
         settings=OpenAIRealtimeLLMService.Settings(
             model=model,
             system_instruction=(
@@ -276,7 +276,7 @@ async def _run_audio_input_pipeline(
 
     llm = LiteLLMRealtimeLLMService(
         api_key=key,
-        base_url=f"{_ws_base_url()}/v1/realtime",
+        base_url=f"{ws_base_url()}/v1/realtime",
         settings=OpenAIRealtimeLLMService.Settings(
             model=model,
             system_instruction=(

tests/e2e/llm_translation/realtime/test_realtime_pipecat_e2e.py

@@ -28,7 +28,7 @@
 from realtime_client import (
     PROVIDERS,
     RealtimeProvider,
-    _ws_base_url,
+    ws_base_url,
     realtime_model,
 )
 
@@ -94,7 +94,7 @@ async def get_weather(params: FunctionCallParams) -> None:
         await params.result_callback({"city": "Paris", "temperature_f": 72})
 
     llm = LiteLLMRealtimeLLMService(
-        api_key=key, base_url=f"{_ws_base_url()}/v1/realtime", model=model
+        api_key=key, base_url=f"{ws_base_url()}/v1/realtime", model=model
     )
     llm.register_function("get_weather", get_weather)
 

tests/e2e/llm_translation/test_cache_control.py

@@ -111,12 +111,11 @@ def _assert_cache_read_on_second_call(
     first = unwrap(_cache_chat(client, key, model, prefix))
     assert first.choices, f"{model}: first cache-priming call returned no choices: {first}"
 
-    read_tokens = 0
     deadline = time.monotonic() + 30.0
-    while time.monotonic() < deadline:
+    while True:
         second = unwrap(_cache_chat(client, key, model, prefix))
         read_tokens = _cached_read_tokens(second.usage)
-        if read_tokens > 0:
+        if read_tokens > 0 or time.monotonic() >= deadline:
             break
         time.sleep(3.0)