diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 20ab02128542..89fa102eb38c 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -63,23 +63,26 @@ The harness is fully typed and new code must not add `Any` or widen the basedpyr The set of tests we want is a registry checked into this repo, one row per behavior; that file is the definition of done and the denominator. Each e2e test declares what it covers with `@pytest.mark.covers("...")`, and a small collector diffs the registry against the tests and ships coverage to the existing Grafana. No Allure, no new dependencies -Coverage is organized as module > feature > test. There are six modules: LLMs, MCPs, Management/UI, Reliability & Performance, Logging & Guardrails, and Other. A feature is either an endpoint (`/chat/completions`) or a behavior (fallbacks, rate limits; config-driven, with no route of its own). A cell reads like `llm.chat_completions.bedrock_converse.tool_use.stream.works` +Coverage is organized as module > feature > test. Dashboard modules are Core LLMs, Non-Core LLMs, MCPs, Management/UI, Reliability & Performance, Logging & Guardrails, and Other. A feature is either an endpoint (`/chat/completions`) or a behavior (fallbacks, rate limits; config-driven, with no route of its own). A cell reads like `llm.chat_completions.bedrock_converse.tool_use.stream.works` The metric is coverage: the share of registry rows that have a passing covering test, reported to Grafana per module so a gap surfaces as an uncovered row rather than a silent absence +Tests do not declare a dashboard module directly. They only declare the registry cell id with `@pytest.mark.covers("...")`; the registry row decides the module, tier, endpoint, and dashboard rollup. Run `python -m coverage_registry.collector --strict` when you want CI to reject unknown marker ids. Add `--fail-on-collection-errors` when the job should also fail on pytest collection errors. + ### Naming grammar per module -LLMs - endpoint features (subject = the route), seeded from the Claude Code compat matrix +LLMs - endpoint features (subject = the route), seeded from the Claude Code compat matrix. `chat_completions`, `messages`, and `responses` are Core LLMs. Other LLM endpoints, including `batches` and `realtime`, roll up as Non-Core LLMs. ``` llm..... endpoint : chat_completions | messages | responses | embeddings | batches | files | rerank | images_generations | audio_speech | audio_transcriptions | moderations - route : openai | azure_openai | anthropic | bedrock_invoke | bedrock_converse | vertex | azure_foundry + | realtime + route : openai | azure_openai | anthropic | bedrock_converse | vertex | azure_foundry + | cohere | together_ai (vocab varies per endpoint; messages is anthropic-format only) - capability : basic | tool_use | prompt_cache_5m | prompt_cache_1h | vision | thinking - | thinking_tool_use | pdf_input | web_search | structured_output | count_tokens - | tool_search | long_context_1m + capability : basic | tool_use | prompt_cache_5m | vision | thinking | structured_output + | service_tier streaming : stream | nonstream (omit where n/a) assertion : works | cost_logged label (not in id): model = haiku-4.5 | sonnet-4.6 | opus-4.7 | gpt-* diff --git a/tests/e2e/coverage_registry/README.md b/tests/e2e/coverage_registry/README.md index a8aab3fcfbc4..4177cba77663 100644 --- a/tests/e2e/coverage_registry/README.md +++ b/tests/e2e/coverage_registry/README.md @@ -9,14 +9,18 @@ note; the naming grammar lives in `tests/e2e/CLAUDE.md`. A **cell** is one customer-noticeable behavior a single e2e test can assert pass/fail on, for example `llm.chat_completions.bedrock_converse.tool_use.stream.works`. Cells are -grouped `module > feature > test`, six dashboard modules in all. Each cell carries a -tier (P0/P1/P2), a source, and a `fail_before_fix` flag. +grouped `module > feature > test`, with LLM cells split into Core LLMs and Non-Core +LLMs for dashboarding. Each cell carries a tier (P0/P1/P2), a source, and a +`fail_before_fix` flag. The rows live in per-prefix YAML files (`llm_*.yaml`, `mgmt.yaml`, `mcp.yaml`, `reliability.yaml`, `logging.yaml`, `guardrail.yaml`, `other.yaml`) and validate against the discriminated union in `schema.py`, so an LLM row cannot carry a guardrail field and -vice versa. `logging` and `guardrail` are two id-prefixes that roll up into the single -"Logging & Guardrails" dashboard module. +vice versa. `llm` rows with `subject_endpoint` of `chat_completions`, `messages`, or +`responses` roll up to "Core LLMs"; all other LLM endpoints roll up to "Non-Core +LLMs". LLM endpoint, route, and capability values are typed in `schema.py`, so new +taxonomy values require an explicit schema change. `logging` and `guardrail` are two +id-prefixes that roll up into the single "Logging & Guardrails" dashboard module. A test declares what it covers with a marker: @@ -36,9 +40,22 @@ proxy. Whether a covered cell currently passes or fails is a separate, live conc cd tests/e2e && PYTHONPATH=. python -m coverage_registry.collector ``` -The headline is P0 coverage. The collector also lists markers that point at ids not in -the registry, so a typo or an unenumerated behavior surfaces instead of being silently -dropped. +Use `--format prometheus` or `--format json` for CI jobs that publish coverage to +Grafana. + +The headline is overall coverage. The collector also lists markers that point at ids +not in the registry, so a typo or an unenumerated behavior surfaces instead of being +silently dropped. + +Use strict mode in CI once existing draft markers are reconciled: + +``` +cd tests/e2e && PYTHONPATH=. python -m coverage_registry.collector --strict +``` + +Strict mode exits non-zero on `@pytest.mark.covers(...)` ids that are not checked into +the registry. Add `--fail-on-collection-errors` when the job should also fail on pytest +collection errors. ## Status: this is a draft for review diff --git a/tests/e2e/coverage_registry/collector.py b/tests/e2e/coverage_registry/collector.py index cc2ce16e3ea3..3b5771066058 100644 --- a/tests/e2e/coverage_registry/collector.py +++ b/tests/e2e/coverage_registry/collector.py @@ -12,14 +12,16 @@ import contextlib import io +import json import sys +from argparse import ArgumentParser from dataclasses import dataclass from pathlib import Path import pytest from .registry import load_registry -from .schema import MODULE_ORDER, ROLLUP, Cell, Tier +from .schema import MODULE_ORDER, Cell, Tier, dashboard_module E2E_DIR = Path(__file__).resolve().parent.parent @@ -46,12 +48,21 @@ def pytest_collectreport(self, report: pytest.CollectReport) -> None: self.collection_errors = (*self.collection_errors, report.nodeid) -def collect_covered_ids(e2e_dir: Path = E2E_DIR) -> tuple[frozenset[str], tuple[str, ...]]: +def collect_covered_ids( + e2e_dir: Path = E2E_DIR, +) -> tuple[frozenset[str], tuple[str, ...]]: """Return (covered cell ids, nodeids that failed to import).""" sink = _CoversSink() with contextlib.redirect_stdout(io.StringIO()): pytest.main( - ["--collect-only", "-qq", "--continue-on-collection-errors", "-p", "no:cacheprovider", str(e2e_dir)], + [ + "--collect-only", + "-qq", + "--continue-on-collection-errors", + "-p", + "no:cacheprovider", + str(e2e_dir), + ], plugins=[sink], ) return sink.covered_ids, sink.collection_errors @@ -65,6 +76,10 @@ class ModuleCoverage: p0_total: int p0_covered: int + @property + def coverage_percent(self) -> float: + return _percent(self.covered, self.total) + @dataclass(frozen=True, slots=True) class CoverageReport: @@ -77,9 +92,19 @@ class CoverageReport: orphan_markers: tuple[str, ...] collection_errors: tuple[str, ...] + @property + def coverage_percent(self) -> float: + return _percent(self.covered, self.total) + + +def _percent(covered: int, total: int) -> float: + return (100.0 * covered / total) if total else 0.0 -def _module_coverage(module: str, cells: tuple[Cell, ...], covered: frozenset[str]) -> ModuleCoverage: - in_module = tuple(c for c in cells if ROLLUP[c.module] == module) + +def _module_coverage( + module: str, cells: tuple[Cell, ...], covered: frozenset[str] +) -> ModuleCoverage: + in_module = tuple(c for c in cells if dashboard_module(c) == module) p0 = tuple(c for c in in_module if c.tier is Tier.P0) return ModuleCoverage( module=module, @@ -109,27 +134,26 @@ def compute_coverage( ) -def _row(label: str, covered: int, total: int, p0_covered: int, p0_total: int) -> str: +def _row(label: str, covered: int, total: int) -> str: frac = f"{covered}/{total}" - p0 = f"{p0_covered}/{p0_total}" - return f"{label:30}{frac:>12}{p0:>14}" + return f"{label:30}{frac:>12}{_percent(covered, total):>11.1f}%" def render(report: CoverageReport) -> str: - rows = tuple(_row(m.module, m.covered, m.total, m.p0_covered, m.p0_total) for m in report.modules) - pct = (100.0 * report.p0_covered / report.p0_total) if report.p0_total else 0.0 + rows = tuple(_row(m.module, m.covered, m.total) for m in report.modules) lines = ( - f"{'MODULE':30}{'COVERED':>12}{'P0 COVERED':>14}", + f"{'MODULE':30}{'COVERED':>12}{'COVERAGE':>12}", *rows, - "-" * 56, - _row("ALL", report.covered, report.total, report.p0_covered, report.p0_total), + "-" * 54, + _row("ALL", report.covered, report.total), "", - f"Headline (P0 coverage): {report.p0_covered}/{report.p0_total} ({pct:.1f}%)", + f"Headline coverage: {report.covered}/{report.total} ({report.coverage_percent:.1f}%)", ) orphans = ( ( f"\n{len(report.orphan_markers)} marker(s) point at ids not in the registry " - f"(reconcile: fix the marker or add the cell):\n " + "\n ".join(report.orphan_markers), + f"(reconcile: fix the marker or add the cell):\n " + + "\n ".join(report.orphan_markers), ) if report.orphan_markers else () @@ -137,7 +161,8 @@ def render(report: CoverageReport) -> str: warning = ( ( f"\nWARNING: {len(report.collection_errors)} node(s) failed to import during " - f"collection, so coverage may undercount:\n " + "\n ".join(report.collection_errors), + f"collection, so coverage may undercount:\n " + + "\n ".join(report.collection_errors), ) if report.collection_errors else () @@ -145,10 +170,109 @@ def render(report: CoverageReport) -> str: return "\n".join((*lines, *orphans, *warning)) +def _report_dict(report: CoverageReport) -> dict[str, object]: + return { + "covered": report.covered, + "total": report.total, + "coverage_percent": report.coverage_percent, + "modules": [ + { + "module": m.module, + "covered": m.covered, + "total": m.total, + "coverage_percent": m.coverage_percent, + "p0_covered": m.p0_covered, + "p0_total": m.p0_total, + } + for m in report.modules + ], + "orphan_markers": list(report.orphan_markers), + "collection_errors": list(report.collection_errors), + } + + +def render_json(report: CoverageReport) -> str: + return json.dumps(_report_dict(report), indent=2, sort_keys=True) + + +def _label_value(value: str) -> str: + return value.replace("\\", "\\\\").replace('"', '\\"').replace("\n", "\\n") + + +def render_prometheus(report: CoverageReport) -> str: + lines = [ + "# HELP litellm_e2e_coverage_cells E2E coverage registry cells by module and state.", + "# TYPE litellm_e2e_coverage_cells gauge", + ] + for module in report.modules: + label = _label_value(module.module) + lines.append( + f'litellm_e2e_coverage_cells{{module="{label}",state="covered"}} {module.covered}' + ) + lines.append( + f'litellm_e2e_coverage_cells{{module="{label}",state="total"}} {module.total}' + ) + lines.extend( + [ + f'litellm_e2e_coverage_cells{{module="ALL",state="covered"}} {report.covered}', + f'litellm_e2e_coverage_cells{{module="ALL",state="total"}} {report.total}', + "# HELP litellm_e2e_coverage_percent E2E coverage percent by module.", + "# TYPE litellm_e2e_coverage_percent gauge", + ] + ) + for module in report.modules: + label = _label_value(module.module) + lines.append( + f'litellm_e2e_coverage_percent{{module="{label}"}} {module.coverage_percent:.6f}' + ) + lines.extend( + [ + f'litellm_e2e_coverage_percent{{module="ALL"}} {report.coverage_percent:.6f}', + "# HELP litellm_e2e_coverage_orphan_markers Coverage markers not found in the registry.", + "# TYPE litellm_e2e_coverage_orphan_markers gauge", + f"litellm_e2e_coverage_orphan_markers {len(report.orphan_markers)}", + "# HELP litellm_e2e_coverage_collection_errors Pytest nodes that failed during collection.", + "# TYPE litellm_e2e_coverage_collection_errors gauge", + f"litellm_e2e_coverage_collection_errors {len(report.collection_errors)}", + ] + ) + return "\n".join(lines) + + def main() -> int: + parser = ArgumentParser() + parser.add_argument( + "--format", + choices=("text", "json", "prometheus"), + default="text", + help="Output format. Use prometheus or json for Grafana ingestion jobs.", + ) + parser.add_argument( + "--strict", + action="store_true", + help="Exit non-zero if markers outside the registry are found.", + ) + parser.add_argument( + "--fail-on-collection-errors", + action="store_true", + help="Exit non-zero if pytest collection errors are found.", + ) + args = parser.parse_args() cells = load_registry() covered, errors = collect_covered_ids() - print(render(compute_coverage(cells, covered, errors))) # noqa: T201 # CLI entrypoint output + report = compute_coverage(cells, covered, errors) + output = { + "text": render, + "json": render_json, + "prometheus": render_prometheus, + }[ + args.format + ](report) + print(output) # noqa: T201 # CLI entrypoint output + if args.strict and report.orphan_markers: + return 1 + if args.fail_on_collection_errors and report.collection_errors: + return 1 return 0 diff --git a/tests/e2e/coverage_registry/llm_conversational.yaml b/tests/e2e/coverage_registry/llm_conversational.yaml index 365776da8bfe..7360aacb916b 100644 --- a/tests/e2e/coverage_registry/llm_conversational.yaml +++ b/tests/e2e/coverage_registry/llm_conversational.yaml @@ -6,6 +6,7 @@ - {id: llm.chat_completions.openai.tool_use.stream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: openai, capability: tool_use, streaming: stream, assertions: [works], source: "model_prices json", rationale: "Tool calls over streaming"} - {id: llm.chat_completions.openai.vision.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: openai, capability: vision, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "gpt-4o vision; high usage"} - {id: llm.chat_completions.openai.prompt_cache_5m.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: openai, capability: prompt_cache_5m, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Prompt caching cost optimization"} +- {id: llm.chat_completions.openai.service_tier.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: service_tier, streaming: nonstream, assertions: [works], source: "OpenAI service_tier param", rationale: "OpenAI scale-tier request option is forwarded and echoed"} - {id: llm.chat_completions.openai.thinking.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: thinking, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "o-series reasoning; emerging"} - {id: llm.chat_completions.openai.structured_output.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: structured_output, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "response_schema extraction"} - {id: llm.chat_completions.anthropic.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: anthropic, capability: basic, streaming: nonstream, assertions: [works], source: "proxy_server.py:8455", rationale: "P0 route translated to Anthropic"} diff --git a/tests/e2e/coverage_registry/schema.py b/tests/e2e/coverage_registry/schema.py index 756bde33e3d5..2e2a00e78ba8 100644 --- a/tests/e2e/coverage_registry/schema.py +++ b/tests/e2e/coverage_registry/schema.py @@ -1,9 +1,9 @@ """Registry row schema: the contract every denominator cell validates against. A cell is one customer-noticeable behavior a single e2e test can assert pass/fail -on. `module` is the id's segment-1 prefix (seven of them); the six-way dashboard -rollup merges logging + guardrail via ROLLUP. The union is discriminated on -`module`, so an LLM row cannot carry a guardrail field and vice versa. +on. `module` is the id's segment-1 prefix (seven of them); dashboard rollups can +split or merge those prefixes. The union is discriminated on `module`, so an LLM +row cannot carry a guardrail field and vice versa. """ from __future__ import annotations @@ -25,6 +25,43 @@ class FailBeforeFix(str, Enum): unproven = "unproven" +LlmEndpoint = Literal[ + "chat_completions", + "messages", + "responses", + "embeddings", + "batches", + "files", + "rerank", + "images_generations", + "audio_speech", + "audio_transcriptions", + "moderations", + "realtime", +] + +LlmRoute = Literal[ + "anthropic", + "azure_foundry", + "azure_openai", + "bedrock_converse", + "cohere", + "openai", + "together_ai", + "vertex", +] + +LlmCapability = Literal[ + "basic", + "prompt_cache_5m", + "service_tier", + "structured_output", + "thinking", + "tool_use", + "vision", +] + + class _Base(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid") @@ -39,9 +76,9 @@ class _Base(BaseModel): class LlmCell(_Base): module: Literal["llm"] - subject_endpoint: str - route: str - capability: str + subject_endpoint: LlmEndpoint + route: LlmRoute + capability: LlmCapability streaming: Literal["stream", "nonstream", "na"] @@ -81,14 +118,27 @@ class OtherCell(_Base): Cell = Annotated[ - LlmCell | MgmtCell | McpCell | ReliabilityCell | LoggingCell | GuardrailCell | OtherCell, + LlmCell + | MgmtCell + | McpCell + | ReliabilityCell + | LoggingCell + | GuardrailCell + | OtherCell, Field(discriminator="module"), ] CELL_ADAPTER: TypeAdapter[Cell] = TypeAdapter(Cell) -ROLLUP: dict[str, str] = { - "llm": "LLMs", +CORE_LLM_ENDPOINTS: frozenset[str] = frozenset( + { + "chat_completions", + "messages", + "responses", + } +) + +PREFIX_ROLLUP: dict[str, str] = { "mcp": "MCPs", "mgmt": "Management/UI", "reliability": "Reliability & Performance", @@ -98,10 +148,20 @@ class OtherCell(_Base): } MODULE_ORDER: tuple[str, ...] = ( - "LLMs", + "Core LLMs", + "Non-Core LLMs", "MCPs", "Management/UI", "Reliability & Performance", "Logging & Guardrails", "Other", ) + + +def dashboard_module(cell: Cell) -> str: + """Return the Grafana/reporting module for a registry cell.""" + if isinstance(cell, LlmCell): + if cell.subject_endpoint in CORE_LLM_ENDPOINTS: + return "Core LLMs" + return "Non-Core LLMs" + return PREFIX_ROLLUP[cell.module] diff --git a/tests/e2e/coverage_registry/test_collector.py b/tests/e2e/coverage_registry/test_collector.py index 065ebafdb3b7..355bc52730d1 100644 --- a/tests/e2e/coverage_registry/test_collector.py +++ b/tests/e2e/coverage_registry/test_collector.py @@ -11,19 +11,32 @@ import pytest -from coverage_registry.collector import compute_coverage +from coverage_registry.collector import ( + compute_coverage, + render, + render_json, + render_prometheus, +) from coverage_registry.registry import load_registry -from coverage_registry.schema import GuardrailCell, LlmCell, LoggingCell, Tier - - -def _llm(cell_id: str, tier: Tier) -> LlmCell: +from coverage_registry.schema import ( + GuardrailCell, + LlmCell, + LlmEndpoint, + LoggingCell, + Tier, +) + + +def _llm( + cell_id: str, tier: Tier, subject_endpoint: LlmEndpoint = "chat_completions" +) -> LlmCell: return LlmCell( id=cell_id, module="llm", tier=tier, assertions=("works",), source="test", - subject_endpoint="chat_completions", + subject_endpoint=subject_endpoint, route="openai", capability="basic", streaming="nonstream", @@ -68,10 +81,74 @@ def test_logging_and_guardrail_roll_up_into_one_module() -> None: ), ) report = compute_coverage(cells, frozenset()) - logging_and_guardrails = next(m for m in report.modules if m.module == "Logging & Guardrails") + logging_and_guardrails = next( + m for m in report.modules if m.module == "Logging & Guardrails" + ) assert logging_and_guardrails.total == 2 +def test_llm_cells_roll_up_by_core_endpoint() -> None: + cells = ( + _llm("llm.chat", Tier.P0, "chat_completions"), + _llm("llm.messages", Tier.P0, "messages"), + _llm("llm.responses", Tier.P1, "responses"), + _llm("llm.batches", Tier.P0, "batches"), + _llm("llm.realtime", Tier.P1, "realtime"), + ) + report = compute_coverage(cells, frozenset({"llm.chat", "llm.batches"})) + + core = next(m for m in report.modules if m.module == "Core LLMs") + non_core = next(m for m in report.modules if m.module == "Non-Core LLMs") + + assert (core.total, core.covered, core.p0_total, core.p0_covered) == (3, 1, 2, 1) + assert ( + non_core.total, + non_core.covered, + non_core.p0_total, + non_core.p0_covered, + ) == (2, 1, 1, 1) + + +def test_text_render_uses_plain_coverage_language() -> None: + report = compute_coverage( + (_llm("llm.chat", Tier.P0), _llm("llm.batches", Tier.P0, "batches")), + frozenset({"llm.chat"}), + ) + + text = render(report) + + assert "COVERAGE" in text + assert "Headline coverage: 1/2 (50.0%)" in text + assert "P0 COVERED" not in text + + +def test_json_render_exposes_module_coverage_for_grafana_jobs() -> None: + report = compute_coverage( + (_llm("llm.chat", Tier.P0), _llm("llm.batches", Tier.P0, "batches")), + frozenset({"llm.chat"}), + ) + + payload = render_json(report) + + assert '"coverage_percent": 50.0' in payload + assert '"module": "Core LLMs"' in payload + assert '"module": "Non-Core LLMs"' in payload + + +def test_prometheus_render_exposes_module_coverage_timeseries() -> None: + report = compute_coverage( + (_llm("llm.chat", Tier.P0), _llm("llm.batches", Tier.P0, "batches")), + frozenset({"llm.chat"}), + ) + + metrics = render_prometheus(report) + + assert 'litellm_e2e_coverage_cells{module="Core LLMs",state="covered"} 1' in metrics + assert 'litellm_e2e_coverage_percent{module="Core LLMs"} 100.000000' in metrics + assert 'litellm_e2e_coverage_percent{module="Non-Core LLMs"} 0.000000' in metrics + assert "litellm_e2e_coverage_orphan_markers 0" in metrics + + def test_real_registry_loads_and_ids_are_unique() -> None: cells = load_registry() ids = [c.id for c in cells] diff --git a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py index 269cb5d6d22b..5cc4ff308fac 100644 --- a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py +++ b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py @@ -33,7 +33,12 @@ class TestChatCompletionsRegression: CHAT_MODELS, ids=[f"{model}-{route}" for model, route in CHAT_MODELS], ) - @pytest.mark.covers("llm.chat_completions.provider.basic.nonstream.works", exercised_on=[]) + @pytest.mark.covers( + "llm.chat_completions.openai.basic.nonstream.works", + "llm.chat_completions.anthropic.basic.nonstream.works", + "llm.chat_completions.vertex.basic.nonstream.works", + exercised_on=[], + ) def test_chat_returns_real_completion( self, client: PassthroughClient, scoped_key: str, model: str, route: str ) -> None: @@ -43,16 +48,23 @@ def test_chat_returns_real_completion( ChatBody( model=model, messages=[ - ChatMessage(role="user", content=f"reply with one word {unique_marker()}") + ChatMessage( + role="user", + content=f"reply with one word {unique_marker()}", + ) ], max_tokens=512, ), ) ) - assert response.model, f"{model} ({route}): response carried no model name: {response}" - assert response.choices, f"{model} ({route}): response had no choices: {response}" + assert ( + response.model + ), f"{model} ({route}): response carried no model name: {response}" + assert ( + response.choices + ), f"{model} ({route}): response had no choices: {response}" message = response.choices[0].message - assert message is not None and message.content and message.content.strip(), ( - f"{model} ({route}): 200 with an empty completion (#28991): {response}" - ) + assert ( + message is not None and message.content and message.content.strip() + ), f"{model} ({route}): 200 with an empty completion (#28991): {response}" diff --git a/tests/e2e/llm_translation/test_provider_features_e2e.py b/tests/e2e/llm_translation/test_provider_features_e2e.py index 9c99c1be161d..cf05a4306b44 100644 --- a/tests/e2e/llm_translation/test_provider_features_e2e.py +++ b/tests/e2e/llm_translation/test_provider_features_e2e.py @@ -76,13 +76,18 @@ def post_chat(client: PassthroughClient, key: str, body: BaseModel) -> ChatRespo class TestServiceTier: - @pytest.mark.covers("llm.chat_completions.openai.service_tier.works", exercised_on=[]) + @pytest.mark.covers( + "llm.chat_completions.openai.service_tier.nonstream.works", exercised_on=[] + ) def test_openai_service_tier_is_echoed( self, client: PassthroughClient, resources: ResourceManager ) -> None: model = f"e2e-service-tier-{unique_marker()}" model_id = client.gateway.create_model( - model, LiteLLMParamsBody(model="openai/gpt-5.5", api_key="os.environ/OPENAI_API_KEY") + model, + LiteLLMParamsBody( + model="openai/gpt-5.5", api_key="os.environ/OPENAI_API_KEY" + ), ) resources.defer(lambda: client.gateway.delete_model(model_id)) key = resources.key() @@ -106,7 +111,8 @@ def test_openai_service_tier_is_echoed( class TestPromptCaching: @pytest.mark.covers( - "llm.chat_completions.bedrock_converse.prompt_cache_5m.nonstream.cache_hit", exercised_on=[] + "llm.chat_completions.bedrock_converse.prompt_cache_5m.nonstream.works", + exercised_on=[], ) def test_bedrock_cache_control_produces_cache_read( self, client: PassthroughClient, resources: ResourceManager @@ -129,7 +135,9 @@ def test_bedrock_cache_control_produces_cache_read( RichMessage( role="user", content=[ - CacheTextBlock(text=cacheable_prefix(), cache_control=CacheControl()), + CacheTextBlock( + text=cacheable_prefix(), cache_control=CacheControl() + ), CacheTextBlock(text="Answer in one word: acknowledged?"), ], ) diff --git a/tests/e2e/management/test_management_e2e.py b/tests/e2e/management/test_management_e2e.py index 3beb039b8bd8..cbd5db0d59ff 100644 --- a/tests/e2e/management/test_management_e2e.py +++ b/tests/e2e/management/test_management_e2e.py @@ -167,7 +167,7 @@ def rejected() -> bool | None: class TestTeamRoutes: - @pytest.mark.covers("management.team.new.persists") + @pytest.mark.covers("mgmt.team.new.persists") def test_new_persists_to_team_info_and_binds_keys( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -212,7 +212,7 @@ def test_member_add_and_delete_persist_to_team_info( class TestUserRoutes: - @pytest.mark.covers("mgmt.user.new.persists") + @pytest.mark.covers("mgmt.user.new.happy_path") def test_new_persists_to_user_info(self, client: ManagementClient, resources: ResourceManager) -> None: email = f"e2e-mgmt-{unique_marker()}@example.com" user_id = _create_user(client, resources, UserNewBody(user_email=email, user_role="internal_user")) @@ -225,7 +225,7 @@ def test_new_persists_to_user_info(self, client: ManagementClient, resources: Re class TestOrganizationRoutes: - @pytest.mark.covers("mgmt.organization.new.persists") + @pytest.mark.covers("mgmt.organization.new.happy_path") def test_new_persists_to_organization_info( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -252,7 +252,7 @@ def _assert_route_forbidden(route: str, outcome: StreamingResponse) -> None: class TestManagementRoutePermissions: - @pytest.mark.covers("mgmt.key.generate.member_forbidden") + @pytest.mark.covers("other.auth.virtual_key.route_permission_enforced") def test_llm_only_key_forbidden_from_management_writes( self, client: ManagementClient, resources: ResourceManager ) -> None: