From 3fb992d2ce15c7cb78f2a3972c54ea4de18dad16 Mon Sep 17 00:00:00 2001 From: Jiong Gong Date: Wed, 23 Sep 2026 14:28:00 +0000 Subject: [PATCH] compass(gates): join the README's exclude-list counts to the list scripts/compass/README.md states cpu_gate_exclude.txt's counts (29 total, 28 GENERATED, 1 MANUAL), and nothing checked them. An entry added to the MANUAL section reddened the in-file pin and left the README wrong in silence. Add test_the_readme_states_the_counts_the_list_holds, parametrised over the three numbers in the README row. Each one is compared with the count derived from the list, the same shape gate_gpu.sh uses for BASE_FAILED against gpu_gate_known_failures.txt. A disagreement names the README, the stated number, the list and the entries it holds. A missing or reworded row fails by name rather than passing vacuously. Nothing under scripts/compass/ changes. Closes #213 Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/compass/test_cpu_gate_exclude.py | 28 ++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/compass/test_cpu_gate_exclude.py b/tests/compass/test_cpu_gate_exclude.py index 868732c094..1835c570b6 100644 --- a/tests/compass/test_cpu_gate_exclude.py +++ b/tests/compass/test_cpu_gate_exclude.py @@ -34,6 +34,7 @@ `regen_cpu_gate_exclude.sh`'s job, not a unit test's. """ +import re from pathlib import Path import pytest @@ -41,6 +42,7 @@ REPO = Path(__file__).resolve().parents[2] EXCLUDE = REPO / "scripts" / "compass" / "cpu_gate_exclude.txt" TRIGGERS = REPO / "scripts" / "compass" / "gpu_gate_triggers.txt" +README = REPO / "scripts" / "compass" / "README.md" GEN_BEGIN, GEN_END = "# BEGIN GENERATED", "# END GENERATED" MAN_BEGIN, MAN_END = "# BEGIN MANUAL", "# END MANUAL" @@ -187,6 +189,32 @@ def test_the_manual_guard_finds_nothing_when_the_section_empties(monkeypatch, tm assert not _manual_entries() +@pytest.mark.parametrize( + "stated, derive", + [ + (r"\*\*(\d+)\*\* excluded test files", _entries), + (r"\*\*(\d+) GENERATED\*\*", lambda: _paths(_section(GEN_BEGIN, GEN_END))), + (r"\*\*(\d+) MANUAL\*\*", _manual_entries), + ], + ids=["total", "generated", "manual"], +) +def test_the_readme_states_the_counts_the_list_holds(stated, derive): + # README.md restates this file's counts, and the pin above holds the MANUAL + # one in this file only. Without this join, an entry added here reddens the + # pin and leaves the README wrong in silence. Same shape as gate_gpu.sh's + # BASE_FAILED against gpu_gate_known_failures.txt: the stated number stays, + # the list is counted, and a disagreement names both rather than picking one. + row = [ln for ln in _lines(README) if ln.startswith(f"| `{EXCLUDE.name}` |")] + assert len(row) == 1, f"{README.name} has {len(row)} rows for {EXCLUDE.name}" + found = re.search(stated, row[0]) + assert found, f"{README.name}'s {EXCLUDE.name} row no longer states /{stated}/" + entries = derive() + assert int(found[1]) == len(entries), ( + f"{README.name} states {found[0]} but {EXCLUDE.name} holds {len(entries)}: " + f"{entries}. Update the README row to match the list." + ) + + @pytest.mark.parametrize("entry", _manual_entries()) def test_every_manual_entry_states_why(entry): # A manual entry is an assertion no script can check, so the reason is the