fix(ci): judge a panic ratchet red by its per-lint rows, not the packages it names

scripts/ci/base_red_excuse.py

@@ -404,6 +404,56 @@ def compile_pass_at(repo: str, sha: str, github) -> str | None:
     return None
 
 
+# The Rust panic ratchet (scripts/cmux-next/rust_panic_ratchet.py, run by .github/scripts/cmux-tui-clippy-changed.py).
+# Its error line names the packages the run checked, and a base run may check the whole workspace while a PR checks
+# only its changed packages, so that line never matches (2026-10-31: cmux#26849's run 41176610742 failed on two rows
+# that base run 41176544319 failed on word for word, and was refused). Its red is judged by the rows of its RED
+# block instead (ratchet_red): each head row `CRATE: clippy::LINT N > WHERE` must be on the base with the same
+# crate, lint and WHERE (`baseline M` or `new crate (must be 0)`) and a count at least the head's; every other
+# line of the block but a hit location must be on the base word for word. A red with no row is never excused.
+RATCHET_SCOPE = re.compile(r"^the panic ratchet is red for .+; its log above names each new hit")
+RATCHET_RED, RATCHET_END = "rust-panic-ratchet: RED", "Fix the hit "
+RATCHET_ROW = re.compile(r"^(\S+): clippy::(\w+) (\d+) > (baseline \d+|new crate \(must be 0\))$")
+RATCHET_HIT = re.compile(r"^[^\s:]+\.rs:\d+:\d+$")
+
+
+def ratchet_red(log: str) -> tuple[dict[tuple[str, str, str], int], set[str]]:
+    """({(crate, lint, where): count} of the ratchet's RED rows, its other RED lines), hit locations left out."""
+    rows: dict[tuple[str, str, str], int] = {}
+    other: set[str] = set()
+    skip = test_reds.echoed(test_reds.log_lines(log))
+    inside = False
+    for n, text in enumerate(lines_of(log)):
+        if n in skip:
+            continue
+        if text == RATCHET_RED:
+            inside = True
+        elif inside and (text.startswith(RATCHET_END) or text.startswith("##[")):
+            inside = False
+        elif inside and text:
+            if row := RATCHET_ROW.match(text):
+                crate, lint, count, where = row.groups()
+                key = (crate, lint, where)
+                rows[key] = max(rows.get(key, 0), int(count))
+            elif not RATCHET_HIT.match(text):
+                other.add(text)
+    return rows, other
+
+
+def ratchet_unmatched(head_log: str, base_log: str) -> list[str]:
+    """The head's panic ratchet failures its base run does not have (empty when every one is covered)."""
+    head_rows, head_other = ratchet_red(head_log)
+    if not head_rows:
+        raise Refused("the panic ratchet is red with no per-lint row to compare with the base "
+                      "(a clippy run that did not finish, say)")
+    base_rows, base_other = ratchet_red(base_log)
+    missing = [f"{crate}: clippy::{lint} {count} > {where}" + (
+                   f" (the base has {base_rows[(crate, lint, where)]})" if (crate, lint, where) in base_rows else "")
+               for (crate, lint, where), count in sorted(head_rows.items())
+               if base_rows.get((crate, lint, where), -1) < count]
+    return missing + sorted(head_other - base_other)
+
+
 def crash_signals(log: str) -> set[str]:
     """The signals a test helper died on, from SwiftPM's or the hang sampler's lines."""
     return {first or second for first, second in CRASH.findall(log)}
@@ -636,13 +686,26 @@ def judge(repo: str, base: str, sha: str, runs: list[int], github, *, checks: li
         if missing := sorted(head_crash - crash_signals(base_log)):
             raise Refused(f"'{name}': the process died on signal {', '.join(missing)}, "
                           f"and {base} run {base_run['id']} did not")
+        # The ratchet's error line names the run's packages, not its red: its rows are compared instead (RATCHET_SCOPE).
+        ratchet = {error for error in head_errors if RATCHET_SCOPE.match(error)}
+        if ratchet:
+            try:
+                unmatched = ratchet_unmatched(head_log, base_log)
+            except Refused as error:
+                raise Refused(f"'{name}': {error}") from None
+            if unmatched:
+                raise Refused(f"'{name}': panic ratchet failures not on {base} run {base_run['id']}: "
+                              f"{'; '.join(unmatched)[:600]}")
+            head_errors = head_errors - ratchet
         compared = [("failed steps", head_steps, base_steps), ("failing tests", head_tests, base_tests)]
         if not head_tests:
             compared.append(("errors", head_errors, base_errors))
         for kind, head, seen in compared:
             if missing := sorted(head - seen):
                 raise Refused(f"'{name}': {kind} not on {base} run {base_run['id']}: {'; '.join(missing)[:600]}")
-        what = "; ".join(sorted(head_tests)) or "; ".join(sorted(head_errors))
+        rows = [f"{crate}: clippy::{lint} {count} > {where}"
+                for (crate, lint, where), count in sorted(ratchet_red(head_log)[0].items())] if ratchet else []
+        what = "; ".join(sorted(head_tests)) or "; ".join(rows + sorted(head_errors))
         url = base_run.get("html_url") or f"https://github.com/{repo}/actions/runs/{base_run['id']}"
         # The base verdict is the newest one, not necessarily the base tip's: the base tier runs at most
         # once every 15 minutes, so name the commit and time it is from.

tests/test_base_red_excuse.py

@@ -1208,6 +1208,109 @@ def test_a_real_fleet_compile_failure_is_compared_with_the_base(self):
         self.assertIn(f"excused '{self.FLEET}'", "\n".join(self.judge()))
 
 
+CLIPPY = "clippy (changed cmux-tui packages)"
+RATCHET_STEP = "Rust panic ratchet on changed cmux-tui packages"
+
+
+def ratchet_log(scope, *rows, other=()):
+    """A panic ratchet step's log, shaped like run 41176610742's (cmux#26849): each ROWS entry is
+    (crate, lint, count, where) with two hit locations under it; OTHER lines are the ratchet's other failures."""
+    lines = [f"cmux-tui-clippy-changed: panic ratchet on {scope}",
+             "##[warning]acpmux: clippy::arithmetic_side_effects 1496 > baseline 1430",
+             "rust-panic-ratchet: 50 crates checked, 17 at deny", "rust-panic-ratchet: RED"]
+    for crate, lint, count, where in rows:
+        lines += [f"  {crate}: clippy::{lint} {count} > {where}",
+                  f"      crates/{crate}/src/lib.rs:{count}:9", f"      crates/{crate}/src/main.rs:{count + 1}:13"]
+    lines += [f"  {line}" for line in other]
+    lines += ["Fix the hit (a typed error handled at a boundary), or put a reviewed",
+              '#[allow(clippy::<lint>, reason = "...")] on the smallest item. Never raise the baseline.',
+              f"##[error]the panic ratchet is red for {scope}; its log above names each new hit "
+              "(plans/cmux-next/rust-panic-free.md)",
+              "##[error]Process completed with exit code 1."]
+    return log(CLIPPY, RATCHET_STEP, *lines)
+
+
+HEAD_ROWS = (("acpmux", "indexing_slicing", 692, "baseline 686"), ("cmux-tui", "indexing_slicing", 836, "baseline 831"))
+RD_LINK = ("cmux-rd-link", "indexing_slicing", 7, "new crate (must be 0)")
+CHANGED = "cmux-remote, cmux-terminal-client, cmux-tui, cmux-tui-core"
+WHOLE = "the whole cmux-tui workspace"
+
+
+class PanicRatchet(unittest.TestCase):
+    """The ratchet's error line names the packages a run checked, and a base run can check the whole workspace
+    while a PR's checks only its changed packages: its red is compared by its per-lint rows instead (cmux#26849's
+    run 41176610742 was refused against base run 41176544319, which failed on the same two rows and one more)."""
+
+    def setUp(self):
+        self.gh = FakeGitHub()
+        self.gh.jobs = {1: failed_job(1, 10, HEAD, CLIPPY, RATCHET_STEP),
+                        2: failed_job(2, 20, BASE_SHA, CLIPPY, RATCHET_STEP)}
+        self.gh.logs[1] = ratchet_log(CHANGED, *HEAD_ROWS)
+        self.gh.logs[2] = ratchet_log(WHOLE, HEAD_ROWS[0], RD_LINK, HEAD_ROWS[1])
+
+    def judge(self):
+        return "\n".join(excuse.judge(REPO, "feat-cmux-next", HEAD, [10], self.gh))
+
+    def test_rows_all_on_a_whole_workspace_base_run_are_excused(self):
+        text = self.judge()
+        self.assertIn(f"excused '{CLIPPY}'", text)
+        self.assertIn("acpmux: clippy::indexing_slicing 692 > baseline 686", text)
+
+    def test_a_base_count_above_the_heads_still_covers_it(self):
+        self.gh.logs[1] = ratchet_log(CHANGED, ("acpmux", "indexing_slicing", 690, "baseline 686"), HEAD_ROWS[1])
+        self.assertIn(f"excused '{CLIPPY}'", self.judge())
+
+    def test_a_head_count_above_the_bases_blocks(self):
+        self.gh.logs[1] = ratchet_log(CHANGED, ("acpmux", "indexing_slicing", 693, "baseline 686"), HEAD_ROWS[1])
+        with self.assertRaisesRegex(excuse.Refused, "acpmux: clippy::indexing_slicing 693 > baseline 686"):
+            self.judge()
+
+    def test_a_head_count_above_the_bases_blocks_with_the_same_scope_too(self):
+        self.gh.logs[1] = ratchet_log(WHOLE, ("acpmux", "indexing_slicing", 693, "baseline 686"), RD_LINK, HEAD_ROWS[1])
+        with self.assertRaisesRegex(excuse.Refused, "693 > baseline 686"):
+            self.judge()
+
+    def test_a_row_not_on_the_base_blocks(self):
+        self.gh.logs[1] = ratchet_log(CHANGED, *HEAD_ROWS, ("cmux-wg", "indexing_slicing", 3, "baseline 1"))
+        with self.assertRaisesRegex(excuse.Refused, "cmux-wg: clippy::indexing_slicing 3 > baseline 1"):
+            self.judge()
+
+    def test_a_row_of_another_lint_blocks(self):
+        self.gh.logs[1] = ratchet_log(CHANGED, HEAD_ROWS[0], ("cmux-tui", "unwrap_used", 836, "baseline 831"))
+        with self.assertRaisesRegex(excuse.Refused, "unwrap_used"):
+            self.judge()
+
+    def test_a_different_baseline_blocks(self):
+        # The PR raised (or set) the row's baseline: base's row is not the same red.
+        self.gh.logs[1] = ratchet_log(CHANGED, ("acpmux", "indexing_slicing", 692, "baseline 680"), HEAD_ROWS[1])
+        with self.assertRaisesRegex(excuse.Refused, "692 > baseline 680"):
+            self.judge()
+
+    def test_another_ratchet_failure_not_on_the_base_blocks(self):
+        reason = 'crates/acpmux/src/lib.rs:12: allow(indexing_slicing) needs reason = "..."'
+        self.gh.logs[1] = ratchet_log(CHANGED, *HEAD_ROWS, other=[reason])
+        with self.assertRaisesRegex(excuse.Refused, "needs reason"):
+            self.judge()
+        self.gh.logs[2] = ratchet_log(WHOLE, HEAD_ROWS[0], RD_LINK, HEAD_ROWS[1], other=[reason])
+        self.assertIn(f"excused '{CLIPPY}'", self.judge())
+
+    def test_a_ratchet_red_with_no_rows_blocks(self):
+        # The ratchet refuses a clippy run that did not finish: nothing in it can be compared.
+        self.gh.logs[1] = log(CLIPPY, RATCHET_STEP,
+                              "rust-panic-ratchet: the clippy run did not finish successfully (no build-finished "
+                              "success line); counts from a partial build prove nothing",
+                              f"##[error]the panic ratchet is red for {WHOLE}; its log above names each new hit "
+                              "(plans/cmux-next/rust-panic-free.md)")
+        self.gh.logs[2] = self.gh.logs[1]
+        with self.assertRaisesRegex(excuse.Refused, "no per-lint row"):
+            self.judge()
+
+    def test_another_error_beside_the_ratchet_is_still_compared(self):
+        self.gh.logs[1] += "\n" + log(CLIPPY, RATCHET_STEP, "error: could not compile `acpmux`")
+        with self.assertRaisesRegex(excuse.Refused, "could not compile"):
+            self.judge()
+
+
 class Script(unittest.TestCase):
     def test_gh_merge_green_asks_the_base_before_refusing_a_red_lane(self):
         text = (ROOT / "scripts/gh-merge-green").read_text()