From fd5ad6bc3e61cbe5f3bce7821aaa1afc925c0b1f Mon Sep 17 00:00:00 2001 From: Leo Li Date: Fri, 30 Oct 2026 23:58:15 -0700 Subject: [PATCH 1/2] test(ci): reproduce the panic ratchet red refused against a whole-workspace base run Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-710a4a518c009a6ad126 --- tests/test_base_red_excuse.py | 103 ++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) diff --git a/tests/test_base_red_excuse.py b/tests/test_base_red_excuse.py index d7a2bda26c47..6b5e89a1ef8b 100644 --- a/tests/test_base_red_excuse.py +++ b/tests/test_base_red_excuse.py @@ -1208,6 +1208,109 @@ def test_a_real_fleet_compile_failure_is_compared_with_the_base(self): self.assertIn(f"excused '{self.FLEET}'", "\n".join(self.judge())) +CLIPPY = "clippy (changed cmux-tui packages)" +RATCHET_STEP = "Rust panic ratchet on changed cmux-tui packages" + + +def ratchet_log(scope, *rows, other=()): + """A panic ratchet step's log, shaped like run 41176610742's (cmux#26849): each ROWS entry is + (crate, lint, count, where) with two hit locations under it; OTHER lines are the ratchet's other failures.""" + lines = [f"cmux-tui-clippy-changed: panic ratchet on {scope}", + "##[warning]acpmux: clippy::arithmetic_side_effects 1496 > baseline 1430", + "rust-panic-ratchet: 50 crates checked, 17 at deny", "rust-panic-ratchet: RED"] + for crate, lint, count, where in rows: + lines += [f" {crate}: clippy::{lint} {count} > {where}", + f" crates/{crate}/src/lib.rs:{count}:9", f" crates/{crate}/src/main.rs:{count + 1}:13"] + lines += [f" {line}" for line in other] + lines += ["Fix the hit (a typed error handled at a boundary), or put a reviewed", + '#[allow(clippy::, reason = "...")] on the smallest item. Never raise the baseline.', + f"##[error]the panic ratchet is red for {scope}; its log above names each new hit " + "(plans/cmux-next/rust-panic-free.md)", + "##[error]Process completed with exit code 1."] + return log(CLIPPY, RATCHET_STEP, *lines) + + +HEAD_ROWS = (("acpmux", "indexing_slicing", 692, "baseline 686"), ("cmux-tui", "indexing_slicing", 836, "baseline 831")) +RD_LINK = ("cmux-rd-link", "indexing_slicing", 7, "new crate (must be 0)") +CHANGED = "cmux-remote, cmux-terminal-client, cmux-tui, cmux-tui-core" +WHOLE = "the whole cmux-tui workspace" + + +class PanicRatchet(unittest.TestCase): + """The ratchet's error line names the packages a run checked, and a base run can check the whole workspace + while a PR's checks only its changed packages: its red is compared by its per-lint rows instead (cmux#26849's + run 41176610742 was refused against base run 41176544319, which failed on the same two rows and one more).""" + + def setUp(self): + self.gh = FakeGitHub() + self.gh.jobs = {1: failed_job(1, 10, HEAD, CLIPPY, RATCHET_STEP), + 2: failed_job(2, 20, BASE_SHA, CLIPPY, RATCHET_STEP)} + self.gh.logs[1] = ratchet_log(CHANGED, *HEAD_ROWS) + self.gh.logs[2] = ratchet_log(WHOLE, HEAD_ROWS[0], RD_LINK, HEAD_ROWS[1]) + + def judge(self): + return "\n".join(excuse.judge(REPO, "feat-cmux-next", HEAD, [10], self.gh)) + + def test_rows_all_on_a_whole_workspace_base_run_are_excused(self): + text = self.judge() + self.assertIn(f"excused '{CLIPPY}'", text) + self.assertIn("acpmux: clippy::indexing_slicing 692 > baseline 686", text) + + def test_a_base_count_above_the_heads_still_covers_it(self): + self.gh.logs[1] = ratchet_log(CHANGED, ("acpmux", "indexing_slicing", 690, "baseline 686"), HEAD_ROWS[1]) + self.assertIn(f"excused '{CLIPPY}'", self.judge()) + + def test_a_head_count_above_the_bases_blocks(self): + self.gh.logs[1] = ratchet_log(CHANGED, ("acpmux", "indexing_slicing", 693, "baseline 686"), HEAD_ROWS[1]) + with self.assertRaisesRegex(excuse.Refused, "acpmux: clippy::indexing_slicing 693 > baseline 686"): + self.judge() + + def test_a_head_count_above_the_bases_blocks_with_the_same_scope_too(self): + self.gh.logs[1] = ratchet_log(WHOLE, ("acpmux", "indexing_slicing", 693, "baseline 686"), RD_LINK, HEAD_ROWS[1]) + with self.assertRaisesRegex(excuse.Refused, "693 > baseline 686"): + self.judge() + + def test_a_row_not_on_the_base_blocks(self): + self.gh.logs[1] = ratchet_log(CHANGED, *HEAD_ROWS, ("cmux-wg", "indexing_slicing", 3, "baseline 1")) + with self.assertRaisesRegex(excuse.Refused, "cmux-wg: clippy::indexing_slicing 3 > baseline 1"): + self.judge() + + def test_a_row_of_another_lint_blocks(self): + self.gh.logs[1] = ratchet_log(CHANGED, HEAD_ROWS[0], ("cmux-tui", "unwrap_used", 836, "baseline 831")) + with self.assertRaisesRegex(excuse.Refused, "unwrap_used"): + self.judge() + + def test_a_different_baseline_blocks(self): + # The PR raised (or set) the row's baseline: base's row is not the same red. + self.gh.logs[1] = ratchet_log(CHANGED, ("acpmux", "indexing_slicing", 692, "baseline 680"), HEAD_ROWS[1]) + with self.assertRaisesRegex(excuse.Refused, "692 > baseline 680"): + self.judge() + + def test_another_ratchet_failure_not_on_the_base_blocks(self): + reason = 'crates/acpmux/src/lib.rs:12: allow(indexing_slicing) needs reason = "..."' + self.gh.logs[1] = ratchet_log(CHANGED, *HEAD_ROWS, other=[reason]) + with self.assertRaisesRegex(excuse.Refused, "needs reason"): + self.judge() + self.gh.logs[2] = ratchet_log(WHOLE, HEAD_ROWS[0], RD_LINK, HEAD_ROWS[1], other=[reason]) + self.assertIn(f"excused '{CLIPPY}'", self.judge()) + + def test_a_ratchet_red_with_no_rows_blocks(self): + # The ratchet refuses a clippy run that did not finish: nothing in it can be compared. + self.gh.logs[1] = log(CLIPPY, RATCHET_STEP, + "rust-panic-ratchet: the clippy run did not finish successfully (no build-finished " + "success line); counts from a partial build prove nothing", + f"##[error]the panic ratchet is red for {WHOLE}; its log above names each new hit " + "(plans/cmux-next/rust-panic-free.md)") + self.gh.logs[2] = self.gh.logs[1] + with self.assertRaisesRegex(excuse.Refused, "no per-lint row"): + self.judge() + + def test_another_error_beside_the_ratchet_is_still_compared(self): + self.gh.logs[1] += "\n" + log(CLIPPY, RATCHET_STEP, "error: could not compile `acpmux`") + with self.assertRaisesRegex(excuse.Refused, "could not compile"): + self.judge() + + class Script(unittest.TestCase): def test_gh_merge_green_asks_the_base_before_refusing_a_red_lane(self): text = (ROOT / "scripts/gh-merge-green").read_text() From 317aa685080385a50376daed8438d9f555710ff8 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Fri, 30 Oct 2026 23:59:27 -0700 Subject: [PATCH 2/2] fix(ci): judge a panic ratchet red by its per-lint rows, not its scope line The ratchet's ::error line names the packages a run checked. A base run can check the whole cmux-tui workspace while a PR checks only its changed packages, so the line never matched and no ratchet red was ever excused (cmux#26849's run 41176610742 failed on two rows that base run 41176544319 failed on word for word). Each head row must now be on the base with the same crate, lint and baseline and a count at least the head's; other RED lines must match word for word, and a red with no row is never excused. A head count above the base's now blocks even when both runs share a scope. Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-710a4a518c009a6ad126 --- scripts/ci/base_red_excuse.py | 65 ++++++++++++++++++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) diff --git a/scripts/ci/base_red_excuse.py b/scripts/ci/base_red_excuse.py index 16ae68b794e1..1fd1a81d647c 100755 --- a/scripts/ci/base_red_excuse.py +++ b/scripts/ci/base_red_excuse.py @@ -404,6 +404,56 @@ def compile_pass_at(repo: str, sha: str, github) -> str | None: return None +# The Rust panic ratchet (scripts/cmux-next/rust_panic_ratchet.py, run by .github/scripts/cmux-tui-clippy-changed.py). +# Its error line names the packages the run checked, and a base run may check the whole workspace while a PR checks +# only its changed packages, so that line never matches (2026-10-31: cmux#26849's run 41176610742 failed on two rows +# that base run 41176544319 failed on word for word, and was refused). Its red is judged by the rows of its RED +# block instead (ratchet_red): each head row `CRATE: clippy::LINT N > WHERE` must be on the base with the same +# crate, lint and WHERE (`baseline M` or `new crate (must be 0)`) and a count at least the head's; every other +# line of the block but a hit location must be on the base word for word. A red with no row is never excused. +RATCHET_SCOPE = re.compile(r"^the panic ratchet is red for .+; its log above names each new hit") +RATCHET_RED, RATCHET_END = "rust-panic-ratchet: RED", "Fix the hit " +RATCHET_ROW = re.compile(r"^(\S+): clippy::(\w+) (\d+) > (baseline \d+|new crate \(must be 0\))$") +RATCHET_HIT = re.compile(r"^[^\s:]+\.rs:\d+:\d+$") + + +def ratchet_red(log: str) -> tuple[dict[tuple[str, str, str], int], set[str]]: + """({(crate, lint, where): count} of the ratchet's RED rows, its other RED lines), hit locations left out.""" + rows: dict[tuple[str, str, str], int] = {} + other: set[str] = set() + skip = test_reds.echoed(test_reds.log_lines(log)) + inside = False + for n, text in enumerate(lines_of(log)): + if n in skip: + continue + if text == RATCHET_RED: + inside = True + elif inside and (text.startswith(RATCHET_END) or text.startswith("##[")): + inside = False + elif inside and text: + if row := RATCHET_ROW.match(text): + crate, lint, count, where = row.groups() + key = (crate, lint, where) + rows[key] = max(rows.get(key, 0), int(count)) + elif not RATCHET_HIT.match(text): + other.add(text) + return rows, other + + +def ratchet_unmatched(head_log: str, base_log: str) -> list[str]: + """The head's panic ratchet failures its base run does not have (empty when every one is covered).""" + head_rows, head_other = ratchet_red(head_log) + if not head_rows: + raise Refused("the panic ratchet is red with no per-lint row to compare with the base " + "(a clippy run that did not finish, say)") + base_rows, base_other = ratchet_red(base_log) + missing = [f"{crate}: clippy::{lint} {count} > {where}" + ( + f" (the base has {base_rows[(crate, lint, where)]})" if (crate, lint, where) in base_rows else "") + for (crate, lint, where), count in sorted(head_rows.items()) + if base_rows.get((crate, lint, where), -1) < count] + return missing + sorted(head_other - base_other) + + def crash_signals(log: str) -> set[str]: """The signals a test helper died on, from SwiftPM's or the hang sampler's lines.""" return {first or second for first, second in CRASH.findall(log)} @@ -636,13 +686,26 @@ def judge(repo: str, base: str, sha: str, runs: list[int], github, *, checks: li if missing := sorted(head_crash - crash_signals(base_log)): raise Refused(f"'{name}': the process died on signal {', '.join(missing)}, " f"and {base} run {base_run['id']} did not") + # The ratchet's error line names the run's packages, not its red: its rows are compared instead (RATCHET_SCOPE). + ratchet = {error for error in head_errors if RATCHET_SCOPE.match(error)} + if ratchet: + try: + unmatched = ratchet_unmatched(head_log, base_log) + except Refused as error: + raise Refused(f"'{name}': {error}") from None + if unmatched: + raise Refused(f"'{name}': panic ratchet failures not on {base} run {base_run['id']}: " + f"{'; '.join(unmatched)[:600]}") + head_errors = head_errors - ratchet compared = [("failed steps", head_steps, base_steps), ("failing tests", head_tests, base_tests)] if not head_tests: compared.append(("errors", head_errors, base_errors)) for kind, head, seen in compared: if missing := sorted(head - seen): raise Refused(f"'{name}': {kind} not on {base} run {base_run['id']}: {'; '.join(missing)[:600]}") - what = "; ".join(sorted(head_tests)) or "; ".join(sorted(head_errors)) + rows = [f"{crate}: clippy::{lint} {count} > {where}" + for (crate, lint, where), count in sorted(ratchet_red(head_log)[0].items())] if ratchet else [] + what = "; ".join(sorted(head_tests)) or "; ".join(rows + sorted(head_errors)) url = base_run.get("html_url") or f"https://github.com/{repo}/actions/runs/{base_run['id']}" # The base verdict is the newest one, not necessarily the base tip's: the base tier runs at most # once every 15 minutes, so name the commit and time it is from.