diff --git a/scripts/ci/base_red_excuse.py b/scripts/ci/base_red_excuse.py index 16ae68b794e1..1fd1a81d647c 100755 --- a/scripts/ci/base_red_excuse.py +++ b/scripts/ci/base_red_excuse.py @@ -404,6 +404,56 @@ def compile_pass_at(repo: str, sha: str, github) -> str | None: return None +# The Rust panic ratchet (scripts/cmux-next/rust_panic_ratchet.py, run by .github/scripts/cmux-tui-clippy-changed.py). +# Its error line names the packages the run checked, and a base run may check the whole workspace while a PR checks +# only its changed packages, so that line never matches (2026-10-31: cmux#26849's run 41176610742 failed on two rows +# that base run 41176544319 failed on word for word, and was refused). Its red is judged by the rows of its RED +# block instead (ratchet_red): each head row `CRATE: clippy::LINT N > WHERE` must be on the base with the same +# crate, lint and WHERE (`baseline M` or `new crate (must be 0)`) and a count at least the head's; every other +# line of the block but a hit location must be on the base word for word. A red with no row is never excused. +RATCHET_SCOPE = re.compile(r"^the panic ratchet is red for .+; its log above names each new hit") +RATCHET_RED, RATCHET_END = "rust-panic-ratchet: RED", "Fix the hit " +RATCHET_ROW = re.compile(r"^(\S+): clippy::(\w+) (\d+) > (baseline \d+|new crate \(must be 0\))$") +RATCHET_HIT = re.compile(r"^[^\s:]+\.rs:\d+:\d+$") + + +def ratchet_red(log: str) -> tuple[dict[tuple[str, str, str], int], set[str]]: + """({(crate, lint, where): count} of the ratchet's RED rows, its other RED lines), hit locations left out.""" + rows: dict[tuple[str, str, str], int] = {} + other: set[str] = set() + skip = test_reds.echoed(test_reds.log_lines(log)) + inside = False + for n, text in enumerate(lines_of(log)): + if n in skip: + continue + if text == RATCHET_RED: + inside = True + elif inside and (text.startswith(RATCHET_END) or text.startswith("##[")): + inside = False + elif inside and text: + if row := RATCHET_ROW.match(text): + crate, lint, count, where = row.groups() + key = (crate, lint, where) + rows[key] = max(rows.get(key, 0), int(count)) + elif not RATCHET_HIT.match(text): + other.add(text) + return rows, other + + +def ratchet_unmatched(head_log: str, base_log: str) -> list[str]: + """The head's panic ratchet failures its base run does not have (empty when every one is covered).""" + head_rows, head_other = ratchet_red(head_log) + if not head_rows: + raise Refused("the panic ratchet is red with no per-lint row to compare with the base " + "(a clippy run that did not finish, say)") + base_rows, base_other = ratchet_red(base_log) + missing = [f"{crate}: clippy::{lint} {count} > {where}" + ( + f" (the base has {base_rows[(crate, lint, where)]})" if (crate, lint, where) in base_rows else "") + for (crate, lint, where), count in sorted(head_rows.items()) + if base_rows.get((crate, lint, where), -1) < count] + return missing + sorted(head_other - base_other) + + def crash_signals(log: str) -> set[str]: """The signals a test helper died on, from SwiftPM's or the hang sampler's lines.""" return {first or second for first, second in CRASH.findall(log)} @@ -636,13 +686,26 @@ def judge(repo: str, base: str, sha: str, runs: list[int], github, *, checks: li if missing := sorted(head_crash - crash_signals(base_log)): raise Refused(f"'{name}': the process died on signal {', '.join(missing)}, " f"and {base} run {base_run['id']} did not") + # The ratchet's error line names the run's packages, not its red: its rows are compared instead (RATCHET_SCOPE). + ratchet = {error for error in head_errors if RATCHET_SCOPE.match(error)} + if ratchet: + try: + unmatched = ratchet_unmatched(head_log, base_log) + except Refused as error: + raise Refused(f"'{name}': {error}") from None + if unmatched: + raise Refused(f"'{name}': panic ratchet failures not on {base} run {base_run['id']}: " + f"{'; '.join(unmatched)[:600]}") + head_errors = head_errors - ratchet compared = [("failed steps", head_steps, base_steps), ("failing tests", head_tests, base_tests)] if not head_tests: compared.append(("errors", head_errors, base_errors)) for kind, head, seen in compared: if missing := sorted(head - seen): raise Refused(f"'{name}': {kind} not on {base} run {base_run['id']}: {'; '.join(missing)[:600]}") - what = "; ".join(sorted(head_tests)) or "; ".join(sorted(head_errors)) + rows = [f"{crate}: clippy::{lint} {count} > {where}" + for (crate, lint, where), count in sorted(ratchet_red(head_log)[0].items())] if ratchet else [] + what = "; ".join(sorted(head_tests)) or "; ".join(rows + sorted(head_errors)) url = base_run.get("html_url") or f"https://github.com/{repo}/actions/runs/{base_run['id']}" # The base verdict is the newest one, not necessarily the base tip's: the base tier runs at most # once every 15 minutes, so name the commit and time it is from. diff --git a/tests/test_base_red_excuse.py b/tests/test_base_red_excuse.py index d7a2bda26c47..6b5e89a1ef8b 100644 --- a/tests/test_base_red_excuse.py +++ b/tests/test_base_red_excuse.py @@ -1208,6 +1208,109 @@ def test_a_real_fleet_compile_failure_is_compared_with_the_base(self): self.assertIn(f"excused '{self.FLEET}'", "\n".join(self.judge())) +CLIPPY = "clippy (changed cmux-tui packages)" +RATCHET_STEP = "Rust panic ratchet on changed cmux-tui packages" + + +def ratchet_log(scope, *rows, other=()): + """A panic ratchet step's log, shaped like run 41176610742's (cmux#26849): each ROWS entry is + (crate, lint, count, where) with two hit locations under it; OTHER lines are the ratchet's other failures.""" + lines = [f"cmux-tui-clippy-changed: panic ratchet on {scope}", + "##[warning]acpmux: clippy::arithmetic_side_effects 1496 > baseline 1430", + "rust-panic-ratchet: 50 crates checked, 17 at deny", "rust-panic-ratchet: RED"] + for crate, lint, count, where in rows: + lines += [f" {crate}: clippy::{lint} {count} > {where}", + f" crates/{crate}/src/lib.rs:{count}:9", f" crates/{crate}/src/main.rs:{count + 1}:13"] + lines += [f" {line}" for line in other] + lines += ["Fix the hit (a typed error handled at a boundary), or put a reviewed", + '#[allow(clippy::, reason = "...")] on the smallest item. Never raise the baseline.', + f"##[error]the panic ratchet is red for {scope}; its log above names each new hit " + "(plans/cmux-next/rust-panic-free.md)", + "##[error]Process completed with exit code 1."] + return log(CLIPPY, RATCHET_STEP, *lines) + + +HEAD_ROWS = (("acpmux", "indexing_slicing", 692, "baseline 686"), ("cmux-tui", "indexing_slicing", 836, "baseline 831")) +RD_LINK = ("cmux-rd-link", "indexing_slicing", 7, "new crate (must be 0)") +CHANGED = "cmux-remote, cmux-terminal-client, cmux-tui, cmux-tui-core" +WHOLE = "the whole cmux-tui workspace" + + +class PanicRatchet(unittest.TestCase): + """The ratchet's error line names the packages a run checked, and a base run can check the whole workspace + while a PR's checks only its changed packages: its red is compared by its per-lint rows instead (cmux#26849's + run 41176610742 was refused against base run 41176544319, which failed on the same two rows and one more).""" + + def setUp(self): + self.gh = FakeGitHub() + self.gh.jobs = {1: failed_job(1, 10, HEAD, CLIPPY, RATCHET_STEP), + 2: failed_job(2, 20, BASE_SHA, CLIPPY, RATCHET_STEP)} + self.gh.logs[1] = ratchet_log(CHANGED, *HEAD_ROWS) + self.gh.logs[2] = ratchet_log(WHOLE, HEAD_ROWS[0], RD_LINK, HEAD_ROWS[1]) + + def judge(self): + return "\n".join(excuse.judge(REPO, "feat-cmux-next", HEAD, [10], self.gh)) + + def test_rows_all_on_a_whole_workspace_base_run_are_excused(self): + text = self.judge() + self.assertIn(f"excused '{CLIPPY}'", text) + self.assertIn("acpmux: clippy::indexing_slicing 692 > baseline 686", text) + + def test_a_base_count_above_the_heads_still_covers_it(self): + self.gh.logs[1] = ratchet_log(CHANGED, ("acpmux", "indexing_slicing", 690, "baseline 686"), HEAD_ROWS[1]) + self.assertIn(f"excused '{CLIPPY}'", self.judge()) + + def test_a_head_count_above_the_bases_blocks(self): + self.gh.logs[1] = ratchet_log(CHANGED, ("acpmux", "indexing_slicing", 693, "baseline 686"), HEAD_ROWS[1]) + with self.assertRaisesRegex(excuse.Refused, "acpmux: clippy::indexing_slicing 693 > baseline 686"): + self.judge() + + def test_a_head_count_above_the_bases_blocks_with_the_same_scope_too(self): + self.gh.logs[1] = ratchet_log(WHOLE, ("acpmux", "indexing_slicing", 693, "baseline 686"), RD_LINK, HEAD_ROWS[1]) + with self.assertRaisesRegex(excuse.Refused, "693 > baseline 686"): + self.judge() + + def test_a_row_not_on_the_base_blocks(self): + self.gh.logs[1] = ratchet_log(CHANGED, *HEAD_ROWS, ("cmux-wg", "indexing_slicing", 3, "baseline 1")) + with self.assertRaisesRegex(excuse.Refused, "cmux-wg: clippy::indexing_slicing 3 > baseline 1"): + self.judge() + + def test_a_row_of_another_lint_blocks(self): + self.gh.logs[1] = ratchet_log(CHANGED, HEAD_ROWS[0], ("cmux-tui", "unwrap_used", 836, "baseline 831")) + with self.assertRaisesRegex(excuse.Refused, "unwrap_used"): + self.judge() + + def test_a_different_baseline_blocks(self): + # The PR raised (or set) the row's baseline: base's row is not the same red. + self.gh.logs[1] = ratchet_log(CHANGED, ("acpmux", "indexing_slicing", 692, "baseline 680"), HEAD_ROWS[1]) + with self.assertRaisesRegex(excuse.Refused, "692 > baseline 680"): + self.judge() + + def test_another_ratchet_failure_not_on_the_base_blocks(self): + reason = 'crates/acpmux/src/lib.rs:12: allow(indexing_slicing) needs reason = "..."' + self.gh.logs[1] = ratchet_log(CHANGED, *HEAD_ROWS, other=[reason]) + with self.assertRaisesRegex(excuse.Refused, "needs reason"): + self.judge() + self.gh.logs[2] = ratchet_log(WHOLE, HEAD_ROWS[0], RD_LINK, HEAD_ROWS[1], other=[reason]) + self.assertIn(f"excused '{CLIPPY}'", self.judge()) + + def test_a_ratchet_red_with_no_rows_blocks(self): + # The ratchet refuses a clippy run that did not finish: nothing in it can be compared. + self.gh.logs[1] = log(CLIPPY, RATCHET_STEP, + "rust-panic-ratchet: the clippy run did not finish successfully (no build-finished " + "success line); counts from a partial build prove nothing", + f"##[error]the panic ratchet is red for {WHOLE}; its log above names each new hit " + "(plans/cmux-next/rust-panic-free.md)") + self.gh.logs[2] = self.gh.logs[1] + with self.assertRaisesRegex(excuse.Refused, "no per-lint row"): + self.judge() + + def test_another_error_beside_the_ratchet_is_still_compared(self): + self.gh.logs[1] += "\n" + log(CLIPPY, RATCHET_STEP, "error: could not compile `acpmux`") + with self.assertRaisesRegex(excuse.Refused, "could not compile"): + self.judge() + + class Script(unittest.TestCase): def test_gh_merge_green_asks_the_base_before_refusing_a_red_lane(self): text = (ROOT / "scripts/gh-merge-green").read_text()