diff --git a/scripts/ci/base_red_excuse.py b/scripts/ci/base_red_excuse.py index de3057529db8..352d0f84f9a1 100755 --- a/scripts/ci/base_red_excuse.py +++ b/scripts/ci/base_red_excuse.py @@ -21,6 +21,11 @@ abort, EXC_*, Fatal error) is the head's whatever the base shows; and a run whose per-test results are cut short (no summary, ended early, stuck) is the head's, since nothing in it can be compared. + +A compile check that gave no verdict (the build fleet's "no result", "No build-fleet verdict") is never a +base red, even with the base's identical line: nothing compiled the head. It is set aside only when the +same head has a real compile pass (a green Mac Release compile or a cmux-next/compile success). Error +lines are compared with commit shas masked, so a real red that names each run's own commit still matches. """ from __future__ import annotations @@ -68,6 +73,33 @@ CRASH = re.compile(r"exited with unexpected signal code (\d+)|the test helper died on signal (\d+)") CRASH_LINE = re.compile(r"exited with unexpected signal code|the test helper died on signal|" r"no \"Test run with\" summary line") +# A commit sha inside an error line: a full 40-hex sha, or 7 to 39 hex digits right after a word that +# names a commit (commit, sha, tree, revision). The same red on the head and the base names each run's +# own commit; any other hex id (an artifact, a hash) is compared as it is. +SHA = re.compile(r"\b[0-9a-f]{40}\b|\b((?:commit|sha|tree|revision) `?)[0-9a-f]{7,39}\b") +# A compile check that gave no verdict (the build-fleet controller unreachable, timed out, or a result +# for another head): nothing compiled the head, so it is never a base red, whatever the base shows +# (2026-10-28: the fleet Release compile's "no result" was red on PRs and on the base alike while the +# controller was wedged). It is set aside only when another job compiled that same head: COMPILE_PASSES. +# Only the job's own ::error annotation counts (`##[error]` at the start of the line): the log also +# echoes the step's script, which spells these words out on every run, a real compile red included. +# Fail closed: of compile-status's own lines, only "(fail)" is a real compile red; every other state (no result, +# unavailable, mismatch, unreadable, pending, "last: ...", a pass it could not post) and a status it could not +# post at all are no verdict. +NO_VERDICT = re.compile(r"^##\[error\](?:cmux-next/compile is not a pass for \S+ \((?!fail\))|" + r"could not post the pending status|bad head sha |No build-fleet verdict for `)") +# The jobs that can end without a verdict: an unreadable log of one is never comparable either. +NO_VERDICT_JOBS = frozenset({"cmux-next fleet Release compile", "cmux-next app compile (build fleet)"}) +# A real compile pass at the head: the Mac Release compile (a check run), or the fleet's cmux-next/compile +# commit status. +COMPILE_PASS_CHECKS = ("cmux-next Release compile (Xcode 26)",) +COMPILE_PASS_STATUS = "cmux-next/compile" +# The Mac Release compile's steps that compile (or reuse a pass for the same inputs). +COMPILE_STEPS = frozenset({"Release compile", "Reuse that pass"}) + + +def mask_shas(text: str) -> str: + return SHA.sub(lambda match: (match.group(1) or "") + "", text) def lines_of(log: str) -> list[str]: @@ -90,10 +122,53 @@ def signature(job: dict, log: str) -> tuple[set[str], set[str], set[str]]: if match := pattern.search(text): tests.add(match.group(1)) if ERROR.search(text) and not GENERIC.search(text) and not WARNING.search(text) and not CRASH_LINE.search(text): - errors.add(re.sub(r"\d+", "#", text.replace("##[error]", "").strip())) + errors.add(re.sub(r"\d+", "#", mask_shas(text.replace("##[error]", "").strip()))) return steps, tests, errors +def no_verdict(log: str) -> str | None: + """The line where a compile check says it gave no verdict, if it does.""" + return next((text for text in lines_of(log) if NO_VERDICT.match(text)), None) + + +def compile_pass_at(repo: str, sha: str, github) -> str | None: + """What compiled SHA itself and passed: the newest Mac Release compile job at SHA that ran (its + `Release compile` or `Reuse that pass` step succeeded), or a cmux-next/compile success.""" + candidates = [] + for page in github.json(f"repos/{repo}/commits/{sha}/check-runs?per_page=100", paginate=True): + for check in page.get("check_runs") or []: + if (check.get("name") in COMPILE_PASS_CHECKS and check.get("head_sha") == sha + and check.get("app", {}).get("slug") == "github-actions" and check.get("status") == "completed" + and check.get("conclusion") != "skipped"): + candidates.append(check) + # Newest first; the same name is also release-compile-fast-admission's (a Linux wrapper that compiles + # nothing), so only a job with the Mac compile's own steps counts, and the newest such attempt decides. + for check in sorted(candidates, key=lambda found: found.get("id", -1), reverse=True): + # The check run's own Actions job, from its details link (a check run id is not a job id). + link = re.fullmatch(r"https://github\.com/" + re.escape(repo) + r"/actions/runs/(\d+)/job/(\d+)", + check.get("details_url") or "") + if not link: + continue + job = github.json(f"repos/{repo}/actions/jobs/{link[2]}") + if (job.get("head_sha") != sha or job.get("name") != check.get("name") + or job.get("run_id") != int(link[1])): + continue + steps = {step.get("name"): step.get("conclusion") for step in job.get("steps") or []} + if not COMPILE_STEPS & steps.keys(): + continue + if job.get("conclusion") == "success" and any(steps.get(name) == "success" for name in COMPILE_STEPS): + return f"'{check['name']}' (job {link[2]})" + break + status = github.json(f"repos/{repo}/commits/{sha}/status") + for found in status.get("statuses") or []: + if found.get("context") == COMPILE_PASS_STATUS: + # The combined status lists the newest per context. + if found.get("state") == "success": + return f"the {COMPILE_PASS_STATUS} status" + break + return None + + def crash_signals(log: str) -> set[str]: """The signals a test helper died on, from SwiftPM's or the hang sampler's lines.""" return {first or second for first, second in CRASH.findall(log)} @@ -155,6 +230,13 @@ def judge(repo: str, base: str, sha: str, runs: list[int], github, *, checks: li head_log = github.log(repo, job) if refused_at_setup(job, head_log): raise Refused(f"'{name}' was refused at setup by its runner; it gets re-run, never excused") + if name in NO_VERDICT_JOBS and (said := no_verdict(head_log)) is not None: + proof = compile_pass_at(repo, sha, github) + if proof is None: + raise Refused(f"'{name}' gave no compile verdict ({said[:300]}) and nothing else compiled {sha}: " + "a no-verdict compile is never a base red; re-run it once the fleet answers") + audit.append(f"ignored '{name}': it gave no compile verdict, but {proof} compiled {sha} and passed") + continue own = test_reds.read(head_log) if test_reds.is_test_red(own, name) and (own.crashes or own.truncated): raise Refused(f"'{name}': " + "; ".join(own.crashes + own.truncated[:1])[:800] diff --git a/scripts/ci/next_batch.py b/scripts/ci/next_batch.py index 2f8be2b0707e..d2191b21309a 100644 --- a/scripts/ci/next_batch.py +++ b/scripts/ci/next_batch.py @@ -1329,9 +1329,13 @@ def expand_job(self, name: str, job_id: int | str, job: dict | None = None) -> s if log is None or not log.strip(): # A test job's unreadable or empty log: nothing can show it is the base's, so it is this job's # own (fails closed). Any other job keeps its bare name, as before. - if base_red_excuse.test_reds.is_test_red(base_red_excuse.test_reds.Result(), name): + if (base_red_excuse.test_reds.is_test_red(base_red_excuse.test_reds.Result(), name) + or name in base_red_excuse.NO_VERDICT_JOBS): return {f"{name}: log unreadable (job {job_id}, never a base red)"} return {name} + if base_red_excuse.no_verdict(log) is not None: + # A compile check with no verdict compiled nothing: keyed by this job's id, it is never the base's. + return {f"{name}: no compile verdict (job {job_id}, never a base red)"} steps, tests, errors = base_red_excuse.signature(job, log) # A crash or a run cut short is never the base's (scripts/ci/test_reds.py): keyed by # this job's id, no base failure can match it, whatever the base shows. diff --git a/tests/test_base_red_excuse.py b/tests/test_base_red_excuse.py index f7f2f982a561..fbfe6545894e 100644 --- a/tests/test_base_red_excuse.py +++ b/tests/test_base_red_excuse.py @@ -337,6 +337,206 @@ def test_a_red_test_the_base_does_not_fail_blocks(self): self.judge() +class NoVerdictCompileIsNeverABaseRed(unittest.TestCase): + """A compile check with no verdict (the build-fleet controller unreachable or timed out) compiled + nothing, so it is never a base red, even when the base shows the same "no result" with only its own + sha differing (2026-10-28, cmux#26798's run against base run 40656801765). It is set aside only when + another job at that same head compiled it and passed. A real red that differs only by the commit + sha it names is still excused.""" + + FLEET = "cmux-next fleet Release compile" + STEP = "Wait for the fleet Release compile and publish cmux-next/compile" + RELEASE = "cmux-next Release compile (Xcode 26)" + + class GitHub(FakeGitHub): + def __init__(self): + super().__init__() + self.check_runs: list[dict] = [] + self.actions_jobs: dict[int, dict] = {} + self.statuses: list[dict] = [] + + def json(self, route, *, paginate=False): + if f"/commits/{HEAD}/check-runs" in route: + return [{"check_runs": copy.deepcopy(self.check_runs)}] + if route.endswith(f"/commits/{HEAD}/status"): + return {"statuses": copy.deepcopy(self.statuses)} + if "/actions/jobs/" in route: + return copy.deepcopy(self.actions_jobs[int(route.rsplit("/", 1)[1])]) + return super().json(route, paginate=paginate) + + def setUp(self): + self.gh = self.GitHub() + self.gh.jobs = {1: failed_job(1, 10, HEAD, self.FLEET, self.STEP), + 2: failed_job(2, 20, BASE_SHA, self.FLEET, self.STEP)} + self.gh.logs[1] = self.no_result(self.FLEET, self.STEP, "df40f81342c8bb1eb135ced40728a4e7a090ed49") + self.gh.logs[2] = self.no_result(self.FLEET, self.STEP, "7ea21d1ce82ffaecc58599ae3f6bbf95afd92c51") + + @staticmethod + def no_result(job, step, sha): + return log(job, step, f"##[warning]no result for {sha} from the controller (attempt 1)", + f"##[error]cmux-next/compile is not a pass for {sha} (no result)", + "##[error]Process completed with exit code 1.") + + def release_run(self, check_id, conclusion="success", *, steps=None, sha=HEAD, check_sha=HEAD): + """A `cmux-next Release compile (Xcode 26)` check run and its job. steps: the Mac compile's by default.""" + steps = steps if steps is not None else [("Checkout", "success"), ("Release compile", conclusion)] + # A check run id is not its job id: the job comes from the details link. + job_id = 1000 + check_id + self.gh.check_runs.append({"id": check_id, "name": self.RELEASE, "status": "completed", + "conclusion": conclusion, "head_sha": check_sha, + "details_url": f"https://github.com/{REPO}/actions/runs/30/job/{job_id}", + "app": {"slug": "github-actions"}}) + self.gh.actions_jobs[job_id] = {"id": job_id, "run_id": 30, "name": self.RELEASE, "head_sha": sha, + "status": "completed", "conclusion": conclusion, + "steps": [{"name": name, "conclusion": state} for name, state in steps]} + + def judge(self): + return excuse.judge(REPO, "feat-cmux-next", HEAD, [10], self.gh) + + def test_no_verdict_with_nothing_else_compiling_the_head_is_refused_even_when_the_base_has_it(self): + with self.assertRaisesRegex(excuse.Refused, r"no compile verdict.*never a base red"): + self.judge() + + def test_the_app_compile_with_no_build_fleet_verdict_is_refused_too(self): + app = "cmux-next app compile (build fleet)" + step = "Compile-check the app at the pull request's head on the build fleet" + self.gh.jobs = {1: failed_job(1, 10, HEAD, app, step), 2: failed_job(2, 20, BASE_SHA, app, step)} + line = ("##[error]No build-fleet verdict for `{}` (controller unreachable or timed out: cmux-ci exit 69), " + "so nothing has compiled this head here.") + self.gh.logs[1] = log(app, step, line.format(HEAD)) + self.gh.logs[2] = log(app, step, line.format(BASE_SHA)) + with self.assertRaisesRegex(excuse.Refused, "never a base red"): + self.judge() + + def test_no_verdict_is_ignored_when_the_mac_release_compile_passed_at_the_head(self): + self.release_run(6) + text = "\n".join(self.judge()) + self.assertIn(f"ignored '{self.FLEET}'", text) + self.assertIn(self.RELEASE, text) + + def test_a_reused_pass_for_the_same_inputs_counts(self): + self.release_run(6, steps=[("Reuse that pass", "success"), ("Release compile", "skipped")]) + self.assertIn(f"ignored '{self.FLEET}'", "\n".join(self.judge())) + + def test_the_admission_job_of_the_same_name_is_not_a_compile(self): + self.release_run(6, steps=[("Record the Release compile admission", "success")]) + with self.assertRaisesRegex(excuse.Refused, "never a base red"): + self.judge() + + def test_an_older_pass_does_not_count_when_a_newer_attempt_failed(self): + self.release_run(6) + self.release_run(9, "failure") + with self.assertRaisesRegex(excuse.Refused, "never a base red"): + self.judge() + + def test_a_skipped_foreign_or_unknown_sha_release_compile_is_not_a_pass(self): + self.release_run(5, "skipped", steps=[]) + self.release_run(6, check_sha=None) + self.release_run(7, sha=BASE_SHA) + self.gh.check_runs.append({"id": 8, "name": self.RELEASE, "status": "completed", "conclusion": "success", + "head_sha": HEAD, "app": {"slug": "someone-else"}}) + with self.assertRaises(excuse.Refused): + self.judge() + + def test_no_verdict_is_ignored_when_cmux_next_compile_succeeded_at_the_head(self): + self.gh.statuses = [{"context": "cmux-next/compile", "state": "success"}] + self.assertIn("cmux-next/compile status", "\n".join(self.judge())) + + def test_another_job_printing_the_marker_is_never_set_aside(self): + """Only the compile jobs that can end without a verdict are set aside by a compile pass; any other + job with the same line is compared with the base like every red.""" + other = "cmux-next swift test" + self.gh.jobs = {1: failed_job(1, 10, HEAD, other, TEST_STEP), 2: failed_job(2, 20, BASE_SHA, other, TEST_STEP)} + self.gh.logs[1] = log(other, TEST_STEP, f"##[error]cmux-next/compile is not a pass for {HEAD} (no result)", OTHER) + self.gh.logs[2] = log(other, TEST_STEP, ISSUE) + self.release_run(6) + with self.assertRaisesRegex(excuse.Refused, "palettePagesReleaseTheRegistry"): + self.judge() + + def test_a_release_compile_without_a_job_link_or_from_another_run_is_not_a_pass(self): + self.release_run(6) + self.gh.check_runs[-1]["details_url"] = "https://example.com/job/1006" + self.release_run(7) + self.gh.actions_jobs[1007]["run_id"] = 31 + with self.assertRaises(excuse.Refused): + self.judge() + + def test_a_failed_cmux_next_compile_status_is_not_a_pass(self): + """The newest status per context decides: an older success behind it is stale.""" + self.gh.statuses = [{"context": "cmux-next/compile", "state": "failure"}, + {"context": "cmux-next/compile", "state": "success"}] + with self.assertRaises(excuse.Refused): + self.judge() + + def test_a_real_fleet_red_whose_log_echoes_the_script_is_not_set_aside(self): + """The log echoes the step's script, no-verdict words included; only the job's own ::error decides.""" + app = "cmux-next app compile (build fleet)" + step = "Compile-check the app at the pull request's head on the build fleet" + self.gh.jobs = {1: failed_job(1, 10, HEAD, app, step), 2: failed_job(2, 20, BASE_SHA, app, step)} + script = ' local summary="No build-fleet verdict for \\`$REF\\` ($1), so nothing has compiled this head here."' + self.gh.logs[1] = log(app, step, script, "##[error]The cmux-next app did not compile on the build fleet at " + f"`{HEAD}`; the errors are in this job's log above.") + self.gh.logs[2] = log(app, step, script, "##[error]something else broke") + self.release_run(6) + with self.assertRaisesRegex(excuse.Refused, "did not compile"): + self.judge() + + def test_a_real_red_differing_only_by_its_embedded_sha_is_excused(self): + scheme = "cmux app scheme compile (Debug)" + self.gh.jobs = {1: failed_job(1, 10, HEAD, scheme, "Build"), 2: failed_job(2, 20, BASE_SHA, scheme, "Build")} + line = "error: cmux-tui for this checkout's tree {} is not published after 0 min." + self.gh.logs[1] = log(scheme, "Build", line.format("5948da759b33dbe312f3108fab7dc39956c3dc56")) + self.gh.logs[2] = log(scheme, "Build", line.format("8f0c2e1")) + self.assertIn(f"excused '{scheme}'", "\n".join(self.judge())) + + def test_a_different_error_is_not_excused(self): + scheme = "cmux app scheme compile (Debug)" + self.gh.jobs = {1: failed_job(1, 10, HEAD, scheme, "Build"), 2: failed_job(2, 20, BASE_SHA, scheme, "Build")} + self.gh.logs[1] = log(scheme, "Build", "error: cmux-tui for this checkout's tree 5948da759b3 is not published.") + self.gh.logs[2] = log(scheme, "Build", "error: cmux-tui for this checkout's tree 8f0c2e1aa is corrupt.") + with self.assertRaisesRegex(excuse.Refused, "not published"): + self.judge() + + def test_other_hex_ids_are_compared_as_they_are(self): + scheme = "cmux app scheme compile (Debug)" + self.gh.jobs = {1: failed_job(1, 10, HEAD, scheme, "Build"), 2: failed_job(2, 20, BASE_SHA, scheme, "Build")} + self.gh.logs[1] = log(scheme, "Build", "error: artifact deadbeef1 missing") + self.gh.logs[2] = log(scheme, "Build", "error: artifact abcdef12 missing") + with self.assertRaisesRegex(excuse.Refused, "artifact"): + self.judge() + + def test_only_commit_shas_are_masked(self): + self.assertEqual(excuse.mask_shas(f"defaced 1234567 accede deadbeef1 {HEAD} at 8f0c2e1 tree `5948da7` " + "commit 1234567 for abcdef12"), + "defaced 1234567 accede deadbeef1 at 8f0c2e1 tree `` commit for abcdef12") + + def test_hex_ids_after_a_generic_word_are_compared_as_they_are(self): + scheme = "cmux app scheme compile (Debug)" + self.gh.jobs = {1: failed_job(1, 10, HEAD, scheme, "Build"), 2: failed_job(2, 20, BASE_SHA, scheme, "Build")} + self.gh.logs[1] = log(scheme, "Build", "error: no artifact for deadbeef1") + self.gh.logs[2] = log(scheme, "Build", "error: no artifact for abcdef12") + with self.assertRaisesRegex(excuse.Refused, "artifact"): + self.judge() + + def test_every_compile_status_state_but_fail_is_no_verdict(self): + """Fail closed: compile-status's own lines are a real red only for "(fail)".""" + for line in ("cmux-next/compile is not a pass for {} (pending)", "cmux-next/compile is not a pass for {} (pass)", + "cmux-next/compile is not a pass for {} (last: queued)", + "cmux-next/compile is not a pass for {} (unavailable)", "could not post the pending status", + "bad head sha '{}'"): + with self.subTest(line=line): + self.gh.logs[1] = log(self.FLEET, self.STEP, "##[error]" + line.format(HEAD)) + self.gh.logs[2] = log(self.FLEET, self.STEP, "##[error]" + line.format(BASE_SHA)) + with self.assertRaisesRegex(excuse.Refused, "never a base red"): + self.judge() + + def test_a_real_fleet_compile_failure_is_compared_with_the_base(self): + line = "##[error]cmux-next/compile is not a pass for {} (fail)" + self.gh.logs[1] = log(self.FLEET, self.STEP, line.format(HEAD)) + self.gh.logs[2] = log(self.FLEET, self.STEP, line.format(BASE_SHA)) + self.assertIn(f"excused '{self.FLEET}'", "\n".join(self.judge())) + + class Script(unittest.TestCase): def test_gh_merge_green_asks_the_base_before_refusing_a_red_lane(self): text = (ROOT / "scripts/gh-merge-green").read_text() diff --git a/tests/test_next_batch.py b/tests/test_next_batch.py index c5b39f128344..7e110516c1e6 100644 --- a/tests/test_next_batch.py +++ b/tests/test_next_batch.py @@ -738,6 +738,33 @@ def test_a_non_test_jobs_crash_signal_still_keys_its_failure(self): self.assertIn("cmux app scheme compile (Debug): crash signal 11", controller.expand_job("cmux app scheme compile (Debug)", 5, {"steps": steps})) + def test_an_unreadable_no_verdict_compile_log_is_keyed_by_its_own_job(self): + controller = self.controller() + steps = [{"name": "Wait", "conclusion": "failure"}] + controller.gh.gh.return_value = subprocess.CompletedProcess([], 1, "", "boom") + for name in ("cmux-next fleet Release compile", "cmux-next app compile (build fleet)"): + with self.subTest(name=name): + self.assertEqual(controller.expand_job(name, 7, {"steps": steps}), + {f"{name}: log unreadable (job 7, never a base red)"}) + + def test_a_compile_with_no_verdict_is_keyed_by_its_own_job(self): + """The fleet Release compile's "no result" compiled nothing: the base showing the same line (its own + sha aside) excuses nothing.""" + controller = self.controller() + name = "cmux-next fleet Release compile" + steps = [{"name": "Wait for the fleet Release compile and publish cmux-next/compile", "conclusion": "failure"}] + + def no_result(sha): + return subprocess.CompletedProcess( + [], 0, f"2026-10-28T08:40:00.1Z ##[error]cmux-next/compile is not a pass for {sha} (no result)\n", "") + + controller.gh.gh.return_value = no_result("7ea21d1ce82ffaecc58599ae3f6bbf95afd92c51") + base = controller.expand_job(name, 4, {"steps": steps}) + controller.gh.gh.return_value = no_result("df40f81342c8bb1eb135ced40728a4e7a090ed49") + stack = controller.expand_job(name, 5, {"steps": steps}) + self.assertEqual(stack, {f"{name}: no compile verdict (job 5, never a base red)"}) + self.assertEqual(stack - base, stack) + def test_an_unreadable_log_fails_closed_only_for_a_test_job(self): controller = self.controller() steps = [{"name": "Build", "conclusion": "failure"}]