ci: a no-verdict compile is never a base red; shas are masked in compared error lines

scripts/ci/base_red_excuse.py

@@ -21,6 +21,11 @@
 abort, EXC_*, Fatal error) is the head's whatever the base shows; and a run
 whose per-test results are cut short (no summary, ended early, stuck) is the
 head's, since nothing in it can be compared.
+
+A compile check that gave no verdict (the build fleet's "no result", "No build-fleet verdict") is never a
+base red, even with the base's identical line: nothing compiled the head. It is set aside only when the
+same head has a real compile pass (a green Mac Release compile or a cmux-next/compile success). Error
+lines are compared with commit shas masked, so a real red that names each run's own commit still matches.
 """
 from __future__ import annotations
 
@@ -68,6 +73,33 @@
 CRASH = re.compile(r"exited with unexpected signal code (\d+)|the test helper died on signal (\d+)")
 CRASH_LINE = re.compile(r"exited with unexpected signal code|the test helper died on signal|"
                         r"no \"Test run with\" summary line")
+# A commit sha inside an error line: a full 40-hex sha, or 7 to 39 hex digits right after a word that
+# names a commit (commit, sha, tree, revision). The same red on the head and the base names each run's
+# own commit; any other hex id (an artifact, a hash) is compared as it is.
+SHA = re.compile(r"\b[0-9a-f]{40}\b|\b((?:commit|sha|tree|revision) `?)[0-9a-f]{7,39}\b")
+# A compile check that gave no verdict (the build-fleet controller unreachable, timed out, or a result
+# for another head): nothing compiled the head, so it is never a base red, whatever the base shows
+# (2026-10-28: the fleet Release compile's "no result" was red on PRs and on the base alike while the
+# controller was wedged). It is set aside only when another job compiled that same head: COMPILE_PASSES.
+# Only the job's own ::error annotation counts (`##[error]` at the start of the line): the log also
+# echoes the step's script, which spells these words out on every run, a real compile red included.
+# Fail closed: of compile-status's own lines, only "(fail)" is a real compile red; every other state (no result,
+# unavailable, mismatch, unreadable, pending, "last: ...", a pass it could not post) and a status it could not
+# post at all are no verdict.
+NO_VERDICT = re.compile(r"^##\[error\](?:cmux-next/compile is not a pass for \S+ \((?!fail\))|"
+                        r"could not post the pending status|bad head sha |No build-fleet verdict for `)")
+# The jobs that can end without a verdict: an unreadable log of one is never comparable either.
+NO_VERDICT_JOBS = frozenset({"cmux-next fleet Release compile", "cmux-next app compile (build fleet)"})
+# A real compile pass at the head: the Mac Release compile (a check run), or the fleet's cmux-next/compile
+# commit status.
+COMPILE_PASS_CHECKS = ("cmux-next Release compile (Xcode 26)",)
+COMPILE_PASS_STATUS = "cmux-next/compile"
+# The Mac Release compile's steps that compile (or reuse a pass for the same inputs).
+COMPILE_STEPS = frozenset({"Release compile", "Reuse that pass"})
+
+
+def mask_shas(text: str) -> str:
+    return SHA.sub(lambda match: (match.group(1) or "") + "<sha>", text)
 
 
 def lines_of(log: str) -> list[str]:
@@ -90,10 +122,53 @@ def signature(job: dict, log: str) -> tuple[set[str], set[str], set[str]]:
             if match := pattern.search(text):
                 tests.add(match.group(1))
         if ERROR.search(text) and not GENERIC.search(text) and not WARNING.search(text) and not CRASH_LINE.search(text):
-            errors.add(re.sub(r"\d+", "#", text.replace("##[error]", "").strip()))
+            errors.add(re.sub(r"\d+", "#", mask_shas(text.replace("##[error]", "").strip())))
     return steps, tests, errors
 
 
+def no_verdict(log: str) -> str | None:
+    """The line where a compile check says it gave no verdict, if it does."""
+    return next((text for text in lines_of(log) if NO_VERDICT.match(text)), None)
+
+
+def compile_pass_at(repo: str, sha: str, github) -> str | None:
+    """What compiled SHA itself and passed: the newest Mac Release compile job at SHA that ran (its
+    `Release compile` or `Reuse that pass` step succeeded), or a cmux-next/compile success."""
+    candidates = []
+    for page in github.json(f"repos/{repo}/commits/{sha}/check-runs?per_page=100", paginate=True):
+        for check in page.get("check_runs") or []:
+            if (check.get("name") in COMPILE_PASS_CHECKS and check.get("head_sha") == sha
+                    and check.get("app", {}).get("slug") == "github-actions" and check.get("status") == "completed"
+                    and check.get("conclusion") != "skipped"):
+                candidates.append(check)
+    # Newest first; the same name is also release-compile-fast-admission's (a Linux wrapper that compiles
+    # nothing), so only a job with the Mac compile's own steps counts, and the newest such attempt decides.
+    for check in sorted(candidates, key=lambda found: found.get("id", -1), reverse=True):
+        # The check run's own Actions job, from its details link (a check run id is not a job id).
+        link = re.fullmatch(r"https://github\.com/" + re.escape(repo) + r"/actions/runs/(\d+)/job/(\d+)",
+                            check.get("details_url") or "")
+        if not link:
+            continue
+        job = github.json(f"repos/{repo}/actions/jobs/{link[2]}")
+        if (job.get("head_sha") != sha or job.get("name") != check.get("name")
+                or job.get("run_id") != int(link[1])):
+            continue
+        steps = {step.get("name"): step.get("conclusion") for step in job.get("steps") or []}
+        if not COMPILE_STEPS & steps.keys():
+            continue
+        if job.get("conclusion") == "success" and any(steps.get(name) == "success" for name in COMPILE_STEPS):
+            return f"'{check['name']}' (job {link[2]})"
+        break
+    status = github.json(f"repos/{repo}/commits/{sha}/status")
+    for found in status.get("statuses") or []:
+        if found.get("context") == COMPILE_PASS_STATUS:
+            # The combined status lists the newest per context.
+            if found.get("state") == "success":
+                return f"the {COMPILE_PASS_STATUS} status"
+            break
+    return None
+
+
 def crash_signals(log: str) -> set[str]:
     """The signals a test helper died on, from SwiftPM's or the hang sampler's lines."""
     return {first or second for first, second in CRASH.findall(log)}
@@ -155,6 +230,13 @@ def judge(repo: str, base: str, sha: str, runs: list[int], github, *, checks: li
         head_log = github.log(repo, job)
         if refused_at_setup(job, head_log):
             raise Refused(f"'{name}' was refused at setup by its runner; it gets re-run, never excused")
+        if name in NO_VERDICT_JOBS and (said := no_verdict(head_log)) is not None:
+            proof = compile_pass_at(repo, sha, github)
+            if proof is None:
+                raise Refused(f"'{name}' gave no compile verdict ({said[:300]}) and nothing else compiled {sha}: "
+                              "a no-verdict compile is never a base red; re-run it once the fleet answers")
+            audit.append(f"ignored '{name}': it gave no compile verdict, but {proof} compiled {sha} and passed")
+            continue
         own = test_reds.read(head_log)
         if test_reds.is_test_red(own, name) and (own.crashes or own.truncated):
             raise Refused(f"'{name}': " + "; ".join(own.crashes + own.truncated[:1])[:800]

scripts/ci/next_batch.py

@@ -1329,9 +1329,13 @@ def expand_job(self, name: str, job_id: int | str, job: dict | None = None) -> s
         if log is None or not log.strip():
             # A test job's unreadable or empty log: nothing can show it is the base's, so it is this job's
             # own (fails closed). Any other job keeps its bare name, as before.
-            if base_red_excuse.test_reds.is_test_red(base_red_excuse.test_reds.Result(), name):
+            if (base_red_excuse.test_reds.is_test_red(base_red_excuse.test_reds.Result(), name)
+                    or name in base_red_excuse.NO_VERDICT_JOBS):
                 return {f"{name}: log unreadable (job {job_id}, never a base red)"}
             return {name}
+        if base_red_excuse.no_verdict(log) is not None:
+            # A compile check with no verdict compiled nothing: keyed by this job's id, it is never the base's.
+            return {f"{name}: no compile verdict (job {job_id}, never a base red)"}
         steps, tests, errors = base_red_excuse.signature(job, log)
         # A crash or a run cut short is never the base's (scripts/ci/test_reds.py): keyed by
         # this job's id, no base failure can match it, whatever the base shows.