ci: leave a merge receipt on each merged pull request

.github/workflows/ci-guards.yml

@@ -368,6 +368,10 @@ jobs:
         if: ${{ matrix.group == 'ci' }}
         run: python3 tests/test_ci_main_regression_attribution.py
 
+      - name: Validate merge receipts
+        if: ${{ matrix.group == 'ci' }}
+        run: python3 tests/test_ci_merge_receipt.py
+
       - name: Validate CI queue janitor policy
         if: ${{ matrix.group == 'ci' }}
         run: python3 tests/test_ci_queue_janitor.py

.github/workflows/merge-receipt.yml

@@ -0,0 +1,41 @@
+name: Merge receipt
+
+# Records on each merged pull request which checks on its head had passed when
+# it merged and which had not (scripts/ci/merge_receipt.py), and labels it
+# merged-unverified when a judging check was not green. It runs after the merge,
+# so it cannot block or slow one. pull_request_target gives fork pull requests
+# a write token; the job only reads the GitHub API and runs the script from the
+# commit this workflow was loaded from, never the pull request's code.
+on:
+  pull_request_target:
+    types: [closed]
+    branches: [main]
+
+permissions: {}
+
+jobs:
+  receipt:
+    if: github.event.pull_request.merged == true
+    runs-on: ubuntu-24.04 # github-hosted-required: trusted pull-request-write token
+    timeout-minutes: 5
+    permissions:
+      contents: read
+      pull-requests: write
+      checks: read
+      statuses: read
+    steps:
+      - name: Checkout trusted script
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          ref: ${{ github.workflow_sha }}
+          fetch-depth: 1
+          persist-credentials: false
+          sparse-checkout: scripts/ci/merge_receipt.py
+          sparse-checkout-cone-mode: false
+
+      - name: Post merge receipt
+        env:
+          GH_TOKEN: ${{ github.token }}
+          REPO: ${{ github.repository }}
+          PR: ${{ github.event.pull_request.number }}
+        run: python3 scripts/ci/merge_receipt.py post --repo "$REPO" --pr "$PR"

scripts/ci/main_regression_attribution.py

@@ -20,7 +20,8 @@
 (test_impact.py), 1 when a changed app declaration or string is named by the
 suite (reverse_test_impact.py), 0 otherwise. The top score names the
 suspects; when every score is 0 the failure is left unattributed rather than
-blaming the whole range.
+blaming the whole range. A tie lists pull requests labeled merged-unverified
+(merge_receipt.py: a judging check was not green when they merged) first.
 
 `report` writes a "New since" markdown section for the tracking issue (read
 by main_full_suite.py report --extra-section) and comments once on each
@@ -45,6 +46,7 @@
 
 sys.path.insert(0, str(Path(__file__).resolve().parent))
 import main_full_suite as suite_run  # noqa: E402
+from merge_receipt import LABEL as UNVERIFIED_LABEL  # noqa: E402
 
 APP_HOST_JOB_RE = re.compile(r"app-host unit tests \((\d+)/\d+\)")
 ANSI_RE = re.compile(r"\x1b\[[0-9;]*m")
@@ -100,6 +102,8 @@ class PullRequest:
     # Suites the diff edits, and suites that name what the diff changes.
     edited_suites: set[str] = field(default_factory=set)
     reached_suites: set[str] = field(default_factory=set)
+    # Labeled by merge_receipt.py: a judging check was not green at merge.
+    unverified: bool = False
 
 
 def log_failures(log_text: str, known: Iterable[str] = ()) -> set[str]:
@@ -215,6 +219,10 @@ def merged_prs(
                     url=str(pr.get("url") or ""),
                     merge_sha=merge_sha,
                     author=str(((pr.get("author") or {}) or {}).get("login") or ""),
+                    unverified=any(
+                        label.get("name") == UNVERIFIED_LABEL
+                        for label in ((pr.get("labels") or {}).get("nodes") or [])
+                    ),
                 )
     merge_order = {sha: index for index, sha in enumerate(reversed(ordered))}
     prs = sorted(found.values(), key=lambda pr: merge_order.get(pr.merge_sha, 0))
@@ -248,7 +256,12 @@ def suspects_for(
     if best == 0:
         return [], "no pull request in the range reaches this suite"
     how = "edits the suite" if best == 2 else "changes code the suite names"
-    return [pr for value, pr in scored if value == best], how
+    tied = [pr for value, pr in scored if value == best]
+    # A pull request that merged before its checks passed is listed first in a
+    # tie; the others stay, since a verified head can still break main
+    # through an interaction with another merge.
+    tied.sort(key=lambda pr: not pr.unverified)
+    return tied, how
 
 
 def tests_digest(tests: Iterable[str]) -> str:
@@ -454,7 +467,8 @@ def associated_prs(repo: str, shas: list[str]) -> dict[str, list[dict]]:
         chunk = shas[start:start + 40]
         fields = " ".join(
             f'c{index}: object(oid: "{sha}") {{ ... on Commit {{ associatedPullRequests(first: 5) '
-            "{ nodes { number title url state baseRefName author { login } mergeCommit { oid } } } } }"
+            "{ nodes { number title url state baseRefName author { login } mergeCommit { oid } "
+            "labels(first: 20) { nodes { name } } } } } }"
             for index, sha in enumerate(chunk)
         )
         query = f'query {{ repository(owner: "{owner}", name: "{name}") {{ {fields} }} }}'