From de35ad30d2ba7f5257913bcd4e9339dedb494a27 Mon Sep 17 00:00:00 2001 From: Leo Date: Fri, 25 Sep 2026 06:50:14 -0400 Subject: [PATCH 1/3] ci: leave a merge receipt on each merged pull request A pull request can merge before every check on its head finishes; on 2026-09-25, 9 of 60 did. A post-merge workflow now comments once on each merged pull request with what was verified at merge, what was still running or missing, and what policy skipped, reading each check as of the merge time. When a judging check (compile admission, app-host unit tests, ci-status, required checks) was not green, the pull request gets the merged-unverified label, and main regression attribution prefers such pull requests in a tie. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci-guards.yml | 4 + .github/workflows/merge-receipt.yml | 41 + scripts/ci/main_regression_attribution.py | 20 +- scripts/ci/merge_receipt.py | 353 ++++++++ scripts/ci/workflow_guard_groups.py | 7 + tests/fixtures/merge_receipt/pr14433.json | 854 +++++++++++++++++++ tests/fixtures/merge_receipt/pr14461.json | 840 ++++++++++++++++++ tests/test-execution.toml | 4 + tests/test_ci_main_regression_attribution.py | 23 +- tests/test_ci_merge_receipt.py | 175 ++++ 10 files changed, 2315 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/merge-receipt.yml create mode 100644 scripts/ci/merge_receipt.py create mode 100644 tests/fixtures/merge_receipt/pr14433.json create mode 100644 tests/fixtures/merge_receipt/pr14461.json create mode 100644 tests/test_ci_merge_receipt.py diff --git a/.github/workflows/ci-guards.yml b/.github/workflows/ci-guards.yml index 6e0f30a1ad29..a935ca793104 100644 --- a/.github/workflows/ci-guards.yml +++ b/.github/workflows/ci-guards.yml @@ -368,6 +368,10 @@ jobs: if: ${{ matrix.group == 'ci' }} run: python3 tests/test_ci_main_regression_attribution.py + - name: Validate merge receipts + if: ${{ matrix.group == 'ci' }} + run: python3 tests/test_ci_merge_receipt.py + - name: Validate CI queue janitor policy if: ${{ matrix.group == 'ci' }} run: python3 tests/test_ci_queue_janitor.py diff --git a/.github/workflows/merge-receipt.yml b/.github/workflows/merge-receipt.yml new file mode 100644 index 000000000000..c57117dbec5c --- /dev/null +++ b/.github/workflows/merge-receipt.yml @@ -0,0 +1,41 @@ +name: Merge receipt + +# Records on each merged pull request which checks on its head had passed when +# it merged and which had not (scripts/ci/merge_receipt.py), and labels it +# merged-unverified when a judging check was not green. It runs after the merge, +# so it cannot block or slow one. pull_request_target gives fork pull requests +# a write token; the job only reads the GitHub API and runs the script from the +# commit this workflow was loaded from, never the pull request's code. +on: + pull_request_target: + types: [closed] + branches: [main] + +permissions: {} + +jobs: + receipt: + if: github.event.pull_request.merged == true + runs-on: ubuntu-24.04 # github-hosted-required: trusted pull-request-write token + timeout-minutes: 5 + permissions: + contents: read + pull-requests: write + checks: read + statuses: read + steps: + - name: Checkout trusted script + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ github.workflow_sha }} + fetch-depth: 1 + persist-credentials: false + sparse-checkout: scripts/ci/merge_receipt.py + sparse-checkout-cone-mode: false + + - name: Post merge receipt + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + PR: ${{ github.event.pull_request.number }} + run: python3 scripts/ci/merge_receipt.py post --repo "$REPO" --pr "$PR" diff --git a/scripts/ci/main_regression_attribution.py b/scripts/ci/main_regression_attribution.py index 56a8c99d9a8d..2f38bb40075e 100644 --- a/scripts/ci/main_regression_attribution.py +++ b/scripts/ci/main_regression_attribution.py @@ -20,7 +20,8 @@ (test_impact.py), 1 when a changed app declaration or string is named by the suite (reverse_test_impact.py), 0 otherwise. The top score names the suspects; when every score is 0 the failure is left unattributed rather than -blaming the whole range. +blaming the whole range. A tie prefers pull requests labeled merged-unverified +(merge_receipt.py): a judging check was not green when they merged. `report` writes a "New since" markdown section for the tracking issue (read by main_full_suite.py report --extra-section) and comments once on each @@ -45,6 +46,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) import main_full_suite as suite_run # noqa: E402 +from merge_receipt import LABEL as UNVERIFIED_LABEL # noqa: E402 APP_HOST_JOB_RE = re.compile(r"app-host unit tests \((\d+)/\d+\)") ANSI_RE = re.compile(r"\x1b\[[0-9;]*m") @@ -100,6 +102,8 @@ class PullRequest: # Suites the diff edits, and suites that name what the diff changes. edited_suites: set[str] = field(default_factory=set) reached_suites: set[str] = field(default_factory=set) + # Labeled by merge_receipt.py: a judging check was not green at merge. + unverified: bool = False def log_failures(log_text: str, known: Iterable[str] = ()) -> set[str]: @@ -215,6 +219,10 @@ def merged_prs( url=str(pr.get("url") or ""), merge_sha=merge_sha, author=str(((pr.get("author") or {}) or {}).get("login") or ""), + unverified=any( + label.get("name") == UNVERIFIED_LABEL + for label in ((pr.get("labels") or {}).get("nodes") or []) + ), ) merge_order = {sha: index for index, sha in enumerate(reversed(ordered))} prs = sorted(found.values(), key=lambda pr: merge_order.get(pr.merge_sha, 0)) @@ -248,7 +256,12 @@ def suspects_for( if best == 0: return [], "no pull request in the range reaches this suite" how = "edits the suite" if best == 2 else "changes code the suite names" - return [pr for value, pr in scored if value == best], how + tied = [pr for value, pr in scored if value == best] + # A pull request that merged before its checks passed breaks a tie. + unverified = [pr for pr in tied if pr.unverified] + if unverified and len(unverified) < len(tied): + return unverified, f"{how}, merged unverified" + return tied, how def tests_digest(tests: Iterable[str]) -> str: @@ -454,7 +467,8 @@ def associated_prs(repo: str, shas: list[str]) -> dict[str, list[dict]]: chunk = shas[start:start + 40] fields = " ".join( f'c{index}: object(oid: "{sha}") {{ ... on Commit {{ associatedPullRequests(first: 5) ' - "{ nodes { number title url state baseRefName author { login } mergeCommit { oid } } } } }" + "{ nodes { number title url state baseRefName author { login } mergeCommit { oid } " + "labels(first: 20) { nodes { name } } } } } }" for index, sha in enumerate(chunk) ) query = f'query {{ repository(owner: "{owner}", name: "{name}") {{ {fields} }} }}' diff --git a/scripts/ci/merge_receipt.py b/scripts/ci/merge_receipt.py new file mode 100644 index 000000000000..9b21fca22a95 --- /dev/null +++ b/scripts/ci/merge_receipt.py @@ -0,0 +1,353 @@ +#!/usr/bin/env python3 +"""Leave a short receipt on each merged pull request: what CI verified at merge. + +main moves fast and pull requests often merge before every check on their head +finishes. That is allowed; this never blocks a merge. It records, once per +pull request, which checks on the head commit had passed when it merged and +which had not, so whoever fixes main later knows where to look. + +Each check's state is read as of the merge time: a run that finished before +the merge counts with its conclusion, one that started but had not finished is +"in progress", and one that started after the merge is "not reported". A check +name that ran several times (a re-run, a superseded run) counts its latest +start before the merge. + +Checks are grouped for reading: reusable-workflow jobs lose their caller +prefix, every guards job folds into "guards", and every app-host shard into +"app-host unit tests". Bots, CLA and other bookkeeping checks are left out +unless they failed. + +A pull request whose judging checks (compile admission, app-host unit tests, +ci-status and required checks) were not all green at merge gets the +`merged-unverified` label, which main_regression_attribution.py uses to break +ties between suspects. The comment is idempotent through a hidden marker and +is edited in place on a re-run. +""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +from collections.abc import Iterable, Mapping +from dataclasses import dataclass, field + +MARKER = "" +LABEL = "merged-unverified" +# Checks that are expected on every pull request even when they never reported. +EXPECTED = ("ci-status",) +# Checks that judge the change itself. Required checks judge it too. +JUDGING_RE = re.compile(r"macOS compile admission|app-host unit tests|^ci-status$") +# Bookkeeping checks from GitHub Actions that say nothing about the change. +NOISE_RE = re.compile( + r"^CLA |^CLA$|CLA Assistant|CLA policy guard|^welcome$|Watch owned pool jobs|\(report only\)|^changes$" +) +ACTIONS_APP = "github-actions" +MAX_LISTED = 12 + +# States, worst first. A group reports its worst member. +FAILURE, CANCELLED, IN_PROGRESS, PENDING, NOT_REPORTED, SUCCESS, SKIPPED = ( + "failure", "cancelled", "in progress", "pending", "not reported", "success", "skipped", +) +ORDER = (FAILURE, CANCELLED, IN_PROGRESS, PENDING, NOT_REPORTED, SUCCESS, SKIPPED) +GREEN = frozenset({SUCCESS, SKIPPED}) +CONCLUSIONS = { + "SUCCESS": SUCCESS, "NEUTRAL": SUCCESS, "SKIPPED": SKIPPED, + "FAILURE": FAILURE, "TIMED_OUT": FAILURE, "STARTUP_FAILURE": FAILURE, "ACTION_REQUIRED": FAILURE, + "CANCELLED": CANCELLED, "STALE": CANCELLED, +} +STATUS_STATES = {"SUCCESS": SUCCESS, "PENDING": IN_PROGRESS, "EXPECTED": PENDING, "FAILURE": FAILURE, "ERROR": FAILURE} + + +@dataclass +class Check: + name: str + state: str + required: bool = False + noise: bool = False + + +@dataclass +class Group: + name: str + checks: list[Check] = field(default_factory=list) + + @property + def state(self) -> str: + states = {check.state for check in self.checks} + bad = [state for state in ORDER if state in states and state not in GREEN] + if bad: + return bad[0] + return SUCCESS if SUCCESS in states else SKIPPED + + @property + def judging(self) -> bool: + return bool(JUDGING_RE.search(self.name)) or any(check.required for check in self.checks) + + def label(self) -> str: + return f"{self.name} ({len(self.checks)})" if len(self.checks) > 1 else self.name + + +def state_at(context: Mapping[str, object], merged_at: str) -> tuple[str, str]: + """(state as of the merge, start time) for one check run or status context. + + Timestamps are ISO-8601 UTC strings from GitHub, so they compare as text. + """ + if context.get("__typename") == "StatusContext": + created = str(context.get("createdAt") or "") + if not created or created > merged_at: + return NOT_REPORTED, created + return STATUS_STATES.get(str(context.get("state") or ""), PENDING), created + started = str(context.get("startedAt") or "") + if not started: + return PENDING, "" + if started > merged_at: + return NOT_REPORTED, started + completed = str(context.get("completedAt") or "") + if completed and completed <= merged_at: + return CONCLUSIONS.get(str(context.get("conclusion") or ""), FAILURE), started + return IN_PROGRESS, started + + +def context_name(context: Mapping[str, object]) -> str: + return str(context.get("name") or context.get("context") or "") + + +def is_noise(context: Mapping[str, object]) -> bool: + if context.get("__typename") == "StatusContext": + return True + app = ((context.get("checkSuite") or {}).get("app") or {}).get("slug") + return app != ACTIONS_APP or bool(NOISE_RE.search(context_name(context))) + + +def checks_at_merge(contexts: Iterable[Mapping[str, object]], merged_at: str) -> list[Check]: + """One check per name: its latest run started before the merge, else its earliest later one.""" + best: dict[str, tuple[bool, str, Check]] = {} + for context in contexts: + name = context_name(context) + state, started = state_at(context, merged_at) + check = Check(name, state, bool(context.get("isRequired")), is_noise(context)) + before = state != NOT_REPORTED + current = best.get(name) + if current is None: + best[name] = (before, started, check) + continue + was_before, was_started, _ = current + if before != was_before: + newer = before + else: + newer = started > was_started if before else started < was_started + if newer: + best[name] = (before, started, check) + # A job that had not started at the merge did not exist yet for whoever + # merged; only required and expected checks are worth naming as missing. + checks = [ + check for _, _, check in best.values() + if check.state != NOT_REPORTED or check.required or check.name in EXPECTED + ] + seen = {check.name for check in checks} + checks += [Check(name, NOT_REPORTED, True) for name in EXPECTED if name not in seen] + return checks + + +def group_name(name: str) -> str: + if "app-host unit tests" in name: + return "app-host unit tests" + if name.startswith("guards / "): + return "guards" + return name.split(" / ", 1)[1] if " / " in name else name + + +def groups(checks: Iterable[Check]) -> list[Group]: + found: dict[str, Group] = {} + for check in checks: + key = group_name(check.name) + found.setdefault(key, Group(key)).checks.append(check) + return sorted(found.values(), key=lambda group: group.name.lower()) + + +@dataclass +class Receipt: + body: str + unverified: bool + + +def listed(items: list[str]) -> str: + if len(items) <= MAX_LISTED: + return ", ".join(items) + return ", ".join(items[:MAX_LISTED]) + f", and {len(items) - MAX_LISTED} more" + + +def receipt(snapshot: Mapping[str, object]) -> Receipt: + """The comment body and whether the pull request merged unverified.""" + merged_at = str(snapshot["mergedAt"]) + sha = str(snapshot["headRefOid"])[:10] + checks = checks_at_merge(snapshot.get("contexts") or [], merged_at) + real = groups(check for check in checks if not check.noise) + noisy = groups(check for check in checks if check.noise) + # Judging groups first so the eye lands on them. + real.sort(key=lambda group: not group.judging) + verified = [group.label() for group in real if group.state == SUCCESS] + skipped = [group.label() for group in real if group.state == SKIPPED] + missing = [f"{group.label()} ({group.state})" for group in real if group.state not in GREEN] + missing += [f"{group.label()} ({group.state})" for group in noisy if group.state in (FAILURE, CANCELLED)] + unverified = any(group.judging and group.state not in GREEN for group in real) + + head = f"**Merge receipt** for `{sha}`" + if not missing: + tail = f"; {len(skipped)} skipped by policy" if skipped else "" + lines = [f"{head}: every check was green at merge ({len(verified)} verified{tail}). " + "Full suite runs on main after merge."] + else: + lines = [f"{head}, merged {merged_at.replace('T', ' ').rstrip('Z')} UTC"] + lines.append(f"- Not verified at merge: {listed(missing)}") + if verified: + lines.append(f"- Verified: {listed(verified)}") + if skipped: + lines.append(f"- Skipped by policy: {listed(skipped)}") + lines.append("- Full suite: runs on main after merge.") + if unverified: + lines.append(f"\nLabeled `{LABEL}`: if main breaks near this merge, look here first.") + lines.append(MARKER) + return Receipt("\n".join(lines), unverified) + + +# --- GitHub --------------------------------------------------------------- + +PR_QUERY = """ +query($owner: String!, $name: String!, $number: Int!) { + repository(owner: $owner, name: $name) { + pullRequest(number: $number) { + number merged mergedAt headRefOid + labels(first: 50) { nodes { name } } + comments(last: 100) { nodes { databaseId body } } + } + } +} +""" + +CONTEXTS_QUERY = """ +query($owner: String!, $name: String!, $oid: GitObjectID!, $number: Int!, $after: String) { + repository(owner: $owner, name: $name) { + object(oid: $oid) { + ... on Commit { + statusCheckRollup { + contexts(first: 100, after: $after) { + pageInfo { hasNextPage endCursor } + nodes { + __typename + ... on CheckRun { + name status conclusion startedAt completedAt + isRequired(pullRequestNumber: $number) + checkSuite { app { slug } } + } + ... on StatusContext { context state createdAt isRequired(pullRequestNumber: $number) } + } + } + } + } + } + } +} +""" + + +def gh(args: list[str]) -> str: + return subprocess.run(["gh", *args], check=True, capture_output=True, text=True).stdout + + +def graphql(query: str, **variables: object) -> dict: + args = ["api", "graphql", "-f", f"query={query}"] + for key, value in variables.items(): + if value is None: + continue + args += ["-F" if isinstance(value, int) else "-f", f"{key}={value}"] + return json.loads(gh(args))["data"] + + +def fetch(repo: str, number: int) -> dict: + """The pull request's merge time, head, labels, receipt comment and head check contexts.""" + owner, name = repo.split("/", 1) + pr = graphql(PR_QUERY, owner=owner, name=name, number=number)["repository"]["pullRequest"] + contexts: list[dict] = [] + after = None + while True: + commit = graphql( + CONTEXTS_QUERY, owner=owner, name=name, oid=pr["headRefOid"], number=number, after=after, + )["repository"]["object"] or {} + page = ((commit.get("statusCheckRollup") or {}).get("contexts")) or {} + contexts += page.get("nodes") or [] + info = page.get("pageInfo") or {} + if not info.get("hasNextPage"): + break + after = info["endCursor"] + return { + "number": pr["number"], "merged": pr["merged"], "mergedAt": pr["mergedAt"], + "headRefOid": pr["headRefOid"], + "labels": [node["name"] for node in (pr.get("labels") or {}).get("nodes") or []], + "comments": (pr.get("comments") or {}).get("nodes") or [], + "contexts": contexts, + } + + +def existing_comment(comments: Iterable[Mapping[str, object]]) -> int | None: + for comment in comments: + if MARKER in str(comment.get("body") or ""): + return int(comment["databaseId"]) + return None + + +def command_post(args: argparse.Namespace) -> int: + snapshot = fetch(args.repo, args.pr) + if args.snapshot_output: + with open(args.snapshot_output, "w", encoding="utf-8") as handle: + json.dump({key: snapshot[key] for key in ("number", "mergedAt", "headRefOid", "contexts")}, handle, indent=1) + if not snapshot["merged"]: + print(f"#{args.pr} is not merged; nothing to record.") + return 0 + result = receipt(snapshot) + print(result.body) + print(f"unverified={str(result.unverified).lower()}") + if args.dry_run: + return 0 + comment_id = existing_comment(snapshot["comments"]) + if comment_id is None: + gh(["api", f"repos/{args.repo}/issues/{args.pr}/comments", "-f", f"body={result.body}"]) + else: + gh(["api", "-X", "PATCH", f"repos/{args.repo}/issues/comments/{comment_id}", "-f", f"body={result.body}"]) + has_label = LABEL in snapshot["labels"] + if result.unverified and not has_label: + gh(["api", f"repos/{args.repo}/issues/{args.pr}/labels", "-f", f"labels[]={LABEL}"]) + elif not result.unverified and has_label: + gh(["api", "-X", "DELETE", f"repos/{args.repo}/issues/{args.pr}/labels/{LABEL}"]) + return 0 + + +def command_render(args: argparse.Namespace) -> int: + with open(args.snapshot, encoding="utf-8") as handle: + result = receipt(json.load(handle)) + print(result.body) + print(f"unverified={str(result.unverified).lower()}") + return 0 + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + sub = parser.add_subparsers(dest="command", required=True) + post = sub.add_parser("post", help="comment on and label one merged pull request") + post.add_argument("--repo", required=True) + post.add_argument("--pr", type=int, required=True) + post.add_argument("--dry-run", action="store_true", help="print the receipt without writing") + post.add_argument("--snapshot-output", help="also save the fetched checks as a fixture") + post.set_defaults(func=command_post) + render = sub.add_parser("render", help="print the receipt for a saved snapshot") + render.add_argument("snapshot") + render.set_defaults(func=command_render) + args = parser.parse_args(argv) + return args.func(args) + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/scripts/ci/workflow_guard_groups.py b/scripts/ci/workflow_guard_groups.py index 3aa77a1d7947..7e2110e5eeed 100644 --- a/scripts/ci/workflow_guard_groups.py +++ b/scripts/ci/workflow_guard_groups.py @@ -87,6 +87,13 @@ "scripts/ci/compile-app-host-test-product.sh": frozenset(("preflight",)), "scripts/ci/find_admitted_build.py": frozenset(("preflight",)), "scripts/ci/main_full_suite.py": frozenset(("ci",)), + # test_ci_merge_receipt.py and test_ci_main_regression_attribution.py load + # these by path; the receipt test also reads its workflow and fixtures. + "scripts/ci/main_regression_attribution.py": frozenset(("ci",)), + "scripts/ci/merge_receipt.py": frozenset(("ci",)), + ".github/workflows/merge-receipt.yml": frozenset(("ci",)), + "tests/fixtures/merge_receipt/pr14433.json": frozenset(("ci",)), + "tests/fixtures/merge_receipt/pr14461.json": frozenset(("ci",)), "scripts/ci/ios_upload_batch_decision.py": frozenset(("release-ios",)), "scripts/ci/peer_product_source.py": frozenset(("preflight",)), diff --git a/tests/fixtures/merge_receipt/pr14433.json b/tests/fixtures/merge_receipt/pr14433.json new file mode 100644 index 000000000000..d9851668194b --- /dev/null +++ b/tests/fixtures/merge_receipt/pr14433.json @@ -0,0 +1,854 @@ +{ + "number": 14433, + "mergedAt": "2026-09-25T09:51:52Z", + "headRefOid": "bef1de725cb4066c4878d18911618de6e6262351", + "contexts": [ + { + "__typename": "CheckRun", + "name": "Claude request", + "status": "COMPLETED", + "conclusion": "SKIPPED", + "startedAt": "2026-09-25T09:44:10Z", + "completedAt": "2026-09-25T09:44:10Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "changes", + "status": "COMPLETED", + "conclusion": "SKIPPED", + "startedAt": "2026-09-25T09:33:19Z", + "completedAt": "2026-09-25T09:33:18Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "changes", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T09:33:57Z", + "completedAt": "2026-09-25T09:34:27Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "transport", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T09:33:20Z", + "completedAt": "2026-09-25T09:33:42Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "receipt-contract", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T09:33:21Z", + "completedAt": "2026-09-25T09:33:39Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "CLA Assistant", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T09:33:19Z", + "completedAt": "2026-09-25T09:33:32Z", + "isRequired": true, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "CLA Assistant", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T09:51:58Z", + "completedAt": "2026-09-25T09:52:03Z", + "isRequired": true, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "CLA policy guard", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T09:33:21Z", + "completedAt": "2026-09-25T09:33:34Z", + "isRequired": true, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "CLA policy guard", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T09:36:47Z", + "completedAt": "2026-09-25T09:36:57Z", + "isRequired": true, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "Testbox broker trust boundary", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T09:33:21Z", + "completedAt": "2026-09-25T09:33:41Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "Web complexity", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T09:33:21Z", + "completedAt": "2026-09-25T09:33:57Z", + "isRequired": true, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "Web complexity", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T09:36:47Z", + "completedAt": "2026-09-25T09:37:25Z", + "isRequired": true, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "web-build", + "status": "COMPLETED", + "conclusion": "SKIPPED", + "startedAt": "2026-09-25T09:33:19Z", + "completedAt": "2026-09-25T09:33:18Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "Fast static checks", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T09:33:57Z", + "completedAt": "2026-09-25T09:34:23Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "web-tests", + "status": "COMPLETED", + "conclusion": "SKIPPED", + "startedAt": "2026-09-25T09:33:19Z", + "completedAt": "2026-09-25T09:33:18Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "Reverse test impact (report only)", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T09:34:29Z", + "completedAt": "2026-09-25T09:34:46Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "Watch owned pool jobs", + "status": "COMPLETED", + "conclusion": "SKIPPED", + "startedAt": "2026-09-25T09:34:27Z", + "completedAt": "2026-09-25T09:34:27Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "web-database-tests", + "status": "COMPLETED", + "conclusion": "SKIPPED", + "startedAt": "2026-09-25T09:33:27Z", + "completedAt": "2026-09-25T09:33:18Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "guards / workflow-guard-tests / preflight", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T09:34:30Z", + "completedAt": "2026-09-25T09:35:25Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "web-validation", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T09:33:21Z", + "completedAt": "2026-09-25T09:33:24Z", + "isRequired": true, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "guards / workflow-guard-tests / ci", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T09:34:30Z", + "completedAt": "2026-09-25T09:35:48Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "guards / workflow-guard-tests / app-host-execution", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T09:34:30Z", + "completedAt": "2026-09-25T09:35:33Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "guards / workflow-guard-tests / app-host-watchdog", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T09:34:40Z", + "completedAt": "2026-09-25T09:35:18Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "guards / workflow-guard-tests / app-host-process", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T09:34:30Z", + "completedAt": "2026-09-25T09:34:59Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "guards / workflow-guard-tests / app-host-cache", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T09:34:29Z", + "completedAt": "2026-09-25T09:34:51Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "guards / workflow-guard-tests / release-ios", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T09:34:30Z", + "completedAt": "2026-09-25T09:35:21Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "guards / workflow-guard-tests / release-notary", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T09:34:29Z", + "completedAt": "2026-09-25T09:35:07Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "guards / workflow-guard-tests / release-tooling", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T09:34:30Z", + "completedAt": "2026-09-25T09:35:10Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "guards / workflow-guard-tests / quality-sharding", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T09:34:30Z", + "completedAt": "2026-09-25T09:35:08Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "guards / workflow-guard-tests / quality-runtime", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T09:34:30Z", + "completedAt": "2026-09-25T09:35:00Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "guards / workflow-guard-tests / quality-determinism", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T09:34:30Z", + "completedAt": "2026-09-25T09:35:03Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "guards / workflow-guard-history", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T09:34:29Z", + "completedAt": "2026-09-25T09:34:54Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "guards / workflow-guard-cli-scripts / tui-resolution", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T09:34:30Z", + "completedAt": "2026-09-25T09:34:47Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "guards / workflow-guard-cli-scripts / profiling", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T09:34:29Z", + "completedAt": "2026-09-25T09:35:03Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "guards / workflow-guard-source-lints / sidebar-layout", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T09:34:29Z", + "completedAt": "2026-09-25T09:35:03Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "guards / workflow-guard-source-lints / dispatch-ownership", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T09:34:30Z", + "completedAt": "2026-09-25T09:35:10Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "guards / Guard status", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T09:35:50Z", + "completedAt": "2026-09-25T09:35:54Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "GhosttyKit release check", + "status": "COMPLETED", + "conclusion": "SKIPPED", + "startedAt": "2026-09-25T09:34:27Z", + "completedAt": "2026-09-25T09:34:27Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "browser", + "status": "COMPLETED", + "conclusion": "SKIPPED", + "startedAt": "2026-09-25T09:34:27Z", + "completedAt": "2026-09-25T09:34:27Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "web", + "status": "COMPLETED", + "conclusion": "SKIPPED", + "startedAt": "2026-09-25T09:34:27Z", + "completedAt": "2026-09-25T09:34:27Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "suite-coverage", + "status": "COMPLETED", + "conclusion": "SKIPPED", + "startedAt": "2026-09-25T09:34:27Z", + "completedAt": "2026-09-25T09:34:27Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "remote-daemon", + "status": "COMPLETED", + "conclusion": "SKIPPED", + "startedAt": "2026-09-25T09:34:27Z", + "completedAt": "2026-09-25T09:34:27Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "Claude wrapper regressions", + "status": "COMPLETED", + "conclusion": "SKIPPED", + "startedAt": "2026-09-25T09:34:27Z", + "completedAt": "2026-09-25T09:34:27Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "macos / macOS compile admission", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T09:35:26Z", + "completedAt": "2026-09-25T09:38:41Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "macos / swift-package-tests", + "status": "COMPLETED", + "conclusion": "SKIPPED", + "startedAt": "2026-09-25T09:34:27Z", + "completedAt": "2026-09-25T09:34:27Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "macos / inputs.unit_selectors != '' && 'app-host unit tests (changed suites)' || format('app-host unit tests ({0}/7)', matrix.shard)", + "status": "COMPLETED", + "conclusion": "SKIPPED", + "startedAt": "2026-09-25T09:38:42Z", + "completedAt": "2026-09-25T09:38:41Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "macos / CLI product tests", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T09:38:44Z", + "completedAt": "2026-09-25T09:40:02Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "macos / tests-build-and-lag", + "status": "COMPLETED", + "conclusion": "SKIPPED", + "startedAt": "2026-09-25T09:38:42Z", + "completedAt": "2026-09-25T09:38:41Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "macos / release-admission", + "status": "COMPLETED", + "conclusion": "SKIPPED", + "startedAt": "2026-09-25T09:38:42Z", + "completedAt": "2026-09-25T09:38:41Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "macos / release-build", + "status": "COMPLETED", + "conclusion": "SKIPPED", + "startedAt": "2026-09-25T09:38:42Z", + "completedAt": "2026-09-25T09:38:42Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "macos / macOS status", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T09:40:11Z", + "completedAt": "2026-09-25T09:40:15Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "linux-preflight", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T09:35:57Z", + "completedAt": "2026-09-25T09:36:00Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "macOS admission gate", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T09:35:57Z", + "completedAt": "2026-09-25T09:36:01Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "tests", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T09:40:30Z", + "completedAt": "2026-09-25T09:40:34Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "ci-status", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T09:40:37Z", + "completedAt": "2026-09-25T09:40:40Z", + "isRequired": true, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "Vercel Agent Review", + "status": "COMPLETED", + "conclusion": "NEUTRAL", + "startedAt": "2026-09-25T09:33:17Z", + "completedAt": "2026-09-25T09:33:17Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "vercel" + } + } + }, + { + "__typename": "CheckRun", + "name": "cubic \u00b7 AI code reviewer", + "status": "COMPLETED", + "conclusion": "NEUTRAL", + "startedAt": "2026-09-25T09:33:31Z", + "completedAt": "2026-09-25T09:33:33Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "cubic-dev-ai" + } + } + }, + { + "__typename": "CheckRun", + "name": "[code]smith", + "status": "COMPLETED", + "conclusion": "SKIPPED", + "startedAt": "2026-09-25T09:33:15Z", + "completedAt": "2026-09-25T09:33:15Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "blacksmith-sh" + } + } + }, + { + "__typename": "StatusContext", + "context": "CodeRabbit", + "state": "SUCCESS", + "createdAt": "2026-09-25T09:44:12Z", + "isRequired": false + }, + { + "__typename": "CheckRun", + "name": "Socket Security: Project Report", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T09:33:21Z", + "completedAt": "2026-09-25T09:33:23Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "socket-security" + } + } + }, + { + "__typename": "CheckRun", + "name": "Socket Security: Pull Request Alerts", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T09:33:23Z", + "completedAt": "2026-09-25T09:33:36Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "socket-security" + } + } + } + ] +} \ No newline at end of file diff --git a/tests/fixtures/merge_receipt/pr14461.json b/tests/fixtures/merge_receipt/pr14461.json new file mode 100644 index 000000000000..05eb206f42da --- /dev/null +++ b/tests/fixtures/merge_receipt/pr14461.json @@ -0,0 +1,840 @@ +{ + "number": 14461, + "mergedAt": "2026-09-25T10:23:09Z", + "headRefOid": "e9426f528e9467cd985859dba1bef88f631ab899", + "contexts": [ + { + "__typename": "CheckRun", + "name": "Web complexity", + "status": "COMPLETED", + "conclusion": "CANCELLED", + "startedAt": "2026-09-25T10:00:33Z", + "completedAt": "2026-09-25T10:00:49Z", + "isRequired": true, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "welcome", + "status": "COMPLETED", + "conclusion": "SKIPPED", + "startedAt": "2026-09-25T10:00:25Z", + "completedAt": "2026-09-25T10:00:24Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "changes", + "status": "COMPLETED", + "conclusion": "SKIPPED", + "startedAt": "2026-09-25T10:00:24Z", + "completedAt": "2026-09-25T10:00:24Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "changes", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T10:00:28Z", + "completedAt": "2026-09-25T10:01:02Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "CLA Assistant", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T10:00:28Z", + "completedAt": "2026-09-25T10:00:37Z", + "isRequired": true, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "CLA Assistant", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T10:23:15Z", + "completedAt": "2026-09-25T10:23:21Z", + "isRequired": true, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "CLA policy guard", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T10:00:27Z", + "completedAt": "2026-09-25T10:00:36Z", + "isRequired": true, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "CLA policy guard", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T10:00:32Z", + "completedAt": "2026-09-25T10:00:42Z", + "isRequired": true, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "CLA policy guard", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T10:00:46Z", + "completedAt": "2026-09-25T10:00:55Z", + "isRequired": true, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "Testbox broker trust boundary", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T10:00:28Z", + "completedAt": "2026-09-25T10:00:49Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "Web complexity", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T10:01:03Z", + "completedAt": "2026-09-25T10:01:21Z", + "isRequired": true, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "Web complexity", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T10:00:52Z", + "completedAt": "2026-09-25T10:01:01Z", + "isRequired": true, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "web-build", + "status": "COMPLETED", + "conclusion": "SKIPPED", + "startedAt": "2026-09-25T10:00:25Z", + "completedAt": "2026-09-25T10:00:24Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "Fast static checks", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T10:00:28Z", + "completedAt": "2026-09-25T10:00:54Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "web-tests", + "status": "COMPLETED", + "conclusion": "SKIPPED", + "startedAt": "2026-09-25T10:00:25Z", + "completedAt": "2026-09-25T10:00:24Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "Reverse test impact (report only)", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T10:01:04Z", + "completedAt": "2026-09-25T10:01:21Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "Watch owned pool jobs", + "status": "COMPLETED", + "conclusion": "SKIPPED", + "startedAt": "2026-09-25T10:01:02Z", + "completedAt": "2026-09-25T10:01:02Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "web-database-tests", + "status": "COMPLETED", + "conclusion": "SKIPPED", + "startedAt": "2026-09-25T10:00:32Z", + "completedAt": "2026-09-25T10:00:24Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "guards / workflow-guard-tests / preflight", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T10:01:05Z", + "completedAt": "2026-09-25T10:01:56Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "web-validation", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T10:00:28Z", + "completedAt": "2026-09-25T10:00:32Z", + "isRequired": true, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "guards / workflow-guard-tests / ci", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T10:01:04Z", + "completedAt": "2026-09-25T10:02:32Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "guards / workflow-guard-tests / app-host-execution", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T10:01:05Z", + "completedAt": "2026-09-25T10:02:10Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "guards / workflow-guard-tests / app-host-watchdog", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T10:01:04Z", + "completedAt": "2026-09-25T10:01:50Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "guards / workflow-guard-tests / app-host-process", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T10:01:05Z", + "completedAt": "2026-09-25T10:01:33Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "guards / workflow-guard-tests / app-host-cache", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T10:01:04Z", + "completedAt": "2026-09-25T10:01:32Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "guards / workflow-guard-tests / release-ios", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T10:01:05Z", + "completedAt": "2026-09-25T10:02:06Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "guards / workflow-guard-tests / release-notary", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T10:01:04Z", + "completedAt": "2026-09-25T10:01:40Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "guards / workflow-guard-tests / release-tooling", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T10:01:04Z", + "completedAt": "2026-09-25T10:01:53Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "guards / workflow-guard-tests / quality-sharding", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T10:01:05Z", + "completedAt": "2026-09-25T10:01:42Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "guards / workflow-guard-tests / quality-runtime", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T10:01:05Z", + "completedAt": "2026-09-25T10:01:40Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "guards / workflow-guard-tests / quality-determinism", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T10:01:05Z", + "completedAt": "2026-09-25T10:01:37Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "guards / workflow-guard-history", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T10:01:04Z", + "completedAt": "2026-09-25T10:01:29Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "guards / workflow-guard-cli-scripts / ${{ matrix.group }}", + "status": "COMPLETED", + "conclusion": "SKIPPED", + "startedAt": "2026-09-25T10:01:02Z", + "completedAt": "2026-09-25T10:01:02Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "guards / workflow-guard-source-lints / sidebar-layout", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T10:01:04Z", + "completedAt": "2026-09-25T10:01:37Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "guards / workflow-guard-source-lints / dispatch-ownership", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T10:01:04Z", + "completedAt": "2026-09-25T10:01:50Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "guards / Guard status", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T10:02:35Z", + "completedAt": "2026-09-25T10:02:39Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "GhosttyKit release check", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T10:01:04Z", + "completedAt": "2026-09-25T10:01:24Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "browser", + "status": "COMPLETED", + "conclusion": "SKIPPED", + "startedAt": "2026-09-25T10:01:02Z", + "completedAt": "2026-09-25T10:01:02Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "web", + "status": "COMPLETED", + "conclusion": "SKIPPED", + "startedAt": "2026-09-25T10:01:02Z", + "completedAt": "2026-09-25T10:01:02Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "suite-coverage", + "status": "COMPLETED", + "conclusion": "SKIPPED", + "startedAt": "2026-09-25T10:01:02Z", + "completedAt": "2026-09-25T10:01:02Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "remote-daemon", + "status": "COMPLETED", + "conclusion": "SKIPPED", + "startedAt": "2026-09-25T10:01:02Z", + "completedAt": "2026-09-25T10:01:02Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "Claude wrapper regressions", + "status": "COMPLETED", + "conclusion": "SKIPPED", + "startedAt": "2026-09-25T10:01:02Z", + "completedAt": "2026-09-25T10:01:02Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "macos / macOS compile admission", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T10:11:07Z", + "completedAt": "2026-09-25T10:27:48Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "macos / swift-package-tests", + "status": "COMPLETED", + "conclusion": "SKIPPED", + "startedAt": "2026-09-25T10:01:02Z", + "completedAt": "2026-09-25T10:01:02Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "macos / inputs.unit_selectors != '' && 'app-host unit tests (changed suites)' || format('app-host unit tests ({0}/7)', matrix.shard)", + "status": "COMPLETED", + "conclusion": "SKIPPED", + "startedAt": "2026-09-25T10:27:49Z", + "completedAt": "2026-09-25T10:27:49Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "macos / CLI product tests", + "status": "COMPLETED", + "conclusion": "SKIPPED", + "startedAt": "2026-09-25T10:27:49Z", + "completedAt": "2026-09-25T10:27:49Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "macos / tests-build-and-lag", + "status": "COMPLETED", + "conclusion": "SKIPPED", + "startedAt": "2026-09-25T10:27:49Z", + "completedAt": "2026-09-25T10:27:49Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "macos / release-admission", + "status": "COMPLETED", + "conclusion": "SKIPPED", + "startedAt": "2026-09-25T10:27:49Z", + "completedAt": "2026-09-25T10:27:49Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "macos / release-build", + "status": "COMPLETED", + "conclusion": "SKIPPED", + "startedAt": "2026-09-25T10:27:49Z", + "completedAt": "2026-09-25T10:27:49Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "macos / macOS status", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T10:27:51Z", + "completedAt": "2026-09-25T10:27:55Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "linux-preflight", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T10:02:42Z", + "completedAt": "2026-09-25T10:02:46Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "macOS admission gate", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T10:02:42Z", + "completedAt": "2026-09-25T10:02:45Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "tests", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T10:27:57Z", + "completedAt": "2026-09-25T10:28:01Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "ci-status", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T10:28:03Z", + "completedAt": "2026-09-25T10:28:07Z", + "isRequired": true, + "checkSuite": { + "app": { + "slug": "github-actions" + } + } + }, + { + "__typename": "CheckRun", + "name": "Vercel Agent Review", + "status": "COMPLETED", + "conclusion": "NEUTRAL", + "startedAt": "2026-09-25T10:00:25Z", + "completedAt": "2026-09-25T10:00:25Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "vercel" + } + } + }, + { + "__typename": "CheckRun", + "name": "cubic \u00b7 AI code reviewer", + "status": "COMPLETED", + "conclusion": "NEUTRAL", + "startedAt": "2026-09-25T10:00:27Z", + "completedAt": "2026-09-25T10:00:31Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "cubic-dev-ai" + } + } + }, + { + "__typename": "CheckRun", + "name": "[code]smith", + "status": "COMPLETED", + "conclusion": "SKIPPED", + "startedAt": "2026-09-25T10:00:23Z", + "completedAt": "2026-09-25T10:00:23Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "blacksmith-sh" + } + } + }, + { + "__typename": "StatusContext", + "context": "CodeRabbit", + "state": "SUCCESS", + "createdAt": "2026-09-25T10:00:32Z", + "isRequired": false + }, + { + "__typename": "CheckRun", + "name": "Socket Security: Project Report", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T10:00:25Z", + "completedAt": "2026-09-25T10:00:28Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "socket-security" + } + } + }, + { + "__typename": "CheckRun", + "name": "Socket Security: Pull Request Alerts", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "startedAt": "2026-09-25T10:00:28Z", + "completedAt": "2026-09-25T10:00:36Z", + "isRequired": false, + "checkSuite": { + "app": { + "slug": "socket-security" + } + } + } + ] +} \ No newline at end of file diff --git a/tests/test-execution.toml b/tests/test-execution.toml index 80e84ea63002..466e7f5511c0 100644 --- a/tests/test-execution.toml +++ b/tests/test-execution.toml @@ -1229,3 +1229,7 @@ lane = "linux-guard" [[test]] path = "tests/test_ci_main_regression_attribution.py" lane = "linux-guard" + +[[test]] +path = "tests/test_ci_merge_receipt.py" +lane = "linux-guard" diff --git a/tests/test_ci_main_regression_attribution.py b/tests/test_ci_main_regression_attribution.py index d59f44cd4785..0c6b55271307 100644 --- a/tests/test_ci_main_regression_attribution.py +++ b/tests/test_ci_main_regression_attribution.py @@ -48,18 +48,19 @@ def run(**overrides): return base -def pr_node(number, merge_sha, state="MERGED", base="main"): +def pr_node(number, merge_sha, state="MERGED", base="main", labels=()): return { "number": number, "title": f"PR {number}", "url": f"https://github.com/{REPO}/pull/{number}", "state": state, "baseRefName": base, "author": {"login": "someone"}, "mergeCommit": {"oid": merge_sha} if merge_sha else None, + "labels": {"nodes": [{"name": name} for name in labels]}, } -def pr(number, edited=(), reached=()): +def pr(number, edited=(), reached=(), unverified=False): return MODULE.PullRequest( number=number, title=f"PR {number}", url=f"u/{number}", merge_sha=f"m{number}", - edited_suites=set(edited), reached_suites=set(reached), + edited_suites=set(edited), reached_suites=set(reached), unverified=unverified, ) @@ -202,6 +203,22 @@ def test_ties_name_every_top_pull_request(self): a, b = pr(1, reached={"Suite"}), pr(2, reached={"Suite"}) self.assertEqual([p.number for p in MODULE.suspects_for("Suite/t()", [a, b, pr(3)])[0]], [1, 2]) + def test_a_tie_prefers_pull_requests_that_merged_unverified(self): + a, b = pr(1, reached={"Suite"}), pr(2, reached={"Suite"}, unverified=True) + suspects, how = MODULE.suspects_for("Suite/t()", [a, b, pr(3, unverified=True)]) + self.assertEqual([p.number for p in suspects], [2]) + self.assertEqual(how, "changes code the suite names, merged unverified") + # The label breaks ties only: a stronger signal still wins, and no signal blames nobody. + edits = pr(4, edited={"Suite"}) + self.assertEqual([p.number for p in MODULE.suspects_for("Suite/t()", [edits, b])[0]], [4]) + self.assertEqual(MODULE.suspects_for("Suite/t()", [pr(1), pr(2, unverified=True)])[0], []) + + def test_merged_prs_reads_the_merged_unverified_label(self): + prs, _ = MODULE.merged_prs(["m1", "m2"], { + "m1": [pr_node(1, "m1", labels=("merged-unverified",))], "m2": [pr_node(2, "m2")], + }) + self.assertEqual([(p.number, p.unverified) for p in prs], [(2, False), (1, True)]) + def test_no_signal_blames_nobody(self): self.assertEqual(MODULE.suspects_for("Suite/t()", [pr(1), pr(2)])[0], []) self.assertEqual(MODULE.suspects_for("Suite/t()", [])[0], []) diff --git a/tests/test_ci_merge_receipt.py b/tests/test_ci_merge_receipt.py new file mode 100644 index 000000000000..80e1a263aabe --- /dev/null +++ b/tests/test_ci_merge_receipt.py @@ -0,0 +1,175 @@ +"""The merge receipt: which checks a pull request's head had passed when it merged.""" + +import importlib.util +import json +import pathlib +import sys +import unittest + +import yaml + +ROOT = pathlib.Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts/ci/merge_receipt.py" +WORKFLOW = ROOT / ".github/workflows/merge-receipt.yml" +FIXTURES = ROOT / "tests/fixtures/merge_receipt" +SPEC = importlib.util.spec_from_file_location("merge_receipt", SCRIPT) +MODULE = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +sys.modules[SPEC.name] = MODULE # dataclasses resolve annotations through it +SPEC.loader.exec_module(MODULE) + +MERGED = "2026-09-25T10:00:00Z" + + +def run(name, conclusion="SUCCESS", started="2026-09-25T09:00:00Z", completed="2026-09-25T09:30:00Z", + required=False, app="github-actions"): + return { + "__typename": "CheckRun", "name": name, "status": "COMPLETED" if completed else "IN_PROGRESS", + "conclusion": conclusion if completed else None, "startedAt": started, "completedAt": completed, + "isRequired": required, "checkSuite": {"app": {"slug": app}}, + } + + +def snapshot(*contexts): + return {"mergedAt": MERGED, "headRefOid": "a" * 40, "contexts": list(contexts)} + + +def load(name): + return json.loads((FIXTURES / name).read_text(encoding="utf-8")) + + +class StateAtMergeTests(unittest.TestCase): + def test_each_check_is_read_as_of_the_merge(self): + self.assertEqual(MODULE.state_at(run("a"), MERGED)[0], "success") + self.assertEqual(MODULE.state_at(run("a", "FAILURE"), MERGED)[0], "failure") + self.assertEqual(MODULE.state_at(run("a", "SKIPPED"), MERGED)[0], "skipped") + self.assertEqual(MODULE.state_at(run("a", "NEUTRAL"), MERGED)[0], "success") + # Finished after the merge: it was still running when the merge happened. + self.assertEqual(MODULE.state_at(run("a", completed="2026-09-25T10:05:00Z"), MERGED)[0], "in progress") + self.assertEqual(MODULE.state_at(run("a", completed=None), MERGED)[0], "in progress") + self.assertEqual(MODULE.state_at(run("a", started="2026-09-25T10:01:00Z"), MERGED)[0], "not reported") + self.assertEqual(MODULE.state_at(run("a", started=None, completed=None), MERGED)[0], "pending") + + def test_status_contexts_count_only_when_posted_before_the_merge(self): + status = {"__typename": "StatusContext", "context": "x", "state": "SUCCESS", "createdAt": "2026-09-25T09:00:00Z"} + self.assertEqual(MODULE.state_at(status, MERGED)[0], "success") + self.assertEqual(MODULE.state_at(dict(status, createdAt="2026-09-25T11:00:00Z"), MERGED)[0], "not reported") + + def test_the_latest_run_started_before_the_merge_wins(self): + checks = MODULE.checks_at_merge([ + run("Web complexity", "CANCELLED", started="2026-09-25T09:00:00Z"), + run("Web complexity", "SUCCESS", started="2026-09-25T09:10:00Z"), + run("Web complexity", "FAILURE", started="2026-09-25T10:10:00Z", completed="2026-09-25T10:20:00Z"), + ], MERGED) + self.assertEqual([(c.name, c.state) for c in checks if c.name == "Web complexity"], [("Web complexity", "success")]) + + def test_jobs_that_had_not_started_are_left_out_but_ci_status_is_expected(self): + checks = MODULE.checks_at_merge([run("macos / CLI product tests", started="2026-09-25T10:30:00Z")], MERGED) + self.assertEqual([(c.name, c.state) for c in checks], [("ci-status", "not reported")]) + + +class GroupingTests(unittest.TestCase): + def test_reusable_jobs_lose_their_prefix_and_guards_and_shards_fold(self): + self.assertEqual(MODULE.group_name("macos / macOS compile admission"), "macOS compile admission") + self.assertEqual(MODULE.group_name("guards / workflow-guard-tests / ci"), "guards") + self.assertEqual(MODULE.group_name("macos / app-host unit tests (3/7)"), "app-host unit tests") + self.assertEqual( + MODULE.group_name("macos / inputs.unit_selectors != '' && 'app-host unit tests (changed suites)' || x"), + "app-host unit tests", + ) + self.assertEqual(MODULE.group_name("ci-status"), "ci-status") + + def test_a_group_reports_its_worst_member(self): + group = MODULE.Group("guards", [MODULE.Check("a", "success"), MODULE.Check("b", "in progress"), + MODULE.Check("c", "skipped")]) + self.assertEqual(group.state, "in progress") + self.assertEqual(MODULE.Group("x", [MODULE.Check("a", "skipped")]).state, "skipped") + self.assertEqual(MODULE.Group("x", [MODULE.Check("a", "skipped"), MODULE.Check("b", "success")]).state, "success") + + +class ReceiptTests(unittest.TestCase): + def test_a_merge_before_compile_admission_finished_is_unverified(self): + # #14461 merged while macOS compile admission was still running. + result = MODULE.receipt(load("pr14461.json")) + self.assertTrue(result.unverified) + lines = result.body.splitlines() + self.assertEqual(lines[0], "**Merge receipt** for `e9426f528e`, merged 2026-09-25 10:23:09 UTC") + self.assertEqual( + lines[1], "- Not verified at merge: ci-status (not reported), macOS compile admission (in progress)", + ) + self.assertIn("guards (17)", lines[2]) + self.assertNotIn("CLA", result.body) + self.assertNotIn("Socket", result.body) + self.assertIn("swift-package-tests", next(line for line in lines if line.startswith("- Skipped by policy"))) + self.assertIn("`merged-unverified`", result.body) + self.assertTrue(result.body.endswith(MODULE.MARKER)) + self.assertLessEqual(len(lines), 8) + + def test_an_all_green_merge_is_one_line(self): + result = MODULE.receipt(load("pr14433.json")) + self.assertFalse(result.unverified) + self.assertEqual(result.body.splitlines(), [ + "**Merge receipt** for `bef1de725c`: every check was green at merge " + "(14 verified; 15 skipped by policy). Full suite runs on main after merge.", + MODULE.MARKER, + ]) + + def test_app_host_skipped_by_policy_is_not_unverified(self): + result = MODULE.receipt(snapshot( + run("ci-status", required=True), run("macos / macOS compile admission"), + run("macos / app-host unit tests (1/7)", "SKIPPED"), + )) + self.assertFalse(result.unverified) + + def test_a_failed_non_judging_job_is_listed_without_the_label(self): + result = MODULE.receipt(snapshot(run("ci-status", required=True), run("linux-preflight", "FAILURE"))) + self.assertFalse(result.unverified) + self.assertIn("- Not verified at merge: linux-preflight (failure)", result.body) + self.assertNotIn("merged-unverified", result.body) + + def test_any_required_check_not_green_is_unverified(self): + result = MODULE.receipt(snapshot(run("ci-status", required=True), run("Web complexity", "FAILURE", required=True))) + self.assertTrue(result.unverified) + + def test_bots_and_cla_show_only_when_they_failed(self): + quiet = MODULE.receipt(snapshot( + run("ci-status", required=True), run("Socket Security: Project Report", app="socket-security"), + run("CLA Assistant", required=True), + )) + self.assertNotIn("Socket", quiet.body) + self.assertNotIn("CLA", quiet.body) + loud = MODULE.receipt(snapshot(run("ci-status", required=True), run("CLA Assistant", "FAILURE", required=True))) + self.assertIn("CLA Assistant (failure)", loud.body) + self.assertFalse(loud.unverified) + + def test_the_existing_receipt_comment_is_found_by_its_marker(self): + comments = [{"databaseId": 1, "body": "hi"}, {"databaseId": 7, "body": f"old\n{MODULE.MARKER}"}] + self.assertEqual(MODULE.existing_comment(comments), 7) + self.assertIsNone(MODULE.existing_comment(comments[:1])) + + +class WorkflowTests(unittest.TestCase): + def setUp(self): + self.workflow = yaml.safe_load(WORKFLOW.read_text(encoding="utf-8")) + self.job = self.workflow["jobs"]["receipt"] + + def test_runs_after_a_merge_into_main_from_any_fork(self): + trigger = self.workflow[True]["pull_request_target"] # YAML reads `on` as True + self.assertEqual(trigger, {"types": ["closed"], "branches": ["main"]}) + self.assertEqual(self.job["if"], "github.event.pull_request.merged == true") + + def test_never_checks_out_pull_request_code(self): + self.assertEqual(self.workflow["permissions"], {}) + self.assertEqual(self.job["permissions"], { + "contents": "read", "pull-requests": "write", "checks": "read", "statuses": "read", + }) + checkout = self.job["steps"][0]["with"] + self.assertEqual(checkout["ref"], "${{ github.workflow_sha }}") + self.assertFalse(checkout["persist-credentials"]) + self.assertEqual(checkout["sparse-checkout"], "scripts/ci/merge_receipt.py") + text = WORKFLOW.read_text(encoding="utf-8") + self.assertNotIn("pull_request.head", text) + + +if __name__ == "__main__": + unittest.main() From fb8ce7c329e06f6b4d9c7b1c7fdd1806ae53586a Mon Sep 17 00:00:00 2001 From: Leo Date: Fri, 25 Sep 2026 06:54:54 -0400 Subject: [PATCH 2/3] ci: harden the merge receipt comment and keep verified suspects in a tie Only a github-actions comment carrying the marker is edited, check names are escaped so a pull request's own job names cannot mention people or hide the receipt, and a failed gh call prints GitHub's error. Attribution now lists merged-unverified pull requests first in a tie instead of dropping the rest, since a verified head can still break main through another merge. Known limits are noted in the script's docstring. Co-Authored-By: Claude Opus 5.5 --- scripts/ci/main_regression_attribution.py | 12 +++--- scripts/ci/merge_receipt.py | 40 +++++++++++++++++--- tests/test_ci_main_regression_attribution.py | 6 +-- tests/test_ci_merge_receipt.py | 18 +++++++-- 4 files changed, 58 insertions(+), 18 deletions(-) diff --git a/scripts/ci/main_regression_attribution.py b/scripts/ci/main_regression_attribution.py index 2f38bb40075e..94e40cac770f 100644 --- a/scripts/ci/main_regression_attribution.py +++ b/scripts/ci/main_regression_attribution.py @@ -20,8 +20,8 @@ (test_impact.py), 1 when a changed app declaration or string is named by the suite (reverse_test_impact.py), 0 otherwise. The top score names the suspects; when every score is 0 the failure is left unattributed rather than -blaming the whole range. A tie prefers pull requests labeled merged-unverified -(merge_receipt.py): a judging check was not green when they merged. +blaming the whole range. A tie lists pull requests labeled merged-unverified +(merge_receipt.py: a judging check was not green when they merged) first. `report` writes a "New since" markdown section for the tracking issue (read by main_full_suite.py report --extra-section) and comments once on each @@ -257,10 +257,10 @@ def suspects_for( return [], "no pull request in the range reaches this suite" how = "edits the suite" if best == 2 else "changes code the suite names" tied = [pr for value, pr in scored if value == best] - # A pull request that merged before its checks passed breaks a tie. - unverified = [pr for pr in tied if pr.unverified] - if unverified and len(unverified) < len(tied): - return unverified, f"{how}, merged unverified" + # A pull request that merged before its checks passed is listed first in a + # tie; the others stay, since a verified head can still break main + # through an interaction with another merge. + tied.sort(key=lambda pr: not pr.unverified) return tied, how diff --git a/scripts/ci/merge_receipt.py b/scripts/ci/merge_receipt.py index 9b21fca22a95..1fc4fc4e0acd 100644 --- a/scripts/ci/merge_receipt.py +++ b/scripts/ci/merge_receipt.py @@ -20,8 +20,21 @@ A pull request whose judging checks (compile admission, app-host unit tests, ci-status and required checks) were not all green at merge gets the `merged-unverified` label, which main_regression_attribution.py uses to break -ties between suspects. The comment is idempotent through a hidden marker and -is edited in place on a re-run. +ties between suspects. The comment is idempotent through a hidden marker on a +github-actions comment and is edited in place on a re-run. + +Known limits: +- Only the last 100 comments are searched for the marker, so re-running this + on a pull request that has since gained more comments posts a second one. +- A required check that never ran on the head has no isRequired to read, so + only EXPECTED checks (ci-status) are named when missing. ci-status depends + on the other jobs today, so that covers them. +- A commit status keeps one context that later updates replace, so a status + shows its current state, not its state at merge. Statuses are noise unless + required. +- With a merge queue, checks run on the merge_group commit, not the pull + request head, so every queued pull request would read as unverified. Gate + this workflow or read the merge_group commit before enabling a queue. """ from __future__ import annotations @@ -45,6 +58,8 @@ r"^CLA |^CLA$|CLA Assistant|CLA policy guard|^welcome$|Watch owned pool jobs|\(report only\)|^changes$" ) ACTIONS_APP = "github-actions" +# The login GraphQL reports for comments this workflow's token posts. +BOT_LOGIN = "github-actions" MAX_LISTED = 12 # States, worst first. A group reports its worst member. @@ -87,7 +102,8 @@ def judging(self) -> bool: return bool(JUDGING_RE.search(self.name)) or any(check.required for check in self.checks) def label(self) -> str: - return f"{self.name} ({len(self.checks)})" if len(self.checks) > 1 else self.name + name = escape(self.name) + return f"{name} ({len(self.checks)})" if len(self.checks) > 1 else name def state_at(context: Mapping[str, object], merged_at: str) -> tuple[str, str]: @@ -174,6 +190,13 @@ class Receipt: unverified: bool +def escape(name: str) -> str: + """A check name as inert Markdown text: a pull request's own workflow can name its checks.""" + name = " ".join(name.split())[:100] + name = name.replace("&", "&").replace("<", "<").replace(">", ">") + return re.sub(r"([\\`*_\[\]#|~!@])", r"\\\1", name) + + def listed(items: list[str]) -> str: if len(items) <= MAX_LISTED: return ", ".join(items) @@ -222,7 +245,7 @@ def receipt(snapshot: Mapping[str, object]) -> Receipt: pullRequest(number: $number) { number merged mergedAt headRefOid labels(first: 50) { nodes { name } } - comments(last: 100) { nodes { databaseId body } } + comments(last: 100) { nodes { databaseId body author { login } } } } } } @@ -255,7 +278,11 @@ def receipt(snapshot: Mapping[str, object]) -> Receipt: def gh(args: list[str]) -> str: - return subprocess.run(["gh", *args], check=True, capture_output=True, text=True).stdout + result = subprocess.run(["gh", *args], capture_output=True, text=True) + if result.returncode: + print(result.stderr, file=sys.stderr) + result.check_returncode() + return result.stdout def graphql(query: str, **variables: object) -> dict: @@ -294,7 +321,8 @@ def fetch(repo: str, number: int) -> dict: def existing_comment(comments: Iterable[Mapping[str, object]]) -> int | None: for comment in comments: - if MARKER in str(comment.get("body") or ""): + author = str(((comment.get("author") or {}) or {}).get("login") or "") + if author == BOT_LOGIN and MARKER in str(comment.get("body") or ""): return int(comment["databaseId"]) return None diff --git a/tests/test_ci_main_regression_attribution.py b/tests/test_ci_main_regression_attribution.py index 0c6b55271307..c02bc3b6da14 100644 --- a/tests/test_ci_main_regression_attribution.py +++ b/tests/test_ci_main_regression_attribution.py @@ -203,11 +203,11 @@ def test_ties_name_every_top_pull_request(self): a, b = pr(1, reached={"Suite"}), pr(2, reached={"Suite"}) self.assertEqual([p.number for p in MODULE.suspects_for("Suite/t()", [a, b, pr(3)])[0]], [1, 2]) - def test_a_tie_prefers_pull_requests_that_merged_unverified(self): + def test_a_tie_lists_pull_requests_that_merged_unverified_first(self): a, b = pr(1, reached={"Suite"}), pr(2, reached={"Suite"}, unverified=True) suspects, how = MODULE.suspects_for("Suite/t()", [a, b, pr(3, unverified=True)]) - self.assertEqual([p.number for p in suspects], [2]) - self.assertEqual(how, "changes code the suite names, merged unverified") + self.assertEqual([p.number for p in suspects], [2, 1]) + self.assertEqual(how, "changes code the suite names") # The label breaks ties only: a stronger signal still wins, and no signal blames nobody. edits = pr(4, edited={"Suite"}) self.assertEqual([p.number for p in MODULE.suspects_for("Suite/t()", [edits, b])[0]], [4]) diff --git a/tests/test_ci_merge_receipt.py b/tests/test_ci_merge_receipt.py index 80e1a263aabe..cf8e8c333eab 100644 --- a/tests/test_ci_merge_receipt.py +++ b/tests/test_ci_merge_receipt.py @@ -142,10 +142,22 @@ def test_bots_and_cla_show_only_when_they_failed(self): self.assertIn("CLA Assistant (failure)", loud.body) self.assertFalse(loud.unverified) - def test_the_existing_receipt_comment_is_found_by_its_marker(self): - comments = [{"databaseId": 1, "body": "hi"}, {"databaseId": 7, "body": f"old\n{MODULE.MARKER}"}] + def test_the_existing_receipt_comment_is_found_by_its_marker_on_a_bot_comment(self): + bot, person = {"login": "github-actions"}, {"login": "someone"} + comments = [ + {"databaseId": 1, "body": "hi", "author": bot}, + {"databaseId": 3, "body": f"quoting\n{MODULE.MARKER}", "author": person}, + {"databaseId": 7, "body": f"old\n{MODULE.MARKER}", "author": bot}, + ] self.assertEqual(MODULE.existing_comment(comments), 7) - self.assertIsNone(MODULE.existing_comment(comments[:1])) + self.assertIsNone(MODULE.existing_comment(comments[:2])) + + def test_check_names_are_inert_markdown(self): + result = MODULE.receipt(snapshot( + run("ci-status", required=True), run("@team