ci: name the merged pull request behind each new main full-suite failure

.github/workflows/ci-guards.yml

@@ -360,6 +360,10 @@ jobs:
         if: ${{ matrix.group == 'ci' }}
         run: python3 tests/test_ci_main_full_suite.py
 
+      - name: Validate main full-suite failure attribution
+        if: ${{ matrix.group == 'ci' }}
+        run: python3 tests/test_ci_main_regression_attribution.py
+
       - name: Validate CI queue janitor policy
         if: ${{ matrix.group == 'ci' }}
         run: python3 tests/test_ci_queue_janitor.py

.github/workflows/ci-main-full-suite.yml

@@ -7,7 +7,8 @@ name: CI main full suite
 # completion dispatches the next on the newest HEAD. A red result therefore
 # covers only the commits that landed during one run. The schedule is a
 # backstop for a missed event. It keeps one tracking issue open while main is
-# red.
+# red, and names the merged pull requests suspected of each new failure, on
+# the issue and on those pull requests (main_regression_attribution.py).
 #
 # It dispatches ci.yml instead of calling it so the run keeps the identity the
 # app-host product transport trusts (path ci.yml, event workflow_dispatch), and
@@ -128,21 +129,51 @@ jobs:
     # report. Reports are idempotent.
     if: ${{ (github.event_name != 'workflow_run' && github.event_name != 'push' && github.ref == 'refs/heads/main') || (github.event.workflow_run.event == 'workflow_dispatch' && github.event.workflow_run.head_branch == 'main' && github.event.workflow_run.path == '.github/workflows/ci.yml') }}
     runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }}
-    timeout-minutes: 5
+    # Attribution reads the failed app-host shard logs of two runs (about
+    # three minutes) and diffs the pull requests merged between them.
+    timeout-minutes: 20
     concurrency:
       group: ci-main-full-suite-report
       cancel-in-progress: false
     permissions:
       actions: read
       contents: read
       issues: write
+      # Comments on the pull requests suspected of a new failure.
+      pull-requests: write
     steps:
       - name: Checkout
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           fetch-depth: 1
           persist-credentials: false
-          sparse-checkout: scripts/ci
+          # The trees attribution ranks pull requests against, besides the scripts.
+          sparse-checkout: |
+            scripts/ci
+            cmuxTests
+            Sources
+            Packages/macOS
+            Packages/Shared
+            CLI
+
+      - name: Attribute new failures to merged pull requests
+        # A report-only heuristic: its failure must not stop the issue sync.
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ github.token }}
+          RUN_ID: ${{ github.event.workflow_run.id }}
+        run: |
+          set -euo pipefail
+          # Main's recent history, without blobs, for the commit range between
+          # two runs and each merged pull request's diff. Continuous runs keep
+          # that range to a few dozen commits.
+          git fetch --no-tags --filter=blob:none --depth=1000 origin main
+          run_args=()
+          if [ -n "$RUN_ID" ]; then
+            run_args=(--run-id "$RUN_ID")
+          fi
+          python3 scripts/ci/main_regression_attribution.py report ${run_args[@]+"${run_args[@]}"} \
+            --section-output "$RUNNER_TEMP/new-failures.md"
 
       - name: Open, update or close the tracking issue
         env:
@@ -154,4 +185,5 @@ jobs:
           if [ -n "$RUN_ID" ]; then
             run_args=(--run-id "$RUN_ID")
           fi
-          python3 scripts/ci/main_full_suite.py report ${run_args[@]+"${run_args[@]}"}
+          python3 scripts/ci/main_full_suite.py report ${run_args[@]+"${run_args[@]}"} \
+            --extra-section "$RUNNER_TEMP/new-failures.md"

scripts/ci/main_full_suite.py

@@ -22,6 +22,8 @@
 
 `report` syncs the single tracking issue with a completed dispatch run on main:
 a red run opens the issue or comments on it once, and a green run closes it.
+A red report carries the "New since" section main_regression_attribution.py
+writes: which tests newly fail and the pull requests suspected of it.
 """
 
 from __future__ import annotations
@@ -135,7 +137,7 @@ def issue_plan(conclusion: str, has_open_issue: bool, already_reported: bool) ->
     return "none"
 
 
-def failure_body(run: Mapping[str, object], jobs: list[Mapping[str, object]]) -> str:
+def failure_body(run: Mapping[str, object], jobs: list[Mapping[str, object]], extra: str = "") -> str:
     lines = [
         f"Full-suite CI on `main` failed at {run.get('head_sha')}: {run.get('html_url')}",
         "",
@@ -148,6 +150,8 @@ def failure_body(run: Mapping[str, object], jobs: list[Mapping[str, object]]) ->
             lines.append(f"- ...and {len(jobs) - MAX_LISTED_JOBS} more")
     else:
         lines.append("No individual job reported failure; see the run summary.")
+    if extra.strip():
+        lines += ["", extra.strip()]
     lines += [
         "",
         "Pull requests run a subset of the suite, so this run is the first place "
@@ -233,6 +237,17 @@ def ensure_label(repo: str) -> None:
     ], check=True, capture_output=True, text=True)
 
 
+def read_extra_section(path: str | None) -> str:
+    """The new-failure attribution, when main_regression_attribution.py wrote one."""
+    if not path:
+        return ""
+    try:
+        with open(path, encoding="utf-8") as handle:
+            return handle.read()
+    except OSError:
+        return ""
+
+
 def command_report(args: argparse.Namespace) -> int:
     if args.run_id:
         run = gh_json_lines([f"repos/{args.repo}/actions/runs/{args.run_id}", "--jq", "tojson"])[0]
@@ -258,7 +273,7 @@ def command_report(args: argparse.Namespace) -> int:
             "-X", "GET", "-f", "filter=latest", "-f", "per_page=100",
             "--jq", ".jobs[] | {name, conclusion, html_url} | tojson",
         ]))
-        body = failure_body(run, jobs)
+        body = failure_body(run, jobs, read_extra_section(args.extra_section))
         if plan == "open":
             ensure_label(args.repo)
             subprocess.run([
@@ -296,6 +311,7 @@ def main(argv: list[str]) -> int:
 
     report = commands.add_parser("report", help="sync the tracking issue with a completed run")
     report.add_argument("--run-id", help="defaults to the newest green or red full-suite run")
+    report.add_argument("--extra-section", help="markdown to add to a failure report, e.g. new-failure attribution")
     report.set_defaults(handler=command_report)
 
     args = parser.parse_args(argv)