ci: optional compile-only pull request runs, and fail-fast merge groups

.github/workflows/ci.yml

@@ -27,6 +27,10 @@ jobs:
       macos: ${{ steps.detect.outputs.macos }}
       web: ${{ steps.detect.outputs.web }}
       agent_session_web: ${{ steps.detect.outputs.agent_session_web }}
+      full_suite: ${{ steps.suite.outputs.full_suite }}
+    permissions:
+      contents: read
+      pull-requests: read
     steps:
       - name: Checkout
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
@@ -147,6 +151,27 @@ jobs:
             --base-sha "$BASE_SHA" \
             --head-sha "$HEAD_SHA"
 
+      # Compile admission runs on every routed push. The rest of the macOS suite
+      # runs on merge groups and dispatches always, and on pull requests unless
+      # the repository variable CI_PULL_REQUEST_SUITE is "compile-only". Under
+      # that policy the "full-ci" label opts one pull request back in. Labels are
+      # read live so a re-run sees a label added after the push.
+      - name: Choose the macOS suite for this run
+        id: suite
+        env:
+          EVENT_NAME: ${{ github.event_name }}
+          PULL_REQUEST_POLICY: ${{ vars.CI_PULL_REQUEST_SUITE }}
+          PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }}
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          set -euo pipefail
+          args=(--event-name "$EVENT_NAME" --pull-request-policy "$PULL_REQUEST_POLICY" --github-output "$GITHUB_OUTPUT")
+          labels="$RUNNER_TEMP/pull-request-labels.txt"
+          if [ -n "$PULL_REQUEST_NUMBER" ] && gh api "repos/$GITHUB_REPOSITORY/issues/$PULL_REQUEST_NUMBER/labels" --paginate --jq '.[].name' > "$labels"; then
+            args+=(--labels-file "$labels")
+          fi
+          python3 scripts/ci/choose_ci_suite.py "${args[@]}"
+
   workflow-guard-tests:
     runs-on: ${{ vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }}
     # This job executes scripts from the pull request. It needs only read
@@ -737,7 +762,7 @@ jobs:
     # jobs that legitimately skip (web/go/agent-session paths), and that
     # transitive skip otherwise marks every macOS job skipped even when
     # linux-preflight itself succeeds. Require the direct needs explicitly.
-    if: ${{ !cancelled() && needs.changes.result == 'success' && needs.linux-preflight.result == 'success' && needs.macos-compile-admission.result == 'success' && needs.changes.outputs.macos == 'true' }}
+    if: ${{ !cancelled() && needs.changes.result == 'success' && needs.linux-preflight.result == 'success' && needs.macos-compile-admission.result == 'success' && needs.changes.outputs.macos == 'true' && needs.changes.outputs.full_suite == 'true' }}
     name: app-host unit tests (${{ matrix.shard }}/6)
     # App-host XCTest needs a runner that can broker testmanagerd control
     # sessions, so route through the shared MACOS_RUNNER_15 var like the other
@@ -747,7 +772,9 @@ jobs:
     runs-on: ${{ vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15' }}
     timeout-minutes: 75
     strategy:
-      fail-fast: false
+      # A pull request wants every shard's failures in one run. A merge group
+      # only needs a verdict, and the first failure already is one.
+      fail-fast: ${{ github.event_name == 'merge_group' }}
       matrix:
         shard: [1, 2, 3, 4, 5, 6]
     env:
@@ -1827,6 +1854,10 @@ jobs:
           changes = needs["changes"]
           tests = needs["app-host-unit-tests"]
           macos = changes.get("outputs", {}).get("macos")
+          # A compile-only pull request run skips the suite on purpose; the
+          # merge group runs it before anything lands.
+          full_suite = changes.get("outputs", {}).get("full_suite") != "false"
+          suite_required = macos == "true" and full_suite
 
           if changes["result"] != "success":
               print(f"changes: {changes['result']}", file=sys.stderr)
@@ -1851,12 +1882,15 @@ jobs:
               )
               sys.exit(1)
 
-          if macos == "true" and tests["result"] != "success":
+          if suite_required and tests["result"] != "success":
               print(f"app-host unit tests were required but did not pass: {tests['result']}", file=sys.stderr)
               sys.exit(1)
 
-          if macos != "true" and tests["result"] not in {"success", "skipped"}:
-              print(f"app-host unit tests had unexpected result for macos={macos}: {tests['result']}", file=sys.stderr)
+          if not suite_required and tests["result"] not in {"success", "skipped"}:
+              print(
+                  f"app-host unit tests had unexpected result for macos={macos} full_suite={full_suite}: {tests['result']}",
+                  file=sys.stderr,
+              )
               sys.exit(1)
 
           # The remaining test/build suites in this workflow gate too. A job that
@@ -1870,6 +1904,7 @@ jobs:
                   sys.exit(1)
 
           print(f"changes.macos={macos}")
+          print(f"changes.full_suite={full_suite}")
           print(f"linux-preflight={preflight['result']}")
           print(f"macos-compile-admission={admission['result']}")
           print(f"app-host unit tests={tests['result']}")
@@ -1886,7 +1921,7 @@ jobs:
     # jobs that legitimately skip (web/go/agent-session paths), and that
     # transitive skip otherwise marks every macOS job skipped even when
     # linux-preflight itself succeeds. Require the direct needs explicitly.
-    if: ${{ !cancelled() && needs.changes.result == 'success' && needs.linux-preflight.result == 'success' && needs.changes.outputs.macos == 'true' }}
+    if: ${{ !cancelled() && needs.changes.result == 'success' && needs.linux-preflight.result == 'success' && needs.changes.outputs.macos == 'true' && needs.changes.outputs.full_suite == 'true' }}
     # Build the release helper with SDK 15, then run package tests with SDK 26.
     runs-on: ${{ vars.MACOS_RUNNER_DUAL_XCODE || 'blacksmith-6vcpu-macos-15' }}
     timeout-minutes: 40
@@ -2395,7 +2430,7 @@ jobs:
     # jobs that legitimately skip (web/go/agent-session paths), and that
     # transitive skip otherwise marks every macOS job skipped even when
     # linux-preflight itself succeeds. Require the direct needs explicitly.
-    if: ${{ !cancelled() && needs.changes.result == 'success' && needs.linux-preflight.result == 'success' && needs.macos-compile-admission.result == 'success' && needs.changes.outputs.macos == 'true' }}
+    if: ${{ !cancelled() && needs.changes.result == 'success' && needs.linux-preflight.result == 'success' && needs.macos-compile-admission.result == 'success' && needs.changes.outputs.macos == 'true' && needs.changes.outputs.full_suite == 'true' }}
     # Build the full cmux scheme once, then run the required display/runtime
     # regressions from the same DerivedData instead of queuing a second display
     # runner for UI-only checks.
@@ -2735,7 +2770,7 @@ jobs:
     # See app-host-unit-tests: explicit direct-needs gate instead of the
     # implicit success() so skipped routed linux jobs upstream of
     # linux-preflight do not skip this job transitively.
-    if: ${{ !cancelled() && needs.changes.result == 'success' && needs.linux-preflight.result == 'success' && needs.swift-package-tests.result == 'success' && needs.macos-compile-admission.result == 'success' && needs.changes.outputs.macos == 'true' }}
+    if: ${{ !cancelled() && needs.changes.result == 'success' && needs.linux-preflight.result == 'success' && needs.swift-package-tests.result == 'success' && needs.macos-compile-admission.result == 'success' && needs.changes.outputs.macos == 'true' && needs.changes.outputs.full_suite == 'true' }}
     # Compile the same unsigned universal Release app that nightly builds before
     # signing, notarization, and publishing. This catches DEBUG/Release boundary
     # mistakes before they reach main.

.github/workflows/merge-group-fail-fast.yml

@@ -0,0 +1,58 @@
+name: Merge-group fail fast
+
+# One failed job already decides a merge group: the queue will remove it. CI's
+# ci-status reports only after every job finishes, so without this the remaining
+# macOS jobs run to the end first and every pull request queued behind waits for
+# them. This workflow cancels a merge group's CI run at its first failed job;
+# ci-status runs on cancellation and reports the failure.
+#
+# It needs actions: write, so it is triggered by workflow_run: the code below
+# always comes from the default branch, and a queued pull request cannot change
+# it. It checks out nothing and runs on a GitHub-hosted runner.
+on:
+  workflow_run:
+    workflows: [CI]
+    types: [requested]
+
+permissions: {}
+
+jobs:
+  watch:
+    name: watch merge group run
+    if: ${{ github.event.workflow_run.event == 'merge_group' }}
+    runs-on: ubuntu-24.04 # github-hosted-required
+    timeout-minutes: 360
+    permissions:
+      actions: write
+    steps:
+      - name: Cancel the CI run when a job fails
+        env:
+          GH_TOKEN: ${{ github.token }}
+          RUN: repos/${{ github.repository }}/actions/runs/${{ github.event.workflow_run.id }}
+        run: |
+          set -euo pipefail
+          errors=0
+          while :; do
+            if status="$(gh api "$RUN" --jq .status)" \
+              && failed="$(gh api "$RUN/jobs?filter=latest" --paginate \
+                --jq '[.jobs[] | select(.conclusion == "failure" or .conclusion == "timed_out" or .conclusion == "startup_failure")] | length' \
+                | awk '{ total += $1 } END { print total + 0 }')"; then
+              errors=0
+              if [ "$status" = "completed" ]; then
+                echo "The CI run finished."
+                exit 0
+              fi
+              if [ "$failed" -gt 0 ]; then
+                echo "$failed job(s) failed; cancelling the CI run so the queue can move on."
+                gh api -X POST "$RUN/cancel"
+                exit 0
+              fi
+            else
+              errors=$((errors + 1))
+              if [ "$errors" -ge 10 ]; then
+                echo "::error::The Actions API failed $errors times in a row; giving up. The CI run continues without fail fast."
+                exit 1
+              fi
+            fi
+            sleep 20
+          done

scripts/ci/choose_ci_suite.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+"""Decide whether a CI run gets the full macOS suite or only compile admission.
+
+The full suite (app-host shards, package tests, the lag build, the Release
+build) is what proves a change. Compile admission is the cheap check that a
+push still builds. With a merge queue the full suite runs on the commit that
+will land, so running it on every push as well spends most Mac time on commits
+that never merge.
+
+The answer is "full" unless everything says otherwise: only a pull_request
+event, under the compile-only policy, without the opt-in label, gets less.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from collections.abc import Iterable
+
+COMPILE_ONLY_POLICY = "compile-only"
+FULL_SUITE_LABEL = "full-ci"
+
+
+def wants_full_suite(event_name: str, pull_request_policy: str, labels: Iterable[str] | None) -> bool:
+    """`labels` is None when they could not be read, which keeps the full suite."""
+    if event_name != "pull_request":
+        return True
+    if pull_request_policy.strip() != COMPILE_ONLY_POLICY:
+        return True
+    if labels is None:
+        return True
+    return FULL_SUITE_LABEL in {label.strip() for label in labels}
+
+
+def main(argv: list[str]) -> int:
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    parser.add_argument("--event-name", required=True)
+    parser.add_argument("--pull-request-policy", default="")
+    parser.add_argument("--labels-file", help="one label per line; omit when labels could not be read")
+    parser.add_argument("--github-output")
+    args = parser.parse_args(argv)
+
+    labels = None
+    if args.labels_file:
+        with open(args.labels_file, encoding="utf-8") as handle:
+            labels = handle.read().splitlines()
+
+    full = wants_full_suite(args.event_name, args.pull_request_policy, labels)
+    line = f"full_suite={'true' if full else 'false'}"
+    print(line)
+    if args.github_output:
+        with open(args.github_output, "a", encoding="utf-8") as handle:
+            handle.write(line + "\n")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

tests/test_ci_change_areas.py

@@ -11,6 +11,7 @@
 import subprocess
 import sys
 import tempfile
+import textwrap
 from pathlib import Path
 
 
@@ -939,7 +940,7 @@ def test_required_tests_status_waits_for_app_host_matrix() -> None:
     assert "      - app-host-unit-tests" in block
     assert "if: ${{ always() }}" in block
     assert 'preflight["result"] != "success"' in block
-    assert 'macos == "true" and tests["result"] != "success"' in block
+    assert 'suite_required and tests["result"] != "success"' in block
     assert 'tests["result"] not in {"success", "skipped"}' in block
 
 
@@ -976,11 +977,79 @@ def test_macos_jobs_wait_for_linux_preflight() -> None:
         expected_if = (
             "if: ${{ !cancelled() && "
             + " && ".join(f"needs.{need}.result == 'success'" for need in expected_needs)
-            + " && needs.changes.outputs.macos == 'true' }}"
+            + " && needs.changes.outputs.macos == 'true'"
+            + ("" if job_name == "macos-compile-admission" else " && needs.changes.outputs.full_suite == 'true'")
+            + " }}"
         )
         assert expected_if in block, f"{job_name} must gate on direct needs explicitly"
 
 
+def run_tests_gate(needs: dict) -> subprocess.CompletedProcess:
+    script = workflow_job_step_script("tests", "Check app-host unit test routing")
+    body = script.split("python3 - <<'PY'\n", 1)[1].rsplit("\nPY", 1)[0]
+    return subprocess.run(
+        [sys.executable, "-c", textwrap.dedent(body)],
+        env={**os.environ, "TESTS_NEEDS": json.dumps(needs)},
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+
+
+def tests_gate_needs(full_suite: str | None, app_host: str, admission: str = "success") -> dict:
+    outputs = {"macos": "true"}
+    if full_suite is not None:
+        outputs["full_suite"] = full_suite
+    return {
+        "changes": {"result": "success", "outputs": outputs},
+        "linux-preflight": {"result": "success"},
+        "macos-compile-admission": {"result": admission},
+        "app-host-unit-tests": {"result": app_host},
+        "swift-package-tests": {"result": "skipped" if app_host == "skipped" else "success"},
+        "agent-session-web-resources": {"result": "skipped"},
+    }
+
+
+def test_compile_only_runs_pass_the_tests_gate_without_the_suite() -> None:
+    assert run_tests_gate(tests_gate_needs("false", app_host="skipped")).returncode == 0
+    # Compile admission still has to pass, and a suite job that ran and failed still blocks.
+    assert run_tests_gate(tests_gate_needs("false", app_host="skipped", admission="failure")).returncode == 1
+    assert run_tests_gate(tests_gate_needs("false", app_host="failure")).returncode == 1
+
+
+def test_full_suite_runs_still_require_the_suite() -> None:
+    assert run_tests_gate(tests_gate_needs("true", app_host="success")).returncode == 0
+    assert run_tests_gate(tests_gate_needs("true", app_host="skipped")).returncode == 1
+    # A missing output means the suite step did not report, which must not relax the gate.
+    assert run_tests_gate(tests_gate_needs(None, app_host="skipped")).returncode == 1
+
+
+def test_only_pull_requests_under_the_compile_only_policy_skip_the_suite() -> None:
+    sys.path.insert(0, str(ROOT / "scripts/ci"))
+    from choose_ci_suite import wants_full_suite
+
+    assert wants_full_suite("pull_request", "compile-only", []) is False
+    assert wants_full_suite("pull_request", "compile-only", ["bug", "full-ci"]) is True
+    assert wants_full_suite("pull_request", "compile-only", None) is True
+    assert wants_full_suite("pull_request", "", []) is True
+    assert wants_full_suite("pull_request", "full", []) is True
+    for event in ("merge_group", "workflow_dispatch", "push"):
+        assert wants_full_suite(event, "compile-only", []) is True
+
+
+def test_merge_groups_stop_at_the_first_failure() -> None:
+    shards = workflow_job_block("app-host-unit-tests")
+    assert "fail-fast: ${{ github.event_name == 'merge_group' }}" in shards
+    # The job that may cancel runs must come from the default branch, where a
+    # queued pull request cannot edit it, and must not run repository code.
+    watcher = (ROOT / ".github/workflows/merge-group-fail-fast.yml").read_text(encoding="utf-8")
+    assert "  workflow_run:\n    workflows: [CI]\n    types: [requested]" in watcher
+    assert "if: ${{ github.event.workflow_run.event == 'merge_group' }}" in watcher
+    assert "permissions: {}" in watcher and "actions: write" in watcher
+    assert "uses:" not in watcher
+    assert "actions: write" not in CI_WORKFLOW.read_text(encoding="utf-8")
+
+
 def test_macos_compile_admission_precedes_expensive_shards() -> None:
     workflow = CI_WORKFLOW.read_text(encoding="utf-8")
     admission = workflow_job_block("macos-compile-admission")