ci: advisory fleet test-compile check for the test targets a change reaches

.github/workflows/cmux-next.yml

@@ -1276,6 +1276,8 @@ jobs:
             python3 tests/test_job_input_fingerprint.py
             # A hand merge of a Swift PR waits for its Release compile (scripts/ci/merge_compile_guard.py).
             python3 tests/test_merge_compile_guard.py
+            # The advisory test-compile check: its plan, the lane phase and red only for a compile failure.
+            python3 tests/test_ci_compile_test_plan.py
             # The merge guards run from the feat-cmux-next tip, never a pull request's checkout.
             python3 tests/test_guard_hooks_from_base.py
             # ci-guards.yml's `ci` group runs for feat-cmux-next too (cmux-next-guards.yml).

.github/workflows/fleet-test-compile.yml

@@ -0,0 +1,131 @@
+name: fleet test-compile
+
+# Advisory: compile the Swift test targets a pull request can break, on a
+# fleet Mac, and fail this check when one does not compile.
+#
+# fleet/compile-check (fleet-compile-check.yml) compiles only the app
+# executable, which links no test target, and merge_compile_guard.py accepts
+# it in place of a queued Release compile. #23360 (a throwing call inside
+# #expect without try) and #23163 (a fake that missed a protocol requirement)
+# passed it and broke feat-cmux-next's test build. This check covers that gap
+# until the Linux pool compiles test targets (its SDK gains Xcode's Testing
+# search paths with the next pool image, cmuxterm-hq #2750).
+#
+# It gates nothing: fleet/compile-check, merge_compile_guard.py and the
+# Release compile neither wait for it nor read it. A red here is a test
+# target that does not compile at the PR's merge commit.
+#
+# The plan (scripts/ci/compile_test_plan.py, over the merge commit's diff from
+# its base) is the CmuxNext test targets that depend on a changed target in
+# the SwiftPM target graph, and the macOS package lane's packages whose
+# sources or path dependencies changed. An empty plan passes on a Linux runner
+# in seconds and queues no Mac; a plan runs `package-test-lane.sh
+# compile-tests BASE` as a light fleet step.
+
+on:
+  pull_request:
+    branches: [feat-cmux-next]
+    paths:
+      - "Packages/**"
+      - "vendor/**"
+      - "scripts/ci/compile_test_plan.py"
+      - "scripts/ci/package-test-lane.sh"
+      - ".github/workflows/fleet-test-compile.yml"
+
+permissions: {}
+
+concurrency:
+  group: fleet-test-compile-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+jobs:
+  plan:
+    name: fleet test-compile plan
+    # The fleet takes same-repository pull requests only (as the Swift canary's fleet job).
+    if: ${{ github.repository_owner == 'manaflow-ai' && github.event.pull_request.head.repo.full_name == github.repository }}
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+    timeout-minutes: 5
+    outputs:
+      needed: ${{ steps.plan.outputs.needed }}
+      base: ${{ steps.plan.outputs.base }}
+    steps:
+      - name: Checkout the merge commit and its base
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+          submodules: false
+          fetch-depth: 2
+
+      - name: Plan the test targets this change reaches
+        id: plan
+        run: |
+          set -euo pipefail
+          base="$(git rev-parse HEAD^1)"
+          git diff --no-renames --name-only "$base" HEAD > "$RUNNER_TEMP/changed.txt"
+          python3 scripts/ci/compile_test_plan.py --changed-files "$RUNNER_TEMP/changed.txt" > "$RUNNER_TEMP/plan.txt"
+          {
+            echo "### fleet test-compile plan"
+            echo
+            if [ -s "$RUNNER_TEMP/plan.txt" ]; then
+              echo 'Package, then its test targets (none listed: every one):'
+              echo
+              echo '```'
+              cat "$RUNNER_TEMP/plan.txt"
+              echo '```'
+            else
+              echo "The change reaches no test target: nothing to compile, no Mac queued."
+            fi
+          } >> "$GITHUB_STEP_SUMMARY"
+          cat "$RUNNER_TEMP/plan.txt"
+          echo "base=$base" >> "$GITHUB_OUTPUT"
+          if [ -s "$RUNNER_TEMP/plan.txt" ]; then echo "needed=true"; else echo "needed=false"; fi >> "$GITHUB_OUTPUT"
+
+  compile:
+    name: fleet/test-compile (advisory)
+    needs: plan
+    # An empty plan reports on a Linux runner at once; only a plan takes the gateway and a fleet Mac.
+    runs-on: ${{ needs.plan.outputs.needed == 'true' && vars.CI_COMPILE_CHECK_GATEWAY || 'ubuntu-24.04' }}
+    if: ${{ needs.plan.outputs.needed == 'false' || (needs.plan.outputs.needed == 'true' && vars.CI_COMPILE_CHECK_GATEWAY != '') }}
+    permissions: {}
+    # Longer than the client's queue (1200 s) plus run (1500 s) timeouts and its
+    # report: a job timeout concludes as a failure, which would read as red.
+    timeout-minutes: 55
+    steps:
+      # The gateway checks nothing out: the fleet fetches the merge commit itself.
+      # Exit 0 is a pass. The lane's "compile-tests: a test target did not
+      # compile" marker is a compile failure (this check goes red) whatever the
+      # exit: the lane prints it as soon as a target fails, and a later
+      # package's setup may end the run with another status. Anything else
+      # (queue or run timeout, setup, the client) is no verdict: a warning,
+      # and this check stays green.
+      - name: Compile the planned test targets on a fleet Mac
+        env:
+          NEEDED: ${{ needs.plan.outputs.needed }}
+          REF: ${{ github.sha }}
+          BASE: ${{ needs.plan.outputs.base }}
+        run: |
+          set -uo pipefail
+          if [[ "$NEEDED" != true ]]; then
+            echo "The change reaches no test target: nothing to compile."
+            exit 0
+          fi
+          client="${CMUX_CI_CLIENT:-$HOME/.local/bin/cmux-ci}"
+          if ! [[ -x "$client" ]]; then
+            echo "::warning title=fleet test-compile::$client is not installed on runner $RUNNER_NAME; no test-compile verdict."
+            exit 0
+          fi
+          log="$RUNNER_TEMP/test-compile.log"
+          status=0
+          "$client" run --class light --script scripts/ci/package-test-lane.sh --ref "$REF" \
+            --arg=compile-tests --arg="$BASE" \
+            --timeout 1500 --queue-timeout 1200 2>&1 | tee "$log" || status=$?
+          if [[ "$status" -eq 0 ]]; then
+            echo "Every planned test target compiled."
+          elif grep -q 'compile-tests: a test target did not compile' "$log"; then
+            echo "::error title=fleet test-compile::A test target this change reaches does not compile (advisory; see the errors above)."
+            exit 1
+          else
+            echo "::warning title=fleet test-compile::No test-compile verdict (exit $status)."
+          fi

scripts/ci/compile_test_plan.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+"""Pick the Swift test targets a change can break, for the fleet test compile.
+
+fleet/compile-check (.github/workflows/fleet-compile-check.yml) compiles the
+app executable, which links no test target. A change can still break a test
+target's compile (#23360: a throwing call inside `#expect` without `try`;
+#23163: a fake that missed a protocol requirement). This picks what the
+advisory fleet-test-compile.yml check builds (`package-test-lane.sh
+compile-tests` on a fleet Mac), from the files the change touches; an empty
+plan queues no Mac:
+
+  CmuxNext    the test targets that depend on a changed target, from the
+              SwiftPM target graph (Packages/macOS/CmuxNext/ci-target-graph.json,
+              the same routing scripts/ci/cmux_next_route.py gives the swift
+              test tier). A manifest or graph change is every test target.
+  packages    each package of the macOS package lane (PACKAGES in
+              scripts/ci/package-test-lane.sh) whose directory or local path
+              dependencies changed: all its test targets.
+
+Only files inside a package or one of its path dependencies count; a Markdown
+file never does. Other local packages (iOS, Shared libraries without a macOS
+lane) build in their own jobs, not with a macOS `swift build`.
+
+Prints one line per package: its directory, then the test targets to build,
+or the directory alone for every test target of the package.
+
+Usage: compile_test_plan.py --changed-files FILE [--root DIR]
+"""
+
+from __future__ import annotations
+
+import argparse
+import re
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from cmux_next_route import PACKAGE, closure, load_graph, route  # noqa: E402,F401
+from select_package_tests import input_prefixes, package_dirs, under  # noqa: E402
+
+ROOT = Path(__file__).resolve().parents[2]
+CMUXNEXT = PACKAGE.rstrip("/")
+LANE = "scripts/ci/package-test-lane.sh"
+LANE_PACKAGES = re.compile(r"^\s*PACKAGES=\(\n(.*?)\n\s*\)", re.S | re.M)
+
+
+def lane_packages(root: Path) -> list[str]:
+    """The packages the macOS package lane tests (its PACKAGES list)."""
+    found = LANE_PACKAGES.search((root / LANE).read_text(encoding="utf-8"))
+    if not found:
+        raise SystemExit(f"no PACKAGES=( ... ) list in {LANE}")
+    return [line.strip() for line in found.group(1).splitlines() if line.strip() and not line.strip().startswith("#")]
+
+
+def cmuxnext_targets(root: Path, changed: list[str]) -> list[str] | None:
+    """The CmuxNext test targets `changed` can break; None for every one."""
+    result = route(root, "pull_request", changed, set())
+    graph = load_graph(root)
+    every = sorted(name for name, target in graph["targets"].items() if target["kind"] == "test")
+    targets = every if result.full else sorted(result.tests)
+    return None if targets == every else targets
+
+
+def plan(root: Path, changed: list[str]) -> list[tuple[str, list[str] | None]]:
+    """[(package directory, test targets or None for all)] for the changed files."""
+    changed = [path for path in changed if path and not path.endswith(".md")]
+    dirs = package_dirs(root)
+    result: list[tuple[str, list[str] | None]] = []
+    for name in [Path(CMUXNEXT).name, *lane_packages(root)]:
+        if name not in dirs or any(directory == dirs[name] for directory, _ in result):
+            continue
+        prefixes = input_prefixes(root, name, dirs)
+        hits = [path for path in changed if any(under(path, prefix) for prefix in prefixes)]
+        if not hits:
+            continue
+        if dirs[name] == CMUXNEXT:
+            targets = cmuxnext_targets(root, hits)
+            if targets == []:
+                continue
+            result.append((dirs[name], targets))
+        else:
+            result.append((dirs[name], None))
+    return result
+
+
+def main(argv: list[str]) -> int:
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    parser.add_argument("--root", type=Path, default=ROOT)
+    parser.add_argument("--changed-files", type=Path, required=True, help="one path per line")
+    args = parser.parse_args(argv)
+    changed = args.changed_files.read_text(encoding="utf-8").splitlines()
+    for directory, targets in plan(args.root, changed):
+        print(" ".join([directory, *(targets or [])]))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))