ci: call a missing pinned Xcode a machine failure

.github/workflows/ci-guards.yml

@@ -103,6 +103,10 @@ jobs:
         if: ${{ matrix.group == 'ci' }}
         run: python3 tests/test_ci_macos_xcode_selection.py
 
+      - name: Validate Xcode pin audit
+        if: ${{ matrix.group == 'ci' }}
+        run: python3 tests/test_ci_xcode_pin_audit.py
+
       - name: Validate v0.64.25 rpath repair workflow contract
         if: ${{ matrix.group == 'ci' }}
         run: bash tests/test_repair_v06425_workflow.sh

scripts/ci/classify_failures.py

@@ -9,7 +9,8 @@
 
   machine   the runner or its products failed: a runner hook refused the job,
             the compiled products did not restore, the CLI loaded package
-            frameworks from another build, the runner went away. The job's
+            frameworks from another build, the runner went away, the runner
+            lacks the Xcode the job pins. The job's
             test failures, if any, are not evidence about the code.
   code      a test recorded an issue, a compile or guard failed, and no
             machine signature matched.
@@ -106,6 +107,13 @@ def sig(name: str, verdict: str, pattern: str, why: str) -> Signature:
         r"|The hosted runner encountered an error",
         "the runner went away mid-job"),
     sig("disk-full", MACHINE, r"No space left on device", "the runner's disk is full"),
+    # scripts/select-ci-xcode.sh on a Mac without the Xcode the job pins. The
+    # marker is today's text; the anchored messages are what a pull request
+    # branched before it prints (the classifier runs main's copy on any head).
+    sig("xcode-pin-missing", MACHINE,
+        r"\[cmux-ci machine: xcode-pin-missing\]|^Pinned Xcode developer dir (?:does not exist|has no usable macOS SDK): "
+        r"|^This macOS \d+ runner has no Xcode \S+, the version scripts/ci/xcode-pins\.txt pins",
+        "the runner does not have the Xcode this job pins (install it: scripts/ci/xcode_pin_audit.py)"),
     sig("swift-testing-issue", CODE, r"^✘ (?:Test|Suite) .+ (?:recorded an issue|failed after)", "a test failed"),
     sig("xctest-failure", CODE, r"\.swift:\d+: error: -\[", "a test failed"),
     sig("ratchet-new-failure", CODE, r"^RATCHET_NEW_FAILURE ", "a test failed that passes on main"),

scripts/ci/machine_failure.py

@@ -22,6 +22,9 @@
     ("screen frame capture failed to start", "screen capture could not start"),
     # Homebrew's refusal when the runner user does not own its prefix.
     ("The following directories are not writable by your user", "the Mac's Homebrew prefix is not writable by the runner user"),
+    # scripts/select-ci-xcode.sh, before any build, on a Mac without the pinned Xcode.
+    ("[cmux-ci machine: xcode-pin-missing]", "the Mac does not have the Xcode the job pins"),
+    ("Pinned Xcode developer dir does not exist", "the Mac does not have the Xcode the job pins"),
 )
 
 # XCTest and Swift Testing lines for a test that began. One of these means the

scripts/ci/owned_pool_rescue.py

@@ -40,7 +40,9 @@
 (compile admission's `always()` metrics steps still succeed after a refusal).
 A failed job that ran no step at all counts too, whatever its length: GitHub
 fails a job whose runner went away only after 10 minutes ("The self-hosted
-runner lost communication with the server"). The watcher lets the rest of the
+runner lost communication with the server"). A failed Select Xcode or Select
+helper Xcode step also counts, without a time limit, because a missing pin is
+a machine fault even when a helper build succeeded first. The watcher lets the rest of the
 run finish, since GitHub re-runs no job of a run in progress and cancelling
 it would kill every healthy sibling, then confirms the head has not moved and
 re-runs its failed jobs. Only a run still going at the watch's end, or main's
@@ -342,6 +344,7 @@ def person_rerun(run: Mapping[str, Any]) -> bool:
             and str((run.get("triggering_actor") or {}).get("login") or "") != RESCUE_ACTOR)
 # The runner's own steps, which run before glaeda's hook decides.
 SETUP_STEPS = frozenset({"Set up job", "Set up runner"})
+XCODE_SELECTION_STEPS = frozenset({"Select Xcode", "Select helper Xcode"})
 # glaeda's hook no longer refuses a job for the mini's capacity: it waits,
 # with no limit, inside the runner's setup ("Set up runner") until the units
 # and tokens it needs are free (glaeda CAPACITY_WAIT, 2026-09-28). A job still
@@ -446,7 +449,7 @@ def job_budget(job: Mapping[str, Any], budget_seconds: int, *, deadline: dt.date
 
 
 def refused(job: Mapping[str, Any]) -> bool:
-    """A job the owned runner refused at job start (see the module docstring), or whose runner was lost."""
+    """An owned runner's setup or Xcode pin failure, or a lost runner."""
     if not job_pool(job) or job.get("status") != "completed" or job.get("conclusion") != "failure":
         return False
     steps = [step for step in job.get("steps") or [] if isinstance(step, Mapping)]
@@ -456,6 +459,11 @@ def refused(job: Mapping[str, Any]) -> bool:
         # server"), which it reports only after 10 minutes, so no length applies
         # (run 36420353579).
         return True
+    # A helper build can precede Xcode selection, so neither elapsed time nor
+    # earlier successful steps make a missing pin a source failure.
+    if any(step.get("name") in XCODE_SELECTION_STEPS and step.get("conclusion") == "failure"
+           for step in steps):
+        return True
     started, completed = parse_time(job.get("started_at")), parse_time(job.get("completed_at"))
     if started is None or completed is None or (completed - started).total_seconds() > REFUSAL_SECONDS:
         return False
@@ -530,7 +538,7 @@ def assess(jobs: Sequence[Mapping[str, Any]], *, now: dt.datetime, budget_second
     turned_away = [job for job in jobs if refused(job)]
     if turned_away:
         names = ", ".join(sorted(str(job.get("name") or job.get("id")) for job in turned_away))
-        return Look("refused", f"{names} refused by {job_pool(turned_away[0])} at job start")
+        return Look("refused", f"{names} refused by {job_pool(turned_away[0])} at job start or Xcode selection")
     if waiting or settling:
         return Look("watch", f"{len(waiting)} job(s) waiting for a persistent runner, {len(settling)} in its setup",
                     waiting=True)

scripts/ci/xcode_pin_audit.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python3
+"""Check that this Mac has every CI Xcode pin and every configured app path.
+
+For a remote Mac, pass --pin and --app values and stream this file over ssh:
+  ssh HOST python3 - --pin 26.3 --pin 26.6 --app NAME=/Applications/Xcode_26.6.app < xcode_pin_audit.py
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+from pathlib import Path
+import re
+import subprocess
+import sys
+
+
+APP_NAME = re.compile(r"^Xcode_(\d+(?:\.\d+)*)\.app$")
+VAR_NAME = re.compile(r"^CMUX_CI_XCODE_APP_[A-Z0-9_]+$")
+DEFAULT_PINS = Path(__file__).with_name("xcode-pins.txt")
+
+
+def pool_pins(path: Path) -> set[str]:
+    pins: set[str] = set()
+    for line in path.read_text().splitlines():
+        fields = line.split("#", 1)[0].split()
+        if fields:
+            if len(fields) != 2 or not fields[0].isdigit() or not re.fullmatch(r"\d+(?:\.\d+)*", fields[1]):
+                raise ValueError(f"invalid pool pin: {line}")
+            pins.add(fields[1])
+    return pins
+
+
+def github_apps(repo: str) -> dict[str, str]:
+    result = subprocess.run(["gh", "variable", "list", "-R", repo, "--json", "name,value"],
+                            check=True, text=True, capture_output=True)
+    return {item["name"]: item["value"] for item in json.loads(result.stdout)
+            if VAR_NAME.fullmatch(item["name"]) and item["value"]}
+
+
+def version(app: Path) -> str | None:
+    developer = app / "Contents" / "Developer"
+    if not developer.is_dir():
+        return None
+    try:
+        result = subprocess.run(["xcodebuild", "-version"], env={**os.environ, "DEVELOPER_DIR": str(developer)},
+                                text=True, capture_output=True, check=True, timeout=20)
+    except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired):
+        return None
+    first = result.stdout.splitlines()[0] if result.stdout else ""
+    return first.removeprefix("Xcode ") if first.startswith("Xcode ") else None
+
+
+def audit(pins: set[str], apps: dict[str, str], applications_dir: Path) -> list[str]:
+    errors: list[str] = []
+    installed = {app: version(app) for app in sorted(applications_dir.glob("Xcode*.app"))}
+    for pin in sorted(pins):
+        if pin not in installed.values():
+            errors.append(f"pool pin Xcode {pin} is missing")
+    for name, path in sorted(apps.items()):
+        app = Path(path)
+        expected = APP_NAME.fullmatch(app.name)
+        if not expected:
+            errors.append(f"{name}={path} must name Xcode_<version>.app")
+            continue
+        found = installed.get(app)
+        if found is None:
+            found = version(app)
+        if found != expected.group(1):
+            errors.append(f"{name}={path} requires Xcode {expected.group(1)}; found {found or 'none'}")
+    return errors
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--pins-file", type=Path, default=DEFAULT_PINS)
+    parser.add_argument("--pin", action="append", default=[], help="pool Xcode version; repeat for remote use")
+    parser.add_argument("--app", action="append", default=[], metavar="NAME=PATH")
+    parser.add_argument("--github-vars", metavar="OWNER/REPO", help="read CMUX_CI_XCODE_APP_* repo variables")
+    parser.add_argument("--applications-dir", type=Path, default=Path("/Applications"))
+    args = parser.parse_args(argv)
+    try:
+        pins = set(args.pin) if args.pin else pool_pins(args.pins_file)
+        apps = {name: value for name, value in os.environ.items() if VAR_NAME.fullmatch(name) and value}
+        if args.github_vars:
+            apps.update(github_apps(args.github_vars))
+        for pair in args.app:
+            name, sep, path = pair.partition("=")
+            if not sep or not VAR_NAME.fullmatch(name) or not path:
+                parser.error(f"invalid --app: {pair}")
+            apps[name] = path
+        if not pins:
+            parser.error("no Xcode pool pins supplied")
+        errors = audit(pins, apps, args.applications_dir)
+    except (OSError, ValueError, subprocess.CalledProcessError) as error:
+        print(f"xcode pin audit: {error}", file=sys.stderr)
+        return 2
+    host = os.uname().nodename
+    if errors:
+        for error in errors:
+            print(f"{host}: {error}")
+        return 1
+    print(f"{host}: Xcode pins present: {', '.join(sorted(pins))}; app paths checked: {len(apps)}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/select-ci-xcode.sh

@@ -17,6 +17,13 @@
 #      pool pin: a newer image Xcode costs a fork cache misses, never a failed
 #      job (docs/ci-runners.md, fork contract). The floor still applies.
 #
+# A pinned Xcode this runner does not have is the machine's fault, not the
+# code's: the error carries MACHINE_MARKER, which scripts/ci/classify_failures.py
+# and scripts/ci/machine_failure.py read as a machine failure, and the step is
+# named "Select Xcode", which scripts/ci/owned_pool_rescue.py treats as a
+# refusal on an owned Mac (re-run as github-actions[bot], so off the fleet).
+# scripts/ci/xcode_pin_audit.py finds such a Mac before a job does.
+#
 # Whatever is selected must be at least the Xcode major in .xcode-version, or
 # the script stops with one ::error:: naming the found and required versions
 # (CMUX_CI_XCODE_ALLOW_BELOW_FLOOR=1 lifts this too). Without the floor, a job
@@ -29,6 +36,8 @@ SELECT_XCODE_REPO_ROOT="$(dirname "$SELECT_XCODE_SCRIPT_DIR")"
 XCODE_VERSION_FILE="${CMUX_XCODE_VERSION_FILE:-$SELECT_XCODE_REPO_ROOT/.xcode-version}"
 XCODE_PINS_FILE="${CMUX_CI_XCODE_PINS_FILE:-$SELECT_XCODE_REPO_ROOT/scripts/ci/xcode-pins.txt}"
 ALLOW_BELOW_FLOOR="${CMUX_CI_XCODE_ALLOW_BELOW_FLOOR:-0}"
+# Keep in step with classify_failures.py's "xcode-pin-missing" signature.
+MACHINE_MARKER="[cmux-ci machine: xcode-pin-missing]"
 
 APPLICATIONS_DIR="${CMUX_XCODE_APPLICATIONS_DIR:-/Applications}"
 REQUIRED_SDK_MAJOR="${CMUX_CI_REQUIRED_MACOS_SDK_MAJOR:-}"
@@ -136,6 +145,10 @@ installed_xcodes() {
   printf '%s' "${listed# }"
 }
 
+this_runner() {
+  printf '%s' "${RUNNER_NAME:-$(hostname -s 2>/dev/null || echo unknown)}"
+}
+
 validate_sdk_constraints() {
   local selected_dir="$1" sdk_version="$2" actual_major
   [ -n "$REQUIRED_SDK_MAJOR$MAX_SDK_MAJOR" ] || return 0
@@ -199,12 +212,12 @@ fi
 
 if [ -n "$PINNED_DEVELOPER_DIR" ]; then
   if [ ! -d "$PINNED_DEVELOPER_DIR" ]; then
-    echo "Pinned Xcode developer dir does not exist: $PINNED_DEVELOPER_DIR" >&2
+    echo "::error::Pinned Xcode developer dir does not exist: $PINNED_DEVELOPER_DIR on runner $(this_runner). $MACHINE_MARKER Installed: $(installed_xcodes)" >&2
     exit 1
   fi
   PINNED_SDK_VER="$(DEVELOPER_DIR="$PINNED_DEVELOPER_DIR" xcrun --sdk macosx --show-sdk-version 2>/dev/null || true)"
   if [ -z "$PINNED_SDK_VER" ]; then
-    echo "Pinned Xcode developer dir has no usable macOS SDK: $PINNED_DEVELOPER_DIR" >&2
+    echo "::error::Pinned Xcode developer dir has no usable macOS SDK: $PINNED_DEVELOPER_DIR on runner $(this_runner). $MACHINE_MARKER" >&2
     exit 1
   fi
   select_developer_dir "$PINNED_DEVELOPER_DIR" "$PINNED_SDK_VER" "Selected pinned Xcode"
@@ -243,7 +256,7 @@ if [ "$ALLOW_BELOW_FLOOR" != "1" ]; then
     if runs_in_fork_repository; then
       echo "::warning::This macOS $POOL_MAJOR runner has no Xcode $POOL_VERSION, the version scripts/ci/xcode-pins.txt pins for its pool; this fork uses the newest stable Xcode on its image instead, so it cannot reuse main's compilation caches. Installed: $(installed_xcodes)"
     else
-      echo "::error::This macOS $POOL_MAJOR runner has no Xcode $POOL_VERSION, the version scripts/ci/xcode-pins.txt pins for its pool. Installed: $(installed_xcodes)" >&2
+      echo "::error::This macOS $POOL_MAJOR runner has no Xcode $POOL_VERSION, the version scripts/ci/xcode-pins.txt pins for its pool. Installed: $(installed_xcodes). Runner $(this_runner). $MACHINE_MARKER" >&2
       exit 1
     fi
   fi
@@ -252,7 +265,7 @@ fi
 if [ "$ALLOW_BELOW_FLOOR" != "1" ] && [ -n "$POOL_DEVELOPER_DIR" ]; then
   POOL_SDK_VER="$(DEVELOPER_DIR="$POOL_DEVELOPER_DIR" xcrun --sdk macosx --show-sdk-version 2>/dev/null || true)"
   if [ -z "$POOL_SDK_VER" ]; then
-    echo "::error::Pool Xcode developer dir has no usable macOS SDK: $POOL_DEVELOPER_DIR" >&2
+    echo "::error::Pool Xcode developer dir has no usable macOS SDK: $POOL_DEVELOPER_DIR on runner $(this_runner). $MACHINE_MARKER" >&2
     exit 1
   fi
   select_developer_dir "$POOL_DEVELOPER_DIR" "$POOL_SDK_VER" "Selected Xcode $POOL_VERSION pinned for macOS $POOL_MAJOR runners"

tests/test-execution.toml

@@ -574,6 +574,10 @@ lane = "linux-guard"
 path = "tests/test_ci_macos_xcode_selection.py"
 lane = "linux-guard"
 
+[[test]]
+path = "tests/test_ci_xcode_pin_audit.py"
+lane = "linux-guard"
+
 [[test]]
 path = "tests/test_ci_required_checks_drift.py"
 lane = "linux-guard"

tests/test_ci_classify_failures.py

@@ -80,6 +80,25 @@
     "2026-09-27T10:40:00.0000000Z macOS admission gate declined: a fast Linux job failed. The product compiled "
     "and was uploaded; re-run failed jobs to collect macOS results anyway.\n"
 )
+# Run 36553044270's swift-package-tests on cmux14 (glaeda-std-xcode-26.6), as
+# the job printed it then (a head branched before the marker), and as
+# scripts/select-ci-xcode.sh prints it now.
+XCODE_PIN_MISSING_LEGACY = textwrap.dedent("""\
+    2026-09-29T12:23:37.0000000Z ##[group]Run set -euo pipefail
+    2026-09-29T12:23:37.0000000Z \x1b[36;1mset -euo pipefail\x1b[0m
+    2026-09-29T12:23:37.0000000Z ##[endgroup]
+    2026-09-29T12:23:37.1000000Z Pinned Xcode developer dir does not exist: /Applications/Xcode_26.3.app/Contents/Developer
+    2026-09-29T12:23:37.1000000Z ##[error]Process completed with exit code 1.
+    """)
+XCODE_PIN_MISSING = (
+    "2026-09-29T12:23:37.1000000Z ##[error]Pinned Xcode developer dir does not exist: "
+    "/Applications/Xcode_26.3.app/Contents/Developer on runner cmux14-glaeda-1. "
+    "[cmux-ci machine: xcode-pin-missing] Installed: Xcode.app=26.3 Xcode_26.6.app=26.6\n"
+)
+POOL_XCODE_MISSING = (
+    "2026-09-29T12:23:37.1000000Z ##[error]This macOS 26 runner has no Xcode 26.6, the version "
+    "scripts/ci/xcode-pins.txt pins for its pool. Installed: Xcode_26.3.app=26.3\n"
+)
 NOISE = textwrap.dedent("""\
     2026-09-27T09:24:56.6949430Z ##[group]Run if [ "$REQUESTED_RUNNER" = ubuntu-24.04 ]; then
     2026-09-27T09:24:56.6949430Z \x1b[36;1m      echo "::error::$REQUESTED_RUNNER resolved outside GitHub-hosted capacity: $RUNNER_CONTEXT_NAME"\x1b[0m
@@ -119,6 +138,21 @@ def test_a_test_failure_on_a_healthy_runner_is_the_code(self) -> None:
         self.assertEqual(self.verdict(COMPILE_FAILED), (cf.CODE, "compile-error"))
         self.assertEqual(self.verdict(STATIC_CHECK_FAILED), (cf.CODE, "static-check-failed"))
 
+    def test_a_missing_pinned_xcode_is_the_machine(self) -> None:
+        for log in (XCODE_PIN_MISSING, XCODE_PIN_MISSING_LEGACY, POOL_XCODE_MISSING):
+            with self.subTest(log=log[:80]):
+                self.assertEqual(self.verdict(log), (cf.MACHINE, "xcode-pin-missing"))
+        # Only every failed job being machine re-runs the run; this one does.
+        jobs = cf.classify_jobs([job(7, "macos / swift-package-tests", step="Select Xcode")],
+                                {7: (XCODE_PIN_MISSING_LEGACY, [])})
+        self.assertTrue(cf.all_machine(jobs))
+
+    def test_a_forks_missing_pool_xcode_warning_is_not_the_machine(self) -> None:
+        # A fork's own CI warns and falls back; that warning is not where it failed.
+        warned = POOL_XCODE_MISSING.replace("##[error]", "##[warning]") + COMPILE_FAILED + \
+            "2026-09-29T12:30:00.0000000Z ##[error]Process completed with exit code 65.\n"
+        self.assertEqual(self.verdict(warned), (cf.CODE, "compile-error"))
+
     def test_a_signature_in_an_echoed_script_or_cleanup_noise_does_not_count(self) -> None:
         self.assertEqual(self.verdict(NOISE), (cf.UNKNOWN, None))
         restore_ok = RESTORE_FAILED.replace('"outcome": "failure"', '"outcome": "success"')

tests/test_ci_machine_failure.py

@@ -28,6 +28,15 @@ def test_an_unwritable_homebrew_prefix_is_a_machine_failure(self):
         log = "##[error]The following directories are not writable by your user:\n  /opt/homebrew\n"
         self.assertEqual(machine_failure.reason(log), "the Mac's Homebrew prefix is not writable by the runner user")
 
+    def test_a_missing_pinned_xcode_is_a_machine_failure(self):
+        for line in (
+            "##[error]Pinned Xcode developer dir does not exist: /Applications/Xcode_26.3.app/Contents/Developer "
+            "on runner cmux14-glaeda-1. [cmux-ci machine: xcode-pin-missing] Installed: Xcode.app=26.3",
+            "Pinned Xcode developer dir does not exist: /Applications/Xcode_26.3.app/Contents/Developer",
+        ):
+            with self.subTest(line=line[:40]):
+                self.assertEqual(machine_failure.reason(PREFIX + line), "the Mac does not have the Xcode the job pins")
+
     def test_a_started_test_makes_it_the_codes_failure(self):
         for started in (
             "Test Case '-[cmuxUITests.SidebarTests testA]' started.",