ci(cmux-next): DEV Release compile builds from one fixed tree and cache per host

.github/workflows/cmux-next.yml

@@ -137,6 +137,10 @@ on:
       - scripts/cmux-next/clear-stale-scheme-build-state.sh
       - scripts/cmux-next/stale-pcm-retry-needed.sh
       - scripts/cmux-next/release-derived-data-path.sh
+      - scripts/cmux-next/compile-dev-release.sh
+      - scripts/cmux-next/host_build_lock.py
+      - scripts/ci/canonical-build-root.sh
+      - scripts/ci/apfs_clone.py
       - scripts/cmux-next/compile-string-catalogs.sh
       - scripts/cmux-next/swift-test-with-hang-sampler.sh
       - scripts/cmux-next/ips-crash-summary.py
@@ -188,6 +192,7 @@ on:
       - scripts/ci/run-checks-concurrently.sh
       - tests/test_run_checks_concurrently.py
       - tests/test_cmux_next_release_build_retention.py
+      - tests/test_cmux_next_dev_release_host_root.py
       - tests/test_cmux_next_dev_release_import_out.py
       - tests/test_cmux_next_generated_catch_up.py
       - .github/workflows/cmux-next-generated-catch-up.yml
@@ -365,6 +370,10 @@ on:
       - scripts/cmux-next/clear-stale-scheme-build-state.sh
       - scripts/cmux-next/stale-pcm-retry-needed.sh
       - scripts/cmux-next/release-derived-data-path.sh
+      - scripts/cmux-next/compile-dev-release.sh
+      - scripts/cmux-next/host_build_lock.py
+      - scripts/ci/canonical-build-root.sh
+      - scripts/ci/apfs_clone.py
       - scripts/cmux-next/compile-string-catalogs.sh
       - scripts/cmux-next/swift-test-with-hang-sampler.sh
       - scripts/cmux-next/ips-crash-summary.py
@@ -418,6 +427,7 @@ on:
       - scripts/ci/run-checks-concurrently.sh
       - tests/test_run_checks_concurrently.py
       - tests/test_cmux_next_release_build_retention.py
+      - tests/test_cmux_next_dev_release_host_root.py
       - tests/test_cmux_next_dev_release_import_out.py
       - tests/test_cmux_next_generated_catch_up.py
       - .github/workflows/cmux-next-generated-catch-up.yml
@@ -1625,6 +1635,7 @@ jobs:
           python3 tests/test_cmux_next_pr_autofix_push_gate.py
           python3 tests/test_run_checks_concurrently.py
           python3 tests/test_cmux_next_release_build_retention.py
+          python3 tests/test_cmux_next_dev_release_host_root.py
           python3 tests/test_cmux_next_dev_release_import_out.py
           python3 tests/test_cmux_next_dev_daily_tree_wait.py
 
@@ -4398,7 +4409,7 @@ jobs:
           # out: the tip job in cmux-next-dev-daily.yml proves it before it
           # notarizes, so a proof failure never holds the DEV zip (#25050).
           CMUX_NEXT_BROWSER_DATA_IMPORT: compiled-out
-          # The DEV app's warm state is the runner's own content-addressed Xcode
+          # The DEV app's warm state is the host's own content-addressed Xcode
           # compilation cache (no kept build products, no remote store:
           # scripts/cmux-next/lib/compilation-cache.sh), built per file with -O
           # (singlefile, Xcode's "Incremental") so a one-file change replays
@@ -4421,15 +4432,13 @@ jobs:
             echo "archive=$archive" >> "$GITHUB_OUTPUT"
             exit 0
           fi
-          release_derived_data="$(scripts/cmux-next/release-derived-data-path.sh dev-daily-release Release)"
           # Only a mini keeps the cache between jobs; a fresh VM would pay to fill it.
           [[ "$CMUX_NEXT_RUNNER" == glaeda-* ]] || export CMUX_NEXT_COMPILATION_CACHE=0
-          scripts/cmux-next/check-cmux-scheme-compile.sh "$release_derived_data"
-          app="$(find "$release_derived_data/Build/Products/Release" -maxdepth 1 -type d -name 'cmux NEXT DEV.app' -print -quit)"
-          test -d "$app"
-          scripts/cmux-next/check-app-personalities.sh "$app"
+          # With the cache on, a mini builds from one fixed tree and DerivedData per host under a
+          # host lock, so the cache's path-holding keys match across its runner instances; a busy
+          # lock or a failed setup builds per runner at once (compile-dev-release.sh).
           archive="$RUNNER_TEMP/cmux-next-dev-daily-unsigned-${DEV_DAILY_SHA}.app.zip"
-          /usr/bin/ditto -c -k --keepParent "$app" "$archive"
+          scripts/cmux-next/compile-dev-release.sh "$archive"
           echo "archive=$archive" >> "$GITHUB_OUTPUT"
 
       - name: Upload DEV daily compile timing

scripts/cmux-next/compile-dev-release.sh

@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+# Compile the cmux NEXT DEV Release app and zip it to ARCHIVE (cmux-next.yml,
+# "Compile the cmux NEXT DEV Release app"; the step's env sets the build).
+#
+# Its warm state is Xcode's compilation cache (lib/compilation-cache.sh). The
+# cache's keys hold the absolute source and DerivedData paths, and a runner's
+# own paths (its workspace, its RUNNER_NAME DerivedData) made one cache per
+# runner instance. The DEV compile lands on 30+ of them (cmuxNN-glaeda-K), so
+# every compile was cold. On a mini with the cache on, this builds from ONE
+# tree per host instead, HOST_ROOT/src, into one DerivedData per host
+# (release-derived-data-path.sh --host), so the keys match whichever instance
+# builds:
+#
+# - The tree is this checkout, cloned whole by canonical-build-root.sh, as the
+#   owned catch-up and compile admission build at a fixed root: the old tree is
+#   removed first (no file from an earlier build survives), the copy is an APFS
+#   clone with an rsync fallback, and it carries .git and the products earlier
+#   steps put in the checkout (the cmux-tui tree, the web bundles).
+# - Only the CAS and SourcePackages carry over in the DerivedData
+#   (check-cmux-scheme-compile.sh starts a cache build from an empty tree
+#   around them; kept build products linked stale objects, #21761).
+# - Two instances on a host never build there at once: host_build_lock.py
+#   takes flock(2) on HOST_ROOT/lock through fd 9, without waiting. Every child
+#   (the clone, xcodebuild, ditto) inherits fd 9, so the kernel holds the lock
+#   until the last of them exits, however this script ends (SIGKILL included),
+#   and no lock outlives them.
+#
+# A busy lock, a lock error or a failed tree setup builds the old way at once,
+# per runner instance from the checkout (release-derived-data-path.sh, still
+# with the cache). Nothing waits for another job and no host is reserved.
+#
+# Usage: compile-dev-release.sh ARCHIVE
+set -euo pipefail
+
+archive="${1:?usage: compile-dev-release.sh ARCHIVE}"
+here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
+workspace="$(cd "$here/../.." && pwd -P)"
+host_root="${HOME:?}/Library/Caches/cmux-next-ci/dev-daily-release/host"
+lock="$host_root/lock"
+
+# Closing this script's fd 9 drops its share of the lock; a child still running
+# keeps it until it exits.
+release_lock() { exec 9>&-; }
+trap 'exit 143' TERM INT HUP
+
+build_root="$workspace"
+derived_data=""
+mode=instance
+reason=""
+
+# The host tree for this commit, or a failure (the caller falls back).
+host_setup() {
+  local src="$host_root/src" head
+  head="$(git -C "$workspace" rev-parse HEAD)" || return 1
+  env -u GITHUB_ENV CMUX_CI_CANONICAL_ROOT="$host_root" \
+    "$workspace/scripts/ci/canonical-build-root.sh" "$workspace" >&2 || return 1
+  [[ "$(git -C "$src" rev-parse HEAD 2>/dev/null)" == "$head" ]] || {
+    echo "compile-dev-release: $src is not at $head after the clone" >&2
+    return 1
+  }
+  derived_data="$("$here/release-derived-data-path.sh" dev-daily-release Release --host)" || return 1
+  build_root="$src"
+}
+
+if [[ "${CMUX_NEXT_RUNNER:-}" != glaeda-* ]]; then
+  reason="not a mini"
+elif [[ "${CMUX_NEXT_COMPILATION_CACHE:-0}" != 1 ]]; then
+  reason="compilation cache off"
+else
+  lock_status=0
+  { mkdir -p "$host_root" && exec 9>>"$lock" && python3 -I "$here/host_build_lock.py" acquire "$lock" --fd 9; } \
+    || lock_status=$?
+  (( lock_status == 0 )) || release_lock 2>/dev/null || true
+  case "$lock_status" in
+    0)
+      if host_setup; then
+        mode=host
+      else
+        reason="host tree setup failed"
+        derived_data=""
+        build_root="$workspace"
+        release_lock
+      fi
+      ;;
+    1) reason="another build holds the host root" ;;
+    *) reason="host lock error ($lock_status)" ;;
+  esac
+fi
+
+if [[ "$mode" == host ]]; then
+  # The acpmux the bundle phase copies, from this tree when it came from the checkout.
+  if [[ "${CMUX_NEXT_ACPMUX_BIN:-}" == "$workspace/"* ]]; then
+    export CMUX_NEXT_ACPMUX_BIN="$build_root/${CMUX_NEXT_ACPMUX_BIN#"$workspace/"}"
+  fi
+else
+  derived_data="$("$here/release-derived-data-path.sh" dev-daily-release Release)"
+fi
+echo "compile-dev-release: $mode build root${reason:+ ($reason)}: $build_root, DerivedData $derived_data"
+if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then
+  echo "DEV Release compile: $mode build root${reason:+ ($reason)}" >> "$GITHUB_STEP_SUMMARY" || true
+fi
+
+"$build_root/scripts/cmux-next/check-cmux-scheme-compile.sh" "$derived_data"
+app="$(find "$derived_data/Build/Products/Release" -maxdepth 1 -type d -name 'cmux NEXT DEV.app' -print -quit)"
+test -d "$app"
+"$build_root/scripts/cmux-next/check-app-personalities.sh" "$app"
+ditto -c -k --keepParent "$app" "$archive"

scripts/cmux-next/host_build_lock.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+"""One build per host in a fixed build root, taken without waiting.
+
+    exec 9>>LOCK; host_build_lock.py acquire LOCK --fd 9
+        exit 0: the open file description behind fd 9 now holds flock(2) on LOCK;
+        exit 1: another build holds it (its record is printed); 2: any other error.
+    host_build_lock.py probe LOCK      exit 0: free; 1: held (for tests and diagnostics)
+
+compile-dev-release.sh builds the DEV Release app from one fixed tree per host, so the
+compilation cache's keys (which hold the absolute paths) match across the host's runner
+instances, and two of them must never build there at once. The lock is the kernel's: flock(2)
+on the caller's descriptor, which the caller and every child it starts (canonical-build-root,
+xcodebuild, ditto) share. It is held while any of them is alive and released by the kernel when
+the last one exits, however it exits: a job killed with SIGKILL while its xcodebuild still runs
+keeps the root until that xcodebuild is gone too, and never leaves a lock behind. There is no
+pid file to go stale. The file's content only names the holder for a busy caller's log.
+
+Nothing waits: a held lock is exit 1 at once, and the caller builds on its own per-runner path.
+"""
+from __future__ import annotations
+
+import argparse
+import errno
+import fcntl
+import json
+import os
+import sys
+import time
+
+
+def holder(lock: str) -> str:
+    try:
+        with open(lock, encoding="utf-8", errors="replace") as f:
+            return f.read(512).strip() or "unknown"
+    except OSError:
+        return "unknown"
+
+
+def try_lock(fd: int) -> bool:
+    try:
+        fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        return True
+    except OSError as e:
+        if e.errno in (errno.EWOULDBLOCK, errno.EAGAIN):
+            return False
+        raise
+
+
+def acquire(lock: str, fd: int) -> int:
+    st, fst = os.stat(lock), os.fstat(fd)
+    if (st.st_dev, st.st_ino) != (fst.st_dev, fst.st_ino):
+        print(f"host_build_lock: fd {fd} is not {lock}", file=sys.stderr)
+        return 2
+    if not try_lock(fd):
+        print(f"host_build_lock: {lock} is held by {holder(lock)}", file=sys.stderr)
+        return 1
+    record = {"pid": os.getppid(), "runner": os.environ.get("RUNNER_NAME", ""),
+              "run": os.environ.get("GITHUB_RUN_ID", ""), "since": int(time.time())}
+    os.ftruncate(fd, 0)
+    os.write(fd, (json.dumps(record) + "\n").encode())
+    return 0
+
+
+def probe(lock: str) -> int:
+    try:
+        fd = os.open(lock, os.O_RDWR | os.O_CLOEXEC)
+    except FileNotFoundError:
+        return 0
+    try:
+        return 0 if try_lock(fd) else 1
+    finally:
+        os.close(fd)  # the only reference: a lock taken here goes with it
+
+
+def main(argv: list[str]) -> int:
+    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    ap.add_argument("command", choices=("acquire", "probe"))
+    ap.add_argument("lock")
+    ap.add_argument("--fd", type=int)
+    args = ap.parse_args(argv)
+    try:
+        if args.command == "probe":
+            return probe(args.lock)
+        if args.fd is None:
+            print("host_build_lock: acquire needs --fd", file=sys.stderr)
+            return 2
+        return acquire(args.lock, args.fd)
+    except OSError as e:
+        print(f"host_build_lock: {args.command} {args.lock}: {e}", file=sys.stderr)
+        return 2
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

scripts/cmux-next/release-derived-data-path.sh

@@ -1,14 +1,21 @@
 #!/usr/bin/env bash
 # Select bounded, worker-local Release build state after Select Xcode. One runner
 # executes one job at a time; separate lanes/workspaces never prune each other.
+# --host selects the lane's one per-host state instead, for a caller that holds
+# the host's build lock (compile-dev-release.sh, host_build_lock.py).
 set -euo pipefail
 
 lane="${1:?Release cache lane required (release-compile or dev-daily-release)}"
 configuration="${2:?Release configuration required}"
+scope="${3:-}"
 case "$lane:$configuration" in
   release-compile:Release|dev-daily-release:Release) ;;
   *) echo "unsupported Release cache lane/configuration: $lane:$configuration" >&2; exit 2 ;;
 esac
+case "$scope" in
+  ""|--host) ;;
+  *) echo "unsupported Release cache scope: $scope (want --host or nothing)" >&2; exit 2 ;;
+esac
 
 if [[ "${CMUX_NEXT_RUNNER:-}" != glaeda-* ]]; then
   printf '%s\n' "${RUNNER_TEMP:?}/cmux-next-$lane"
@@ -17,8 +24,13 @@ fi
 
 # RUNNER_NAME isolates concurrent root/side workers on the same host. Xcode's
 # build database also contains absolute source paths, so a workspace move is a
-# new cache. Never share a fallback worker name or an unknown workspace.
-worker_key="$(printf '%s\n' "${RUNNER_NAME:?}" "${GITHUB_WORKSPACE:?}" | shasum -a 256 | cut -c1-16)"
+# new cache. Never share a fallback worker name or an unknown workspace. The
+# host scope is shared by design: its caller's host lock keeps it to one build.
+if [[ "$scope" == --host ]]; then
+  worker_key=host
+else
+  worker_key="$(printf '%s\n' "${RUNNER_NAME:?}" "${GITHUB_WORKSPACE:?}" | shasum -a 256 | cut -c1-16)"
+fi
 xcode_version="$(xcodebuild -version)"
 sdk_build="$(xcrun --sdk macosx --show-sdk-build-version)"
 toolchain_key="$(printf '%s\n' "${DEVELOPER_DIR:?}" "$xcode_version" "$sdk_build" | shasum -a 256 | cut -c1-16)"

tests/test-execution.toml

@@ -1252,6 +1252,10 @@ path = "tests/test_run_checks_concurrently.py"
 lane = "linux-guard"
 serial = true
 
+[[test]]
+path = "tests/test_cmux_next_dev_release_host_root.py"
+lane = "linux-guard"
+
 [[test]]
 path = "tests/test_cmux_next_release_build_retention.py"
 lane = "linux-guard"