ci(cmux-next media): let tours run side by side on one Mac

.github/workflows/ci-guards.yml

@@ -1139,6 +1139,10 @@ jobs:
         if: ${{ matrix.group == 'release-notary' }}
         run: python3 tests/test_ci_prune_media_derived_data.py
 
+      - name: Validate a second cmux-next tour of a tag waits for the tour holding it
+        if: ${{ matrix.group == 'release-notary' }}
+        run: python3 tests/test_media_tag_lock.py
+
       - name: Validate the cmux-next tour drops stale prebuilt modules when a binary pin changes
         if: ${{ matrix.group == 'release-notary' }}
         run: python3 tests/test_ci_media_module_cache_stamp.py

.github/workflows/cmux-next-media.yml

@@ -249,6 +249,35 @@ jobs:
         with:
           node-version: "24.11.1"
 
+      # Tours run side by side on a Mac's session runners, each capturing only
+      # its own app's window. One tag's DerivedData, sockets and daemon session
+      # take one tour at a time: a cancelled run still stopping, a dispatch, or
+      # a merge tour beside a dispatch could otherwise build into the same
+      # DerivedData at once. Other tags never wait.
+      # A pull request branched before the lock existed checks out a head
+      # without the lock script, and with a prune that ignores held tags; this
+      # step comes from the merge ref, so it takes both from that commit and
+      # hands the prune step their dir (MEDIA_CI_SCRIPTS).
+      # A tour waits an hour for its tag: longer than a warm merge tour holds
+      # feat-cmux-next (54 min seen), and short enough that the waiter keeps
+      # 45+ min of its own job timeout for a cold build and the tour. A
+      # longer wait would end in a job-timeout cancel, not the "held" error.
+      - name: Hold this tour's tag
+        if: steps.scope.outputs.is_ui == 'true'
+        run: |
+          set -euo pipefail
+          ci=scripts/ci
+          if [ ! -f "$ci/media_tag_lock.py" ] || ! grep -q -- '--lock-dir' "$ci/prune_media_derived_data.py"; then
+            git fetch --no-tags origin "$GITHUB_SHA"
+            ci="$RUNNER_TEMP/media-ci"
+            mkdir -p "$ci"
+            for script in media_tag_lock.py prune_media_derived_data.py; do
+              git show "$GITHUB_SHA:scripts/ci/$script" > "$ci/$script"
+            done
+          fi
+          echo "MEDIA_CI_SCRIPTS=$ci" >> "$GITHUB_ENV"
+          python3 "$ci/media_tag_lock.py" hold --tag "$TOUR_TAG" --wait 3600
+
       # Each tag's DerivedData is about 9 GiB and nothing else removes it: a
       # mini holding 8 failed reload.sh's 40 GiB floor (run 38614931315).
       # Keep this run's pair and the most recently used other pair; a dir
@@ -257,7 +286,7 @@ jobs:
         if: steps.scope.outputs.is_ui == 'true'
         continue-on-error: true
         run: |
-          python3 scripts/ci/prune_media_derived_data.py \
+          python3 "${MEDIA_CI_SCRIPTS:-scripts/ci}/prune_media_derived_data.py" \
             --keep-tag "${TOUR_TAG}-base" --keep-tag "${TOUR_TAG}-head" --keep-others 2
 
       - name: Download pre-built GhosttyKit.xcframework for both revisions

scripts/ci/media_tag_lock.py

@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+"""Hold a cmux-next media tour's tag for the rest of its job.
+
+    media_tag_lock.py hold --tag ci-media-pr-123 [--wait SECONDS]
+
+A tour builds into per-tag DerivedData (cmux-<tag>-base, cmux-<tag>-head)
+kept warm across runs of one pull request, and launches apps whose sockets
+and daemon session come from the tag. Several tours run on one Mac at once,
+each on its own runner; the workflow's concurrency group keeps two runs of a
+pull request apart, but not a cancelled run that is still stopping, a
+dispatch, or a merge tour beside a dispatch on the shared feat-cmux-next tag.
+Two tours building into one DerivedData at once break its module cache
+("ghostty.h has been modified since the module file was built").
+
+`hold` waits for the tag's lock (one file per tag under --lock-dir), then
+leaves a background process holding it and returns. The holder lets go when
+the job's Runner.Worker exits (the runner also ends a job's leftover
+processes) or after --max-hold seconds, so a lost job never keeps a tag.
+"""
+
+from __future__ import annotations
+
+import argparse
+import fcntl
+import os
+from pathlib import Path
+import re
+import subprocess
+import sys
+import time
+
+DEFAULT_LOCK_DIR = Path.home() / "Library/Caches/cmux-ci-media/locks"
+WORKER = "Runner.Worker"
+
+
+def worker_pid(start: int) -> int | None:
+    """The nearest Runner.Worker among start's ancestors, from ps."""
+    pid = start
+    for _ in range(64):
+        if pid <= 1:
+            return None
+        done = subprocess.run(["ps", "-o", "ppid=,comm=", "-p", str(pid)], capture_output=True, text=True)
+        fields = done.stdout.strip().split(None, 1)
+        if len(fields) != 2:
+            return None
+        if os.path.basename(fields[1]) == WORKER:
+            return pid
+        pid = int(fields[0])
+    return None
+
+
+def alive(pid: int) -> bool:
+    try:
+        os.kill(pid, 0)
+    except ProcessLookupError:
+        return False
+    except PermissionError:
+        return True
+    return True
+
+
+def acquire(path: Path, wait: float) -> int | None:
+    """An fd holding path's exclusive lock, or None after wait seconds."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    fd = os.open(path, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o644)
+    deadline = time.monotonic() + wait
+    said = False
+    while True:
+        try:
+            fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            return fd
+        except BlockingIOError:
+            pass
+        if not said:
+            holder = path.read_text(errors="replace").strip() or "unknown"
+            print(f"waiting for {path.name}: held by {holder}", flush=True)
+            said = True
+        if time.monotonic() >= deadline:
+            os.close(fd)
+            return None
+        time.sleep(2)
+
+
+def hold(fd: int, path: Path, worker: int | None, max_hold: float) -> None:
+    """Fork a holder that keeps fd until the worker exits or max_hold passes; the caller returns."""
+    os.ftruncate(fd, 0)
+    os.pwrite(fd, f"run {os.environ.get('GITHUB_RUN_ID', '?')} on {os.environ.get('RUNNER_NAME', '?')}\n".encode(), 0)
+    if os.fork():
+        os.close(fd)
+        return
+    os.setsid()
+    devnull = os.open(os.devnull, os.O_RDWR)
+    for target in (0, 1, 2):
+        os.dup2(devnull, target)
+    deadline = time.monotonic() + max_hold
+    while time.monotonic() < deadline and (worker is None or alive(worker)):
+        time.sleep(5)
+    os._exit(0)
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    sub = parser.add_subparsers(dest="command", required=True)
+    held = sub.add_parser("hold")
+    held.add_argument("--tag", required=True)
+    held.add_argument("--lock-dir", type=Path, default=DEFAULT_LOCK_DIR)
+    held.add_argument("--wait", type=float, default=7200)
+    held.add_argument("--max-hold", type=float, default=3 * 3600)
+    args = parser.parse_args(argv)
+    if not re.fullmatch(r"[A-Za-z0-9._-]{1,100}", args.tag):
+        parser.error("--tag must be a simple tag name")
+    path = args.lock_dir / f"{args.tag}.lock"
+    fd = acquire(path, args.wait)
+    if fd is None:
+        print(f"::error::{path.name} was still held after {args.wait:.0f}s", flush=True)
+        return 1
+    worker = worker_pid(os.getppid())
+    if worker is None:
+        print(f"no {WORKER} above this step; the tag is held for at most {args.max_hold:.0f}s", flush=True)
+    hold(fd, path, worker, args.max_hold)
+    print(f"holding {path.name}", flush=True)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/ci/prune_media_derived_data.py

@@ -19,6 +19,9 @@
     nothing is removed,
   - a build touched it within --min-idle-minutes (longer than the tour's
     timeout, so a tour still building elsewhere on this Mac is safe),
+  - a tour holds its tag (media_tag_lock.py in --lock-dir): it may be about
+    to build into the dir it left idle. The prune takes the tag's lock itself
+    while it deletes, so a tour that asks for the tag meanwhile waits,
   - it is a symlink or not a directory, or its name is not cmux-ci-media-<tag>.
 
 Nothing outside cmux-ci-media-* is listed, so other tags' DerivedData is never
@@ -28,6 +31,7 @@
 from __future__ import annotations
 
 import argparse
+import fcntl
 import os
 import re
 import shutil
@@ -96,6 +100,27 @@ def is_open(path: Path, opened: list[str]) -> bool:
     return any(entry in exact or entry.startswith(tuple(prefixes)) for entry in opened)
 
 
+def claim_tag(path: Path, lock_dir: Path) -> int | None:
+    """Take this dir's tag lock (media_tag_lock.py) without waiting, or None when a tour holds it or the lock
+    cannot be opened. The caller keeps the returned fd until the dir is gone: a tour that asks for the tag in
+    the meantime waits, so it never builds into a dir that is being removed."""
+    tag = path.name[len(PREFIX):]
+    for side in ("-base", "-head"):
+        if tag.endswith(side):
+            tag = tag[: -len(side)]
+    try:
+        lock_dir.mkdir(parents=True, exist_ok=True)
+        fd = os.open(lock_dir / f"{tag}.lock", os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o644)
+    except OSError:
+        return None
+    try:
+        fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+    except OSError:
+        os.close(fd)
+        return None
+    return fd
+
+
 def allocated_gib(path: Path) -> float:
     total = 0
     for current, dirs, files in os.walk(path):
@@ -132,6 +157,8 @@ def main() -> int:
                         help="a reload.sh tag this run builds; its DerivedData is kept")
     parser.add_argument("--keep-others", type=int, default=2,
                         help="media dirs to keep besides this run's own (default 2, one pair)")
+    parser.add_argument("--lock-dir", type=Path, default=Path.home() / "Library/Caches/cmux-ci-media/locks",
+                        help="media_tag_lock.py's lock dir: a dir whose tag a tour holds is kept")
     parser.add_argument("--min-idle-minutes", type=int, default=150,
                         help="never remove a dir a build touched more recently (default 150)")
     args = parser.parse_args()
@@ -162,12 +189,19 @@ def main() -> int:
         if is_open(path, opened):
             print(f"{path.name}: open in a process; keeping it")
             continue
-        size = allocated_gib(path)
+        claim = claim_tag(path, args.lock_dir)
+        if claim is None:
+            # Tours of other tags run beside this one; that tour holds its tag before it builds into it.
+            print(f"{path.name}: a tour holds its tag (or its lock cannot be opened); keeping it")
+            continue
         try:
+            size = allocated_gib(path)
             shutil.rmtree(path)
         except OSError as error:
             print(f"{path.name}: could not remove ({error}); keeping it")
             continue
+        finally:
+            os.close(claim)
         remaining -= 1
         freed += size
         print(f"{path.name}: removed ({size:.1f} GiB)")