ci: bound the SwiftPM scratch holder and cache scratch sizes

docs/ci-runners.md

@@ -488,9 +488,11 @@ checkout's clean would delete every package's `.build`, so
 `/Users/Shared/cmux-build-fleet/ci/spm-scratch/` outside the workspace, keyed
 by a hash of `xcodebuild -version`, `swift -version` and the workspace path,
 and SwiftPM rebuilds only what the change touched. The job holds its directory
-with a shared flock until it ends. The mini's scratch stays under 24 GiB, least
-recently built first out, whichever runner or Xcode left it, skipping the ones
-a job holds; a dropped directory is renamed to `.trash-*` before it is
+with a shared flock until it ends, or 65 minutes at most (past the job's
+60-minute timeout) if the runner dies mid-job. The mini's scratch stays under
+24 GiB, least recently used first out, whichever runner or Xcode left it,
+skipping the ones a job holds. Each directory's size is cached in a `.size`
+file beside it and measured again only after a later job used it; a dropped directory is renamed to `.trash-*` before it is
 deleted. `keep` out of space and `owned_spm_scratch.py evict` drop every
 directory no job holds.
 

scripts/ci/owned_spm_scratch.py

@@ -22,9 +22,13 @@
 A scratch directory in use is held by a shared flock on
 STORE/spm-scratch/<fingerprint>.lock: `link` starts a small holder process that
 keeps it until the runner ends the job (the runner kills a job's leftover
-processes). Before linking, `link` keeps the whole mini's scratch under
-MAX_BYTES, dropping the least recently built directories first (newest file
-inside), whatever runner or Xcode left them, and skipping any another job holds.
+processes), or HOLD_SECONDS at most, past the job's timeout, should the runner
+die mid-job and leave it. Before linking, `link` keeps the whole mini's scratch
+under MAX_BYTES, dropping the least recently used directories first, whatever
+runner or Xcode left them, and skipping any another job holds. `link` touches
+the lock file, so its time is the directory's last use, and a directory's size
+is kept in <fingerprint>.size, measured again only once the directory was used
+after it and no job holds it, so a link walks only what changed since.
 A directory is dropped by renaming it to .trash-* first, so a half-deleted one
 is never reused; the next run sweeps what a killed removal left. `evict` drops
 every directory no job holds, for disk tooling and for
@@ -49,6 +53,8 @@
 SCRATCH = "spm-scratch"
 TRASH = ".trash-"
 HELD = "held"
+# swift-package-tests' timeout-minutes is 60: a holder outliving it belongs to a job the runner lost.
+HOLD_SECONDS = 65 * 60
 
 
 def packages(workspace: Path) -> list[Path]:
@@ -69,29 +75,51 @@ def toolchain_fingerprint(workspace: Path) -> str:
     return hashlib.sha256("\n".join(parts).encode()).hexdigest()[:24]
 
 
-def tree_stats(root: Path) -> tuple[int, float]:
-    """Bytes under ROOT and the newest modification time in it. A build writes
-    inside its package's directory, so the newest time is when a job last used it."""
+def tree_bytes(root: Path) -> int:
     total = 0
-    try:
-        newest = root.stat().st_mtime
-    except OSError:
-        newest = 0.0
     for base, _, files in os.walk(root):
         for name in files:
-            try:
-                info = Path(base, name).lstat()
-            except OSError:
-                continue
-            total += info.st_size
-            newest = max(newest, info.st_mtime)
-    return total, newest
+            with contextlib.suppress(OSError):
+                total += Path(base, name).lstat().st_size
+    return total
 
 
 def lock_path(entry: Path) -> Path:
     return entry.with_name(entry.name + ".lock")
 
 
+def size_path(entry: Path) -> Path:
+    return entry.with_name(entry.name + ".size")
+
+
+def mtime(path: Path) -> float | None:
+    try:
+        return path.stat().st_mtime
+    except OSError:
+        return None
+
+
+def tree_stats(entry: Path) -> tuple[int, float]:
+    """ENTRY's bytes and its last use (its lock file's time, which `link` touches).
+
+    The size comes from ENTRY.size unless the directory was used after it was
+    written; then it is measured again, and recorded only when no job holds the
+    directory (a held one may still be growing)."""
+    used = mtime(lock_path(entry)) or mtime(entry) or 0.0
+    recorded = size_path(entry)
+    measured = mtime(recorded)
+    with contextlib.suppress(OSError, ValueError):
+        if measured is not None and measured > used:  # a tie re-measures: coarse clocks
+            return int(recorded.read_text()), used
+    size = tree_bytes(entry)
+    with contextlib.suppress(OSError), open(lock_path(entry), "a") as handle:
+        fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        staged = recorded.with_name(f"{recorded.name}.{os.getpid()}")
+        staged.write_text(str(size))
+        staged.replace(recorded)
+    return size, used
+
+
 def drop(entry: Path) -> bool:
     """Remove ENTRY unless a job holds its lock: rename it to .trash-* under an exclusive lock, then delete it."""
     try:
@@ -108,6 +136,8 @@ def drop(entry: Path) -> bool:
             entry.rename(aside)
         except OSError:
             return False
+        with contextlib.suppress(OSError):
+            size_path(entry).unlink()
     shutil.rmtree(aside, ignore_errors=True)
     return True
 
@@ -127,7 +157,7 @@ def entries(scratch: Path) -> list[Path]:
 
 
 def prune(scratch: Path, max_bytes: int = MAX_BYTES) -> list[str]:
-    """Drop the least recently built directories no job holds until the mini's scratch fits MAX_BYTES."""
+    """Drop the least recently used directories no job holds until the mini's scratch fits MAX_BYTES."""
     sweep_trash(scratch)
     stats = {entry: tree_stats(entry) for entry in entries(scratch)}
     total = sum(size for size, _ in stats.values())
@@ -174,6 +204,7 @@ def link(workspace: Path, store: Path, runner: str, fingerprint: str | None = No
     with open(lock_path(target_root), "a") as own:
         # Held from here, so prune skips this job's directory; the holder keeps it for the job.
         fcntl.flock(own, fcntl.LOCK_SH)
+        os.utime(lock_path(target_root))  # its last use, for prune's order
         prune(scratch)
         hold(lock_path(target_root))
     linked = []
@@ -199,6 +230,8 @@ def main(argv: list[str]) -> int:
         handle = open(argv[2], "a")
         fcntl.flock(handle, fcntl.LOCK_SH)
         print(HELD, flush=True)
+        # Bounded: a runner that dies mid-job never kills its leftovers, and the lock would last until reboot.
+        signal.alarm(HOLD_SECONDS)
         while True:
             signal.pause()
     if len(argv) in (3, 4) and argv[1] == "link":

tests/test_ci_owned_spm_scratch.py

@@ -5,6 +5,7 @@
 
 import fcntl
 import os
+import subprocess
 import sys
 import tempfile
 import unittest
@@ -25,6 +26,8 @@ def make_entry(root: Path, name: str, size: int, built: float) -> Path:
     (entry / "pkg").mkdir(parents=True)
     (entry / "pkg" / "blob").write_bytes(b"x" * size)
     os.utime(entry / "pkg" / "blob", (built, built))
+    scratch.lock_path(entry).touch()
+    os.utime(scratch.lock_path(entry), (built, built))  # its last use
     return entry
 
 
@@ -105,7 +108,36 @@ def test_a_half_deleted_directory_is_never_reused_and_is_swept(self):
         make_entry(self.scratch, f"{scratch.TRASH}old-123", 10, 1)
         self.assertEqual(scratch.entries(self.scratch), [])
         scratch.prune(self.scratch)
-        self.assertEqual(list(self.scratch.glob(f"{scratch.TRASH}*")), [])
+        self.assertEqual([path for path in self.scratch.glob(f"{scratch.TRASH}*") if path.is_dir()], [])
+
+    def test_a_size_is_measured_once_until_the_directory_is_used_again(self):
+        entry = make_entry(self.scratch, "a", 100, 1)
+        self.assertEqual(scratch.tree_stats(entry), (100, 1))
+        with unittest.mock.patch.object(scratch, "tree_bytes", side_effect=AssertionError("walked")):
+            self.assertEqual(scratch.tree_stats(entry), (100, 1))
+        (entry / "pkg" / "more").write_bytes(b"x" * 50)
+        os.utime(scratch.lock_path(entry))  # a later link
+        self.assertEqual(scratch.tree_stats(entry)[0], 150)
+
+    def test_a_held_directory_is_measured_but_its_size_not_recorded(self):
+        entry = make_entry(self.scratch, "busy", 100, 1)
+        with open(scratch.lock_path(entry), "a") as handle:
+            fcntl.flock(handle, fcntl.LOCK_SH)
+            self.assertEqual(scratch.tree_stats(entry)[0], 100)
+        self.assertFalse(scratch.size_path(entry).exists())
+
+    def test_the_holder_gives_up_its_lock_after_its_bound(self):
+        lock = self.scratch / "x.lock"
+        self.scratch.mkdir()
+        code = ("import sys; sys.path.insert(0, sys.argv[1]); import owned_spm_scratch as s; "
+                "s.HOLD_SECONDS = 1; s.main(['x', 'hold', sys.argv[2]])")
+        holder = subprocess.Popen([sys.executable, "-c", code, str(ROOT / "scripts/ci"), str(lock)],
+                                  stdout=subprocess.PIPE, text=True)
+        self.assertEqual(holder.stdout.readline().strip(), scratch.HELD)
+        holder.wait(timeout=30)
+        with open(lock, "a") as handle:
+            fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)  # free again
+        holder.stdout.close()
 
     def test_the_workflow_links_before_the_package_tests(self):
         text = WORKFLOW.read_text()