diff --git a/docs/CMUX_MINI_RUNNER.md b/docs/CMUX_MINI_RUNNER.md index 472347e00..614eec600 100644 --- a/docs/CMUX_MINI_RUNNER.md +++ b/docs/CMUX_MINI_RUNNER.md @@ -943,6 +943,18 @@ rebuild-tier from main's head, and each mini had sat fully idle 17 to 33% of the Check one mini: `glaeda-idle-warm` (plan) says whether it would warm now and which root, or why not. +**Idle UI fuzzing.** With `~/.config/glaeda/idle-fuzz.enabled`, an idle spell with nothing left to warm runs +cmux's UI fuzzer (`scripts/fuzz` from the kept main checkout) instead of skipping. It clones the newest main +build a root keeps into `fuzz/builds/` (APFS `cp -c`, no root token: the copy counts only if the root's +stamp is unchanged after it), fuzzes it for up to 10 minutes and minimizes up to two failures, 26 minutes at most; +runs land in `/Users/Shared/cmux-build-fleet/fuzz/runs`. Same gates and yield as a catch-up, plus: every capacity +unit free (no admitted job at all, since a gui-step job takes the gui token only later with take-gui), this user +owns an unlocked console, no Xcode test runs, and 90 GiB free. It holds one unit through +`capacity/idle-warm.json`, never the gui token or a root (either would stop a gui runner's listener). A job's +SIGTERM ends the fuzzer and its app together: the app is the fuzzer's child, in the catch-up's process group. +`glaeda-idle-warm --apply --fuzz` runs it now. cmuxterm-hq's `build-fleet/fuzz/collect.py` files the findings +as cmux issues. + ## 2l. Mini health: heal what the runner user can, report the rest `glaeda-mini-health` (the 2-minute mini-health LaunchAgent from glaeda-mini-setup, a no-op without glaeda diff --git a/scripts/glaeda-idle-warm b/scripts/glaeda-idle-warm index e164075b7..4b7674a27 100644 --- a/scripts/glaeda-idle-warm +++ b/scripts/glaeda-idle-warm @@ -3,6 +3,7 @@ glaeda-idle-warm plan: would this mini warm now, which root, and why not glaeda-idle-warm --apply warm (the 1-minute LaunchAgent from glaeda-mini-setup) + glaeda-idle-warm --apply --fuzz run the idle UI fuzzer now instead of warming (same gates and yield) cmux compile admission on an owned mini starts from the root's kept build (cmux owned_build_state.py) or a kept seed. What it compiles depends on how far that start is from the job's tree (cmux warm_distance.py): @@ -34,6 +35,9 @@ kills the build's process group at once and exits, which releases them. cmux's s from a checkout of main's head kept under ci/.catch-up, does the build: the same check, prefer, adopt, record, compile and keep steps a main dispatch's compile admission runs, and every step is safe to kill. +With ~/.config/glaeda/idle-fuzz.enabled, an idle spell with nothing left to warm runs cmux's UI fuzzer instead +(see fuzz() below): the same yield to every job, one capacity unit, no gui token. + The build is trusted main code run as the build user on a PR mini. Pull requests admitted to that root already share its kept state, so it adds no new trust boundary. One run at a time (flock on ci/.catch-up/lock). Prints one JSON line. @@ -98,6 +102,7 @@ GIT = shutil.which("git", path=PATH) or "/usr/bin/git" _child: subprocess.Popen | None = None _holder: Path | None = None _grouped = False # we lead the process group the build runs in +_fuzzing = False # a fuzz run (and its app) is under way def record(**fields: Any) -> dict[str, Any]: @@ -505,6 +510,8 @@ def stop(signum: int, _frame: Any) -> None: if _grouped: kill_build(passes=1) # within the hook's WARM_YIELD_S; the killpg below takes the rest forget_inode_override() + if _fuzzing: + dismiss_crash_dialogs() os.killpg(os.getpid(), signal.SIGKILL) os._exit(128 + signum) @@ -593,6 +600,308 @@ def trim_logs(logs: Path) -> None: stale.unlink() +# ---------------------------------------------------------------- fuzz (idle gap fill) + +# The idle UI fuzzer: cmux's scripts/fuzz (dogfood/fuzz) drives a copy of the newest main build this mini kept +# through random but valid action sequences (splits, tabs, workspaces, drags, palette, settings, windows) and +# records every crash, hang, error log or broken layout invariant with a minimized repro and frames under +# FUZZ_DIR/runs, for the collector to file as cmux issues. Opt-in per mini (FUZZ_ENABLE). It runs only when +# nothing is left to warm, the catch-up's gates hold, no job holds the gui token and no Xcode test runs, and this +# user owns an unlocked console. It holds one capacity unit (plus, while it copies a build, that root's token) +# through WARM_HOLDER, so a job's admission stops it the way it stops a catch-up: SIGTERM, then the process group, +# which the app is in (the fuzzer starts it as its own child). It takes no gui token: holding it would stop a gui +# runner's listener, and the fuzzer must never keep a job away. +# A run holds the catch-up lock, so a main push waits for it: at most FUZZ_TIMEOUT_S (26 minutes). +FUZZ_MINUTES = 10 +FUZZ_MINIMIZE_MINUTES = 5 # per finding +FUZZ_MAX_FINDINGS = 2 +FUZZ_TIMEOUT_S = (FUZZ_MINUTES + FUZZ_MAX_FINDINGS * FUZZ_MINIMIZE_MINUTES + 6) * 60 +FUZZ_KEEP_BUILDS = 2 +FUZZ_KEEP_QUIET_RUNS = 10 # runs without a finding +FUZZ_KEEP_FINDING_RUNS = 30 # runs with one, until the collector files them; the oldest go first past this +# Runs and staged builds add bytes (a catch-up rebuilds in place), so the fuzzer stays well above the runner's +# 50 GiB admission floor: it must never be why a job is refused for disk. +FUZZ_MIN_FREE_BYTES = 90 * 1024 ** 3 +FUZZ_APP = "cmux DEV.app" +XCODE_TEST = re.compile(r"(^|/)(xctest|XCTRunner)(\s|$)|-Runner\.app/") + + +def fuzz_enable_file() -> Path: + return HOME / ".config/glaeda/idle-fuzz.enabled" + + +def fuzz_dir() -> Path: + return FLEET_DIR / "fuzz" + + +def console_refusal(hook: Any) -> str: + """Why this user cannot drive the console now: another user (or nobody) owns it, or it is screen-locked. + The hook's own reading (its listener gate's), so both agree on what an usable console is.""" + try: + if os.stat("/dev/console").st_uid != os.getuid(): + return "another user owns the console" + except OSError: + return "cannot read the console owner" + try: + state = hook.console_state() + except Exception: # noqa: BLE001 - an older hook + state = None + if state is None: + return "cannot read the console session" + return "" if state.state == "unlocked" else f"the console session is {state.state}" + + +def gui_refusal(hook: Any, capacity: Path) -> str: + """A GUI job on this mini: the gui token held, or an Xcode test process running.""" + if not hook.probe(capacity / "gui.token"): + return "a job holds the gui token" + try: + out = subprocess.run(["/bin/ps", "-axww", "-o", "command="], capture_output=True, text=True, timeout=60) + except (OSError, subprocess.SubprocessError): + return "cannot list processes" + for command in out.stdout.splitlines(): + if XCODE_TEST.search(command): + return "an Xcode test is running" + return "" + + +def fuzz_engine(state: Path) -> Path | None: + """The cmux checkout whose scripts/fuzz runs: main's, kept for the catch-up, or GLAEDA_IDLE_FUZZ_ENGINE.""" + override = os.environ.get("GLAEDA_IDLE_FUZZ_ENGINE") + checkout = Path(override) if override else state / ".catch-up" / "cmux" + return checkout if (checkout / "scripts" / "fuzz").is_file() and (checkout / "dogfood" / "fuzz").is_dir() else None + + +def main_roots(hook: Any, roots: int, state: Path) -> list[tuple[int, str, float]]: + """(root, main commit, when kept) for each root whose kept build is a main build, newest first.""" + found = [] + for root in range(1, roots + 1): + try: + stamp = hook.root_stamp(root, os.fspath(state)) + except Exception: # noqa: BLE001 - an older hook or an unreadable stamp + continue + sha = stamp.get("merged_onto") if isinstance(stamp, dict) and not stamp.get("pr") else None + if not isinstance(sha, str) or not re.fullmatch(r"[0-9a-f]{40}", sha): + continue + store = state if root == 1 else state / f"cmux-ci-{root}" + if not (store / "derived-data" / "Build" / "Products" / "Debug" / FUZZ_APP).is_dir(): + continue + try: + found.append((root, sha, (store / "stamp.json").stat().st_mtime)) + except OSError: + continue + return sorted(found, key=lambda item: -item[2]) + + +def staged_builds() -> list[Path]: + """Complete staged main builds (FUZZ_DIR/builds// with its app and build.json), newest first. A copy a + kill cut short is a dot directory without build.json, and never counts.""" + builds = [] + for path in (fuzz_dir() / "builds").glob("*"): + if path.name.startswith(".") or not (path / FUZZ_APP).is_dir() or not (path / "build.json").is_file(): + continue + with contextlib.suppress(OSError): + builds.append((path.stat().st_mtime, path)) + return [path for _, path in sorted(builds, key=lambda item: -item[0])] + + +def stage_build(state: Path, root: int, sha: str) -> Path | str: + """Copy root ROOT's kept main app (an APFS clone: seconds, no space) to FUZZ_DIR/builds/SHA, or why not. + + Without the root's token, which would stop a gui runner's listener while it is held: the kept build changes + only when a job's `keep` rewrites stamp.json, so a stamp read the same before and after the copy means the + copy is of one build. The copy is the fuzzer's own; a running app must not sit under a job's adopt.""" + store = state if root == 1 else state / f"cmux-ci-{root}" + source = store / "derived-data" / "Build" / "Products" / "Debug" / FUZZ_APP + builds = fuzz_dir() / "builds" + builds.mkdir(parents=True, exist_ok=True) + for stale in builds.glob(".*"): # copies a kill cut short + shutil.rmtree(stale, ignore_errors=True) + before = (store / "stamp.json").read_bytes() + final = builds / sha + incoming = builds / f".{sha}.{os.getpid()}" + incoming.mkdir() + subprocess.run(["/bin/cp", "-Rc", os.fspath(source), os.fspath(incoming / FUZZ_APP)], check=True, + capture_output=True, timeout=600) + if (store / "stamp.json").read_bytes() != before: + shutil.rmtree(incoming, ignore_errors=True) + return f"root-{root}'s kept build changed while it was copied" + (incoming / "build.json").write_text(json.dumps({"sha": sha, "root": root, "at": int(time.time())}) + "\n") + shutil.rmtree(final, ignore_errors=True) + incoming.rename(final) + for stale in staged_builds()[FUZZ_KEEP_BUILDS:]: + shutil.rmtree(stale, ignore_errors=True) + return final + + +def take_fuzz(hook: Any, capacity: Path, setup: dict[str, Any]) -> tuple[list[int], list[str]] | str: + """One capacity unit with the host lock shared, under admission.lock, and WARM_HOLDER written before + admission.lock is released; or why not. Only when every unit is free: any admitted job holds one, and a + gui-step job (app-host tests, test-e2e's test job) takes the gui token only later, from its step, with + take-gui, which does not stop a yielding holder. So the fuzzer starts only on a mini with no job at all, + and every job after it is admitted through take_capacity, which stops it first.""" + global _holder + capacity.mkdir(parents=True, exist_ok=True) + admission = os.open(capacity / "admission.lock", os.O_RDONLY | os.O_CREAT | os.O_NOFOLLOW, 0o644) + fds: list[int] = [] + try: + try: + fcntl.flock(admission, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + return "an admission is running" + host = os.open(FLEET_DIR / "host.lock", os.O_RDONLY | os.O_CREAT | os.O_NOFOLLOW, 0o660) + fds.append(host) + try: + fcntl.flock(host, fcntl.LOCK_SH | fcntl.LOCK_NB) + except OSError: + return "a fleet build holds the host lock" + held = [] + for slot in range(setup["units"]): + fd = hook.lock_file(capacity / f"unit-{slot}", fcntl.LOCK_EX) + if fd is None: + return f"a job holds unit-{slot}" + if held: + os.close(fd) # only probing: every unit must be free, one is enough to hold + else: + fds.append(fd) + held.append(f"unit-{slot}") + if not held: + return "no capacity unit is free" + holder = capacity / hook.WARM_HOLDER + incoming = capacity / f".{hook.WARM_HOLDER}.{os.getpid()}" + incoming.write_text(json.dumps({"pid": os.getpid(), "held": held, "fuzz": True}) + "\n") + incoming.rename(holder) + _holder = holder + taken, fds = fds, [] + return taken, held + except OSError as error: + return f"cannot take the ledger ({type(error).__name__})" + finally: + for fd in fds: + with contextlib.suppress(OSError): + os.close(fd) + os.close(admission) + + +def trim_fuzz_runs(runs: Path) -> None: + """Keep the newest FUZZ_KEEP_QUIET_RUNS finished runs without a finding and FUZZ_KEEP_FINDING_RUNS with one.""" + quiet, found = [], [] + now = time.time() + for run_dir in runs.glob("*"): + try: + stamp = run_dir.stat().st_mtime + except OSError: + continue + try: + summary = json.loads((run_dir / "summary.json").read_text()) + except (OSError, ValueError): + # A job's preemption kills the fuzzer before it writes its summary: past the longest run, it is done. + if now - stamp < FUZZ_TIMEOUT_S: + continue + summary = {"findings": [os.fspath(p.parent) for p in run_dir.glob("session-*/finding.json")]} + findings = summary.get("findings") if isinstance(summary, dict) else None + (found if isinstance(findings, list) and findings else quiet).append((stamp, run_dir)) + for kept, group in ((FUZZ_KEEP_QUIET_RUNS, quiet), (FUZZ_KEEP_FINDING_RUNS, found)): + for _, stale in sorted(group, key=lambda item: -item[0])[kept:]: + shutil.rmtree(stale, ignore_errors=True) + + +def free_bytes(path: Path) -> int | None: + try: + stat = os.statvfs(path) + except OSError: + return None + return stat.f_bavail * stat.f_frsize + + +def fuzz(hook: Any, scope: dict[str, Any], state: Path, head: str, why_not_warm: str) -> dict[str, Any]: + """Fuzz the newest main build this mini kept, as a preemptible idle gap fill; one JSON record.""" + global _child, _grouped, _fuzzing + base = record(head=head, warm=why_not_warm, mode="fuzz") + capacity = FLEET_DIR / "capacity" + why = console_refusal(hook) or gui_refusal(hook, capacity) + if why: + return {**base, "state": "skip", "reason": why} + free = free_bytes(FLEET_DIR) + if free is None or free < FUZZ_MIN_FREE_BYTES: + return {**base, "state": "skip", "reason": f"{(free or 0) // 1024 ** 3} GiB free, the fuzzer needs " + f"{FUZZ_MIN_FREE_BYTES // 1024 ** 3}"} + engine = fuzz_engine(state) + if engine is None: + return {**base, "state": "skip", "reason": "no scripts/fuzz in the cmux checkout"} + roots = main_roots(hook, scope["roots"], state) + staged = {path.name: path for path in staged_builds()} + newest = roots[0] if roots else None + if newest is None and not staged: + return {**base, "state": "skip", "reason": "no main build kept or staged on this mini"} + _grouped = own_group() + if not _grouped: + return {**base, "state": "skip", "reason": "cannot lead a process group for the fuzzer"} + for signum in (signal.SIGTERM, signal.SIGHUP, signal.SIGINT): + signal.signal(signum, stop) + taken = take_fuzz(hook, capacity, scope) + if isinstance(taken, str): + return {**base, "state": "skip", "reason": taken} + fds, held = taken + _fuzzing = True + try: + if newest is not None and newest[1] not in staged: + build = stage_build(state, newest[0], newest[1]) + if isinstance(build, str): + return {**base, "state": "skip", "reason": build} + else: + build = staged[newest[1]] if newest else next(iter(staged.values())) + why = running() or gui_refusal(hook, capacity) # a job that started while the locks were taken + if why: + return {**base, "state": "skip", "reason": why} + runs = fuzz_dir() / "runs" + runs.mkdir(parents=True, exist_ok=True) + seed = int.from_bytes(os.urandom(4), "big") >> 1 + out = runs / f"{time.strftime('%Y%m%dT%H%M%SZ', time.gmtime())}-{build.name[:12]}-{seed}" + out.mkdir() + env = {key: value for key, value in os.environ.items() if not key.startswith(("GITHUB_", "RUNNER_", "CMUX_"))} + env["PATH"] = PATH + started = time.monotonic() + with open(out / "fuzz.log", "ab") as log: + _child = subprocess.Popen( + [sys.executable, os.fspath(engine / "scripts" / "fuzz"), "run", "--app", os.fspath(build / FUZZ_APP), + "--minutes", str(FUZZ_MINUTES), "--minimize-minutes", str(FUZZ_MINIMIZE_MINUTES), + "--max-findings", str(FUZZ_MAX_FINDINGS), "--seed", str(seed), "--out", os.fspath(out), + "--label", "main", "--sha", build.name], + cwd=engine, env=env, stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT) + try: + code = _child.wait(timeout=FUZZ_TIMEOUT_S) + except subprocess.TimeoutExpired: + code = None + finally: + _child = None + kill_build() # the app and anything else the fuzzer left in our group + dismiss_crash_dialogs() + try: + summary = json.loads((out / "summary.json").read_text()) + except (OSError, ValueError): + summary = None + summary = summary if isinstance(summary, dict) else {} + findings = summary.get("findings") + trim_fuzz_runs(runs) + return {**base, "state": "fuzzed", "held": held, "build": build.name, "run": out.name, + "exit": code, "steps": summary.get("steps"), + "findings": len(findings) if isinstance(findings, list) else 0, + "wall_seconds": round(time.monotonic() - started, 1)} + except (OSError, ValueError, subprocess.SubprocessError) as error: + return {**base, "state": "error", "reason": f"{type(error).__name__}: {str(error)[:160]}"} + finally: + _fuzzing = False + release(fds) + + +def dismiss_crash_dialogs() -> None: + """A crash the fuzzer caused leaves macOS's crash dialog (not in our process group) on the console, over the + next GUI job: close it.""" + with contextlib.suppress(OSError, subprocess.SubprocessError): + subprocess.run(["/usr/bin/pkill", "-9", "-x", "Problem Reporter"], capture_output=True, timeout=2) + + # ---------------------------------------------------------------- run @@ -633,7 +942,7 @@ def save_memory(path: Path, memory: dict[str, Any]) -> None: incoming.rename(path) -def run(apply: bool, state: Path, now: float | None = None) -> dict[str, Any]: +def run(apply: bool, state: Path, now: float | None = None, fuzz_now: bool = False) -> dict[str, Any]: global _grouped now = time.time() if now is None else now if platform.system() != "Darwin": @@ -671,8 +980,10 @@ def run(apply: bool, state: Path, now: float | None = None) -> dict[str, Any]: head = main_head(state) if not head: return record(state="skip", reason="no main head in the seed prefetch mirror") - picked = pick_root(hook, scope["roots"], head, state, memory) + picked = "asked to fuzz" if fuzz_now else pick_root(hook, scope["roots"], head, state, memory) if isinstance(picked, str): + if apply and (fuzz_now or fuzz_enable_file().exists()): # nothing to warm: fill the idle gap with the fuzzer + return fuzz(hook, scope, state, head, picked) return record(state="skip", reason=picked, head=head) root, predicted = picked base = record(head=head, root=root, predicted=predicted) @@ -728,9 +1039,11 @@ def run(apply: bool, state: Path, now: float | None = None) -> dict[str, Any]: def main(argv: list[str]) -> int: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument("--apply", action="store_true") + parser.add_argument("--fuzz", action="store_true", + help="with --apply: skip warming and run the idle fuzzer now, under the same gates") args = parser.parse_args(argv) state = Path(os.environ.get("GLAEDA_SEED_STATE") or FLEET_DIR / "ci") - print(json.dumps(run(args.apply, state)), flush=True) + print(json.dumps(run(args.apply, state, fuzz_now=args.fuzz)), flush=True) return 0 diff --git a/scripts/test-glaeda-idle-warm.py b/scripts/test-glaeda-idle-warm.py index 152ab10c1..81a3b6b1b 100644 --- a/scripts/test-glaeda-idle-warm.py +++ b/scripts/test-glaeda-idle-warm.py @@ -52,7 +52,7 @@ def setUp(self) -> None: self.capacity = self.dir / "fleet" / "capacity" self.saved = {name: getattr(warm, name) for name in ("FLEET_DIR", "HOME", "KILL_SWITCH", "runner_setup", "idle_refusal", "main_head", - "prepare_checkout", "running", "host_denied")} + "prepare_checkout", "running", "host_denied", "console_refusal", "free_bytes")} self.saved_env = dict(os.environ) warm.FLEET_DIR = self.dir / "fleet" warm.HOME = self.dir @@ -384,6 +384,191 @@ def fake_setup_on_disk(self) -> None: script.chmod(0o755) +# Stands in for cmux's scripts/fuzz: records its arguments, starts a child (the app) in its process group, sleeps, +# then writes the run summary the way `scripts/fuzz run` does. +FAKE_FUZZ = """#!/usr/bin/env python3 +import json, os, subprocess, sys, time +args = sys.argv[1:] +with open(os.environ["FAKE_CALLS"], "a") as calls: + calls.write(" ".join(args) + "\\n") +out = args[args.index("--out") + 1] +app = subprocess.Popen(["/bin/sleep", "300"]) +open(os.path.join(out, "app.pid"), "w").write(str(app.pid)) +time.sleep(float(os.environ.get("FAKE_SLEEP", "0"))) +app.kill() +json.dump({"seed": 1, "sessions": 1, "steps": 42, "findings": []}, open(os.path.join(out, "summary.json"), "w")) +""" + + +class FuzzTest(Base): + def fuzz_setup(self, sleep: str = "0") -> tuple[types.SimpleNamespace, dict]: + """A mini with nothing to warm: root 2 keeps a main build, the engine is main's checkout.""" + (self.state / "seed-source.json").write_text("{}") + (self.dir / "fleet" / "host.lock").touch() + engine = self.state / ".catch-up" / "cmux" + (engine / "scripts").mkdir(parents=True) + (engine / "dogfood" / "fuzz").mkdir(parents=True) + (engine / "scripts" / "fuzz").write_text(FAKE_FUZZ) + store = self.state / "cmux-ci-2" + app = store / "derived-data" / "Build" / "Products" / "Debug" / "cmux DEV.app" / "Contents" + app.mkdir(parents=True) + (app / "Info.plist").write_text("main build") + (store / "stamp.json").write_text(json.dumps({"merged_onto": HEAD})) + (self.dir / ".config" / "glaeda").mkdir(parents=True) + os.environ.update(FAKE_CALLS=os.fspath(self.dir / "calls"), FAKE_SLEEP=sleep) + scope = {"units": 4, "roots": 2, "compile_slots": 2, "xcode": "/Applications/Xcode_26.6.app"} + stamps = {1: {"merged_onto": "b" * 40, "pr": 15000}, 2: {"merged_onto": HEAD}} + stub = types.SimpleNamespace(**{name: getattr(hook, name) for name in + ("lock_file", "CLASS_COST", "WARM_HOLDER", "probe")}, + warm_root_costs=lambda order, base, number, state: (order, {}), + root_stamp=lambda k, state: stamps.get(k)) + warm.host_denied = lambda names=None: "" + warm.runner_setup = lambda dirs: (stub, scope) + warm.idle_refusal = lambda hook_module, now, state: "" + warm.main_head = lambda state: HEAD + warm.running = lambda: "" + warm.console_refusal = lambda hook_module: "" + warm.free_bytes = lambda path: 500 * 1024 ** 3 + return stub, scope + + def test_off_unless_enabled(self) -> None: + if sys.platform != "darwin": + return + self.fuzz_setup() + memory = self.state / ".catch-up" / "state.json" + memory.parent.mkdir(parents=True, exist_ok=True) + memory.write_text(json.dumps({"roots": {"1": {"head": HEAD}, "2": {"head": HEAD}}})) + result = warm.run(True, self.state) + self.assertEqual(result["state"], "skip", result) + self.assertFalse((self.dir / "calls").exists(), "no fuzz run without the enable file") + + @unittest.skipUnless(sys.platform == "darwin", "runs only on macOS") + def test_nothing_to_warm_fuzzes_a_staged_copy_of_the_main_build_and_releases(self) -> None: + self.fuzz_setup() + (self.dir / ".config" / "glaeda" / "idle-fuzz.enabled").touch() + memory = self.state / ".catch-up" / "state.json" + memory.parent.mkdir(parents=True, exist_ok=True) + memory.write_text(json.dumps({"roots": {"1": {"head": HEAD}, "2": {"head": HEAD}}})) + result = warm.run(True, self.state) + self.assertEqual(result["state"], "fuzzed", result) + self.assertEqual(result["held"], ["unit-0"], "one unit, never a root or the gui token") + self.assertEqual((result["build"], result["steps"], result["findings"]), (HEAD, 42, 0)) + staged = self.dir / "fleet" / "fuzz" / "builds" / HEAD / "cmux DEV.app" / "Contents" / "Info.plist" + self.assertEqual(staged.read_text(), "main build") + call = (self.dir / "calls").read_text() + self.assertIn(f"run --app {staged.parents[1]} --minutes {warm.FUZZ_MINUTES}", call) + self.assertIn(f"--label main --sha {HEAD}", call) + self.assertFalse((self.capacity / "idle-warm.json").exists()) + fd = hook.lock_file(self.capacity / "unit-0", fcntl.LOCK_EX) + self.assertIsNotNone(fd, "released") + os.close(fd) + (self.dir / "calls").unlink() + again = warm.run(True, self.state, fuzz_now=True) # staged already: the same copy again + self.assertEqual((again["state"], again["build"]), ("fuzzed", HEAD), again) + + @unittest.skipUnless(sys.platform == "darwin", "runs only on macOS") + def test_never_beside_a_gui_job_and_needs_an_engine_and_a_main_build(self) -> None: + stub, scope = self.fuzz_setup() + self.capacity.mkdir(parents=True) + gui = hook.lock_file(self.capacity / "gui.token", fcntl.LOCK_EX) + self.assertIn("gui token", warm.fuzz(stub, scope, self.state, HEAD, "x")["reason"]) + os.close(gui) + (self.state / ".catch-up" / "cmux" / "scripts" / "fuzz").unlink() + self.assertIn("no scripts/fuzz", warm.fuzz(stub, scope, self.state, HEAD, "x")["reason"]) + (self.state / ".catch-up" / "cmux" / "scripts" / "fuzz").write_text(FAKE_FUZZ) + unit = hook.lock_file(self.capacity / "unit-3", fcntl.LOCK_EX) # any admitted job holds a unit + self.assertEqual(warm.fuzz(stub, scope, self.state, HEAD, "x")["reason"], "a job holds unit-3") + os.close(unit) + self.assertFalse((self.capacity / "idle-warm.json").exists()) + stub.root_stamp = lambda k, state: {"merged_onto": HEAD, "pr": 15000} # pull request builds only + self.assertIn("no main build", warm.fuzz(stub, scope, self.state, HEAD, "x")["reason"]) + # a copy a kill cut short is never taken for a build + partial = self.dir / "fleet" / "fuzz" / "builds" / f".{HEAD}.123" / "cmux DEV.app" + partial.mkdir(parents=True) + self.assertEqual(warm.staged_builds(), []) + self.assertIn("no main build", warm.fuzz(stub, scope, self.state, HEAD, "x")["reason"]) + warm.free_bytes = lambda path: 10 * 1024 ** 3 + stub.root_stamp = lambda k, state: {"merged_onto": HEAD} + self.assertIn("GiB free", warm.fuzz(stub, scope, self.state, HEAD, "x")["reason"]) + self.assertFalse((self.dir / "calls").exists()) + + @unittest.skipUnless(sys.platform == "darwin", "runs only on macOS") + def test_a_jobs_sigterm_ends_the_fuzzer_and_its_app_at_once(self) -> None: + """What a job's admission (yield_idle_warm) does to a fuzz run: the app goes with the process group.""" + self.fuzz_setup(sleep="120") + (self.dir / ".config" / "glaeda" / "idle-fuzz.enabled").touch() + driver = self.dir / "glaeda-idle-warm-driver.py" + driver.write_text(textwrap.dedent(f""" + import importlib.machinery, importlib.util, json, os, sys, types + from pathlib import Path + def load(name, path): + loader = importlib.machinery.SourceFileLoader(name, path) + spec = importlib.util.spec_from_loader(name, loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + return module + warm = load("w", {os.fspath(ROOT / 'scripts' / 'glaeda-idle-warm')!r}) + hook = load("h", {os.fspath(ROOT / 'scripts' / 'glaeda-cmux-runner-hook')!r}) + state = Path({os.fspath(self.state)!r}) + warm.FLEET_DIR = state.parent + warm.HOME = Path({os.fspath(self.dir)!r}) + hook.root_stamp = lambda k, s: {{"merged_onto": "{HEAD}"}} if k == 2 else None + warm.host_denied = lambda names=None: "" + warm.runner_setup = lambda dirs: (hook, {{"units": 4, "roots": 2, "compile_slots": 1, "xcode": "/x.app"}}) + warm.idle_refusal = lambda *a: "" + warm.main_head = lambda s: "{HEAD}" + warm.running = lambda: "" + warm.console_refusal = lambda h: "" + warm.gui_refusal = lambda h, c: "" # other agents' Xcode tests on this Mac are not this test's + warm.free_bytes = lambda p: 500 * 1024 ** 3 + print(warm.run(True, state, fuzz_now=True), flush=True) + """)) + proc = subprocess.Popen([sys.executable, os.fspath(driver)], stdout=subprocess.PIPE, text=True, + env={**os.environ}) + self.addCleanup(proc.kill) + runs = self.dir / "fleet" / "fuzz" / "runs" + deadline = time.monotonic() + 30 + pid_files: list[Path] = [] + while not pid_files and time.monotonic() < deadline: + pid_files = list(runs.glob("*/app.pid")) if runs.is_dir() else [] + time.sleep(0.1) + if not pid_files: + proc.kill() + self.fail(f"the fuzzer started no app: {proc.communicate(timeout=10)[0]}") + app_pid = int(pid_files[0].read_text()) + holder = json.loads((self.capacity / "idle-warm.json").read_text()) + self.assertEqual((holder["pid"], holder["held"]), (proc.pid, ["unit-0"])) + self.assertIsNone(hook.lock_file(self.capacity / "unit-0", fcntl.LOCK_EX)) + started = time.monotonic() + proc.send_signal(signal.SIGTERM) # yield_idle_warm's first signal + self.assertEqual(proc.wait(timeout=10), -signal.SIGKILL) + self.assertLess(time.monotonic() - started, 5) + self.assertFalse(hook.pid_alive(app_pid), "the app went with the fuzzer's process group") + self.assertIsNotNone(hook.lock_file(self.capacity / "unit-0", fcntl.LOCK_EX)) + + +class TrimTest(Base): + def test_runs_a_preemption_cut_short_are_trimmed_once_old(self) -> None: + runs = self.dir / "runs" + old = time.time() - warm.FUZZ_TIMEOUT_S - 60 + for n in range(warm.FUZZ_KEEP_QUIET_RUNS + 3): + run = runs / f"cut-{n:02d}" + (run / "session-000").mkdir(parents=True) # no summary.json: SIGKILLed mid-run + os.utime(run, (old - n, old - n)) + live = runs / "live" + live.mkdir() # no summary yet and young: still running + found = runs / "found" / "session-000" + found.mkdir(parents=True) + (found / "finding.json").write_text("{}") + os.utime(runs / "found", (old, old)) + warm.trim_fuzz_runs(runs) + left = sorted(p.name for p in runs.iterdir()) + self.assertIn("live", left) + self.assertIn("found", left, "a cut-short run with a finding waits for the collector") + self.assertEqual(len([n for n in left if n.startswith("cut-")]), warm.FUZZ_KEEP_QUIET_RUNS) + self.assertIn("cut-00", left, "the newest are kept") + + class CheckoutTest(unittest.TestCase): def test_first_clone_is_shallow_blobless_and_checks_out_head(self) -> None: with tempfile.TemporaryDirectory() as tmp: