diff --git a/docs/CMUX_MINI_RUNNER.md b/docs/CMUX_MINI_RUNNER.md index ee2dc198b..b4de1a2c4 100644 --- a/docs/CMUX_MINI_RUNNER.md +++ b/docs/CMUX_MINI_RUNNER.md @@ -120,7 +120,8 @@ What `--apply` does: token (one writer of the kept DerivedData at a time), `app-host-unit-tests` and `tests-build-and-lag` 1 unit plus the `gui` token (one console session), test-e2e's `build` (compile, then the selected tests in the console session) 2 units - plus the `gui` token, test-e2e's `test` 1 unit plus the `gui` token, + (it takes the `gui` token itself before its tests, with take-gui), test-e2e's `test` 1 + unit plus the `gui` token, `cli-product-tests`, `swift-package-tests` and the side lanes `cli-pipe-regressions`, `remote-daemon-macos-tests` and `claude-wrapper` 1 unit, and any other job counts as a compile. When units or a token are taken it refuses at once with @@ -406,8 +407,8 @@ and app-host test consumers restore a product there with `rm -rf /src`, be in at the producer's root. So one root job per root per mini: - Root jobs are compile (macos-compile-admission and any unknown job id), compile-gui (test-e2e's `build`: - a producer that holds the gui token as well, and no persistent-dd: it only clones its root's kept state, - which the root token already guards), gui (app-host-unit-tests, + a producer that takes the gui token later, in its own step with take-gui, and no persistent-dd: it only + clones its root's kept state, which the root token already guards), gui (app-host-unit-tests, tests-build-and-lag, app-host-test-rerun's `rerun`, test-e2e's `test`) and product (cli-product-tests). Each also takes an exclusive `capacity/root-k.token` (k = 1 to `canonicalRoots`), and the hook writes `CMUX_CI_CANONICAL_ROOT=` to `$GITHUB_ENV` and `$RUNNER_TEMP/glaeda-canonical-root`. The root follows the token, never the runner instance. diff --git a/scripts/glaeda-cmux-runner-hook b/scripts/glaeda-cmux-runner-hook index 23a751954..4ddcf5ea2 100755 --- a/scripts/glaeda-cmux-runner-hook +++ b/scripts/glaeda-cmux-runner-hook @@ -135,12 +135,12 @@ WORKFLOW_JOB_CLASSES = {("cmux-tui.yml", "lint"): "isolated", ("cmux-tui.yml", " ("test-e2e.yml", "build"): "compile-gui", ("test-e2e.yml", "test"): "gui"} # root: the canonical root (/private/tmp/cmux-ci) that compiles build in and that app-host test consumers # restore a product into with rm -rf of its src, so one root job per root per mini (cmux#14338 adds a second). -# compile-gui holds the gui token for its whole job, compile included: the tests it runs afterwards start from -# a step the hook never sees, so the token has to be taken at job start. gui comes before root, so a refusal -# for the gui token is the one GUI_WAIT_S waits for. +# compile-gui takes no gui token at job start: the step that decides whether the build tests takes it with +# glaeda-canonical-root take-gui (cmux#14705), and leaves the tests to test-e2e's `test` job when take-gui gives +# way or times out. So its compile leaves the console session to other GUI jobs. CLASS_COST: dict[str, tuple[int, tuple[str, ...]]] = { "compile": (2, ("persistent-dd", "root")), "gui": (1, ("gui", "root")), "product": (1, ("root",)), - "compile-gui": (2, ("gui", "root")), + "compile-gui": (2, ("root",)), "isolated": (2, ()), "simulator": (2, ("simulator",)), "light": (1, ())} # Guest repositories (any repo other than --home-repo, admitted by an org-scoped runner's --allowed-owner) # never touch cmux's canonical roots or persistent DerivedData, so they are costed by job-id suffix alone: @@ -150,12 +150,12 @@ DEFAULT_HOME_REPO = "manaflow-ai/cmux" GUEST_WAIT_S = 600.0 # guests have no rescue workflow: a full mini holds their job this long before refusing GUEST_POLL_S = 5.0 -# A cmux GUI job (app-host shard, tests-build-and-lag, test-e2e's build and test) refused only because another +# A cmux GUI job (app-host shard, tests-build-and-lag, test-e2e's test) refused only because another # GUI job holds this mini's one gui token waits this long for it instead. Two root runners share one gui token, # so a full suite's shards often meet on a mini, and a GUI job there runs about 90 s, while a refusal's rescue # round trip took 71 s to 18 min (median 126 s, 16 refusals, cmux PR CI 2026-09-25 07:00-09:00Z) and sent the -# re-run shards to Blacksmith. test-e2e's build holds the token for its compile too, so a job that meets it -# usually waits out GUI_WAIT_S and is refused: the rescue then moves it, as before this wait existed. +# re-run shards to Blacksmith. test-e2e's build takes the token only for its tests (take-gui), so it holds it +# about as long as a shard does. # It must fit inside cmux's owned_pool_rescue.REFUSAL_SECONDS (120) with room to spare: a job that fails later # than that after starting no longer counts as refused, so the rescue would not re-run it. Around the wait go # about 5 s of runner setup before the hook, the last try (normally under a second, but up to tens of seconds diff --git a/scripts/test-glaeda-cmux-runner.py b/scripts/test-glaeda-cmux-runner.py index ba07e087e..540cb13be 100644 --- a/scripts/test-glaeda-cmux-runner.py +++ b/scripts/test-glaeda-cmux-runner.py @@ -636,7 +636,7 @@ def test_switch_swaps_a_producers_root_without_holding_two(self) -> None: state = self.dir / "state" try: build = self.job("build", "e0", 8, None, *two, env=e2e) - self.assertIn("+gui+root-1 for build (compile-gui", build.stdout) + self.assertIn("units+root-1 for build (compile-gui", build.stdout) self.assertTrue((state / "host-lock-holder-e0-root-1.pid").exists(), "the root has a holder of its own") refused = self.take("2", "e0", "--wait", "0") self.assertEqual(refused.returncode, 2, "without --switch a second root is still refused") @@ -656,8 +656,8 @@ def test_switch_swaps_a_producers_root_without_holding_two(self) -> None: self.assertEqual(env_file.read_text(), "CMUX_CI_CANONICAL_ROOT=/private/tmp/cmux-ci-2\n") self.assertEqual((state / "host-lock-holder-e0.roots").read_text().split(), ["root-2"]) self.assertEqual(self.take("1", "c0", "--wait", "5").returncode, 0, "root 1 is free again") - gui = self.job("app-host-unit-tests", "g0", 8, None, *two, "--gui-wait", "0") - self.assertIn("the gui token is taken", gui.stdout, "the admission holder keeps units and gui") + full = self.job("swift-package-tests", "g0", 2, None, *two) # e0 still holds unit-0 and unit-1 + self.assertIn("0 of 2 units free", full.stdout, "the admission holder keeps its units") self.assertEqual(self.take("2", "e0").returncode, 0, "a re-take of the new root is a no-op") self.assertIn("canonical root holder(s) released", self.finish("e0")) self.assertEqual(self.take("2", "c1", "--wait", "5").returncode, 0, "released with the job") @@ -947,44 +947,49 @@ def test_capacity_gui_job_waits_for_the_gui_token(self) -> None: for runner in ("w0", "w1", "w2", "w3", "w4", "w5"): self.finish(runner) - def test_capacity_e2e_jobs_hold_the_gui_token(self) -> None: - # test-e2e's build compiles and then runs its tests in the console session; its test job is the fallback + def test_capacity_e2e_build_takes_the_gui_token_for_its_tests_only(self) -> None: + # test-e2e's build compiles, then takes the gui token itself (take-gui) before its console-session tests self.fleet() e2e = {"GITHUB_WORKFLOW_REF": "manaflow-ai/cmux/.github/workflows/test-e2e.yml@refs/heads/main"} two = ("--canonical-roots", "2", "--compile-slots", "2") env_file = self.dir / "github_env" + state = self.dir / "state" try: build = self.job("build", "e0", 8, None, *two, env={**e2e, "GITHUB_ENV": os.fspath(env_file)}) self.assertEqual(build.returncode, 0, build.stdout) - self.assertIn("holding 2/8 units+gui+root-1 for build (compile-gui", build.stdout) + self.assertIn("holding 2/8 units+root-1 for build (compile-gui", build.stdout) + self.assertNotIn("gui+", build.stdout) self.assertNotIn("persistent-dd", build.stdout) + self.assertFalse((state / "host-lock-holder-e0.gui").exists()) self.assertEqual(env_file.read_text(), "CMUX_CI_CANONICAL_ROOT=/private/tmp/cmux-ci\n") # its own restore step re-takes the root it holds: a no-op self.assertEqual(self.take("/private/tmp/cmux-ci", "e0").returncode, 0) - # a compile admission still runs beside it, in the other root + # while it compiles, a GUI job runs beside it, and so does a compile admission in the other root + shard = self.job("app-host-unit-tests", "g0", 8, None, *two, "--gui-wait", "0") + self.assertIn("+gui for app-host-unit-tests (gui", shard.stdout) self.assertIn("persistent-dd+root-2", self.job("macos-compile-admission", "e1", 8, None, *two).stdout) - for n, (job, ref) in enumerate((("app-host-unit-tests", {}), ("test", e2e), ("build", e2e))): - waited = time.monotonic() - refused = self.job(job, f"g{n}", 8, None, *two, "--gui-wait", "1", env=ref) - self.assertEqual(refused.returncode, 1, refused.stdout) - self.assertIn("refused: capacity: the gui token is taken", refused.stdout) - self.assertGreaterEqual(time.monotonic() - waited, 1, f"{job} waits for the gui token") - self.finish("e0") - # the fallback test job is a consumer: no root at job start, the producer's root from its restore step + # the build's tests wait for that shard's token, then take it + self.assertEqual(self.take_gui("e0", "--wait", "1").returncode, 1, "the shard still holds it") + self.finish("g0") + self.assertEqual(self.take_gui("e0", "--wait", "5").returncode, 0) + refused = self.job("tests-build-and-lag", "g1", 8, None, *two, "--gui-wait", "0") + self.assertIn("refused: capacity: the gui token is taken", refused.stdout) + self.assertIn("the gui token holder released", self.finish("e0")) # with root 1 free, a build on the second root runner still takes it: root 1's seeds and caches # are the ones main publishes, and it keeps no per-root state to prefer its own root for first = self.job("build", "e3", 8, None, *two, "--instance", "1", env=e2e) - self.assertIn("+gui+root-1 for build (compile-gui", first.stdout) + self.assertIn("units+root-1 for build (compile-gui", first.stdout) self.finish("e3") + # the fallback test job is a consumer: gui at job start, the producer's root from its restore step test = self.job("test", "e2", 8, None, *two, env=e2e) self.assertEqual(test.returncode, 0, test.stdout) - self.assertIn("for test (gui", test.stdout) + self.assertIn("+gui for test (gui", test.stdout) self.assertNotIn("root-", test.stdout.split("holding", 1)[1]) self.assertEqual(self.take("/private/tmp/cmux-ci-2", "e2", "--wait", "0").returncode, 1, "the compile admission still holds root 2") self.assertEqual(self.take("/private/tmp/cmux-ci", "e2").returncode, 0) finally: - for runner in ("e0", "e1", "e2", "e3", "g0", "g1", "g2"): + for runner in ("e0", "e1", "e2", "e3", "g0", "g1"): self.finish(runner) self.assertTrue(self.lock_free()) @@ -1031,7 +1036,7 @@ def test_job_class_keys_on_workflow_file_and_job_id(self) -> None: self.assertEqual(hook.job_class("test", home, home, "test-e2e.yml"), ("gui", False)) self.assertEqual(hook.job_class("lint", home, home, "test-e2e.yml"), ("compile", True)) self.assertEqual(hook.job_class("build", "someone/else", home, "test-e2e.yml"), ("isolated", False)) - self.assertEqual(hook.CLASS_COST["compile-gui"], (2, ("gui", "root")), + self.assertEqual(hook.CLASS_COST["compile-gui"], (2, ("root",)), "no persistent-dd: the E2E build never writes compile admission's kept DerivedData") self.assertNotIn("compile-gui", hook.ROOT_CONSUMERS, "a producer takes a token-chosen root") for klass in {*hook.JOB_CLASSES.values(), *hook.WORKFLOW_JOB_CLASSES.values()}: