From 523f855e1cb008abb1c3d6bb46df9fd41ad0122d Mon Sep 17 00:00:00 2001 From: Leo Li Date: Wed, 4 Nov 2026 04:35:53 -0800 Subject: [PATCH] ci(dev publisher): verify takes an idle owned runner before it waits on the fleet The DEV publisher's verify (1.5 min of work) waited a median of 7 min and up to 67 min on glaeda-std-xcode-26.6 behind long fleet jobs (run 41814294095: 0 runners online, 22 jobs queued) while glaeda-root-std-xcode-26.6 runners with no queued job sat idle on another Mac. The pool step now asks e2e_runner_pool.py --free-now first: an owned std label (pool or root label, the lane's Xcode pin) with an online idle runner whose labels have nothing queued in the janitor snapshot. Without one, the existing pick stands. Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-fb75b7af9902231028aa --- .github/workflows/cmux-next-dev-daily.yml | 15 ++++- scripts/ci/e2e_runner_pool.py | 70 +++++++++++++++++++++++ tests/test_ci_pr_runner_pool.py | 66 +++++++++++++++++++++ 3 files changed, 149 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cmux-next-dev-daily.yml b/.github/workflows/cmux-next-dev-daily.yml index 861fa670f8e1..bdc183d66838 100644 --- a/.github/workflows/cmux-next-dev-daily.yml +++ b/.github/workflows/cmux-next-dev-daily.yml @@ -318,6 +318,8 @@ jobs: # tip-dmg-build, also secret-free, takes `dmg_label`: the same owned pool, # but only when it has two machines free now, so it never queues behind # verify on one mini; otherwise it stays on Blacksmith macOS. + # Before any wait, verify takes an owned std label (a pool or its root label) + # with an idle runner no queued job waits for (e2e_runner_pool.py --free-now). # Only the std class (--owned-classes std): both jobs download the 188 MB # signed app, which took 736-974 s on the light pool's minis (24 h to # 2026-10-30), so verify ran 895 s median there against 41 s on std and @@ -476,10 +478,19 @@ jobs: glaeda-*) ;; *) echo "No owned Mac to wait on (picker said '${label:-nothing}'); Blacksmith macOS 26 takes verify."; label="" ;; esac + # verify (1.5 min of work) takes an owned label with an idle runner that no queued job + # waits for, when one exists, instead of the label picked above: on 2026-11-04 it waited + # 67 min on glaeda-std-xcode-26.6 (0 runners online, 22 jobs queued; run 41814294095) + # while glaeda-root-std-xcode-26.6 runners with nothing queued sat idle. Same std class + # (the download-bound rule above) and Xcode pin; tip-dmg-build keeps its own pick. + free="$(pick 1 --free-now)" || free="" + case "$free" in + glaeda-*) echo "verify takes $free: an idle owned runner with no job queued on its labels."; label="$free" ;; + esac echo "label=$label" >> "$GITHUB_OUTPUT" echo "dmg_label=$dmg_label" >> "$GITHUB_OUTPUT" echo "owned=$owned" >> "$GITHUB_OUTPUT" - [ -z "$label" ] || echo "Runner: $label for verify${dmg_label:+ and tip-dmg-build} (a re-run takes Blacksmith macOS 26)" + [ -z "$label" ] || echo "Runner: $label for verify${dmg_label:+,$dmg_label for tip-dmg-build} (a re-run takes Blacksmith macOS 26)" # ci-owned-pool-rescue.yml's sweeper watches a run with these markers and # re-runs verify (and tip-dmg-build) when an owned Mac leaves it queued or refuses it @@ -500,7 +511,7 @@ jobs: continue-on-error: true uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: macos-pool-persistent-${{ github.run_id }}-${{ github.run_attempt }}-${{ steps.pool.outputs.dmg_label != '' && '2' || '1' }}-${{ steps.pool.outputs.label }} + name: macos-pool-persistent-${{ github.run_id }}-${{ github.run_attempt }}-${{ steps.pool.outputs.dmg_label == steps.pool.outputs.label && '2' || '1' }}-${{ steps.pool.outputs.label }} path: ${{ steps.marker.outputs.path }} retention-days: 1 diff --git a/scripts/ci/e2e_runner_pool.py b/scripts/ci/e2e_runner_pool.py index fcbcf768c7c6..5760b17739f5 100644 --- a/scripts/ci/e2e_runner_pool.py +++ b/scripts/ci/e2e_runner_pool.py @@ -614,6 +614,48 @@ def owned_from_slots(env: Mapping[str, str], classes: set[str]) -> tuple[simple_ return tuple(pools) +def free_now(runners: Sequence[Mapping[str, Any]] | None, snapshot: Mapping[str, Any] | None, *, + owned_slots: str | None, xcode_app: str | None, classes: set[str]) -> tuple[str, str]: + """An owned label a one-job run starts on now, and why: (label, reason), the label "" when there is none. + + The candidates are the class pools CI_OWNED_POOL_SLOTS configures for the Xcode pin (and `classes`, when + given), each followed by its root label. A label is free now when one of its runners is online and idle and + the janitor snapshot shows no job queued on any label that runner carries, so no queued job takes it first. + simple_pool_picker.state_from() counts every job queued on a pool's family against each runner of that + family, which reads an idle runner that carries none of the queued labels as taken: on 2026-11-04 the DEV + publisher's verify waited 67 min on glaeda-std-xcode-26.6 (0 runners online, 22 jobs queued; run + 41814294095) while three glaeda-root-std-xcode-26.6 runners on another Mac sat idle with nothing queued + on their labels. Without a runners read or a readable snapshot nothing is free now.""" + if runners is None: + return "", "no runners read" + pools = (snapshot or {}).get("pools") if isinstance(snapshot, Mapping) else None + if not isinstance(pools, Mapping): + return "", "no readable pool snapshot" + + def queued(label: str) -> int: + entry = pools.get(label) + try: + return max(0, int(entry.get("queued", 0))) if isinstance(entry, Mapping) else 0 + except (TypeError, ValueError): + return 0 + + candidates: list[str] = [] + for label in sorted(simple_pool_picker._slots(owned_slots), key=simple_pool_picker.owned_order): + if not simple_pool_picker.owned_matches_xcode(label, xcode_app or ""): + continue + if classes and simple_pool_picker.owned_class(label) not in classes: + continue + candidates += [name for name in (label, pr_runner_pool.root_label(label)) if name and name not in candidates] + for label in candidates: + for runner in runners: + names = [str(item.get("name", "")) for item in runner.get("labels", ()) if isinstance(item, Mapping)] + if runner.get("status") != "online" or runner.get("busy") or label not in names: + continue + if not any(queued(name) for name in names): + return label, f"{runner.get('name') or 'a runner'} is idle with no job queued on its labels" + return "", f"no idle runner on {', '.join(candidates) or 'no configured owned label'} without a queue" + + def main(argv: Sequence[str] | None = None, env: Mapping[str, str] | None = None) -> int: env = os.environ if env is None else env parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) @@ -645,6 +687,10 @@ def main(argv: Sequence[str] | None = None, env: Mapping[str, str] | None = None parser.add_argument("--owned-classes", default="", help="comma-separated owned classes (std, light, xl) an auto pick may name or wait on " "(default: all); cmux-next-dev-daily.yml keeps its download-bound jobs off light") + parser.add_argument("--free-now", action="store_true", + help="print an owned label (a configured class pool for the Xcode pin and --owned-classes, " + "or its root label) with an idle runner no queued job waits for, or nothing; " + "cmux-next-dev-daily.yml's verify takes it before it waits on the fleet") args = parser.parse_args(argv) classes = {name.strip() for name in args.owned_classes.split(",") if name.strip()} unknown = sorted(classes - set(simple_pool_picker.RUN_CLASSES)) @@ -658,6 +704,30 @@ def main(argv: Sequence[str] | None = None, env: Mapping[str, str] | None = None print(fleet[0].label if fleet else "") return 0 + if args.free_now: + label, reason = "", "owned pools are off" + if (args.owned or "").strip() == "1": + repository = env.get("GH_REPO") or env.get("GITHUB_REPOSITORY") or "" + runners = snapshot = None + # The runners need the routing App (ROUTE_TOKEN); the janitor snapshot is a repository artifact + # the job's own token reads. + if (env.get("ROUTE_TOKEN") or "").strip() and repository: + try: + runners = simple_pool_picker.LiveState(env["ROUTE_TOKEN"].strip(), repository).runners() + except Exception as error: # noqa: BLE001 - nothing is free now + print(f"could not list runners ({error})", file=sys.stderr) + token = (env.get("GH_TOKEN") or env.get("ROUTE_TOKEN") or "").strip() + if runners is not None and token: + try: + snapshot = simple_pool_picker.LiveState(token, repository).snapshot() + except Exception as error: # noqa: BLE001 - nothing is free now + print(f"could not read the pool snapshot ({error})", file=sys.stderr) + label, reason = free_now(runners, snapshot, owned_slots=args.owned_slots, + xcode_app=args.pr_xcode_app, classes=classes) + print(f"free now: {label or 'nothing'} ({reason})", file=sys.stderr) + print(label) + return 0 + # Auto choices use the shared live rule. Explicit labels retain the # workflow's direct-request contract and its validation below. # The shared simple picker has no test-filter or console-session diff --git a/tests/test_ci_pr_runner_pool.py b/tests/test_ci_pr_runner_pool.py index 5d0f8da73480..1ebfdcb1fe53 100644 --- a/tests/test_ci_pr_runner_pool.py +++ b/tests/test_ci_pr_runner_pool.py @@ -2183,6 +2183,72 @@ def fallback(*argv): self.assertIn('label="$owned"', step) self.assertIn('echo "owned=$owned" >> "$GITHUB_OUTPUT"', step) + def test_free_now_finds_an_idle_runner_no_queued_job_waits_for(self): + """DEV publisher run 41814294095 (2026-11-04): verify waited 67 min on glaeda-std-xcode-26.6 (0 runners + online, 22 jobs queued) while three glaeda-root-std-xcode-26.6 runners on another Mac, which carry no + label with a queue, sat idle. --free-now names that root label; a runner whose labels have a queue, a + busy or offline runner, another class or Xcode pin, or a missing read names nothing.""" + std, root = "glaeda-std-xcode-26.6", "glaeda-root-std-xcode-26.6" + slots = json.dumps({std: 42, "glaeda-light-xcode-26.6": 4, root: 19, "glaeda-root-light-xcode-26.6": 2, + "glaeda-ios-sim": 8, "glaeda-gui-std-xcode-26.6": 9, "glaeda-std-xcode-26.3": 5}) + + def runner(name, *labels, busy=False, status="online"): + return {"name": name, "status": status, "busy": busy, "labels": [{"name": label} for label in labels]} + + mini = runner("cmux14-glaeda", std, root, "glaeda-ios-sim") + ultra = runner("ultra2-glaeda", root, "glaeda-root-xl-xcode-26.6", "glaeda-xl-xcode-26.6") + light = runner("cmux-austin-mini-0-glaeda-1", "glaeda-light-xcode-26.6") + incident = {"pools": {std: {"queued": 22}, "glaeda-side-std-xcode-26.6": {"queued": 61}}} + + def free(runners, snapshot=incident, classes=frozenset({"std"}), xcode=PR_XCODE): + return e2e_pool.free_now(runners, snapshot, owned_slots=slots, xcode_app=xcode, classes=set(classes))[0] + + self.assertEqual(free([mini, ultra, light]), root) + # The idle mini carries the queued std label: the queue takes it first. + self.assertEqual(free([mini, light]), "") + # With no queue anywhere the pool's own label comes first. + self.assertEqual(free([mini, ultra], snapshot={"pools": {}}), std) + # A queue on any label the idle runner carries (here its xl label) means it is not free. + self.assertEqual(free([ultra], snapshot={"pools": {"glaeda-xl-xcode-26.6": {"queued": 1}}}), "") + self.assertEqual(free([dict(ultra, busy=True)]), "") + self.assertEqual(free([dict(ultra, status="offline")]), "") + # Light is not a class verify takes; another Xcode pin matches nothing. + self.assertEqual(free([light], snapshot={"pools": {}}), "") + self.assertEqual(free([ultra], xcode="/Applications/Xcode_26.3.app"), "") + # No runners read or no snapshot: nothing is free now. + self.assertEqual(free(None), "") + self.assertEqual(free([ultra], snapshot=None), "") + + def main(*argv, env): + with unittest.mock.patch("sys.stdout", io.StringIO()) as output, \ + unittest.mock.patch("sys.stderr", io.StringIO()): + e2e_pool.main(["--free-now", "--owned-slots", slots, "--pr-xcode-app", PR_XCODE, + "--owned-classes", "std", *argv], env=env) + return output.getvalue().strip() + + live = unittest.mock.Mock() + live.runners.return_value = [mini, ultra] + live.snapshot.return_value = incident + env = {"GITHUB_REPOSITORY": "manaflow-ai/cmux", "ROUTE_TOKEN": "route", "GH_TOKEN": "job"} + with unittest.mock.patch.object(e2e_pool.simple_pool_picker, "LiveState", return_value=live) as state: + self.assertEqual(main("--owned", "1", env=env), root) + self.assertEqual([call.args[0] for call in state.call_args_list], ["route", "job"]) + # Owned pools off, or no routing token to list runners with: nothing, and no read. + state.reset_mock() + self.assertEqual(main("--owned", "0", env=env), "") + self.assertEqual(main("--owned", "1", env={**env, "ROUTE_TOKEN": ""}), "") + state.assert_not_called() + live.runners.side_effect = urllib.error.URLError("403") + with unittest.mock.patch.object(e2e_pool.simple_pool_picker, "LiveState", return_value=live): + self.assertEqual(main("--owned", "1", env=env), "") + + dev = (ROOT / ".github/workflows/cmux-next-dev-daily.yml").read_text() + step = dev[dev.index("Pick the macOS pool"):dev.index("Mark a run on a persistent")] + self.assertIn('free="$(pick 1 --free-now)"', step) + self.assertLess(step.index('free="$(pick 1 --free-now)"'), step.index('echo "label=$label"')) + self.assertIn('label="$free"', step) + self.assertNotIn('dmg_label="$free"', step) + def test_ui_auto_route_skips_simple_picker_and_uses_gui_label(self): """A media tour's auto pick must reach the UI-aware E2E picker.""" with unittest.mock.patch.object(