From 73f13d939539cd4214ed2e5a1780669e7951a5c4 Mon Sep 17 00:00:00 2001 From: teamleaderleo Date: Tue, 29 Sep 2026 06:37:47 -0400 Subject: [PATCH] ci: live runners decide root, gui and side routing; CI_OWNED_POOL_SLOTS is the fallback only With the runners API read, the picker already took each owned label's capacity from its online runners, but CI_OWNED_POOL_SLOTS still decided whether a pool routed its root jobs to the root label, its GUI jobs to the gui label and its side lanes to the side/light side labels. A hand-set count could keep jobs off labels every mini carries or send them to a label no runner online carries. routing_slots() returns the online runners per owned label (pool, root, gui) when the runners were read, and the variable only when they could not be. late_placement.py routes GUI jobs to the gui label while any runner carries it (offline included, so a drained mini never sends them to the root label). Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/ci-runners.md | 12 ++++--- scripts/ci/late_placement.py | 12 ++++--- scripts/ci/pr_runner_pool.py | 55 +++++++++++++++++++++++---------- tests/test_ci_late_placement.py | 18 +++++++---- tests/test_ci_pr_runner_pool.py | 33 +++++++++++++++----- 5 files changed, 91 insertions(+), 39 deletions(-) diff --git a/docs/ci-runners.md b/docs/ci-runners.md index 28aa865b938e..f5c9f9de310d 100644 --- a/docs/ci-runners.md +++ b/docs/ci-runners.md @@ -225,7 +225,7 @@ names no owned pool. | `CI_PR_POOL_OWNED` | unset (off) | `1` puts owned pools first and turns on the rescue below | | `CI_OWNED_POOL_SLOTS` | unset (no slots) | JSON, owned pool label to machine count, the `conforming_count` from `glaeda-mini-fleet pools --json`: `{"glaeda-std-xcode-26.6": 12, "glaeda-light-xcode-26.6": 2}`. A class (`{"std": 12, "light": 2}`) or a bare count (`12`, the std class) means that class at the lane's Xcode pin | | `CI_OWNED_MAIN_RESERVE` | `0` | machines, and root runners, main's full-suite dispatch leaves free for pull requests; above 0 it takes an owned pool only whole (below) | -| `GLAEDA_ROUTE_APP_ID` + secret `GLAEDA_ROUTE_APP_KEY` | unset (snapshot only) | the org's `manaflow-glaeda-route` App. `ci.yml`'s `changes` job mints a token with `administration: read` for same-repository pull requests and main's full-suite dispatch only, on its ephemeral Linux runner, and the picker lists the repository's runners: the online runners carrying an owned label are that pool's capacity, and the idle ones its free runners, less what runs of the last `LIVE_WINDOW_MINUTES` took. That replaces the counts of `CI_OWNED_POOL_SLOTS` (which still turns a pool's root routing on) and the snapshot's owned counts and age. Any failure falls back to them | +| `GLAEDA_ROUTE_APP_ID` + secret `GLAEDA_ROUTE_APP_KEY` | unset (snapshot only) | the org's `manaflow-glaeda-route` App. `ci.yml`'s `changes` job mints a token with `administration: read` for same-repository pull requests and main's full-suite dispatch only, on its ephemeral Linux runner, and the picker lists the repository's runners: the online runners carrying an owned label are that pool's capacity, and the idle ones its free runners, less what runs of the last `LIVE_WINDOW_MINUTES` took. That replaces `CI_OWNED_POOL_SLOTS` and the snapshot's owned counts and age: capacity, and which labels route (a pool's root, gui and side labels route while an online runner carries them, `routing_slots()`). Any failure falls back to them | | `CI_OWNED_LIGHT_RETRY` | unset (off) | `1` lets attempt 2, the full re-run the rescue starts for a job stuck on a full `std` pool, take the `light` pool when the run's whole owned peak is free there and `github-actions[bot]` started the re-run (a person's re-run of attempt 2 stays on Blacksmith). The rescue watches that attempt like attempt 1, and a job stuck or refused there goes to Blacksmith on attempt 3. Only while it is on do the janitor and the picker look up attempt 2's marker. Order: std, light, Blacksmith | Main's full suite: `ci-main-full-suite.yml` dispatches `ci.yml` on main about @@ -259,8 +259,9 @@ A class has `canonicalRoots` roots per mini (two on a std mini, root-1 and root-2), and a compile takes any free one. The first `canonicalRoots` runners of each mini are its root runners and carry `glaeda-root--xcode-`; the others are its side runners and -carry `glaeda-side--xcode-`. A root count in -`CI_OWNED_POOL_SLOTS` (`"root-std": 10`, or the full root label) sends those +carry `glaeda-side--xcode-`. An online runner carrying the +root label (or, when the runners cannot be read, a root count in +`CI_OWNED_POOL_SLOTS`: `"root-std": 10`, or the full root label) sends those jobs to the root label, where they wait for a free root instead of being refused, and the picker places no more of them than the root runners free. The janitor counts a root job toward the root label and its pool. Without a @@ -505,8 +506,9 @@ pick is an owned pool, one side lane per light side runner (`macos_pr_light_side_runner`, for the lanes in `macos_pr_light_side_jobs`), and the picked pool counts the rest beside admission and what follows it (`pr_runner_pool.light_side_lanes()`). The other lanes take the picked -pool's side label as before. Giving the light pool no machines beyond its -root runners in `CI_OWNED_POOL_SLOTS` turns this off. +pool's side label as before. It reads the runners live, so a light pool +with no side runner idle takes none; `CI_OWNED_POOL_SLOTS` no longer turns it +off. | Variable | Default | Effect | | --- | --- | --- | diff --git a/scripts/ci/late_placement.py b/scripts/ci/late_placement.py index aadb301bfdae..d6989c9c394e 100644 --- a/scripts/ci/late_placement.py +++ b/scripts/ci/late_placement.py @@ -16,8 +16,8 @@ test-without-building on the mini against admission's uploaded products, as they do after an owned admission; they never compile. -When OWNED_SLOTS (vars.CI_OWNED_POOL_SLOTS) gives the pool's gui label a count -(pr_runner_pool.gui_label(): one gui runner per mini), the jobs that hold the +When a runner carries the pool's gui label (pr_runner_pool.gui_label(): one +gui runner per mini), the jobs that hold the gui token (pr_runner_pool.gui_token_job(): the shards, tests-build-and-lag, cli-product-tests) take that label instead, one per idle gui runner, and the other jobs the root label, one per idle root runner: each mini runs one GUI job at a time. @@ -208,10 +208,12 @@ def decide(env: Mapping[str, str], runners: Sequence[Mapping[str, Any]] | None, return {}, f"no owned pool runs admission's Xcode ({env.get('ADMISSION_XCODE_APP') or 'unknown'})" if runners is None: return {}, "owned runners could not be read live" - # From the slots, not the picker's gui_runner: a run the picker sent to Blacksmith has none, - # and its GUI jobs must still never take the root label once the minis have gui runners. + # From the runners, not the picker's gui_runner: a run the picker sent to Blacksmith has none, + # and its GUI jobs must still never take the root label once the minis have gui runners. Any + # runner carrying the label counts, online or not (drained minis' runners go offline), so the + # jobs never fall back to the root label; CI_OWNED_POOL_SLOTS no longer decides it. gui_label = pool.gui_label(pool.pool_label(root)) - if pool.slots(env.get("OWNED_SLOTS"), env.get("ADMISSION_XCODE_APP")).get(gui_label, 0) <= 0: + if not any(gui_label in pool.runner_labels(runner) for runner in runners): gui_label = "" free = pool.live_owned_free(runners, [root, *([gui_label] if gui_label else [])]) idle, gui_idle = free[root], free.get(gui_label, 0) diff --git a/scripts/ci/pr_runner_pool.py b/scripts/ci/pr_runner_pool.py index f66f154273c9..a53c758bfd13 100644 --- a/scripts/ci/pr_runner_pool.py +++ b/scripts/ci/pr_runner_pool.py @@ -534,8 +534,9 @@ def gui_runner(choice: "Choice", owned_slots: Mapping[str, int]) -> str: (guiRunners) carrying `glaeda-gui--xcode-`, whose listener stops while the gui token or every root is taken, so a GUI job waits in GitHub's queue for a mini that can run it. Only on a pool with a root - count, and only while CI_OWNED_POOL_SLOTS gives the gui label a count, - so a GUI job never waits on a label no runner carries. + count, and only while an online runner carries the gui label (routing_slots(): the + variable only when the runners cannot be read), so a GUI job never waits on a label no + runner carries. """ if not choice.root_runner or not persistent(choice.runner): return "" @@ -547,8 +548,9 @@ def side_runner(choice: "Choice", owned_slots: Mapping[str, int]) -> str: """The label a pick's side lanes take: the pool's side label, or "" to keep the pool label. Only on a pool with a root count (the root and side runners are split), - and only while CI_OWNED_POOL_SLOTS leaves it machines beyond its root - runners, so a side lane never waits on a label no runner carries. + and only while the pool has machines beyond its root runners (routing_slots(): its + online runners, or CI_OWNED_POOL_SLOTS when they cannot be read), so a side lane never + waits on a label no runner carries. """ if not choice.root_runner or not persistent(choice.runner): return "" @@ -562,9 +564,9 @@ def light_side_lanes(plan: "RunJobs", runners: Sequence[Mapping[str, Any]], owne """The light pool's side label and the side lanes of `plan` its idle side runners take now, one per runner. release-build (RELEASE_BUILD_JOB), a universal Release compile, is never - one of them. ("", ()) when none is idle, and always while CI_OWNED_POOL_SLOTS gives + one of them. ("", ()) when none is idle, and always while `owned_slots` (routing_slots()) gives the light pool no machines beyond its root runners (side_runner()'s - rule), so removing that count turns it off. + rule). """ light = next((label for label in owned_pools(pr_xcode_app) if label.startswith(f"glaeda-{LIGHT_CLASS}-")), "") label = side_label(light) @@ -913,6 +915,24 @@ def slots(raw: str | None, pr_xcode_app: str | None = None) -> dict[str, int]: return _slots(raw, pr_xcode_app)[0] +def routing_slots(raw: str | None, pr_xcode_app: str | None, + runners: Sequence[Mapping[str, Any]] | None) -> dict[str, int]: + """The owned labels that route, with their machines: the online runners carrying each when the runners + were read (`runners`), else CI_OWNED_POOL_SLOTS (slots()). + + The variable used to decide whether a pool routes its root jobs to the root label, its GUI jobs to the + gui label and its side lanes to the side label even when the runners API gave the live answer, so a + hand-set count could route jobs to a label no runner online carries, or keep them off one that every + mini carries. With the runners read, a label routes while one online runner carries it; the variable + is only the fallback when they cannot be read (the capacity counts in live_pools() already were). + """ + if runners is None: + return slots(raw, pr_xcode_app) + labels = [label for pool_name in owned_pools(pr_xcode_app) + for label in (pool_name, root_label(pool_name), gui_label(pool_name))] + return {label: count for label, count in live_online(runners, labels).items() if count > 0} + + def capability_slots(raw: str | None) -> dict[str, int]: """CI_OWNED_POOL_SLOTS: capability label -> machines carrying it (`{"glaeda-ios-sim": 2}`).""" try: @@ -1335,8 +1355,9 @@ def live_pools(snapshot: Mapping[str, Any], idle: Mapping[str, int], slot_counts what holds the label now plus what choose() passes: the peaks of runs younger than DEFAULT_JOB_MINUTES, less what they already hold. - A root label counts only while CI_OWNED_POOL_SLOTS gives it a root count, - which is what turns root routing on (root_label()). + A root label counts only while `slot_counts` (routing_slots(): the online + runners carrying it, or CI_OWNED_POOL_SLOTS when they cannot be read) has + it, which is what turns root routing on (root_label()). """ pools = dict(snapshot.get("pools") or {}) capacity: dict[str, int] = {} @@ -2296,10 +2317,6 @@ def count_routed(since: str) -> int: # jobs at their peak. gui = (env.get("POOL_OWNED_GUI") or "").strip() != "0" jobs = owned_peak(plan, gui) - # The slots name gui runners (gui_runner()): the GUI jobs then hold no root runner. The pool is not - # picked yet, so any gui count counts here; place() below checks the picked pool's own. - gui_runners = any(label.startswith(GUI_PREFIX) - for label in slots(env.get("OWNED_SLOTS"), env.get(PR_XCODE_VARIABLE))) # The org App's token (ci.yml mints it for same-repository pull requests # only) reads which owned runners are idle now. Without it, or on any # error, the slot counts and the snapshot decide as before. @@ -2317,12 +2334,18 @@ def count_routed(since: str) -> int: except Exception as error: # noqa: BLE001 - the snapshot path still decides print(f"::warning title=live owned capacity::could not list runners ({error}); using the snapshot") live_owned = online = live_runners = None + # Which owned labels route, and their machines: the online runners when they were read, the + # variable only when they could not be (routing_slots()). + routing = routing_slots(env.get("OWNED_SLOTS"), env.get(PR_XCODE_VARIABLE), live_runners) + routing_raw = env.get("OWNED_SLOTS") if live_runners is None else json.dumps(routing) + # Gui runners route (gui_runner()): the GUI jobs then hold no root runner. The pool is not + # picked yet, so any gui label counts here; place() below checks the picked pool's own. + gui_runners = any(label.startswith(GUI_PREFIX) for label in routing) # As many side lanes as the light minis' side runners idle now (light_side_lanes()) take them: the pool # picked below then holds admission, what follows it and the other side lanes. light_side, side_lanes = "", () if live_runners is not None and attempt in ("", "1") and event == "pull_request" and env.get("HEAD_REPO") == repo: - light_side, side_lanes = light_side_lanes( - plan, live_runners, slots(env.get("OWNED_SLOTS"), env.get(PR_XCODE_VARIABLE)), env.get(PR_XCODE_VARIABLE)) + light_side, side_lanes = light_side_lanes(plan, live_runners, routing, env.get(PR_XCODE_VARIABLE)) if side_lanes: plan = dataclasses.replace(plan, side=tuple(key for key in plan.side if key not in side_lanes)) jobs = owned_peak(plan, gui) @@ -2341,7 +2364,7 @@ def count_routed(since: str) -> int: order=env.get("POOL_ORDER"), max_queued=env.get("POOL_MAX_QUEUED"), owned=env.get("POOL_OWNED"), - owned_slots=env.get("OWNED_SLOTS"), + owned_slots=routing_raw, jobs=jobs, split=env.get("POOL_OWNED_SPLIT"), root_jobs=root_peak(plan, gui, gui_runners), @@ -2371,7 +2394,7 @@ def count_routed(since: str) -> int: print(f"::error title={SLOTS_VARIABLE}::{problem}") # A persistent pick names the jobs that take it; every other job of the # run takes retry_runner. The marker's jobs are the owned machines held. - owned_slots = slots(env.get("OWNED_SLOTS"), pr_xcode_app) + owned_slots = routing gui_label_out = gui_runner(choice, owned_slots) if choice.runner.startswith(f"glaeda-{LIGHT_CLASS}-"): # The light pool's own pick places no universal Release compile; it keeps MACOS_RUNNER_26. diff --git a/tests/test_ci_late_placement.py b/tests/test_ci_late_placement.py index 3f770fe7877e..603ab67db586 100644 --- a/tests/test_ci_late_placement.py +++ b/tests/test_ci_late_placement.py @@ -65,13 +65,18 @@ def test_gui_jobs_take_idle_gui_runners_and_the_rest_idle_roots(self): # cli-product-tests holds the gui token too, so it queues behind the shards for a gui runner. self.assertEqual(placed, {"shard-1": gui, "shard-2": gui}) self.assertIn(f"2 idle `{gui}`", why) - # No gui count yet: the GUI jobs take the root label as before. + # The runners decide, not CI_OWNED_POOL_SLOTS: without a gui count the gui runners still route. self.assertEqual(late.decide(dict(FULL, OWNED_SLOTS='{"std": 40, "root-std": 19}'), runners)[0], - {"shard-1": ROOT_STD, "shard-2": ROOT_STD, "shard-3": ROOT_STD}) - self.assertEqual(late.decide(FULL, runners)[0], + {"shard-1": gui, "shard-2": gui}) + self.assertEqual(late.decide(FULL, runners)[0], {"shard-1": gui, "shard-2": gui}) + # No runner carries the gui label: the GUI jobs take the root label as before, whatever the slots. + self.assertEqual(late.decide(dict(FULL, OWNED_SLOTS=slots), roots(idle=3))[0], {"shard-1": ROOT_STD, "shard-2": ROOT_STD, "shard-3": ROOT_STD}) # No idle gui runner: the gui-token jobs stay where the picker put them. - self.assertEqual(late.decide(dict(FULL, OWNED_SLOTS=slots), roots(idle=3))[0], {}) + self.assertEqual(late.decide(dict(FULL, OWNED_SLOTS=slots), + [*roots(idle=3), runner("gui-busy", gui, busy=True)])[0], {}) + # An offline gui runner (a drained mini) still keeps them off the root label. + self.assertEqual(late.decide(FULL, [*roots(idle=3), runner("gui-off", gui, status="offline")])[0], {}) # Enough gui runners: cli-product-tests takes one, never the root label. many = [*roots(idle=3), *(runner(f"gui-{i}", "self-hosted", gui) for i in range(10))] self.assertEqual(late.decide(dict(FULL, OWNED_SLOTS=slots), many)[0]["cli-product"], gui) @@ -162,7 +167,8 @@ def test_an_empty_blacksmith_pool_takes_its_machines_at_once(self): def test_no_gui_runner_online_moves_every_owned_gui_job_without_a_read(self): count, calls = self.backlog(queued=0, retry_queued=99) - placed, _ = late.decide(OWNED, roots(idle=2), count) + offline = [runner(f"gui-off-{i}", GUI, status="offline") for i in range(10)] + placed, _ = late.decide(OWNED, [*roots(idle=2), *offline], count) self.assertEqual(set(placed.values()), {RETRY}) self.assertEqual(len(placed), 9) self.assertEqual(calls, []) @@ -224,7 +230,7 @@ def test_gui_off_or_no_gui_label_moves_no_owned_job(self): count, _ = self.backlog(queued=40) busy = [*roots(idle=0, busy=16), *guis(idle=0, busy=10)] self.assertEqual(late.decide(dict(OWNED, POOL_OWNED_GUI="0"), busy, count)[0], {}) - self.assertEqual(late.decide(dict(OWNED, OWNED_SLOTS='{"std": 40, "root-std": 19}'), busy, count)[0], {}) + self.assertEqual(late.decide(OWNED, roots(idle=0, busy=16), count)[0], {}) def test_owned_jobs_that_stay_take_the_idle_gui_runners_before_unowned_ones(self): count, _ = self.backlog(queued=0) diff --git a/tests/test_ci_pr_runner_pool.py b/tests/test_ci_pr_runner_pool.py index e02b35ed1a7d..c6cc161a11e3 100644 --- a/tests/test_ci_pr_runner_pool.py +++ b/tests/test_ci_pr_runner_pool.py @@ -2055,12 +2055,16 @@ def test_side_lanes_take_the_light_side_runners_when_enough_are_idle(self): self.assertIn(key, values["owned_jobs"]) self.assertEqual(values["jobs"], "2") self.assertIn("claude-wrapper take `" + light_side + "`", values["summary"]) - # None idle, or no light side count: the std side label as before. - for runners, count in ((std, slots), (std + idle, '{"std": 40, "root-std": 10, "light": 2, "root-light": 2}')): - values = self.outputs(runners, slots=count, extra=lanes) - self.assertEqual((values["runner"], values["side_runner"], values["light_side_jobs"]), - (MINI, SIDE_MINI, ""), runners) - self.assertIn(" claude-wrapper ", values["owned_jobs"]) + # No light side runner online: the std side label as before. + values = self.outputs(std, slots=slots, extra=lanes) + self.assertEqual((values["runner"], values["side_runner"], values["light_side_jobs"]), + (MINI, SIDE_MINI, "")) + self.assertIn(" claude-wrapper ", values["owned_jobs"]) + # The runners read live decide, not CI_OWNED_POOL_SLOTS: a variable that gives the light pool no + # machines beyond its root runners no longer keeps the lanes off the idle light side runners. + values = self.outputs(std + idle, slots='{"std": 40, "root-std": 10, "light": 2, "root-light": 2}', + extra=lanes) + self.assertEqual(values["light_side_jobs"], " claude-wrapper remote-daemon ") # A retry attempt keeps its own route. self.assertEqual(self.outputs(std + idle, slots=slots, attempt="2", extra=lanes)["light_side_jobs"], "") @@ -2125,7 +2129,10 @@ def test_main_names_the_warm_runner_for_admission(self): self.assertEqual(busy_values["admission_runner"], "") # admission-placement still gets the warm names; it re-reads which are idle. self.assertEqual(json.loads(busy_values["admission_warm"]), [["cmux2-glaeda"]]) - self.assertEqual(self.outputs(runners, slots='{"std": 40}')["admission_runner"], "") + # No root runner online carries the root label: no root routing, whatever the variable says. + self.assertEqual(self.outputs([live_runner(1, MINI), live_runner(2, MINI)])["admission_runner"], "") + # The runners read live turn root routing on without a root count in CI_OWNED_POOL_SLOTS. + self.assertEqual(json.loads(self.outputs(runners, slots='{"std": 40}')["admission_runner"]), [ROOT_MINI, own]) # A snapshot without `warm` (the janitor's sweep off or failed). self.assertEqual(self.outputs(runners, state={})["admission_warm"], "") # Without the route token the runners are never read. @@ -2324,6 +2331,18 @@ def test_online_runners_count_per_label(self): mini_runner("mini-b", 0, MINI, ROOT_MINI, status="offline"), mini_runner("mini-c", 0, LIGHT)] self.assertEqual(pool.live_online(runners, (MINI, ROOT_MINI, LIGHT)), {MINI: 2, ROOT_MINI: 1, LIGHT: 1}) + def test_routing_labels_come_from_the_online_runners_and_the_variable_only_without_them(self): + gui = pool.gui_label(MINI) + runners = [mini_runner("mini-a", 0, MINI, ROOT_MINI), mini_runner("mini-a", 1, gui), + mini_runner("mini-b", 0, MINI, ROOT_MINI, status="offline"), mini_runner("mini-c", 0, MINI)] + # Live: a label routes while an online runner carries it; the variable's counts and omissions do not count. + self.assertEqual(pool.routing_slots('{"std": 40}', PR_XCODE, runners), {MINI: 2, ROOT_MINI: 1, gui: 1}) + self.assertEqual(pool.routing_slots('{"std": 40, "root-std": 19, "gui-std": 10}', PR_XCODE, + [mini_runner("mini-c", 0, MINI)]), {MINI: 1}) + # Unreadable runners: the variable, as before. + self.assertEqual(pool.routing_slots('{"std": 40, "root-std": 19}', PR_XCODE, None), + {MINI: 40, ROOT_MINI: 19}) + def test_capacity_is_the_online_count_when_listed(self): snap = fleet(busy=0) slot_counts = {MINI: 40, ROOT_MINI: 18}