ci: explicit owned E2E runs take root runners; rescue jobs waiting in setup

scripts/ci/e2e_runner_pool.py

@@ -468,6 +468,14 @@ def resolve(
     requested = (requested or "").strip()
     if requested and requested != "auto":
         if requested not in BLACKSMITH_NO_UI:
+            # An owned pool asked for by name still takes its root runners, as auto_runner() does: glaeda gives
+            # the build a canonical root, and only the root runners' gate keeps a root free for what they take.
+            # On the pool label a non-root runner took the build, and E2E builds on two of them held both of a
+            # mini's roots while its root runner's compile admission waited (2026-09-28, cmux10s).
+            root = pr_runner_pool.root_label(requested)
+            if root and pr_runner_pool.slots(owned_slots, pr_xcode_app).get(root, 0) > 0:
+                log(f"{requested} -> {root} (an E2E build takes a canonical root)")
+                return root
             return requested
         return ui_owned_runner(requested, test_filter=test_filter, owned=owned, owned_ui=owned_ui, order=order,
                                owned_slots=owned_slots, pr_xcode_app=pr_xcode_app, log=log) or requested

scripts/ci/owned_pool_rescue.py

@@ -53,6 +53,14 @@
 Xcode 26.6 build 17F113. If those builds ever differ, re-run the whole run
 here instead (rescue with failed_only=False).
 
+glaeda's hook does not refuse a job for the mini's capacity: it waits, with
+no limit, inside the runner's setup until the units and tokens the job needs
+are free. A job on the persistent pool still in its runner's setup
+SETUP_WAIT_SECONDS after it started is stuck like a queued one: once no
+sibling is still running (or the watch is about to end), the run is cancelled
+and re-run the same way. Until then the job is not accepted, so the watch goes
+on.
+
 A refused job never goes back to the fleet: this rescue re-runs as
 github-actions[bot], and every runs-on sends the bot's re-run to retry_runner
 (Blacksmith), so it cannot land on the mini that refused it, and one re-run
@@ -310,10 +318,9 @@
 RERUN_MARGIN_SECONDS = 60
 # A refused job fails within the runner's setup; a real failure of the first
 # step after checkout takes longer than this, and one that does not is cheap to
-# retry. glaeda's hook may wait up to 240 s inside that setup for the mini's one
-# gui token (GUI_WAIT_S) before refusing, and an app-host shard waiting there
-# costs far less than a refusal's rescue round trip and a Blacksmith re-run
-# (cmuxterm-hq#661 Workstream 7), so the window covers that wait with room.
+# retry. glaeda's hook once waited up to 240 s inside that setup for the mini's
+# capacity before refusing; it now waits with no limit (SETUP_WAIT_SECONDS
+# covers that), so a refusal is the hook's other checks, which answer at once.
 REFUSAL_SECONDS = 360
 # The last attempt of the bot's own re-runs that may run on an owned pool: a
 # stuck run's full re-run on the light tier (CI_OWNED_LIGHT_RETRY). A person's
@@ -332,6 +339,13 @@ def person_rerun(run: Mapping[str, Any]) -> bool:
             and str((run.get("triggering_actor") or {}).get("login") or "") != RESCUE_ACTOR)
 # The runner's own steps, which run before glaeda's hook decides.
 SETUP_STEPS = frozenset({"Set up job", "Set up runner"})
+# glaeda's hook no longer refuses a job for the mini's capacity: it waits,
+# with no limit, inside the runner's setup ("Set up runner") until the units
+# and tokens it needs are free (glaeda CAPACITY_WAIT, 2026-09-28). A job still
+# there this long after it started is stuck like a queued one and is moved the
+# same way (Look "rescue"). A compile waiting for a canonical root normally
+# waits for one running compile, about 7 minutes.
+SETUP_WAIT_SECONDS = 15 * 60
 MAX_JOB_PAGES = 3
 # Main's full-suite dispatch (ci-main-full-suite.yml) runs ci.yml on this branch.
 MAIN_BRANCH = "main"
@@ -382,6 +396,23 @@ def waiting_for_runner(job: Mapping[str, Any]) -> bool:
     return job.get("status") == "queued" and not job.get("runner_name")
 
 
+def in_setup(job: Mapping[str, Any]) -> bool:
+    """A job its runner took that is still in the runner's setup, where glaeda's hook waits for capacity."""
+    if job.get("status") != "in_progress":
+        return False
+    steps = [step for step in job.get("steps") or [] if isinstance(step, Mapping)]
+    return any(step.get("name") in SETUP_STEPS and step.get("status") == "in_progress" for step in steps)
+
+
+def setup_seconds(job: Mapping[str, Any], now: dt.datetime) -> float:
+    """How long the job has been in its current setup step (glaeda's hook runs in "Set up runner")."""
+    steps = [step for step in job.get("steps") or [] if isinstance(step, Mapping)]
+    step = next((step for step in steps if step.get("name") in SETUP_STEPS and step.get("status") == "in_progress"),
+                None)
+    started = parse_time((step or {}).get("started_at")) or parse_time(job.get("started_at"))
+    return 0.0 if started is None else max(0.0, (now - started).total_seconds())
+
+
 def wait_start(job: Mapping[str, Any], first_seen: dt.datetime | None = None) -> dt.datetime | None:
     created = parse_time(job.get("created_at"))
     return max(filter(None, (created, first_seen)), default=None)
@@ -428,11 +459,12 @@ def refused(job: Mapping[str, Any]) -> bool:
 
 
 def accepted(job: Mapping[str, Any], now: dt.datetime) -> bool:
-    """An owned job its runner took and has not refused: started over REFUSAL_SECONDS ago, or done."""
+    """An owned job its runner took and has not refused: past its setup and started over REFUSAL_SECONDS ago,
+    or done."""
     if job.get("status") == "completed":
         return not refused(job)
     started = parse_time(job.get("started_at"))
-    return job.get("status") == "in_progress" and started is not None and \
+    return job.get("status") == "in_progress" and not in_setup(job) and started is not None and \
         (now - started).total_seconds() > REFUSAL_SECONDS
 
 
@@ -452,6 +484,16 @@ class Look:
     waiting: bool = False  # a persistent-pool job has no runner yet
 
 
+def setup_budget(job: Mapping[str, Any], now: dt.datetime, budget_seconds: int,
+                 deadline: dt.datetime | None) -> float:
+    """SETUP_WAIT_SECONDS, cut so a job that entered setup late is still judged before the watch ends (as
+    job_budget() cuts a queued job's), and never below the queued budget."""
+    if deadline is None:
+        return SETUP_WAIT_SECONDS
+    left = setup_seconds(job, now) + (deadline - now).total_seconds() - END_MARGIN_SECONDS
+    return min(SETUP_WAIT_SECONDS, max(budget_seconds, left))
+
+
 def assess(jobs: Sequence[Mapping[str, Any]], *, now: dt.datetime, budget_seconds: int,
            first_seen: Mapping[Any, dt.datetime] | None = None, deadline: dt.datetime | None = None,
            floor_seconds: int | None = None) -> Look:
@@ -465,12 +507,24 @@ def assess(jobs: Sequence[Mapping[str, Any]], *, now: dt.datetime, budget_second
         names = ", ".join(sorted(str(job.get("name") or job.get("id")) for job in stuck))
         return Look("rescue", f"{names} queued on {job_pool(stuck[0])} for at least "
                               f"{min(budgets[id(job)] for job in stuck)}s with no runner")
+    settling = [job for job in jobs if job_pool(job) and in_setup(job)]
+    held = [job for job in settling if setup_seconds(job, now) >= setup_budget(job, now, budget_seconds, deadline)]
+    # Cancelling the run would kill siblings still running (run 36198335113 lost five
+    # shards that way to a refusal), so a job held in setup waits for them, as a
+    # refusal does, until the watch is about to end.
+    running = [job for job in jobs if job.get("status") == "in_progress" and not in_setup(job)]
+    closing = deadline is not None and now >= deadline - dt.timedelta(seconds=END_MARGIN_SECONDS)
+    if held and (not running or closing):
+        names = ", ".join(sorted(str(job.get("name") or job.get("id")) for job in held))
+        return Look("rescue", f"{names} waited in {job_pool(held[0])}'s runner setup for capacity for at least "
+                              f"{SETUP_WAIT_SECONDS}s")
     turned_away = [job for job in jobs if refused(job)]
     if turned_away:
         names = ", ".join(sorted(str(job.get("name") or job.get("id")) for job in turned_away))
         return Look("refused", f"{names} refused by {job_pool(turned_away[0])} at job start")
-    if waiting:
-        return Look("watch", f"{len(waiting)} job(s) waiting for a persistent runner", waiting=True)
+    if waiting or settling:
+        return Look("watch", f"{len(waiting)} job(s) waiting for a persistent runner, {len(settling)} in its setup",
+                    waiting=True)
     return Look("watch", "no job is waiting for a persistent runner")
 
 

tests/test_ci_owned_pool_rescue.py

@@ -188,6 +188,54 @@ def jobs(seconds):
     return jobs
 
 
+def setup_job(*, started=41, past_setup=False):
+    """An owned job its runner took, still in glaeda's hook ("Set up runner") unless past_setup."""
+    found = job("macos / macOS compile admission", status="in_progress", labels=[MINI], created=40, runner="mini-1")
+    found.update(started_at=stamp(started), steps=[
+        {"name": "Set up job", "status": "completed", "conclusion": "success"},
+        {"name": "Set up runner", "status": "completed" if past_setup else "in_progress", "conclusion": None,
+         "started_at": stamp(started + 3)},
+        {"name": "Checkout", "status": "in_progress" if past_setup else "queued", "conclusion": None}])
+    return found
+
+
+class SetupWait(unittest.TestCase):
+    def test_a_job_waiting_in_setup_is_watched_then_rescued(self):
+        waiting = setup_job()
+        self.assertTrue(rescue.in_setup(waiting))
+        self.assertFalse(rescue.in_setup(setup_job(past_setup=True)))
+        early = START + dt.timedelta(seconds=41 + rescue.REFUSAL_SECONDS + 60)
+        self.assertFalse(rescue.accepted(waiting, early), "a job in setup has not been accepted yet")
+        self.assertTrue(rescue.accepted(setup_job(past_setup=True), early))
+        look = rescue.assess([changes()(60), waiting], now=early, budget_seconds=90)
+        self.assertEqual((look.action, look.waiting), ("watch", True))
+        # measured from the setup step the hook waits in, not from the job's start
+        self.assertEqual(rescue.assess([changes()(60), waiting], budget_seconds=90,
+                                       now=START + dt.timedelta(seconds=41 + rescue.SETUP_WAIT_SECONDS)).action,
+                         "watch")
+        late = START + dt.timedelta(seconds=44 + rescue.SETUP_WAIT_SECONDS)
+        look = rescue.assess([changes()(60), waiting], now=late, budget_seconds=90)
+        self.assertEqual(look.action, "rescue")
+        self.assertIn("runner setup", look.reason)
+        self.assertEqual(rescue.assess([changes()(60), setup_job(past_setup=True)], now=late,
+                                       budget_seconds=90).action, "watch")
+        # a job that entered setup late is judged before the watch ends, but not before the queued budget
+        soon = START + dt.timedelta(seconds=44 + 300)
+        self.assertEqual(rescue.assess([changes()(60), waiting], now=soon, budget_seconds=90,
+                                       deadline=soon + dt.timedelta(seconds=rescue.END_MARGIN_SECONDS)).action,
+                         "rescue")
+        early_close = START + dt.timedelta(seconds=44 + 30)
+        self.assertEqual(rescue.assess([changes()(60), waiting], now=early_close, budget_seconds=90,
+                                       deadline=early_close).action, "watch")
+        # a sibling still running is not cancelled for it, until the watch is about to end
+        shard = job("macos / shard", status="in_progress", labels=[MINI], runner="mini-2")
+        look = rescue.assess([changes()(60), waiting, shard], now=late, budget_seconds=90)
+        self.assertEqual((look.action, look.waiting), ("watch", True))
+        look = rescue.assess([changes()(60), waiting, shard], now=late, budget_seconds=90,
+                             deadline=late + dt.timedelta(seconds=rescue.END_MARGIN_SECONDS - 1))
+        self.assertEqual(look.action, "rescue")
+
+
 class Refusal(unittest.TestCase):
     def test_what_counts_as_a_refusal(self):
         self.assertTrue(rescue.refused(refused_job()))

tests/test_run_e2e.py

@@ -1406,6 +1406,19 @@ def test_an_explicit_choice_or_admin_variable_is_never_rerouted(self):
         label, calls, _ = self.decide(queue(), variable=OLD)
         self.assertEqual((label, calls), (OLD, []))
 
+    def test_an_explicit_owned_pool_takes_its_root_runners(self):
+        # glaeda gives an E2E build a canonical root: on the pool label a non-root runner took it, and two such
+        # builds held both of a mini's roots while its root runner's compile admission waited (2026-09-28)
+        root = "glaeda-root-std-xcode-26.6"
+        for slots, want in (({MINI: 8, root: 4}, root), ({MINI: 8}, MINI), ({MINI: 8, root: 0}, MINI)):
+            with self.subTest(slots=slots):
+                label = self.pool.resolve(MINI, "", overflow="", order="", max_queued="", measure=lambda: None,
+                                          now=NOW, owned_slots=json.dumps(slots),
+                                          pr_xcode_app="/Applications/Xcode_26.6.app")
+                self.assertEqual(label, want)
+        self.assertEqual(self.pool.resolve(root, "", overflow="", order="", max_queued="", measure=lambda: None,
+                                           now=NOW, owned_slots=json.dumps({root: 4})), root)
+
     def test_the_commit_does_not_decide(self):
         for commit in self.COMMITS:
             with self.subTest(commit=commit):