Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions scripts/ci/e2e_runner_pool.py
Original file line number Diff line number Diff line change
Expand Up @@ -468,6 +468,14 @@ def resolve(
requested = (requested or "").strip()
if requested and requested != "auto":
if requested not in BLACKSMITH_NO_UI:
# An owned pool asked for by name still takes its root runners, as auto_runner() does: glaeda gives
# the build a canonical root, and only the root runners' gate keeps a root free for what they take.
# On the pool label a non-root runner took the build, and E2E builds on two of them held both of a
# mini's roots while its root runner's compile admission waited (2026-09-28, cmux10s).
root = pr_runner_pool.root_label(requested)
if root and pr_runner_pool.slots(owned_slots, pr_xcode_app).get(root, 0) > 0:
log(f"{requested} -> {root} (an E2E build takes a canonical root)")
return root
return requested
return ui_owned_runner(requested, test_filter=test_filter, owned=owned, owned_ui=owned_ui, order=order,
owned_slots=owned_slots, pr_xcode_app=pr_xcode_app, log=log) or requested
Expand Down
50 changes: 42 additions & 8 deletions scripts/ci/owned_pool_rescue.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,13 @@
Xcode 26.6 build 17F113. If those builds ever differ, re-run the whole run
here instead (rescue with failed_only=False).

glaeda's hook does not refuse a job for the mini's capacity: it waits, with
no limit, inside the runner's setup until the units and tokens the job needs
are free. A job on the persistent pool still in its runner's setup
SETUP_WAIT_SECONDS after it started is stuck like a queued one: the run is
cancelled and re-run the same way, and until then the job is not accepted, so
the watch goes on.

A refused job never goes back to the fleet: this rescue re-runs as
github-actions[bot], and every runs-on sends the bot's re-run to retry_runner
(Blacksmith), so it cannot land on the mini that refused it, and one re-run
Expand Down Expand Up @@ -310,10 +317,9 @@
RERUN_MARGIN_SECONDS = 60
# A refused job fails within the runner's setup; a real failure of the first
# step after checkout takes longer than this, and one that does not is cheap to
# retry. glaeda's hook may wait up to 240 s inside that setup for the mini's one
# gui token (GUI_WAIT_S) before refusing, and an app-host shard waiting there
# costs far less than a refusal's rescue round trip and a Blacksmith re-run
# (cmuxterm-hq#661 Workstream 7), so the window covers that wait with room.
# retry. glaeda's hook once waited up to 240 s inside that setup for the mini's
# capacity before refusing; it now waits with no limit (SETUP_WAIT_SECONDS
# covers that), so a refusal is the hook's other checks, which answer at once.
REFUSAL_SECONDS = 360
# The last attempt of the bot's own re-runs that may run on an owned pool: a
# stuck run's full re-run on the light tier (CI_OWNED_LIGHT_RETRY). A person's
Expand All @@ -332,6 +338,13 @@ def person_rerun(run: Mapping[str, Any]) -> bool:
and str((run.get("triggering_actor") or {}).get("login") or "") != RESCUE_ACTOR)
# The runner's own steps, which run before glaeda's hook decides.
SETUP_STEPS = frozenset({"Set up job", "Set up runner"})
# glaeda's hook no longer refuses a job for the mini's capacity: it waits,
# with no limit, inside the runner's setup ("Set up runner") until the units
# and tokens it needs are free (glaeda CAPACITY_WAIT, 2026-09-28). A job still
# there this long after it started is stuck like a queued one and is moved the
# same way (Look "rescue"). A compile waiting for a canonical root normally
# waits for one running compile, about 7 minutes.
SETUP_WAIT_SECONDS = 15 * 60
MAX_JOB_PAGES = 3
# Main's full-suite dispatch (ci-main-full-suite.yml) runs ci.yml on this branch.
MAIN_BRANCH = "main"
Expand Down Expand Up @@ -382,6 +395,19 @@ def waiting_for_runner(job: Mapping[str, Any]) -> bool:
return job.get("status") == "queued" and not job.get("runner_name")


def in_setup(job: Mapping[str, Any]) -> bool:
"""A job its runner took that is still in the runner's setup, where glaeda's hook waits for capacity."""
if job.get("status") != "in_progress":
return False
steps = [step for step in job.get("steps") or [] if isinstance(step, Mapping)]
return any(step.get("name") in SETUP_STEPS and step.get("status") == "in_progress" for step in steps)


def setup_seconds(job: Mapping[str, Any], now: dt.datetime) -> float:
started = parse_time(job.get("started_at"))
return 0.0 if started is None else max(0.0, (now - started).total_seconds())


Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
def wait_start(job: Mapping[str, Any], first_seen: dt.datetime | None = None) -> dt.datetime | None:
created = parse_time(job.get("created_at"))
return max(filter(None, (created, first_seen)), default=None)
Expand Down Expand Up @@ -428,11 +454,12 @@ def refused(job: Mapping[str, Any]) -> bool:


def accepted(job: Mapping[str, Any], now: dt.datetime) -> bool:
"""An owned job its runner took and has not refused: started over REFUSAL_SECONDS ago, or done."""
"""An owned job its runner took and has not refused: past its setup and started over REFUSAL_SECONDS ago,
or done."""
if job.get("status") == "completed":
return not refused(job)
started = parse_time(job.get("started_at"))
return job.get("status") == "in_progress" and started is not None and \
return job.get("status") == "in_progress" and not in_setup(job) and started is not None and \
(now - started).total_seconds() > REFUSAL_SECONDS


Expand Down Expand Up @@ -465,12 +492,19 @@ def assess(jobs: Sequence[Mapping[str, Any]], *, now: dt.datetime, budget_second
names = ", ".join(sorted(str(job.get("name") or job.get("id")) for job in stuck))
return Look("rescue", f"{names} queued on {job_pool(stuck[0])} for at least "
f"{min(budgets[id(job)] for job in stuck)}s with no runner")
settling = [job for job in jobs if job_pool(job) and in_setup(job)]
held = [job for job in settling if setup_seconds(job, now) >= SETUP_WAIT_SECONDS]
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
if held:
names = ", ".join(sorted(str(job.get("name") or job.get("id")) for job in held))
return Look("rescue", f"{names} waited in {job_pool(held[0])}'s runner setup for capacity for at least "
f"{SETUP_WAIT_SECONDS}s")
turned_away = [job for job in jobs if refused(job)]
if turned_away:
names = ", ".join(sorted(str(job.get("name") or job.get("id")) for job in turned_away))
return Look("refused", f"{names} refused by {job_pool(turned_away[0])} at job start")
if waiting:
return Look("watch", f"{len(waiting)} job(s) waiting for a persistent runner", waiting=True)
if waiting or settling:
return Look("watch", f"{len(waiting)} job(s) waiting for a persistent runner, {len(settling)} in its setup",
waiting=True)
return Look("watch", "no job is waiting for a persistent runner")


Expand Down
28 changes: 28 additions & 0 deletions tests/test_ci_owned_pool_rescue.py
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,34 @@ def jobs(seconds):
return jobs


def setup_job(*, started=41, past_setup=False):
"""An owned job its runner took, still in glaeda's hook ("Set up runner") unless past_setup."""
found = job("macos / macOS compile admission", status="in_progress", labels=[MINI], created=40, runner="mini-1")
found.update(started_at=stamp(started), steps=[
{"name": "Set up job", "status": "completed", "conclusion": "success"},
{"name": "Set up runner", "status": "completed" if past_setup else "in_progress", "conclusion": None},
{"name": "Checkout", "status": "in_progress" if past_setup else "queued", "conclusion": None}])
return found


class SetupWait(unittest.TestCase):
def test_a_job_waiting_in_setup_is_watched_then_rescued(self):
waiting = setup_job()
self.assertTrue(rescue.in_setup(waiting))
self.assertFalse(rescue.in_setup(setup_job(past_setup=True)))
early = START + dt.timedelta(seconds=41 + rescue.REFUSAL_SECONDS + 60)
self.assertFalse(rescue.accepted(waiting, early), "a job in setup has not been accepted yet")
self.assertTrue(rescue.accepted(setup_job(past_setup=True), early))
look = rescue.assess([changes()(60), waiting], now=early, budget_seconds=90)
self.assertEqual((look.action, look.waiting), ("watch", True))
late = START + dt.timedelta(seconds=41 + rescue.SETUP_WAIT_SECONDS)
look = rescue.assess([changes()(60), waiting], now=late, budget_seconds=90)
self.assertEqual(look.action, "rescue")
self.assertIn("runner setup", look.reason)
self.assertEqual(rescue.assess([changes()(60), setup_job(past_setup=True)], now=late,
budget_seconds=90).action, "watch")


class Refusal(unittest.TestCase):
def test_what_counts_as_a_refusal(self):
self.assertTrue(rescue.refused(refused_job()))
Expand Down
13 changes: 13 additions & 0 deletions tests/test_run_e2e.py
Original file line number Diff line number Diff line change
Expand Up @@ -1406,6 +1406,19 @@ def test_an_explicit_choice_or_admin_variable_is_never_rerouted(self):
label, calls, _ = self.decide(queue(), variable=OLD)
self.assertEqual((label, calls), (OLD, []))

def test_an_explicit_owned_pool_takes_its_root_runners(self):
# glaeda gives an E2E build a canonical root: on the pool label a non-root runner took it, and two such
# builds held both of a mini's roots while its root runner's compile admission waited (2026-09-28)
root = "glaeda-root-std-xcode-26.6"
for slots, want in (({MINI: 8, root: 4}, root), ({MINI: 8}, MINI), ({MINI: 8, root: 0}, MINI)):
with self.subTest(slots=slots):
label = self.pool.resolve(MINI, "", overflow="", order="", max_queued="", measure=lambda: None,
now=NOW, owned_slots=json.dumps(slots),
pr_xcode_app="/Applications/Xcode_26.6.app")
self.assertEqual(label, want)
self.assertEqual(self.pool.resolve(root, "", overflow="", order="", max_queued="", measure=lambda: None,
now=NOW, owned_slots=json.dumps({root: 4})), root)

def test_the_commit_does_not_decide(self):
for commit in self.COMMITS:
with self.subTest(commit=commit):
Expand Down
Loading