teamleaderleo · Sep 25, 2026 · Sep 25, 2026 · Sep 25, 2026
@@ -65,6 +65,7 @@ jobs:
           CMUX_CI_E2E_LARGE_POOL_OVERFLOW: ${{ vars.CI_E2E_LARGE_POOL_OVERFLOW }}
           CMUX_CI_PR_POOL_ORDER: ${{ vars.CI_PR_POOL_ORDER }}
           CMUX_CI_PR_POOL_MAX_QUEUED: ${{ vars.CI_PR_POOL_MAX_QUEUED }}
+          CMUX_CI_PR_POOL_QUEUE_ROUNDS: ${{ vars.CI_PR_POOL_QUEUE_ROUNDS }}
           CMUX_CI_PR_POOL_OWNED: ${{ vars.CI_PR_POOL_OWNED }}
           CMUX_CI_OWNED_POOL_SLOTS: ${{ vars.CI_OWNED_POOL_SLOTS }}
           CMUX_CMUX_CI_XCODE_APP_PR: ${{ vars.CMUX_CI_XCODE_APP_PR }}

@@ -73,8 +73,8 @@ jobs:
     # back. Any error reading the queue stays on 6vcpu, and
     # CI_E2E_LARGE_POOL_OVERFLOW=0 turns the choice off. run-e2e.sh applies the
     # same rule from the same script. With CI_PR_POOL_OWNED=1 an owned Mac
-    # (glaeda-*) with a free slot in CI_OWNED_POOL_SLOTS comes first, as for
-    # pull requests. A job an owned Mac leaves queued or refuses is re-run by
+    # (glaeda-*) comes first while its job starts within CI_PR_POOL_QUEUE_ROUNDS
+    # job lengths and no later than on Blacksmith, as for pull requests. A job an owned Mac leaves queued or refuses is re-run by
     # ci-owned-pool-rescue.yml, and every re-run attempt of build or test takes
     # retry_label (Blacksmith) instead, so a busy Mac never strands a run.
     runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }}
@@ -113,6 +113,7 @@ jobs:
           LARGE_POOL_OVERFLOW: ${{ vars.CI_E2E_LARGE_POOL_OVERFLOW }}
           POOL_ORDER: ${{ vars.CI_PR_POOL_ORDER }}
           POOL_MAX_QUEUED: ${{ vars.CI_PR_POOL_MAX_QUEUED }}
+          POOL_QUEUE_ROUNDS: ${{ vars.CI_PR_POOL_QUEUE_ROUNDS }}
           POOL_OWNED: ${{ vars.CI_PR_POOL_OWNED }}
           OWNED_SLOTS: ${{ vars.CI_OWNED_POOL_SLOTS }}
           PR_XCODE_APP: ${{ vars.CMUX_CI_XCODE_APP_PR }}
@@ -126,6 +127,7 @@ jobs:
             --overflow "$LARGE_POOL_OVERFLOW" \
             --order "$POOL_ORDER" \
             --max-queued "$POOL_MAX_QUEUED" \
+            --queue-rounds "$POOL_QUEUE_ROUNDS" \
             --owned "$POOL_OWNED" \
             --owned-slots "$OWNED_SLOTS" \
             --pr-xcode-app "$PR_XCODE_APP" \

@@ -95,6 +95,7 @@ jobs:
           CMUX_CI_E2E_LARGE_POOL_OVERFLOW: ${{ vars.CI_E2E_LARGE_POOL_OVERFLOW }}
           CMUX_CI_PR_POOL_ORDER: ${{ vars.CI_PR_POOL_ORDER }}
           CMUX_CI_PR_POOL_MAX_QUEUED: ${{ vars.CI_PR_POOL_MAX_QUEUED }}
+          CMUX_CI_PR_POOL_QUEUE_ROUNDS: ${{ vars.CI_PR_POOL_QUEUE_ROUNDS }}
           UNIT_TEST_SUITES: ${{ inputs.unit_test_suites }}
           TEST_REF: ${{ inputs.ref || github.sha }}
           TEST_TIMEOUT: ${{ inputs.test_timeout }}

diff --git a/scripts/ci/dispatch-focused-test.py b/scripts/ci/dispatch-focused-test.py
@@ -40,6 +40,7 @@
 OVERFLOW_ENV = "CMUX_" + pool.OVERFLOW_VARIABLE
 ORDER_ENV = "CMUX_" + pool.ORDER_VARIABLE
 MAX_QUEUED_ENV = "CMUX_" + pool.MAX_QUEUED_VARIABLE
+QUEUE_ROUNDS_ENV = "CMUX_" + pool.QUEUE_ROUNDS_VARIABLE
 OWNED_ENV = "CMUX_" + pool.OWNED_VARIABLE
 SLOTS_ENV = "CMUX_" + pool.SLOTS_VARIABLE
 PR_XCODE_ENV = "CMUX_" + pool.PR_XCODE_VARIABLE
@@ -388,6 +389,8 @@ def routed_runner(default: str | None, test_target: str | None = None) -> str |
             if test_target in (None, "cmuxTests")
             or (repository_variable(pool.OWNED_UI_VARIABLE, OWNED_UI_ENV) or "").strip() == "1" else "",
             repository_variable(pool.PR_XCODE_VARIABLE, PR_XCODE_ENV),
+            # Unset is pull request CI's default rounds, as test-e2e.yml passes it.
+            repository_variable(pool.QUEUE_ROUNDS_VARIABLE, QUEUE_ROUNDS_ENV) or "",
         ),
         measure=lambda: pool.measure_load(GhApi(), now=now),
         now=now,

diff --git a/scripts/ci/e2e_runner_pool.py b/scripts/ci/e2e_runner_pool.py
@@ -48,14 +48,25 @@
 Owned Macs (glaeda-<class>-xcode-<version>, pr_runner_pool.persistent) join
 the choice exactly as they do for pull requests: only when
 `vars.CI_PR_POOL_OWNED == '1'`, only the labels for the lane's Xcode pin
-(vars.CMUX_CI_XCODE_APP_PR), ahead of Blacksmith, and only while
-vars.CI_OWNED_POOL_SLOTS leaves a machine free on a snapshot younger than
-pr_runner_pool.MAX_SNAPSHOT_MINUTES. An E2E run holds one machine at a time
+(vars.CMUX_CI_XCODE_APP_PR), ahead of Blacksmith, on a snapshot younger than
+pr_runner_pool.MAX_SNAPSHOT_MINUTES, and by pull request CI's queue rule
+(vars.CI_PR_POOL_QUEUE_ROUNDS, `--queue-rounds`): an owned pool takes the run
+while its job starts there within that many job lengths and no later than on
+the best Blacksmith pool, and the queue stays within machines x (1 + rounds)
+(pr_runner_pool.owned_room()). Without the rounds the picker used the kill
+switch rule, which counts every in-flight run's whole future peak (the
+janitor's `committed`) as taken now: on 2026-09-25 that read 43 of 32 std
+machines taken while 8 ran (run 36136190497, an iOS run on this rule).
+`--queue-rounds 0` restores it; a caller that omits the flag gets it too.
+The rounds decide only whether an owned pool takes the run: when none does,
+the Blacksmith pool is chosen by the headroom rule above, as before.
+ci-owned-pool-rescue.yml gives a test-e2e.yml run's owned jobs the same
+queue allowance as a CI run's before it moves them. An E2E run holds one machine at a time
 (build, then test), so it needs one free machine. glaeda gives both jobs the
 mini's canonical-root token, so when CI_OWNED_POOL_SLOTS gives the pool a root
 count (pr_runner_pool.root_label()) the run takes the root label and needs a
 free root runner as well. An owned pool is never the
-fewest-queued fallback: with no free machine the run takes Blacksmith. A job
+fewest-queued fallback: with no room within the rounds the run takes Blacksmith. A job
 that waits on, or is refused by, an owned Mac is re-run on Blacksmith by
 ci-owned-pool-rescue.yml; every re-run attempt takes retry_runner(). That
 holds for an explicit owned runner too: it is the one pick that is moved.
@@ -98,6 +109,7 @@
 OVERFLOW_VARIABLE = "CI_E2E_LARGE_POOL_OVERFLOW"
 ORDER_VARIABLE = pr_runner_pool.ORDER_VARIABLE
 MAX_QUEUED_VARIABLE = pr_runner_pool.MAX_QUEUED_VARIABLE
+QUEUE_ROUNDS_VARIABLE = pr_runner_pool.QUEUE_ROUNDS_VARIABLE
 OWNED_VARIABLE = pr_runner_pool.OWNED_VARIABLE
 SLOTS_VARIABLE = pr_runner_pool.SLOTS_VARIABLE
 PR_XCODE_VARIABLE = pr_runner_pool.PR_XCODE_VARIABLE
@@ -138,8 +150,8 @@ def settings(order: str | None, max_queued: str | None, owned: str | None = None
 
     Owned pools stay in the order only when `owned` is "1", and only for the
     lane's Xcode pin, exactly as for pull requests. `queue_rounds` is
-    CI_PR_POOL_QUEUE_ROUNDS for a caller that reads it (ios_runner_pool.py);
-    None, as E2E passes, means no rounds (pr_runner_pool.settings()).
+    CI_PR_POOL_QUEUE_ROUNDS ("" when unset, the default); None, from a caller
+    that never reads it, means no rounds (pr_runner_pool.settings()).
     """
     return pr_runner_pool.settings(None, order, max_queued, owned, pr_xcode_app, queue_rounds)
 
@@ -221,12 +233,25 @@ def decide(load: PoolLoad | None, limits: pr_runner_pool.Settings, *, now: dt.da
         # Pull request runs are not being routed, so each stays on its lane.
         placed[lane] = placed.get(lane, 0) + routed
         routed = 0
-    return pr_runner_pool.decide(
-        load.snapshot, limits, now=now, xcode_pins={},
-        routed_since=routed, auto_xcode=True,
-        placed=placed, choose_from=pools,
-        owned_slots=owned_slots or {}, jobs=jobs, root_jobs=jobs,
-    )
+    def rule(settings: pr_runner_pool.Settings, choose_from: list[str]) -> pr_runner_pool.Choice:
+        return pr_runner_pool.decide(
+            load.snapshot, settings, now=now, xcode_pins={},
+            routed_since=routed, auto_xcode=True,
+            placed=placed, choose_from=choose_from,
+            owned_slots=owned_slots or {}, jobs=jobs, root_jobs=jobs,
+        )
+
+    choice = rule(limits, pools)
+    blacksmith = [label for label in pools if not pr_runner_pool.persistent(label)]
+    if limits.queue_rounds and blacksmith and choice.runner and not pr_runner_pool.persistent(choice.runner):
+        # The rounds decide only whether an owned pool takes the run; the
+        # Blacksmith pool is the headroom rule's, as without them (the first
+        # pool with a free machine, then the shorter queue in rounds).
+        choice = rule(dataclasses.replace(limits, queue_rounds=0), blacksmith)
+        if len(blacksmith) < len(pools):
+            choice = dataclasses.replace(choice, reason=f"no owned pool within {limits.queue_rounds} queue "
+                                                        f"round(s); {choice.reason}")
+    return choice
 
 
 def pr_routing_off(snapshot: Mapping[str, Any]) -> str | None:
@@ -268,7 +293,7 @@ def auto_runner(
         log(f"{OVERFLOW_VARIABLE}=0; staying on {SMALL_RUNNER}")
         return default
     if limits is None:
-        log(f"invalid {ORDER_VARIABLE} or {MAX_QUEUED_VARIABLE}; staying on {SMALL_RUNNER}")
+        log(f"invalid {ORDER_VARIABLE}, {MAX_QUEUED_VARIABLE} or {QUEUE_ROUNDS_VARIABLE}; staying on {SMALL_RUNNER}")
         return default
     if not any(e2e_pool(label) for label in limits.order):
         log(f"{ORDER_VARIABLE} names no macOS 26 pool; staying on {SMALL_RUNNER}")
@@ -306,6 +331,7 @@ def resolve(
     pr_xcode_app: str | None = None,
     test_filter: str | None = None,
     owned_ui: str | None = None,
+    queue_rounds: str | None = None,
 ) -> str:
     """The runner label for a workflow run, from its inputs and variables."""
     requested = (requested or "").strip()
@@ -318,7 +344,7 @@ def resolve(
     return auto_runner(
         default,
         enabled=enabled(overflow),
-        limits=settings(order, max_queued, owned, pr_xcode_app),
+        limits=settings(order, max_queued, owned, pr_xcode_app, queue_rounds),
         measure=measure,
         now=now,
         log=log,
@@ -334,6 +360,8 @@ def main(argv: Sequence[str] | None = None, env: Mapping[str, str] | None = None
     parser.add_argument("--overflow", default="", help=f"vars.{OVERFLOW_VARIABLE}")
     parser.add_argument("--order", default="", help=f"vars.{ORDER_VARIABLE}")
     parser.add_argument("--max-queued", default="", help=f"vars.{MAX_QUEUED_VARIABLE}")
+    parser.add_argument("--queue-rounds", default=None,
+                        help=f"vars.{QUEUE_ROUNDS_VARIABLE} (\"\" is its default; omitted is 0)")
     parser.add_argument("--owned", default="", help=f"vars.{OWNED_VARIABLE}")
     parser.add_argument("--owned-slots", default="", help=f"vars.{SLOTS_VARIABLE}")
     parser.add_argument("--pr-xcode-app", default="", help=f"vars.{PR_XCODE_VARIABLE}")
@@ -360,7 +388,7 @@ def measure() -> PoolLoad | None:
         args.requested, args.variable,
         overflow=args.overflow, order=args.order, max_queued=args.max_queued,
         owned=args.owned, owned_slots=args.owned_slots, pr_xcode_app=args.pr_xcode_app,
-        test_filter=args.test_filter, owned_ui=args.owned_ui,
+        test_filter=args.test_filter, owned_ui=args.owned_ui, queue_rounds=args.queue_rounds,
         measure=measure, now=now,
         log=lambda message: print(message, file=sys.stderr),
     ))

diff --git a/scripts/ci/owned_pool_rescue.py b/scripts/ci/owned_pool_rescue.py
@@ -153,9 +153,9 @@
 (queue_seconds(), 930 seconds by default), under the watch limit so a stuck
 job is still moved. With the rounds at 0 the picker takes an owned pool
 only with machines free now, and the budget is the configured one. A
-test-ios.yml run's picker queues by the same rounds, so it gets the same
-allowance. The configured budget alone is an E2E, iOS screenshots or
-side-lane run's (#14391: no picker; the side lanes share the
+test-ios.yml or test-e2e.yml run's picker queues by the same rounds, so it
+gets the same allowance. The configured budget alone is an iOS screenshots
+or side-lane run's (#14391: no picker; the side lanes share the
 runners PR runs queue on, so they are moved to Blacksmith more often), and a
 re-run of failed jobs'.
 """
@@ -187,8 +187,9 @@
 # picks the pool and uploads the marker.
 DISPATCH_WORKFLOW_PATHS = (E2E_WORKFLOW_PATH, IOS_TEST_WORKFLOW_PATH, IOS_SCREENSHOTS_WORKFLOW_PATH)
 # Workflows whose picker may queue a run's jobs on an owned pool within
-# CI_PR_POOL_QUEUE_ROUNDS (ios_runner_pool.py reads it since run 36136190497).
-QUEUEING_WORKFLOW_PATHS = (CI_WORKFLOW_PATH, IOS_TEST_WORKFLOW_PATH)
+# CI_PR_POOL_QUEUE_ROUNDS (ios_runner_pool.py and e2e_runner_pool.py read it
+# since run 36136190497).
+QUEUEING_WORKFLOW_PATHS = (CI_WORKFLOW_PATH, IOS_TEST_WORKFLOW_PATH, E2E_WORKFLOW_PATH)
 # Side-lane workflows: no picker job. Their light macOS jobs take
 # vars.CI_SIDE_LANE_RUNNER (a glaeda-side-* label) on attempt 1 of a same-repo
 # pull request run, and every later attempt takes their Blacksmith default.
@@ -876,7 +877,7 @@ def capped(deadline: dt.datetime) -> dt.datetime:
         else f"pull request #{target.pr_number}"
     if target.side:
         subject += " (side lane)"
-    # ci.yml's and test-ios.yml's pickers queue on purpose, within the queue
+    # ci.yml's, test-ios.yml's and test-e2e.yml's pickers queue on purpose, within the queue
     # rounds: their owned jobs may wait up to the pool's expected wait (see the docstring).
     queue_extra = queue_seconds(queue_rounds) if target.path in QUEUEING_WORKFLOW_PATHS else 0
     log(f"watching run {target.run_id} of {subject} (budget {seconds + queue_extra}s"

diff --git a/tests/test_ci_owned_pool_rescue.py b/tests/test_ci_owned_pool_rescue.py
@@ -557,16 +557,17 @@ def test_with_queueing_off_the_rescue_fires_at_30_seconds(self):
         self.assertIn("for at least 30s", summary)
         self.assertLess(api.cancelled_at, 40 + 30 + rescue.POLL_SECONDS + 1)
 
-    def test_only_ci_and_test_ios_runs_get_the_expected_wait(self):
-        # E2E, iOS screenshots and side-lane runs have no queueing picker.
-        for payload in (e2e_event(), e2e_event(path=".github/workflows/ios-screenshots.yml")):
+    def test_only_ci_test_ios_and_e2e_runs_get_the_expected_wait(self):
+        # iOS screenshots and side-lane runs have no queueing picker.
+        for payload in (e2e_event(path=".github/workflows/ios-screenshots.yml"),):
             clock = Clock()
             api = FakeAPI(clock, lambda s: [e2e_runner()(s)])
             _, summary = run_main(api, clock, payload=payload,
                                   env_extra={"RESCUE_SECONDS": "30", "QUEUE_ROUNDS": ""})
             self.assertIn("(budget 30s)", summary, payload["workflow_run"]["path"])
-        # ios_runner_pool.py queues within CI_PR_POOL_QUEUE_ROUNDS, dispatch or pull request.
-        for payload in (e2e_event(path=".github/workflows/test-ios.yml"), event(path=".github/workflows/test-ios.yml")):
+        # ios_runner_pool.py and e2e_runner_pool.py queue within CI_PR_POOL_QUEUE_ROUNDS.
+        for payload in (e2e_event(path=".github/workflows/test-ios.yml"), event(path=".github/workflows/test-ios.yml"),
+                        e2e_event()):
             clock = Clock()
             api = FakeAPI(clock, lambda s: [e2e_runner()(s)])
             _, summary = run_main(api, clock, payload=payload,

diff --git a/tests/test_ci_pr_runner_pool.py b/tests/test_ci_pr_runner_pool.py
@@ -2341,6 +2341,77 @@ def test_main_prints_outputs(self):
         outputs = dict(line.split("=", 1) for line in out.getvalue().splitlines())
         self.assertEqual((outputs["jobs"], outputs["sim_jobs"]), ("1", "0"))
 
+class E2EQueueRounds(unittest.TestCase):
+    """e2e_runner_pool.py queues for an owned pool within CI_PR_POOL_QUEUE_ROUNDS, as pull requests do."""
+
+    LIGHT = "glaeda-light-xcode-26.6"
+    ROOT_LIGHT = "glaeda-root-light-xcode-26.6"
+    SLOTS = {MINI: 32, "glaeda-light-xcode-26.6": 4, ROOT_MINI: 15, "glaeda-root-light-xcode-26.6": 2}
+
+    def snapshot(self):
+        """The janitor snapshot of 2026-09-25 12:34:57 UTC that run 36136190497 read."""
+        snap = backlog(small=62, large=23, old=29)
+        snap["pools"][SMALL]["running"] = 5
+        snap["pools"][OLD]["reserved_queued"] = 1
+        snap["pools"][MINI] = {"queued": 15, "running": 8, "committed": 43}
+        snap["pools"][ROOT_MINI] = {"queued": 15, "running": 8, "committed": 42}
+        snap["pools"][self.LIGHT] = {"queued": 3, "running": 1, "committed": 5}
+        snap["pools"][self.ROOT_LIGHT] = {"queued": 3, "running": 1, "committed": 5}
+        return snap
+
+    def choice(self, queue_rounds, pull_requests_since, snap=None):
+        limits = e2e_pool.settings("", "", "1", PR_XCODE, queue_rounds)
+        return e2e_pool.decide(e2e_pool.PoolLoad(snap or self.snapshot(), {}, pull_requests_since), limits,
+                               now=NOW, owned_slots=self.SLOTS)
+
+    def test_the_incident_snapshot_queues_for_a_mini_with_the_rounds(self):
+        # Without the rounds `committed` reads every owned pool full: Blacksmith.
+        for rounds in (None, "0"):
+            choice = self.choice(rounds, 13)
+            self.assertFalse(pool.persistent(choice.runner), rounds)
+            self.assertIn("every pool is full", choice.reason)
+        # With 2 rounds the run joins an owned root queue that starts it within 20 minutes.
+        choice = self.choice("2", 13)
+        self.assertTrue(pool.persistent(choice.runner))
+        self.assertTrue(choice.root_runner.startswith(pool.ROOT_PREFIX))
+        self.assertIn("queue places", choice.reason)
+
+    def test_with_no_owned_room_the_blacksmith_pick_is_the_headroom_rules(self):
+        # The rounds decide only whether an owned pool takes the run.
+        for pull_requests_since in (22, 40):
+            without = self.choice("0", pull_requests_since)
+            within = self.choice("2", pull_requests_since)
+            self.assertFalse(pool.persistent(within.runner))
+            self.assertEqual(within.runner, without.runner, pull_requests_since)
+            self.assertIn("no owned pool within 2 queue round(s)", within.reason)
+        # A free 12vcpu machine still wins on the headroom rule, not the least expected wait.
+        snap = backlog(small=0, large=0)
+        snap["pools"][LARGE]["running"] = 4
+        for label in (MINI, ROOT_MINI, self.LIGHT, self.ROOT_LIGHT):
+            snap["pools"][label] = {"queued": 60, "running": 40, "committed": 99}
+        self.assertEqual(self.choice("2", 0, snap).runner, LARGE)
+        # With owned pools off the reason names no owned pool.
+        choice = e2e_pool.decide(e2e_pool.PoolLoad(snap, {}, 0), e2e_pool.settings("", "", "", PR_XCODE, "2"),
+                                 now=NOW, owned_slots=self.SLOTS)
+        self.assertEqual(choice.runner, LARGE)
+        self.assertNotIn("owned", choice.reason)
+        # A stale snapshot's empty pick passes through.
+        stale = self.snapshot()
+        stale["generated_at"] = "2026-09-24T08:00:00Z"
+        self.assertEqual(self.choice("2", 0, stale).runner, "")
+
+    def test_the_workflow_and_launchers_pass_the_rounds(self):
+        doc = yaml.safe_load((WORKFLOWS / "test-e2e.yml").read_text())
+        step = next(step for step in doc["jobs"]["runner"]["steps"] if step.get("id") == "pool")
+        self.assertEqual(step["env"]["POOL_QUEUE_ROUNDS"], "${{ vars.CI_PR_POOL_QUEUE_ROUNDS }}")
+        self.assertIn('--queue-rounds "$POOL_QUEUE_ROUNDS"', step["run"])
+        for name in ("test-macos-suite.yml", "main-regression-bisect.yml"):
+            text = (WORKFLOWS / name).read_text()
+            self.assertIn("CMUX_CI_PR_POOL_QUEUE_ROUNDS: ${{ vars.CI_PR_POOL_QUEUE_ROUNDS }}", text, name)
+        self.assertIn("pool.QUEUE_ROUNDS_VARIABLE, QUEUE_ROUNDS_ENV",
+                      (ROOT / "scripts/ci/dispatch-focused-test.py").read_text())
+
+
 class IOSWiring(unittest.TestCase):
     """The unsigned iOS jobs read ios_runner_pool.py; everything that signs or leaks stays on Blacksmith."""
 

diff --git a/tests/test_run_e2e.py b/tests/test_run_e2e.py
@@ -1237,6 +1237,7 @@ def run_pool_step(self, *, requested="auto", variable="", overflow="", order="",
             "${{ vars.CI_E2E_LARGE_POOL_OVERFLOW }}": overflow,
             "${{ vars.CI_PR_POOL_ORDER }}": order,
             "${{ vars.CI_PR_POOL_MAX_QUEUED }}": max_queued,
+            "${{ vars.CI_PR_POOL_QUEUE_ROUNDS }}": "",
             "${{ vars.CI_PR_POOL_OWNED }}": "1",
             "${{ vars.CI_OWNED_POOL_SLOTS }}": json.dumps({MINI: 8}),
             "${{ vars.CMUX_CI_XCODE_APP_PR }}": "/Applications/Xcode_26.6.app",