ci: run the owned side lanes on side runners, not root runners

.github/workflows/ci.yml

@@ -111,6 +111,12 @@ jobs:
       # macos_pr_owned_jobs (admission, shards, lag, cli-product) take it in
       # place of the pool label, on attempt 1 and after a refusal.
       macos_pr_root_runner: ${{ steps.macos-pool.outputs.root_runner }}
+      # Set only beside a root runner, when the pool has machines beyond its
+      # root runners: glaeda-side-<class>-xcode-<version>, the minis' other
+      # runners. The side lanes in macos_pr_owned_jobs (claude-wrapper,
+      # remote-daemon) take it in place of the pool label, so they never hold
+      # a root runner.
+      macos_pr_side_runner: ${{ steps.macos-pool.outputs.side_runner }}
       # Set only beside a root runner, when an idle one carries the warm label
       # of this run's merge base: ["<root label>", "glaeda-warm-<sha12>"],
       # the labels compile admission's attempt 1 takes instead.
@@ -1090,6 +1096,7 @@ jobs:
       pr_retry_runner: ${{ needs.changes.outputs.macos_pr_retry_runner }}
       pr_owned_jobs: ${{ needs.changes.outputs.macos_pr_owned_jobs }}
       pr_refused_retry_runner: ${{ needs.changes.outputs.macos_pr_refused_retry_runner }}
+      pr_side_runner: ${{ needs.changes.outputs.macos_pr_side_runner }}
       native_tests: ${{ needs.changes.outputs.remote_daemon_native == 'true' || contains(github.event.pull_request.labels.*.name, 'full-ci') }}
 
   web:
@@ -1106,7 +1113,7 @@ jobs:
     name: Claude wrapper regressions
     needs: [changes, static-preflight]
     if: ${{ !cancelled() && needs.changes.result == 'success' && needs.static-preflight.result == 'success' && (needs.changes.outputs.claude_wrapper == 'true' || (needs.changes.outputs.macos == 'true' && needs.changes.outputs.full_suite == 'true')) }}
-    runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (startsWith(needs.changes.outputs.macos_pr_runner, 'blacksmith-') && needs.changes.outputs.macos_pr_runner || 'blacksmith-6vcpu-macos-15') || github.event_name == 'pull_request' && github.run_attempt == 2 && github.triggering_actor == 'github-actions[bot]' && contains(needs.changes.outputs.macos_pr_owned_jobs, ' claude-wrapper ') && needs.changes.outputs.macos_pr_refused_retry_runner || github.event_name == 'pull_request' && (github.run_attempt > 1 || !contains(needs.changes.outputs.macos_pr_owned_jobs, ' claude-wrapper ')) && needs.changes.outputs.macos_pr_retry_runner || github.event_name == 'pull_request' && (needs.changes.outputs.macos_pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15') || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15') }}
+    runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (startsWith(needs.changes.outputs.macos_pr_runner, 'blacksmith-') && needs.changes.outputs.macos_pr_runner || 'blacksmith-6vcpu-macos-15') || github.event_name == 'pull_request' && github.run_attempt == 2 && github.triggering_actor == 'github-actions[bot]' && contains(needs.changes.outputs.macos_pr_owned_jobs, ' claude-wrapper ') && (needs.changes.outputs.macos_pr_side_runner || needs.changes.outputs.macos_pr_refused_retry_runner) || github.event_name == 'pull_request' && (github.run_attempt > 1 || !contains(needs.changes.outputs.macos_pr_owned_jobs, ' claude-wrapper ')) && needs.changes.outputs.macos_pr_retry_runner || github.event_name == 'pull_request' && (needs.changes.outputs.macos_pr_side_runner || needs.changes.outputs.macos_pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15') || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15') }}
     timeout-minutes: 10
     steps:
       - name: Checkout wrapper and test inputs

.github/workflows/remote-daemon.yml

@@ -38,6 +38,14 @@ on:
         required: false
         default: ""
         type: string
+      # Set only when pr_runner is persistent with a root count and machines
+      # beyond its root runners: the pool's side label
+      # (glaeda-side-<class>-xcode-<version>). The macOS tests take it in
+      # place of pr_runner, so they never hold a mini's root runner.
+      pr_side_runner:
+        required: false
+        default: ""
+        type: string
   push:
     branches: [main]
     paths:
@@ -111,7 +119,7 @@ jobs:
     # Plain `go test` with no Xcode or GUI: any Mac will do. Follow the same
     # lanes as the other pull-request macOS jobs instead of pinning the
     # contended macOS 26 pool.
-    runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (startsWith(inputs.pr_runner, 'blacksmith-') && inputs.pr_runner || 'blacksmith-6vcpu-macos-15') || github.event_name == 'pull_request' && (github.run_attempt == 2 && github.triggering_actor == 'github-actions[bot]' && contains(inputs.pr_owned_jobs, ' remote-daemon ') && inputs.pr_refused_retry_runner || (github.run_attempt > 1 || !contains(inputs.pr_owned_jobs, ' remote-daemon ')) && inputs.pr_retry_runner || inputs.pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15') || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15' }}
+    runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (startsWith(inputs.pr_runner, 'blacksmith-') && inputs.pr_runner || 'blacksmith-6vcpu-macos-15') || github.event_name == 'pull_request' && (github.run_attempt == 2 && github.triggering_actor == 'github-actions[bot]' && contains(inputs.pr_owned_jobs, ' remote-daemon ') && (inputs.pr_side_runner || inputs.pr_refused_retry_runner) || (github.run_attempt > 1 || !contains(inputs.pr_owned_jobs, ' remote-daemon ')) && inputs.pr_retry_runner || inputs.pr_side_runner || inputs.pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15') || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15' }}
     timeout-minutes: 15
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

scripts/ci/pr_runner_pool.py

@@ -126,9 +126,18 @@
 a root count sends its placed root jobs (ROOT_JOBS) to the `root_runner`
 output, and place() puts no more of them there than its root runners have
 room for, by the same expected wait; a
-pool without one keeps the pool label for every job. The side lanes keep the
-pool label either way. A root job also holds one of the pool's machines, so
-it counts against both.
+pool without one keeps the pool label for every job. A root job also holds
+one of the pool's machines, so it counts against both.
+
+Side lanes (the Claude wrapper and remote daemon lanes, light jobs that never
+touch a canonical root) take the pool's side label,
+`glaeda-side-<class>-xcode-<version>` (side_label()), the other runners of
+each mini, whenever the pool has a root count and more machines than root
+runners (the `side_runner` output). On the pool label a side lane landed on a
+root runner about half the time (11 of 21 on 2026-09-25, 06:30 to 09:00Z,
+3,300 s of root-runner time) and kept a compile or product consumer off that
+mini's root while it ran; on cmux7s and cmux9s, with one root, it blocked the
+mini's only compile. A pool without a root count keeps the pool label.
 
 Warm affinity: an owned Mac keeps compile admission's DerivedData
 (owned_build_state.py), and ci-owned-warm-labels.yml labels its root runner
@@ -244,8 +253,8 @@
 RUN_CLASSES = ("std", "light")
 # `glaeda-root-...` is the one runner per mini that may take a root job (ROOT_JOBS).
 # `glaeda-side-...` are the other runners: the light side-lane workflows take it
-# (vars.CI_SIDE_LANE_RUNNER, owned_pool_rescue.SIDE_WORKFLOW_PATHS). No picker
-# routes to it, but its jobs hold its pool's machines.
+# (vars.CI_SIDE_LANE_RUNNER, owned_pool_rescue.SIDE_WORKFLOW_PATHS), and so do
+# this picker's side lanes (side_runner()). Its jobs hold its pool's machines.
 OWNED_LABEL = re.compile(r"glaeda-(?:root-|side-)?(?:xl|std|light)-xcode-[0-9]+(?:\.[0-9]+)*")
 ROOT_PREFIX = "glaeda-root-"
 SIDE_PREFIX = "glaeda-side-"
@@ -387,6 +396,27 @@ def root_label(label: str) -> str:
     return ROOT_PREFIX + label.removeprefix("glaeda-")
 
 
+def side_label(label: str) -> str:
+    """The side runners' label for an owned pool label, or "" for any other label."""
+    if not persistent(label) or label.startswith((ROOT_PREFIX, SIDE_PREFIX)):
+        return ""
+    return SIDE_PREFIX + label.removeprefix("glaeda-")
+
+
+def side_runner(choice: "Choice", owned_slots: Mapping[str, int]) -> str:
+    """The label a pick's side lanes take: the pool's side label, or "" to keep the pool label.
+
+    Only on a pool with a root count (the root and side runners are split),
+    and only while CI_OWNED_POOL_SLOTS leaves it machines beyond its root
+    runners, so a side lane never waits on a label no runner carries.
+    """
+    if not choice.root_runner or not persistent(choice.runner):
+        return ""
+    if owned_slots.get(choice.runner, 0) <= owned_slots.get(choice.root_runner, 0):
+        return ""
+    return side_label(choice.runner)
+
+
 def pool_label(label: str) -> str:
     """The owned pool a root or side label's runners belong to; any other label unchanged."""
     for prefix in (ROOT_PREFIX, SIDE_PREFIX):
@@ -515,6 +545,8 @@ def run_jobs(**routing: str | None) -> int:
 # glaeda's canonical-root jobs: admission and every job after it (RunJobs.after:
 # the shards, tests-build-and-lag, cli-product-tests). The side lanes are not.
 ROOT_JOBS = "admission, shards, lag, cli-product"
+# The side lanes (RunJobs.side): light, no canonical root; they take side_runner() on a pool with a root count.
+SIDE_LANE_JOBS = ("claude-wrapper", "remote-daemon")
 
 
 def gui_job(key: str) -> bool:
@@ -704,8 +736,9 @@ def _slots(raw: str | None, pr_xcode_app: str | None = None) -> tuple[dict[str,
         if not isinstance(count, int) or isinstance(count, bool) or count <= 0:
             problems.append(f"{SLOTS_VARIABLE} entry {label!r} has {count!r} machines, not a positive whole number")
         elif label.startswith(("side-", SIDE_PREFIX)):
-            # No picker routes to side runners (vars.CI_SIDE_LANE_RUNNER does), so a count is a mistake.
-            problems.append(f"{SLOTS_VARIABLE} entry {label!r} names side runners, which take no picked run")
+            # Side runners are a pool's machines less its root runners (side_runner()), so a count is a mistake.
+            problems.append(f"{SLOTS_VARIABLE} entry {label!r} names side runners, which are counted "
+                            "as the pool's machines less its root runners")
         elif label in CAPABILITY_LABELS:
             continue
         elif persistent(label):
@@ -1565,7 +1598,7 @@ def pull_request_runs_since(self, since: str, *, exclude_run_id: int | None) ->
 
 def summary(choice: Choice, snapshot: Mapping[str, Any] | None, *, now: dt.datetime,
             owned_slots: Mapping[str, int] | None = None, problems: Sequence[str] = (),
-            owned_jobs: Sequence[str] = (), admission_runner: str = "") -> str:
+            owned_jobs: Sequence[str] = (), admission_runner: str = "", side: str = "") -> str:
     runner = choice.runner or "each job's default (MACOS_RUNNER_PR or its fallback)"
     lines = ["### macOS pool for this run", "", f"- Pool: `{runner}`", f"- Why: {choice.reason}"]
     if choice.xcode_app:
@@ -1575,6 +1608,8 @@ def summary(choice: Choice, snapshot: Mapping[str, Any] | None, *, now: dt.datet
                      f"and a re-run of failed jobs, goes to: `{choice.retry_runner}`")
     if choice.root_runner:
         lines.append(f"- Root jobs among them ({ROOT_JOBS}) take `{choice.root_runner}`")
+    if side:
+        lines.append(f"- Side lanes among them ({', '.join(SIDE_LANE_JOBS)}) take `{side}`")
     if admission_runner:
         labels = " + ".join(f"`{label}`" for label in json.loads(admission_runner))
         lines.append(f"- Compile admission takes {labels}: an idle root runner kept a build of this run's merge base")
@@ -1704,8 +1739,10 @@ def count_routed(since: str) -> int:
     if (env.get("WARM_LABELS") == "1" and choice.root_runner and ADMISSION_JOB in owned_jobs
             and live_runners is not None):
         admission_runner = warm_admission_runner(live_runners, choice.root_runner, env.get("MERGED_ONTO"))
-    text = summary(choice, snapshot, now=now, owned_slots=slots(env.get("OWNED_SLOTS"), pr_xcode_app), problems=problems,
-                   owned_jobs=owned_jobs, admission_runner=admission_runner)
+    owned_slots = slots(env.get("OWNED_SLOTS"), pr_xcode_app)
+    side = side_runner(choice, owned_slots)
+    text = summary(choice, snapshot, now=now, owned_slots=owned_slots, problems=problems,
+                   owned_jobs=owned_jobs, admission_runner=admission_runner, side=side)
     print(text)
     if env.get("GITHUB_STEP_SUMMARY"):
         with open(env["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as handle:
@@ -1722,6 +1759,9 @@ def count_routed(since: str) -> int:
                          # What the root jobs in owned_jobs take instead of
                          # the pool label, on attempt 1 and on that attempt 2.
                          f"root_runner={choice.root_runner}\n"
+                         # What the side lanes in owned_jobs take instead of
+                         # the pool label, on attempt 1 and on that attempt 2.
+                         f"side_runner={side}\n"
                          # JSON labels for admission's attempt 1: the root label
                          # and the warm label of this run's merge base, or "".
                          f"admission_runner={admission_runner}\n"

scripts/ci/queue_janitor.py

@@ -85,7 +85,7 @@
 sys.path.insert(0, str(Path(__file__).resolve().parent))
 from pr_runner_pool import MAX_RUN_JOBS  # noqa: E402
 from pr_runner_pool import persistent as owned_pool  # noqa: E402
-from pr_runner_pool import CAPABILITY_LABELS, pool_label, root_label  # noqa: E402
+from pr_runner_pool import CAPABILITY_LABELS, pool_label, root_label, side_label  # noqa: E402
 
 
 API = "https://api.github.com"
@@ -293,15 +293,16 @@ def marker_peaks(marker: tuple[str, int], owned_jobs: Sequence[Mapping[str, Any]
 
     ci.yml's marker names the pool label and every owned machine the run
     placed, root jobs and side lanes alike. Its root jobs' share is that peak
-    less the jobs it put on the pool label itself (the side lanes, which
-    start beside admission); a side lane not listed yet only reserves more.
+    less the jobs it put on the pool label itself or on its side label (the
+    side lanes, which start beside admission and take the side label when
+    the picker named one); a side lane not listed yet only reserves more.
     An E2E marker names the root label when the run took one, which is also
     one of the pool's machines.
     """
     pool, peak = marker
     if pool_label(pool) != pool:
         return [(pool, peak), (pool_label(pool), peak)]
-    side = sum(1 for job in owned_jobs if owned_label(job) == pool)
+    side = sum(1 for job in owned_jobs if owned_label(job) in (pool, side_label(pool)))
     return [(pool, peak)] + ([(root_label(pool), peak - side)] if root_label(pool) and peak > side else [])