From 22fd014ee6cb924cec19e255e354c347a3b128f9 Mon Sep 17 00:00:00 2001
From: Leo Li <cheerleaderleo@outlook.com>
Date: Fri, 25 Sep 2026 05:59:52 -0400
Subject: [PATCH 1/4] ci: move post-admission jobs onto root runners that are
 idle once admission finishes

The picker places every job at run start, but the app-host shards,
tests-build-and-lag and cli-product-tests start only after compile
admission. When the owned pool was full at the start they stayed on
Blacksmith even after roots drained (09-25: 19 jobs queued on
blacksmith-12vcpu-macos-26, cap 5, while 9 of 16 std roots were idle).

A late-placement job reads the idle root runners live through the route
App after admission succeeds and gives the not-yet-owned jobs the root
label for admission's Xcode, one per idle runner. They fetch admission's
products as they do after an owned admission. Attempt 1 of same-repo PRs
only; any failure leaves the run-start placement.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .github/workflows/ci-guards.yml              |   4 +
 .github/workflows/ci-macos.yml               |  78 ++++++++++-
 .github/workflows/ci.yml                     |   3 +
 scripts/ci/late_placement.py                 | 110 ++++++++++++++++
 tests/test_ci_change_areas.py                |  36 ++++-
 tests/test_ci_late_placement.py              | 130 +++++++++++++++++++
 tests/test_ci_parallel_artifact_transport.py |   5 +-
 tests/test_ci_pr_runner_pool.py              |   3 +-
 tests/test_ci_self_hosted_guard.sh           |   4 +-
 9 files changed, 361 insertions(+), 12 deletions(-)
 create mode 100644 scripts/ci/late_placement.py
 create mode 100644 tests/test_ci_late_placement.py

diff --git a/.github/workflows/ci-guards.yml b/.github/workflows/ci-guards.yml
index bc0a888e5edd..c92ce41460a0 100644
--- a/.github/workflows/ci-guards.yml
+++ b/.github/workflows/ci-guards.yml
@@ -368,6 +368,10 @@ jobs:
         if: ${{ matrix.group == 'ci' }}
         run: python3 tests/test_ci_pr_runner_pool.py
 
+      - name: Validate late placement onto idle root runners
+        if: ${{ matrix.group == 'ci' }}
+        run: python3 tests/test_ci_late_placement.py
+
       - name: Validate owned Mac build state
         if: ${{ matrix.group == 'ci' }}
         run: python3 tests/test_ci_owned_build_state.py
diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml
index 2d0d8eefad82..094e0b1a069f 100644
--- a/.github/workflows/ci-macos.yml
+++ b/.github/workflows/ci-macos.yml
@@ -139,6 +139,12 @@ on:
         required: false
         default: ""
         type: string
+    secrets:
+      # The org's manaflow-glaeda-route App key (read-only on runners), for
+      # late-placement's live read of idle root runners. Optional: without it
+      # the jobs keep their run-start placement.
+      GLAEDA_ROUTE_APP_KEY:
+        required: false
 
 permissions:
   contents: read
@@ -1352,6 +1358,63 @@ jobs:
               ;;
           esac
 
+  late-placement:
+    # The picker (pr_runner_pool.py) places every job when the run starts, but
+    # the shards, tests-build-and-lag and cli-product-tests start only after
+    # compile admission, often ten minutes later. When the owned pool was full
+    # at the start they are committed to Blacksmith and queue there even after
+    # root runners drain. This re-reads the idle root runners live once
+    # admission succeeds and moves the jobs that are not already owned onto
+    # them, one per idle runner (scripts/ci/late_placement.py). They reuse
+    # admission's uploaded products there, as after an owned admission. Any
+    # failure outputs {} and leaves every job where the picker put it.
+    # Pull requests only: main's full-suite dispatch already runs on the owned
+    # pools and keeps CI_OWNED_MAIN_RESERVE's machines for pull requests.
+    needs:
+      - macos-compile-admission
+    if: ${{ !cancelled() && needs.macos-compile-admission.result == 'success' && github.run_attempt == 1 && github.repository_owner == 'manaflow-ai' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && vars.CI_PR_POOL_OWNED == '1' && vars.GLAEDA_ROUTE_APP_ID != '' && (!startsWith(needs.macos-compile-admission.outputs.runner, 'glaeda-') || !contains(inputs.pr_owned_jobs, ' cli-product ')) }}
+    runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }}
+    timeout-minutes: 3
+    permissions:
+      contents: read
+    outputs:
+      runners: ${{ steps.place.outputs.runners || '{}' }}
+    steps:
+      # The picker's code from the base branch, as ci.yml's changes job uses it.
+      - name: Check out the trusted picker
+        continue-on-error: true
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          ref: ${{ github.event.pull_request.base.sha }}
+          sparse-checkout: scripts/ci
+          persist-credentials: false
+      - name: Mint the owned-pool routing token
+        id: route-token
+        continue-on-error: true
+        timeout-minutes: 1
+        uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
+        with:
+          app-id: ${{ vars.GLAEDA_ROUTE_APP_ID }}
+          private-key: ${{ secrets.GLAEDA_ROUTE_APP_KEY }}
+          permission-administration: read
+      - name: Place the jobs after admission
+        id: place
+        continue-on-error: true
+        timeout-minutes: 1
+        env:
+          ROUTE_TOKEN: ${{ steps.route-token.outputs.token }}
+          MACOS: ${{ inputs.macos }}
+          CLI: ${{ inputs.cli }}
+          FULL_SUITE: ${{ inputs.full_suite }}
+          UNIT_SUITE: ${{ inputs.unit_suite }}
+          UNIT_IN_ADMISSION: ${{ inputs.unit_in_admission }}
+          UNIT_SELECTORS: ${{ inputs.unit_selectors }}
+          OWNED_JOBS: ${{ inputs.pr_owned_jobs }}
+          POOL_OWNED_GUI: ${{ vars.CI_PR_POOL_OWNED_GUI }}
+          ADMISSION_RUNNER: ${{ needs.macos-compile-admission.outputs.runner }}
+          ADMISSION_XCODE_APP: ${{ needs.macos-compile-admission.outputs.xcode_app }}
+        run: python3 scripts/ci/late_placement.py
+
   app-host-unit-tests:
     permissions:
       contents: read
@@ -1359,6 +1422,7 @@ jobs:
       id-token: write
     needs:
       - macos-compile-admission
+      - late-placement
     # !cancelled() disables the implicit success() gate, which GitHub evaluates
     # over the transitive needs chain: linux-preflight runs behind routed linux
     # jobs that legitimately skip (web/go/agent-session paths), and that
@@ -1386,7 +1450,7 @@ jobs:
     # Xcode with more room, and admission's own pool when it is empty. A
     # placed shard takes the admission's runner, which is pr_root_runner
     # when the pool has one.
-    runs-on: ${{ github.run_attempt == 2 && github.triggering_actor == 'github-actions[bot]' && contains(inputs.pr_owned_jobs, format(' shard-{0} ', matrix.shard)) && (inputs.pr_root_runner || inputs.pr_refused_retry_runner) || (github.run_attempt > 1 || !contains(inputs.pr_owned_jobs, format(' shard-{0} ', matrix.shard))) && inputs.pr_retry_runner || inputs.pr_shard_runner || needs.macos-compile-admission.outputs.runner }}
+    runs-on: ${{ github.run_attempt == 1 && fromJSON(needs.late-placement.outputs.runners || '{}')[format('shard-{0}', matrix.shard)] || github.run_attempt == 2 && github.triggering_actor == 'github-actions[bot]' && contains(inputs.pr_owned_jobs, format(' shard-{0} ', matrix.shard)) && (inputs.pr_root_runner || inputs.pr_refused_retry_runner) || (github.run_attempt > 1 || !contains(inputs.pr_owned_jobs, format(' shard-{0} ', matrix.shard))) && inputs.pr_retry_runner || inputs.pr_shard_runner || needs.macos-compile-admission.outputs.runner }}
     timeout-minutes: 75
     strategy:
       # A pull request wants every shard's failures in one run. A merge group
@@ -1470,7 +1534,7 @@ jobs:
       - name: Verify GitHub-hosted route
         env:
           RUNNER_ENVIRONMENT: ${{ runner.environment }}
-          REQUESTED_RUNNER: ${{ github.run_attempt == 2 && github.triggering_actor == 'github-actions[bot]' && contains(inputs.pr_owned_jobs, format(' shard-{0} ', matrix.shard)) && (inputs.pr_root_runner || inputs.pr_refused_retry_runner) || (github.run_attempt > 1 || !contains(inputs.pr_owned_jobs, format(' shard-{0} ', matrix.shard))) && inputs.pr_retry_runner || inputs.pr_shard_runner || needs.macos-compile-admission.outputs.runner }}
+          REQUESTED_RUNNER: ${{ github.run_attempt == 1 && fromJSON(needs.late-placement.outputs.runners || '{}')[format('shard-{0}', matrix.shard)] || github.run_attempt == 2 && github.triggering_actor == 'github-actions[bot]' && contains(inputs.pr_owned_jobs, format(' shard-{0} ', matrix.shard)) && (inputs.pr_root_runner || inputs.pr_refused_retry_runner) || (github.run_attempt > 1 || !contains(inputs.pr_owned_jobs, format(' shard-{0} ', matrix.shard))) && inputs.pr_retry_runner || inputs.pr_shard_runner || needs.macos-compile-admission.outputs.runner }}
           RUNNER_CONTEXT_NAME: ${{ runner.name }}
         run: |
           set -euo pipefail
@@ -2379,6 +2443,7 @@ jobs:
       id-token: write
     needs:
       - macos-compile-admission
+      - late-placement
     # Targeted CLI routing consumes the shared compiled product even when prior
     # compile admission exists; it does not enable the app-host/full-suite lanes.
     # Same !cancelled() reasoning as app-host-unit-tests: the implicit success()
@@ -2392,7 +2457,7 @@ jobs:
     # on the pool and Xcode that built it: the test bundle only loads under
     # the Xcode that linked it. On an owned-pool run it may take
     # pr_retry_runner instead, as the shards do.
-    runs-on: ${{ github.run_attempt == 2 && github.triggering_actor == 'github-actions[bot]' && contains(inputs.pr_owned_jobs, ' cli-product ') && (inputs.pr_root_runner || inputs.pr_refused_retry_runner) || (github.run_attempt > 1 || !contains(inputs.pr_owned_jobs, ' cli-product ')) && inputs.pr_retry_runner || needs.macos-compile-admission.outputs.runner }}
+    runs-on: ${{ github.run_attempt == 1 && fromJSON(needs.late-placement.outputs.runners || '{}')['cli-product'] || github.run_attempt == 2 && github.triggering_actor == 'github-actions[bot]' && contains(inputs.pr_owned_jobs, ' cli-product ') && (inputs.pr_root_runner || inputs.pr_refused_retry_runner) || (github.run_attempt > 1 || !contains(inputs.pr_owned_jobs, ' cli-product ')) && inputs.pr_retry_runner || needs.macos-compile-admission.outputs.runner }}
     timeout-minutes: 40
     env:
       CMUX_NODE_PRODUCT_CACHE_ROOT: ${{ vars.CMUX_NODE_PRODUCT_CACHE_ROOT }}
@@ -2408,7 +2473,7 @@ jobs:
       - name: Verify GitHub-hosted route
         env:
           RUNNER_ENVIRONMENT: ${{ runner.environment }}
-          REQUESTED_RUNNER: ${{ github.run_attempt == 2 && github.triggering_actor == 'github-actions[bot]' && contains(inputs.pr_owned_jobs, ' cli-product ') && (inputs.pr_root_runner || inputs.pr_refused_retry_runner) || (github.run_attempt > 1 || !contains(inputs.pr_owned_jobs, ' cli-product ')) && inputs.pr_retry_runner || needs.macos-compile-admission.outputs.runner }}
+          REQUESTED_RUNNER: ${{ github.run_attempt == 1 && fromJSON(needs.late-placement.outputs.runners || '{}')['cli-product'] || github.run_attempt == 2 && github.triggering_actor == 'github-actions[bot]' && contains(inputs.pr_owned_jobs, ' cli-product ') && (inputs.pr_root_runner || inputs.pr_refused_retry_runner) || (github.run_attempt > 1 || !contains(inputs.pr_owned_jobs, ' cli-product ')) && inputs.pr_retry_runner || needs.macos-compile-admission.outputs.runner }}
           RUNNER_CONTEXT_NAME: ${{ runner.name }}
         run: |
           set -euo pipefail
@@ -3145,6 +3210,7 @@ jobs:
       id-token: write
     needs:
       - macos-compile-admission
+      - late-placement
     # !cancelled() disables the implicit success() gate, which GitHub evaluates
     # over the transitive needs chain: linux-preflight runs behind routed linux
     # jobs that legitimately skip (web/go/agent-session paths), and that
@@ -3159,7 +3225,7 @@ jobs:
     # full-suite dispatch follows admission onto the pull-request pool and
     # Xcode. The product consumer guard in the CI change-area tests fails when
     # the two drift.
-    runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (startsWith(inputs.pr_runner, 'blacksmith-') && inputs.pr_runner || 'blacksmith-6vcpu-macos-15') || (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && (github.run_attempt == 2 && github.triggering_actor == 'github-actions[bot]' && contains(inputs.pr_owned_jobs, ' lag ') && (inputs.pr_root_runner || inputs.pr_refused_retry_runner) || (github.run_attempt > 1 || !contains(inputs.pr_owned_jobs, ' lag ')) && inputs.pr_retry_runner || inputs.pr_root_runner || inputs.pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15') || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_DISPLAY || 'blacksmith-6vcpu-macos-15') }}
+    runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || github.run_attempt == 1 && fromJSON(needs.late-placement.outputs.runners || '{}')['lag'] || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (startsWith(inputs.pr_runner, 'blacksmith-') && inputs.pr_runner || 'blacksmith-6vcpu-macos-15') || (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && (github.run_attempt == 2 && github.triggering_actor == 'github-actions[bot]' && contains(inputs.pr_owned_jobs, ' lag ') && (inputs.pr_root_runner || inputs.pr_refused_retry_runner) || (github.run_attempt > 1 || !contains(inputs.pr_owned_jobs, ' lag ')) && inputs.pr_retry_runner || inputs.pr_root_runner || inputs.pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15') || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_DISPLAY || 'blacksmith-6vcpu-macos-15') }}
     timeout-minutes: 75
     env:
       CMUX_NODE_PRODUCT_CACHE_ROOT: ${{ vars.CMUX_NODE_PRODUCT_CACHE_ROOT }}
@@ -3172,7 +3238,7 @@ jobs:
     steps:
       - name: Validate display runner identity
         env:
-          REQUESTED_RUNNER: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (startsWith(inputs.pr_runner, 'blacksmith-') && inputs.pr_runner || 'blacksmith-6vcpu-macos-15') || (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && (github.run_attempt == 2 && github.triggering_actor == 'github-actions[bot]' && contains(inputs.pr_owned_jobs, ' lag ') && (inputs.pr_root_runner || inputs.pr_refused_retry_runner) || (github.run_attempt > 1 || !contains(inputs.pr_owned_jobs, ' lag ')) && inputs.pr_retry_runner || inputs.pr_root_runner || inputs.pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15') || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_DISPLAY || 'blacksmith-6vcpu-macos-15') }}
+          REQUESTED_RUNNER: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || github.run_attempt == 1 && fromJSON(needs.late-placement.outputs.runners || '{}')['lag'] || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (startsWith(inputs.pr_runner, 'blacksmith-') && inputs.pr_runner || 'blacksmith-6vcpu-macos-15') || (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && (github.run_attempt == 2 && github.triggering_actor == 'github-actions[bot]' && contains(inputs.pr_owned_jobs, ' lag ') && (inputs.pr_root_runner || inputs.pr_refused_retry_runner) || (github.run_attempt > 1 || !contains(inputs.pr_owned_jobs, ' lag ')) && inputs.pr_retry_runner || inputs.pr_root_runner || inputs.pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15') || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_DISPLAY || 'blacksmith-6vcpu-macos-15') }}
           RUNNER_CONTEXT_NAME: ${{ runner.name }}
         run: |
           set -euo pipefail
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 51f31c3732c8..cbeafba6f0f2 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -1255,6 +1255,9 @@ jobs:
       pr_root_runner: ${{ needs.changes.outputs.macos_pr_root_runner }}
       pr_admission_runner: ${{ needs.changes.outputs.macos_pr_admission_runner }}
       pr_xcode_app: ${{ needs.changes.outputs.macos_pr_xcode_app }}
+    # ci-macos.yml's late-placement reads idle root runners with it.
+    secrets:
+      GLAEDA_ROUTE_APP_KEY: ${{ secrets.GLAEDA_ROUTE_APP_KEY }}
 
   tests:
     name: tests
diff --git a/scripts/ci/late_placement.py b/scripts/ci/late_placement.py
new file mode 100644
index 000000000000..581d0acd3ef4
--- /dev/null
+++ b/scripts/ci/late_placement.py
@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+"""Move a run's post-admission jobs onto owned root runners that are idle now.
+
+pr_runner_pool.py places every macOS job of a run when the run starts. The
+jobs after compile admission (the app-host shards, tests-build-and-lag and
+cli-product-tests) only start once admission finishes, often ten minutes
+later. If the owned pool was full at the start, they are committed to
+Blacksmith (admission's pool, or pr_retry_runner) and wait in its queue even
+when root runners have drained in the meantime.
+
+ci-macos.yml's late-placement job runs this after admission succeeds, on
+attempt 1 of a same-repository pull request.
+It reads the idle root runners live through the org route App and gives
+each job that is not already owned the root label, in owned priority order,
+up to that many idle runners. The shards and friends then run
+test-without-building on the mini against admission's uploaded products, as
+they do after an owned admission; they never compile.
+
+Output `runners` is a JSON object from job key (shard-N, lag, cli-product)
+to label. Any failure prints a warning and outputs {} (no change).
+"""
+from __future__ import annotations
+
+import importlib.util
+import json
+import os
+import sys
+from pathlib import Path
+from typing import Any, Mapping, Sequence
+
+
+def _picker():
+    path = Path(__file__).with_name("pr_runner_pool.py")
+    spec = importlib.util.spec_from_file_location("pr_runner_pool", path)
+    module = importlib.util.module_from_spec(spec)
+    sys.modules.setdefault("pr_runner_pool", module)
+    spec.loader.exec_module(module)
+    return module
+
+
+pool = _picker()
+
+
+def late_jobs(*, macos: str | None, cli: str | None, full_suite: str | None, unit_suite: str | None,
+              unit_in_admission: str | None, unit_selectors: str | None) -> tuple[str, ...]:
+    """The jobs that run after compile admission in this run (pr_runner_pool.run_plan)."""
+    plan = pool.run_plan(macos=macos, full_suite=full_suite, unit_suite=unit_suite,
+                         unit_in_admission=unit_in_admission, claude_wrapper=None, cli=cli,
+                         remote_daemon=None, unit_selectors=unit_selectors)
+    return plan.after
+
+
+def root_for(xcode_app: str | None) -> str:
+    """The std root label for admission's Xcode; "" when no owned pool pins it."""
+    std = [label for label in pool.owned_pools(xcode_app) if label.startswith("glaeda-std-")]
+    return pool.root_label(std[0]) if std else ""
+
+
+def place(jobs: Sequence[str], *, owned_jobs: str, idle: int, root: str, gui: bool = True) -> dict[str, str]:
+    """Give the not-yet-owned jobs the root label, highest priority first, one per idle runner."""
+    if not root or idle <= 0:
+        return {}
+    owned = f" {owned_jobs.strip()} " if owned_jobs.strip() else " "
+    waiting = [key for key in jobs if f" {key} " not in owned and (gui or not pool.gui_job(key))]
+    return {key: root for key in sorted(waiting, key=pool.priority)[:idle]}
+
+
+def decide(env: Mapping[str, str], runners: Sequence[Mapping[str, Any]] | None) -> tuple[dict[str, str], str]:
+    jobs = late_jobs(macos=env.get("MACOS"), cli=env.get("CLI"), full_suite=env.get("FULL_SUITE"),
+                     unit_suite=env.get("UNIT_SUITE"), unit_in_admission=env.get("UNIT_IN_ADMISSION"),
+                     unit_selectors=env.get("UNIT_SELECTORS"))
+    if not jobs:
+        return {}, "no job runs after compile admission"
+    root = root_for(env.get("ADMISSION_XCODE_APP"))
+    if not root:
+        return {}, f"no owned pool runs admission's Xcode ({env.get('ADMISSION_XCODE_APP') or 'unknown'})"
+    if runners is None:
+        return {}, "owned runners could not be read live"
+    idle = pool.live_owned_free(runners, [root])[root]
+    placed = place(jobs, owned_jobs=env.get("OWNED_JOBS", ""), idle=idle, root=root,
+                   gui=env.get("POOL_OWNED_GUI", "").strip() != "0")
+    if not placed:
+        return {}, f"{idle} idle `{root}` runner(s); nothing to move"
+    return placed, (f"{idle} idle `{root}` runner(s) now; moved {', '.join(placed)} there "
+                    f"(admission ran on `{env.get('ADMISSION_RUNNER') or 'unknown'}`)")
+
+
+def main(env: Mapping[str, str] = os.environ) -> int:
+    runners = None
+    token, repo = env.get("ROUTE_TOKEN", ""), env.get("GITHUB_REPOSITORY", "")
+    if token and repo:
+        try:
+            runners = pool.GitHub(token, repo).runners()
+        except Exception as error:  # noqa: BLE001 - fail open: keep the run-start placement
+            print(f"::warning title=late placement::could not list runners ({error})")
+    placed, why = decide(env, runners)
+    print(f"late placement: {why}")
+    output = env.get("GITHUB_OUTPUT")
+    if output:
+        with open(output, "a", encoding="utf-8") as handle:
+            handle.write(f"runners={json.dumps(placed, sort_keys=True)}\n")
+    summary = env.get("GITHUB_STEP_SUMMARY")
+    if summary:
+        with open(summary, "a", encoding="utf-8") as handle:
+            handle.write(f"### Late placement\n\n{why}\n")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/tests/test_ci_change_areas.py b/tests/test_ci_change_areas.py
index b2399893084a..db87b064c15e 100755
--- a/tests/test_ci_change_areas.py
+++ b/tests/test_ci_change_areas.py
@@ -4528,6 +4528,26 @@ def admission_route(runs_on: str) -> str:
     """Compile admission's runs-on without its warm labels: the route its consumers restate."""
     return runs_on.replace(WARM_ADMISSION + " || ", "")
 PRODUCT_XCODE_OUTPUT = "${{ needs.macos-compile-admission.outputs.xcode_app }}"
+# Attempt 1 may take the root label late-placement chose once admission
+# finished (scripts/ci/late_placement.py). That label is derived from the
+# admission's own xcode_app output, so it keeps the consumer on the producer's
+# Xcode; late_placement_route strips it only while that stays true.
+LATE_KEYS = {
+    "app-host-unit-tests": "format('shard-{0}', matrix.shard)",
+    "cli-product-tests": "'cli-product'",
+    "tests-build-and-lag": "'lag'",
+}
+
+
+def late_placement_route(workflow: dict, name: str, runs_on: str) -> str:
+    """The consumer's runs-on without its late-placement branch, when that branch is Xcode-safe."""
+    late = workflow["jobs"].get("late-placement") or {}
+    steps = [step for step in late.get("steps", []) if step.get("id") == "place"]
+    if not steps or (steps[0].get("env") or {}).get("ADMISSION_XCODE_APP") != PRODUCT_XCODE_OUTPUT:
+        return runs_on
+    prefix = ("${{ github.run_attempt == 1 && fromJSON(needs.late-placement.outputs.runners || '{}')["
+              + LATE_KEYS.get(name, "") + "] || ")
+    return runs_on.replace(prefix.removeprefix("${{ "), "", 1)
 
 
 def product_consumer_route_violations(workflow: dict) -> list[str]:
@@ -4549,7 +4569,7 @@ def product_consumer_route_violations(workflow: dict) -> list[str]:
     if outputs.get("xcode_app") != "${{ env.CMUX_CI_XCODE_APP }}":
         violations.append("macos-compile-admission: missing xcode_app output")
     for name, job in app_host_product_consumers(workflow).items():
-        runs_on = job.get("runs-on", "")
+        runs_on = late_placement_route(workflow, name, job.get("runs-on", ""))
         xcode = (job.get("env") or {}).get("CMUX_CI_XCODE_APP")
         if name in PRODUCT_RUNNER_KEYS and runs_on == product_runner_output(PRODUCT_RUNNER_KEYS[name]) \
                 and xcode == PRODUCT_XCODE_OUTPUT:
@@ -4581,10 +4601,22 @@ def test_app_host_product_consumers_run_on_the_producers_pool_and_xcode() -> Non
     # The app-host shards read the outputs, so a route added to the admission
     # moves them without an edit here.
     shards = workflow["jobs"]["app-host-unit-tests"]
-    assert shards["runs-on"] == PRODUCT_RUNNER_OUTPUT
+    assert late_placement_route(workflow, "app-host-unit-tests", shards["runs-on"]) == PRODUCT_RUNNER_OUTPUT
     assert shards["env"]["CMUX_CI_XCODE_APP"] == PRODUCT_XCODE_OUTPUT
 
 
+def test_late_placement_must_keep_the_admissions_xcode() -> None:
+    # late-placement picks the owned root label for the admission's Xcode. If it
+    # stopped reading that output, its label could name another Xcode, and
+    # every consumer taking it would be reported.
+    workflow = yaml.safe_load(MACOS_WORKFLOW.read_text(encoding="utf-8"))
+    assert product_consumer_route_violations(workflow) == []
+    place = next(step for step in workflow["jobs"]["late-placement"]["steps"] if step.get("id") == "place")
+    place["env"]["ADMISSION_XCODE_APP"] = "${{ inputs.pr_xcode_app }}"
+    reported = {line.split(":", 1)[0] for line in product_consumer_route_violations(workflow)}
+    assert {"app-host-unit-tests", "cli-product-tests", "tests-build-and-lag"} <= reported, reported
+
+
 def test_product_consumer_guard_follows_a_new_admission_route() -> None:
     # Give the admission a new route, as sending merge groups to the
     # pull-request pool and Xcode would. Consumers that read the outputs stay
diff --git a/tests/test_ci_late_placement.py b/tests/test_ci_late_placement.py
new file mode 100644
index 000000000000..57483069c2b6
--- /dev/null
+++ b/tests/test_ci_late_placement.py
@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+"""Late placement: jobs after compile admission move onto idle root runners."""
+from __future__ import annotations
+
+import importlib.util
+import sys
+import unittest
+from pathlib import Path
+
+import yaml
+
+ROOT = Path(__file__).resolve().parents[1]
+SCRIPT = ROOT / "scripts/ci/late_placement.py"
+WORKFLOW = ROOT / ".github/workflows/ci-macos.yml"
+XCODE = "/Applications/Xcode_26.6.app"
+ROOT_STD = "glaeda-root-std-xcode-26.6"
+FULL = {"MACOS": "true", "CLI": "false", "FULL_SUITE": "true", "UNIT_SUITE": "false",
+        "UNIT_IN_ADMISSION": "false", "UNIT_SELECTORS": "", "ADMISSION_XCODE_APP": XCODE,
+        "ADMISSION_RUNNER": "blacksmith-12vcpu-macos-26", "OWNED_JOBS": ""}
+
+
+def load():
+    spec = importlib.util.spec_from_file_location("late_placement", SCRIPT)
+    module = importlib.util.module_from_spec(spec)
+    sys.modules["late_placement"] = module
+    spec.loader.exec_module(module)
+    return module
+
+
+late = load()
+
+
+def runner(name: str, *labels: str, busy: bool = False, status: str = "online") -> dict:
+    return {"name": name, "status": status, "busy": busy, "labels": [{"name": label} for label in labels]}
+
+
+def roots(idle: int, busy: int = 0) -> list[dict]:
+    return ([runner(f"idle-{i}", "self-hosted", ROOT_STD) for i in range(idle)]
+            + [runner(f"busy-{i}", ROOT_STD, busy=True) for i in range(busy)])
+
+
+class Decide(unittest.TestCase):
+    def test_a_full_suite_off_blacksmith_takes_the_idle_roots_shards_first(self):
+        placed, why = late.decide(FULL, roots(idle=3, busy=5))
+        self.assertEqual(placed, {"shard-1": ROOT_STD, "shard-2": ROOT_STD, "shard-3": ROOT_STD})
+        self.assertIn("3 idle", why)
+
+    def test_enough_idle_roots_move_every_job_after_admission(self):
+        placed, _ = late.decide(FULL, roots(idle=16))
+        self.assertEqual(set(placed), {*(f"shard-{i}" for i in range(1, 8)), "lag", "cli-product"})
+
+    def test_jobs_the_picker_already_owned_stay_put(self):
+        env = dict(FULL, OWNED_JOBS=" admission shard-1 shard-2 ")
+        placed, _ = late.decide(env, roots(idle=2))
+        self.assertEqual(placed, {"shard-3": ROOT_STD, "shard-4": ROOT_STD})
+
+    def test_no_idle_root_changes_nothing(self):
+        self.assertEqual(late.decide(FULL, roots(idle=0, busy=16))[0], {})
+
+    def test_offline_runners_do_not_count(self):
+        self.assertEqual(late.decide(FULL, [runner("off", ROOT_STD, status="offline")])[0], {})
+
+    def test_another_xcode_has_no_owned_pool(self):
+        env = dict(FULL, ADMISSION_XCODE_APP="/Applications/Xcode_26.3.app")
+        placed, why = late.decide(env, roots(idle=8))
+        self.assertEqual(placed, {})
+
+    def test_unreadable_runners_change_nothing(self):
+        placed, why = late.decide(FULL, None)
+        self.assertEqual(placed, {})
+        self.assertIn("could not be read", why)
+
+    def test_gui_off_moves_only_cli_product(self):
+        env = dict(FULL, POOL_OWNED_GUI="0")
+        self.assertEqual(late.decide(env, roots(idle=8))[0], {"cli-product": ROOT_STD})
+
+    def test_a_changed_suites_run_moves_its_one_worker(self):
+        env = dict(FULL, FULL_SUITE="false", UNIT_SUITE="true", UNIT_SELECTORS="cmuxTests/FooTests")
+        self.assertEqual(late.decide(env, roots(idle=4))[0], {"shard-8": ROOT_STD})
+
+    def test_a_compile_only_run_has_nothing_after_admission(self):
+        env = dict(FULL, FULL_SUITE="false")
+        self.assertEqual(late.decide(env, roots(idle=4))[0], {})
+
+
+class Output(unittest.TestCase):
+    def test_main_writes_an_empty_object_without_a_token(self):
+        import tempfile
+        with tempfile.NamedTemporaryFile("r+", suffix=".out") as out:
+            self.assertEqual(late.main(dict(FULL, GITHUB_OUTPUT=out.name)), 0)
+            self.assertEqual(Path(out.name).read_text(), "runners={}\n")
+
+
+class Workflow(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.jobs = yaml.safe_load(WORKFLOW.read_text())["jobs"]
+
+    def test_the_consumers_wait_for_late_placement_and_read_it_first_on_attempt_one(self):
+        keys = {"app-host-unit-tests": "format('shard-{0}', matrix.shard)",
+                "tests-build-and-lag": "'lag'", "cli-product-tests": "'cli-product'"}
+        prefix = "${{ github.run_attempt == 1 && fromJSON(needs.late-placement.outputs.runners || '{}')[%s] || "
+        for job, key in keys.items():
+            with self.subTest(job=job):
+                spec = self.jobs[job]
+                self.assertIn("late-placement", spec["needs"])
+                late = (prefix % key).removeprefix("${{ ")
+                owner = "${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || "
+                # tests-build-and-lag keeps the fork-owner branch first (test_ci_fork_runner_routing).
+                self.assertTrue(spec["runs-on"].startswith("${{ " + late) or spec["runs-on"].startswith(owner + late),
+                                spec["runs-on"][:200])
+                # The job-level if never requires late-placement, so a skipped or failed one
+                # leaves the consumer running where the picker put it.
+                self.assertNotIn("late-placement", spec["if"])
+                requested = [step["env"]["REQUESTED_RUNNER"] for step in spec["steps"]
+                             if "REQUESTED_RUNNER" in (step.get("env") or {})]
+                self.assertEqual(requested, [spec["runs-on"]])
+
+    def test_late_placement_runs_only_where_the_picker_may_use_owned_runners(self):
+        spec = self.jobs["late-placement"]
+        for clause in ("github.run_attempt == 1", "vars.CI_PR_POOL_OWNED == '1'",
+                       "github.event.pull_request.head.repo.full_name == github.repository",
+                       "needs.macos-compile-admission.result == 'success'"):
+            self.assertIn(clause, spec["if"])
+        self.assertTrue(all(step.get("continue-on-error") for step in spec["steps"]))
+        self.assertEqual(spec["outputs"]["runners"], "${{ steps.place.outputs.runners || '{}' }}")
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_ci_parallel_artifact_transport.py b/tests/test_ci_parallel_artifact_transport.py
index 6da04318074a..3abba94d8c6e 100644
--- a/tests/test_ci_parallel_artifact_transport.py
+++ b/tests/test_ci_parallel_artifact_transport.py
@@ -133,8 +133,9 @@ def test_cli_product_lane_keeps_the_consumer_transport_chain(self):
         shard_route = step_block(job_block("app-host-unit-tests"), "Verify GitHub-hosted route")
         self.assertIn("inputs.pr_shard_runner || ", shard_route)
         self.assertEqual(
-            step_block(block, "Verify GitHub-hosted route").replace("' cli-product '", "KEY"),
-            shard_route.replace("format(' shard-{0} ', matrix.shard)", "KEY").replace("inputs.pr_shard_runner || ", ""),
+            step_block(block, "Verify GitHub-hosted route").replace("' cli-product '", "KEY").replace("'cli-product'", "LATE"),
+            shard_route.replace("format(' shard-{0} ', matrix.shard)", "KEY").replace("format('shard-{0}', matrix.shard)", "LATE")
+            .replace("inputs.pr_shard_runner || ", ""),
         )
 
     def test_layer_transport_prefers_parallel_reads_and_keeps_the_stream_fallback(self):
diff --git a/tests/test_ci_pr_runner_pool.py b/tests/test_ci_pr_runner_pool.py
index 2dabee524505..47fb07953c54 100644
--- a/tests/test_ci_pr_runner_pool.py
+++ b/tests/test_ci_pr_runner_pool.py
@@ -1733,7 +1733,8 @@ def test_every_pr_route_in_the_run_reads_the_choice(self):
 
     def test_a_rerun_of_failed_shards_leaves_the_owned_pool(self):
         shards = self.workflow("ci-macos.yml")["jobs"]["app-host-unit-tests"]
-        self.assertEqual(shards["runs-on"], "${{ github.run_attempt == 2 && github.triggering_actor == 'github-actions[bot]' && contains(inputs.pr_owned_jobs, "
+        self.assertEqual(shards["runs-on"], "${{ github.run_attempt == 1 && fromJSON(needs.late-placement.outputs.runners || '{}')"
+                                            "[format('shard-{0}', matrix.shard)] || github.run_attempt == 2 && github.triggering_actor == 'github-actions[bot]' && contains(inputs.pr_owned_jobs, "
                                             "format(' shard-{0} ', matrix.shard)) && (inputs.pr_root_runner || inputs.pr_refused_retry_runner) "
                                             "|| (github.run_attempt > 1 || !contains(inputs.pr_owned_jobs, "
                                             "format(' shard-{0} ', matrix.shard))) && inputs.pr_retry_runner "
diff --git a/tests/test_ci_self_hosted_guard.sh b/tests/test_ci_self_hosted_guard.sh
index eab83ff1de63..cfb7f651fe6d 100755
--- a/tests/test_ci_self_hosted_guard.sh
+++ b/tests/test_ci_self_hosted_guard.sh
@@ -48,7 +48,9 @@ check_macos_runner() {
     # place this shard on the owned pool, the Blacksmith pool the pull
     # request picker named for a run on an owned pool (pr_retry_runner), or
     # on attempt 2 of a refused owned shard, the owned pool once more.
-    in_job && /runs-on:[[:space:]]*\$\{\{ (github\.run_attempt == 2 && github\.triggering_actor == .github-actions\[bot\]. && contains\(inputs\.pr_owned_jobs, format\(. shard-\{0\} ., matrix\.shard\)\) && \(inputs\.pr_root_runner \|\| inputs\.pr_refused_retry_runner\) \|\| )?(\(github\.run_attempt > 1 \|\| !contains\(inputs\.pr_owned_jobs, format\(. shard-\{0\} ., matrix\.shard\)\)\) && inputs\.pr_retry_runner \|\| )?(inputs\.pr_shard_runner \|\| )?needs\.macos-compile-admission\.outputs\.runner \}\}/ { saw=1 }
+    # On attempt 1 it may first take the root label late-placement chose (an owned
+    # root runner found idle once admission finished; late_placement.py).
+    in_job && /runs-on:[[:space:]]*\$\{\{ (github\.run_attempt == 1 && fromJSON\(needs\.late-placement\.outputs\.runners \|\| .\{\}.\)\[format\(.shard-\{0\}., matrix\.shard\)\] \|\| )?(github\.run_attempt == 2 && github\.triggering_actor == .github-actions\[bot\]. && contains\(inputs\.pr_owned_jobs, format\(. shard-\{0\} ., matrix\.shard\)\) && \(inputs\.pr_root_runner \|\| inputs\.pr_refused_retry_runner\) \|\| )?(\(github\.run_attempt > 1 \|\| !contains\(inputs\.pr_owned_jobs, format\(. shard-\{0\} ., matrix\.shard\)\)\) && inputs\.pr_retry_runner \|\| )?(inputs\.pr_shard_runner \|\| )?needs\.macos-compile-admission\.outputs\.runner \}\}/ { saw=1 }
     in_job && /os:.*(vars\.MACOS_RUNNER|blacksmith-[0-9]+vcpu-macos-|warp-macos-[0-9]+-arm64|depot-macos-)/ { saw=1 }
     END { exit !(saw) }
   ' "$file"; then

From 347207c83dd5355e5a4f92add92b98465c9cd2d2 Mon Sep 17 00:00:00 2001
From: Leo Li <cheerleaderleo@outlook.com>
Date: Fri, 25 Sep 2026 06:16:15 -0400
Subject: [PATCH 2/4] ci: watch the jobs late placement moves, through the
 owned-pool rescue

Review: a moved job waits for a root runner that another run may take
first, and a run whose picker owned nothing had no rescue watch, so it
could sit queued with nothing to move it back to Blacksmith.

late-placement now uploads a macos-pool-late-<run>-<attempt> marker when
it moves jobs. ci.yml's owned-pool-watch also starts the rescue for a
same-repo PR whose picker owned nothing but which has jobs after
admission (full or unit suite, or the CLI lane), with late=1. That watch
reads the picker's marker once, then waits at IDLE_POLL_SECONDS for
ci-macos.yml's late-placement job, and follows the run only if its marker
exists. Existing watches are unchanged.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .github/workflows/ci-macos.yml             | 18 +++++++++
 .github/workflows/ci-owned-pool-rescue.yml |  7 ++++
 .github/workflows/ci.yml                   | 10 ++++-
 scripts/ci/owned_pool_rescue.py            | 41 ++++++++++++++++---
 tests/test_ci_late_placement.py            | 12 ++++++
 tests/test_ci_owned_pool_rescue.py         | 47 +++++++++++++++++++++-
 6 files changed, 127 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml
index 094e0b1a069f..159e89912c42 100644
--- a/.github/workflows/ci-macos.yml
+++ b/.github/workflows/ci-macos.yml
@@ -1414,6 +1414,24 @@ jobs:
           ADMISSION_RUNNER: ${{ needs.macos-compile-admission.outputs.runner }}
           ADMISSION_XCODE_APP: ${{ needs.macos-compile-admission.outputs.xcode_app }}
         run: python3 scripts/ci/late_placement.py
+      # A moved job waits for a root runner, which another run may take first.
+      # ci.yml's owned-pool-watch starts the rescue for a run that might get
+      # here, and this marker tells it jobs moved (owned_pool_rescue.py,
+      # LATE_MARKER_PREFIX), so it watches them like any owned job.
+      - name: Record the moved jobs
+        if: steps.place.outputs.runners != '' && steps.place.outputs.runners != '{}'
+        continue-on-error: true
+        env:
+          RUNNERS: ${{ steps.place.outputs.runners }}
+        run: mkdir -p "$RUNNER_TEMP/late-placement" && printf '%s\n' "$RUNNERS" > "$RUNNER_TEMP/late-placement/runners.json"
+      - name: Upload the late placement marker
+        if: steps.place.outputs.runners != '' && steps.place.outputs.runners != '{}'
+        continue-on-error: true
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: macos-pool-late-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/late-placement/runners.json
+          retention-days: 1
 
   app-host-unit-tests:
     permissions:
diff --git a/.github/workflows/ci-owned-pool-rescue.yml b/.github/workflows/ci-owned-pool-rescue.yml
index d0f1329398a0..ffd8837e8d9c 100644
--- a/.github/workflows/ci-owned-pool-rescue.yml
+++ b/.github/workflows/ci-owned-pool-rescue.yml
@@ -65,6 +65,11 @@ on:
         required: false
         default: "1"
         type: string
+      late:
+        description: 1 when the picker placed nothing owned, so the watch waits for ci-macos.yml's late placement
+        required: false
+        default: "0"
+        type: string
 
 permissions: {}
 
@@ -111,6 +116,8 @@ jobs:
           RESCUE_SECONDS: ${{ vars.CI_OWNED_POOL_RESCUE_SECONDS }}
           POOL_OWNED: ${{ vars.CI_PR_POOL_OWNED }}
           WATCH_RUN_ID: ${{ inputs.run_id }}
+          # 1: wait for ci-macos.yml's late-placement before calling the run ephemeral.
+          LATE_PLACEMENT: ${{ inputs.late }}
           # 1: a stuck run's full re-run may take the light pool; watch it.
           OWNED_LIGHT_RETRY: ${{ vars.CI_OWNED_LIGHT_RETRY }}
         run: python3 scripts/ci/owned_pool_rescue.py
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index cbeafba6f0f2..dc6dca64b42b 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -1000,7 +1000,11 @@ jobs:
     # same-repository pull request, so it is watched too.
     name: Watch owned pool jobs
     needs: changes
-    if: ${{ needs.changes.outputs.macos_pr_owned_jobs != '' && (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && github.run_attempt == 1 && vars.CI_PR_POOL_OWNED == '1' && (vars.CI_OWNED_POOL_RESCUE || '1') != '0' }}
+    # A same-repository pull request whose picker placed nothing owned is
+    # watched too when ci-macos.yml's late-placement may still move its jobs
+    # after compile admission (a full or unit suite, or the CLI lane): the
+    # watch waits for that job and follows the run only if it moved some.
+    if: ${{ (needs.changes.outputs.macos_pr_owned_jobs != '' || github.event_name == 'pull_request' && vars.GLAEDA_ROUTE_APP_ID != '' && (needs.changes.outputs.macos == 'true' && (needs.changes.outputs.full_suite == 'true' || needs.changes.outputs.unit_suite == 'true') || needs.changes.outputs.cli == 'true')) && (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && github.run_attempt == 1 && vars.CI_PR_POOL_OWNED == '1' && (vars.CI_OWNED_POOL_RESCUE || '1') != '0' }}
     runs-on: ubuntu-24.04 # github-hosted-required: one API call; keeps CI's Linux pool free
     timeout-minutes: 5
     # Job level, like reverse-test-impact, so macos-admission-gate never
@@ -1015,7 +1019,9 @@ jobs:
           GH_REPO: ${{ github.repository }}
           RUN_ID: ${{ github.run_id }}
           RUN_ATTEMPT: ${{ github.run_attempt }}
-        run: gh workflow run ci-owned-pool-rescue.yml --ref main -f run_id="$RUN_ID" -f run_attempt="$RUN_ATTEMPT"
+          # 1 when the picker placed nothing owned: wait for late placement.
+          LATE: ${{ needs.changes.outputs.macos_pr_owned_jobs == '' && '1' || '0' }}
+        run: gh workflow run ci-owned-pool-rescue.yml --ref main -f run_id="$RUN_ID" -f run_attempt="$RUN_ATTEMPT" -f late="$LATE"
 
   static-preflight:
     name: Fast static checks
diff --git a/scripts/ci/owned_pool_rescue.py b/scripts/ci/owned_pool_rescue.py
index 7eabde3b880b..4bcb6ab5ad94 100644
--- a/scripts/ci/owned_pool_rescue.py
+++ b/scripts/ci/owned_pool_rescue.py
@@ -122,7 +122,10 @@
 - on the attempt 2 it re-ran from failed jobs, no job runs on an owned label;
 - on the attempt 2 it re-ran in full, `changes` finished without that
   attempt's marker, or CI_OWNED_LIGHT_RETRY is off (not watched at all);
-- `changes` finished without a marker: the run is on an ephemeral pool;
+- `changes` finished without a marker: the run is on an ephemeral pool.
+  When ci.yml started the watch for late placement (LATE_PLACEMENT=1), it
+  first waits for ci-macos.yml's late-placement job and follows the run if
+  that job uploaded its marker (it moved jobs onto owned root runners);
 - the run finished, or the watch limit passed.
 
 Request budget: the GITHUB_TOKEN allows about 1000 requests an hour for the
@@ -217,6 +220,10 @@
 READ_ATTEMPTS = 3
 READ_RETRY_SECONDS = 10
 MARKER_PREFIX = "macos-pool-persistent"
+# ci-macos.yml's late-placement moved jobs after compile admission onto idle
+# owned root runners (late_placement.py), and started this watch itself.
+LATE_MARKER_PREFIX = "macos-pool-late"
+LATE_JOB = "macos / late-placement"
 # A cancelled run is only useful re-run: giving up leaves the pull request's
 # run cancelled for good. A Mac job mid-compile has taken over 5 minutes to
 # settle after a force-cancel (run 36074561333, 2026-09-24), so wait long, and
@@ -460,6 +467,9 @@ class Target:
     full_rerun: bool = False
     side: bool = False  # a side-lane workflow (SIDE_WORKFLOW_PATHS): no picker job
     main: bool = False  # main's full-suite dispatch of ci.yml: no pull request, main's HEAD instead
+    # ci.yml started this watch because late-placement may move jobs onto owned
+    # root runners after compile admission (LATE_PLACEMENT=1); the picker placed none.
+    late: bool = False
 
     @property
     def picker_job(self) -> str:
@@ -509,6 +519,11 @@ def marker_name(target: Target) -> str:
     return f"{MARKER_PREFIX}-{target.run_id}-{target.attempt}-"
 
 
+def late_marker_name(target: Target) -> str:
+    """The marker late-placement uploads when it moved jobs onto owned root runners."""
+    return f"{LATE_MARKER_PREFIX}-{target.run_id}-{target.attempt}"
+
+
 READ_ERRORS = (urllib.error.URLError, http.client.HTTPException, OSError, ValueError)
 
 
@@ -542,6 +557,7 @@ def watch(api: GitHub, target: Target, *, budget_seconds: int,
     sleep(FIRST_LOOK_SECONDS)
     looks = 0
     on_persistent = False
+    picker_marker: bool | None = None
     first_seen: dict[Any, dt.datetime] = {}
     while True:
         looks += 1
@@ -565,13 +581,25 @@ def watch(api: GitHub, target: Target, *, budget_seconds: int,
                 return "stop", "no job of this attempt asked for a persistent pool"
         elif not on_persistent:
             if picker_finished(jobs, target.picker_job):
-                if not read(lambda: api.has_artifact(target.run_id, marker_name(target)), sleep, log):
+                if picker_marker is None:
+                    picker_marker = bool(read(lambda: api.has_artifact(target.run_id, marker_name(target)),
+                                              sleep, log))
+                if picker_marker:
+                    log("the picker chose a persistent pool")
+                    on_persistent = True
+                elif not target.late:
                     return "stop", "the run is on an ephemeral pool"
-                on_persistent = True
-                log("the picker chose a persistent pool")
+                elif picker_finished(jobs, LATE_JOB):
+                    if not read(lambda: api.has_artifact(target.run_id, late_marker_name(target)), sleep, log):
+                        return "stop", "late placement moved no job onto a persistent pool"
+                    log("late placement moved jobs onto a persistent pool")
+                    on_persistent = True
+                elif run_finished(jobs):
+                    return "stop", "the run finished on an ephemeral pool"
             elif run_finished(jobs):
                 return "stop", "the run finished before the pool choice"
-        interval = POLL_SECONDS
+        # Waiting for compile admission and late placement: nothing can be stuck yet.
+        interval = POLL_SECONDS if on_persistent or picker_marker is None else IDLE_POLL_SECONDS
         if on_persistent:
             if any(refused(job) for job in jobs):
                 look = assess(jobs, now=now(), budget_seconds=budget_seconds, first_seen=first_seen,
@@ -755,6 +783,9 @@ def finish(outcome: str) -> int:
     target = target_from_event(event, repository)
     if isinstance(target, str):
         return finish(f"not watched: {target}")
+    if ((env.get("LATE_PLACEMENT") or "").strip() == "1" and target.attempt == 1
+            and not (target.e2e or target.main or target.side)):
+        target = dataclasses.replace(target, late=True)
     subject = ("an E2E dispatch" if target.path == E2E_WORKFLOW_PATH else f"a dispatch of {target.path}") \
         if target.e2e else f"main's full-suite dispatch at {target.head_sha[:12]}" if target.main \
         else f"pull request #{target.pr_number}"
diff --git a/tests/test_ci_late_placement.py b/tests/test_ci_late_placement.py
index 57483069c2b6..aeeb5daa850c 100644
--- a/tests/test_ci_late_placement.py
+++ b/tests/test_ci_late_placement.py
@@ -125,6 +125,18 @@ def test_late_placement_runs_only_where_the_picker_may_use_owned_runners(self):
         self.assertTrue(all(step.get("continue-on-error") for step in spec["steps"]))
         self.assertEqual(spec["outputs"]["runners"], "${{ steps.place.outputs.runners || '{}' }}")
 
+    def test_moved_jobs_leave_the_marker_the_rescue_watch_looks_for(self):
+        steps = {step["name"]: step for step in self.jobs["late-placement"]["steps"]}
+        marker = steps["Upload the late placement marker"]
+        self.assertIn("steps.place.outputs.runners != '{}'", marker["if"])
+        rescue = (ROOT / "scripts/ci/owned_pool_rescue.py").read_text()
+        # owned_pool_rescue.late_marker_name() and LATE_JOB: the names the watch reads.
+        self.assertIn('LATE_MARKER_PREFIX = "macos-pool-late"', rescue)
+        self.assertEqual(marker["with"]["name"], "macos-pool-late-${{ github.run_id }}-${{ github.run_attempt }}")
+        self.assertIn('LATE_JOB = "macos / late-placement"', rescue)
+        # It starts nothing itself: ci.yml's owned-pool-watch holds the only actions: write.
+        self.assertEqual(self.jobs["late-placement"]["permissions"], {"contents": "read"})
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_ci_owned_pool_rescue.py b/tests/test_ci_owned_pool_rescue.py
index 3461c95e4be8..c0e16b30309b 100644
--- a/tests/test_ci_owned_pool_rescue.py
+++ b/tests/test_ci_owned_pool_rescue.py
@@ -401,6 +401,45 @@ def test_ephemeral_run_stops_after_the_marker_check(self):
         self.assertEqual(api.calls, ["jobs", f"artifact:macos-pool-persistent-{RUN_ID}-1-"])
         self.assertIn("the run is on an ephemeral pool", summary)
 
+    def test_jobs_late_placement_moved_are_watched_and_rescued(self):
+        # The picker put everything on Blacksmith (no marker), so ci.yml started the watch
+        # for late placement, which moved a shard onto an owned root runner that another
+        # run took first.
+        def jobs(seconds):
+            found = [changes()(seconds),
+                     job("macos / macOS compile admission", status="completed", labels=[BLACKSMITH], created=5),
+                     job("macos / swift-package-tests", status="in_progress", labels=[BLACKSMITH], created=5,
+                         runner="bs-1")]
+            if seconds >= 600:
+                found.append(job(rescue.LATE_JOB, status="completed", created=590))
+                found.append(job("macos / app-host unit tests (1/7)", labels=[MINI], created=600))
+            return found
+        clock = Clock()
+        api = FakeAPI(clock, jobs, marker=lambda name: name.startswith("macos-pool-late-"))
+        _, summary = run_main(api, clock, env_extra={"RESCUE_SECONDS": "30", "QUEUE_ROUNDS": "",
+                                                     "LATE_PLACEMENT": "1"})
+        self.assertIn(f"artifact:macos-pool-late-{RUN_ID}-1", api.calls)
+        self.assertEqual(api.calls[-4:], ["cancel", "run", "pull", "rerun"])
+        # The picker's marker is read once, not on every look while admission runs.
+        self.assertEqual(api.calls.count(f"artifact:macos-pool-persistent-{RUN_ID}-1-"), 1)
+
+    def test_late_placement_that_moved_nothing_ends_the_watch(self):
+        def jobs(seconds):
+            found = [changes()(seconds),
+                     job("macos / macOS compile admission", status="completed", labels=[BLACKSMITH], created=5),
+                     job("macos / swift-package-tests", status="in_progress", labels=[BLACKSMITH], created=5,
+                         runner="bs-1")]
+            if seconds >= 300:
+                found.append(job(rescue.LATE_JOB, status="completed", created=290))
+            return found
+        clock = Clock()
+        api = FakeAPI(clock, jobs, marker=False)
+        _, summary = run_main(api, clock, env_extra={"LATE_PLACEMENT": "1"})
+        self.assertIn("late placement moved no job", summary)
+        self.assertNotIn("cancel", api.calls)
+        # Admission's minutes pass at IDLE_POLL_SECONDS: looks at 45, 165, 285 and 405 s.
+        self.assertLessEqual(api.calls.count("jobs"), 5)
+
     def test_waits_for_the_picker_before_looking_for_the_marker(self):
         clock = Clock()
         api = FakeAPI(clock, lambda s: [changes(done_at=100)(s)])
@@ -1097,9 +1136,15 @@ def test_ci_and_e2e_dispatch_it_only_for_an_owned_placement(self):
             # Fail-safe: ci.yml at job level (macos-admission-gate skips a job
             # that cannot fail); the dispatch workflows at step level.
             self.assertIs(job.get("continue-on-error", job["steps"][0].get("continue-on-error")), True, path)
+            # ci.yml also says whether to wait for late placement (the picker owned nothing).
+            late = ' -f late="$LATE"' if path.endswith("/ci.yml") else ""
             self.assertEqual(job["steps"][0]["run"],
                              'gh workflow run ci-owned-pool-rescue.yml --ref main '
-                             '-f run_id="$RUN_ID" -f run_attempt="$RUN_ATTEMPT"', path)
+                             '-f run_id="$RUN_ID" -f run_attempt="$RUN_ATTEMPT"' + late, path)
+        watch = yaml.safe_load((ROOT / ".github/workflows/ci.yml").read_text(encoding="utf-8"))["jobs"]["owned-pool-watch"]
+        self.assertEqual(watch["steps"][0]["env"]["LATE"], "${{ needs.changes.outputs.macos_pr_owned_jobs == '' && '1' || '0' }}")
+        # A run the picker placed nothing owned is watched only where late placement can move jobs.
+        self.assertIn("github.event_name == 'pull_request' && vars.GLAEDA_ROUTE_APP_ID != ''", watch["if"])
         ios = yaml.safe_load((ROOT / ".github/workflows/test-ios.yml").read_text(encoding="utf-8"))["jobs"]
         self.assertEqual(ios["runner"]["outputs"]["owned_marker"], "${{ steps.marker.outputs.path != '' }}")
         screenshots = yaml.safe_load((ROOT / ".github/workflows/ios-screenshots.yml").read_text(encoding="utf-8"))

From 55ba6ccab1bc622244f0256b06e1ec03f8ae5fe9 Mon Sep 17 00:00:00 2001
From: Leo Li <cheerleaderleo@outlook.com>
Date: Fri, 25 Sep 2026 06:20:29 -0400
Subject: [PATCH 3/4] tests: register test_ci_late_placement.py on the
 linux-guard lane

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 tests/test-execution.toml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/test-execution.toml b/tests/test-execution.toml
index 4513493c5321..050c4348dd64 100644
--- a/tests/test-execution.toml
+++ b/tests/test-execution.toml
@@ -1211,3 +1211,7 @@ lane = "macos-shell"
 [[test]]
 path = "tests/test_shell_surface_scoped_keys.py"
 lane = "linux-guard"
+
+[[test]]
+path = "tests/test_ci_late_placement.py"
+lane = "linux-guard"

From a1c9fe94c3feacf8dfc04b666caea8a38e6b2e17 Mon Sep 17 00:00:00 2001
From: Leo Li <cheerleaderleo@outlook.com>
Date: Fri, 25 Sep 2026 06:32:32 -0400
Subject: [PATCH 4/4] ci: read the owned-pool rescue's Actions API through the
 route App

The watch polls jobs and artifacts on GITHUB_TOKEN, about 1000 requests
an hour for the whole repository. The org's manaflow-glaeda-route App
already has actions access (cmuxterm-hq#639) and its own 5000 an hour.

The rescue workflow mints an App token (actions, contents and
pull-requests read) and the script sends every GET with it. Cancels and
re-runs keep GITHUB_TOKEN, because ci-macos.yml's attempt-2 routing
requires the re-run's triggering actor to be github-actions[bot]. A 401
on a read (the token lasts an hour, a watch may run longer) drops back to
GITHUB_TOKEN; a failed mint leaves every request on GITHUB_TOKEN.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .github/workflows/ci-owned-pool-rescue.yml | 17 ++++++
 scripts/ci/owned_pool_rescue.py            | 52 ++++++++++++++-----
 tests/test_ci_owned_pool_rescue.py         | 60 ++++++++++++++++++++++
 3 files changed, 116 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/ci-owned-pool-rescue.yml b/.github/workflows/ci-owned-pool-rescue.yml
index ffd8837e8d9c..82ff8d87fd55 100644
--- a/.github/workflows/ci-owned-pool-rescue.yml
+++ b/.github/workflows/ci-owned-pool-rescue.yml
@@ -108,8 +108,25 @@ jobs:
           ref: main
           persist-credentials: false
 
+      # Reads from the org's manaflow-glaeda-route App, on its own 5000 requests
+      # an hour; cancels and re-runs stay on GITHUB_TOKEN (owned_pool_rescue.py,
+      # GitHub). Any failure leaves READ_TOKEN empty and GITHUB_TOKEN reads.
+      - name: Mint the read token
+        id: read-token
+        if: vars.GLAEDA_ROUTE_APP_ID != ''
+        continue-on-error: true
+        timeout-minutes: 1
+        uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
+        with:
+          app-id: ${{ vars.GLAEDA_ROUTE_APP_ID }}
+          private-key: ${{ secrets.GLAEDA_ROUTE_APP_KEY }}
+          permission-actions: read
+          permission-contents: read
+          permission-pull-requests: read
+
       - name: Watch the run's persistent pool jobs
         env:
+          READ_TOKEN: ${{ steps.read-token.outputs.token }}
           # A CI run's budget grows by a round per queue round the picker allows.
           QUEUE_ROUNDS: ${{ vars.CI_PR_POOL_QUEUE_ROUNDS }}
           GH_TOKEN: ${{ github.token }}
diff --git a/scripts/ci/owned_pool_rescue.py b/scripts/ci/owned_pool_rescue.py
index 4bcb6ab5ad94..7a310c33476e 100644
--- a/scripts/ci/owned_pool_rescue.py
+++ b/scripts/ci/owned_pool_rescue.py
@@ -128,8 +128,10 @@
   that job uploaded its marker (it moved jobs onto owned root runners);
 - the run finished, or the watch limit passed.
 
-Request budget: the GITHUB_TOKEN allows about 1000 requests an hour for the
-whole repository. A run on an ephemeral pool costs a jobs listing every
+Request budget: the reads use a manaflow-glaeda-route App token when the
+workflow could mint one (READ_TOKEN; its own 5000 requests an hour), else
+GITHUB_TOKEN, which allows about 1000 requests an hour for the whole
+repository. Cancels and re-runs always use GITHUB_TOKEN (GitHub.__doc__). A run on an ephemeral pool costs a jobs listing every
 POLL_SECONDS until `changes` finishes (usually two or three) plus one artifact
 listing. A run on a persistent pool adds a jobs listing every POLL_SECONDS
 while one of its jobs waits for a runner and every IDLE_POLL_SECONDS otherwise,
@@ -395,20 +397,43 @@ class Aborted(Exception):
     pass
 
 
+def _headers(token: str) -> dict[str, str]:
+    return {
+        "Accept": "application/vnd.github+json",
+        "Authorization": f"Bearer {token}",
+        "X-GitHub-Api-Version": "2022-11-28",
+        "User-Agent": "cmux-ci-owned-pool-rescue",
+    }
+
+
 class GitHub:
-    def __init__(self, token: str, repo: str) -> None:
+    """The Actions API. Reads may use `read_token`, writes always use `token`.
+
+    `read_token` is a manaflow-glaeda-route App installation token, so the
+    watch's polling draws on the App's own 5000 requests an hour instead of
+    the repository's GITHUB_TOKEN budget. Cancels and re-runs keep
+    GITHUB_TOKEN: a re-run's triggering actor must stay github-actions[bot],
+    which ci-macos.yml's attempt-2 routing checks. An installation token
+    lasts an hour and a watch may outlive it, so a 401 on a read drops back to
+    `token` for the rest of the watch.
+    """
+
+    def __init__(self, token: str, repo: str, read_token: str = "") -> None:
         self.repo = repo
-        self.headers = {
-            "Accept": "application/vnd.github+json",
-            "Authorization": f"Bearer {token}",
-            "X-GitHub-Api-Version": "2022-11-28",
-            "User-Agent": "cmux-ci-owned-pool-rescue",
-        }
+        self.headers = _headers(token)
+        self.read_headers = _headers(read_token) if read_token else self.headers
 
     def request(self, method: str, path: str) -> Any:
-        request = urllib.request.Request(f"{API}/repos/{self.repo}{path}", method=method, headers=self.headers)
-        with urllib.request.urlopen(request, timeout=20) as response:
-            body = response.read()
+        headers = self.read_headers if method == "GET" else self.headers
+        request = urllib.request.Request(f"{API}/repos/{self.repo}{path}", method=method, headers=headers)
+        try:
+            with urllib.request.urlopen(request, timeout=20) as response:
+                body = response.read()
+        except urllib.error.HTTPError as error:
+            if error.code != 401 or headers is self.headers:
+                raise
+            self.read_headers = self.headers
+            return self.request(method, path)
         return json.loads(body) if body else None
 
     def run(self, run_id: int) -> Mapping[str, Any]:
@@ -765,7 +790,8 @@ def finish(outcome: str) -> int:
         return finish(f"CI_OWNED_POOL_RESCUE_SECONDS must be {MIN_BUDGET_SECONDS} to {MAX_BUDGET_SECONDS}; "
                       "nothing to watch")
     repository = env.get("GITHUB_REPOSITORY") or ""
-    client = api or GitHub(env.get("GH_TOKEN") or env.get("GITHUB_TOKEN") or "", repository)
+    client = api or GitHub(env.get("GH_TOKEN") or env.get("GITHUB_TOKEN") or "", repository,
+                           read_token=env.get("READ_TOKEN") or "")
     run_id = (env.get("WATCH_RUN_ID") or "").strip()
     if run_id:
         # Dispatched by the picker's job: read the run it names and check it
diff --git a/tests/test_ci_owned_pool_rescue.py b/tests/test_ci_owned_pool_rescue.py
index c0e16b30309b..148e3cefb1ed 100644
--- a/tests/test_ci_owned_pool_rescue.py
+++ b/tests/test_ci_owned_pool_rescue.py
@@ -1065,6 +1065,66 @@ def test_a_refused_main_job_reruns_the_failed_jobs(self):
         self.assertIn("rerun-failed", api.calls)
 
 
+class Tokens(unittest.TestCase):
+    """Reads may use the App's token; writes always use GITHUB_TOKEN."""
+
+    def open_with(self, fail_first_read=False):
+        seen = []
+
+        class Response(io.BytesIO):
+            def __enter__(self):
+                return self
+
+            def __exit__(self, *exc):
+                return False
+
+        def urlopen(request, timeout):
+            seen.append((request.get_method(), request.headers["Authorization"]))
+            if fail_first_read and len(seen) == 1:
+                raise rescue.urllib.error.HTTPError(request.full_url, 401, "expired", {}, None)
+            return Response(b"{}")
+        return seen, unittest.mock.patch.object(rescue.urllib.request, "urlopen", urlopen)
+
+    def test_reads_use_the_app_token_and_writes_keep_github_token(self):
+        seen, patch = self.open_with()
+        with patch:
+            api = rescue.GitHub("repo-token", "o/r", read_token="app-token")
+            api.run(1)
+            api.rerun_failed(1)
+            api.cancel(1)
+        # A re-run started by the App would not be github-actions[bot], which
+        # ci-macos.yml's attempt-2 routing requires.
+        self.assertEqual(seen, [("GET", "Bearer app-token"), ("POST", "Bearer repo-token"),
+                                ("POST", "Bearer repo-token")])
+
+    def test_an_expired_app_token_falls_back_for_the_rest_of_the_watch(self):
+        seen, patch = self.open_with(fail_first_read=True)
+        with patch:
+            api = rescue.GitHub("repo-token", "o/r", read_token="app-token")
+            api.run(1)
+            api.run(1)
+        self.assertEqual(seen, [("GET", "Bearer app-token"), ("GET", "Bearer repo-token"),
+                                ("GET", "Bearer repo-token")])
+
+    def test_without_an_app_token_everything_uses_github_token(self):
+        seen, patch = self.open_with()
+        with patch:
+            rescue.GitHub("repo-token", "o/r").run(1)
+        self.assertEqual(seen, [("GET", "Bearer repo-token")])
+
+    def test_the_workflow_mints_a_read_only_token_and_passes_it(self):
+        steps = yaml.safe_load((ROOT / ".github/workflows/ci-owned-pool-rescue.yml").read_text(
+            encoding="utf-8"))["jobs"]["rescue"]["steps"]
+        mint = next(step for step in steps if step.get("id") == "read-token")
+        self.assertTrue(mint["continue-on-error"])
+        self.assertEqual({key: value for key, value in mint["with"].items() if key.startswith("permission-")},
+                         {"permission-actions": "read", "permission-contents": "read",
+                          "permission-pull-requests": "read"})
+        watch = next(step for step in steps if step.get("name") == "Watch the run's persistent pool jobs")
+        self.assertEqual(watch["env"]["READ_TOKEN"], "${{ steps.read-token.outputs.token }}")
+        self.assertEqual(watch["env"]["GH_TOKEN"], "${{ github.token }}")
+
+
 class Workflow(unittest.TestCase):
     def setUp(self):
         self.text = (ROOT / ".github/workflows/ci-owned-pool-rescue.yml").read_text(encoding="utf-8")