ci: route iOS kit tests through the runner pool

.github/runners.json

@@ -203,10 +203,7 @@
     "cmux-next-frame-pacing.yml#bench": "apple-gui",
     "cmux-next-guards.yml#ci-guard-group": "portable-checks",
     "cmux-next-guards.yml#tooling-tests": "portable-checks",
-    "cmux-next-ios-kit.yml#kit-tests": {
-      "class": "apple-swift-tests",
-      "picker_followup": "runs-on names a fixed macOS label; #28145 takes it from ios_runner_pool.py and removes this followup"
-    },
+    "cmux-next-ios-kit.yml#kit-tests": "apple-swift-tests",
     "cmux-next-media.yml#tour": "apple-gui",
     "cmux-next-perf-bench.yml#bench": {
       "class": "apple-gui",

.github/workflows/ci-owned-pool-rescue.yml

@@ -57,6 +57,7 @@ on:
       - Nightly macOS build
       - CI
       - iOS simulator tests
+      - cmux-next-ios kit tests
       - Iroh release gate
       - cmux NEXT DEV daily
       # Side lanes can create a gated Mac job after workflow_run(requested).
@@ -112,6 +113,7 @@ env:
     .github/workflows/nightly.yml
     .github/workflows/ci.yml
     .github/workflows/test-ios.yml
+    .github/workflows/cmux-next-ios-kit.yml
     .github/workflows/iroh-release-gate.yml
     .github/workflows/cmux-next-dev-daily.yml
     .github/workflows/agent-brands.yml

.github/workflows/cmux-next-ios-kit.yml

@@ -16,11 +16,8 @@ name: cmux-next-ios kit tests
 # - runs the tests on an iOS simulator (`xcodebuild test`): the live link
 #   coordinator, its banner and the FFI transport are iOS only, and #23829 left an
 #   ambiguous type that only the iOS build saw.
-# It runs on Blacksmith macOS 26 (Xcode 26, iOS simulator SDK), as the other
-# mobile FFI job does (cmux-mobile-ffi.yml's apple job, on macOS 15 there): an
-# owned side-lane label needs a placement job (scripts/ci/runner_routing_table.py),
-# which only cmux-next.yml has. CI_PAID_MACOS_OVERFLOW and MACOS_RUNNER_26 are
-# the existing overflow switch and label cmux-next.yml reads.
+# The Linux picker offers an idle owned simulator Mac first, with Blacksmith
+# as fallback. The persistent-pool rescue watches stalled owned jobs.
 on:
   pull_request:
     branches: [feat-cmux-next]
@@ -36,6 +33,7 @@ on:
       - "Packages/Shared/CmuxChiefVoiceCore/**"
       - "Packages/iOS/CmuxTabsKit/**"
       - "Packages/iOS/CmuxConversation/**"
+      - "Packages/iOS/CmuxMobileSupport/**"
       - "vendor/stack-auth-swift-sdk-prerelease/**"
       # The toolchains it uses.
       - ".github/actions/setup-cmux-tui-rust/**"
@@ -58,6 +56,7 @@ on:
       - "Packages/Shared/CmuxChiefVoiceCore/**"
       - "Packages/iOS/CmuxTabsKit/**"
       - "Packages/iOS/CmuxConversation/**"
+      - "Packages/iOS/CmuxMobileSupport/**"
       - "vendor/stack-auth-swift-sdk-prerelease/**"
       # The toolchains it uses.
       - ".github/actions/setup-cmux-tui-rust/**"
@@ -77,10 +76,118 @@ concurrency:
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
+  # Pick an idle owned Mac with an iOS simulator, then fall back to Blacksmith
+  # on retries. Keeping this picker ahead of the test job lets the rescue
+  # sweeper re-admit a job that stalls on a dead owned label.
+  runner:
+    environment: ${{ !contains(fromJSON('["pull_request","merge_group"]'), github.event_name) && 'ci-routing-agent' || '' }}
+    runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && 'blacksmith-4vcpu-ubuntu-2404' || !startsWith(vars.LINUX_RUNNER, 'blacksmith-') && vars.LINUX_RUNNER || fromJSON(vars.RUNNER_POOLS || '{}').linux || vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }}
+    timeout-minutes: 5
+    permissions:
+      contents: read
+      actions: read
+    outputs:
+      runs_on: ${{ steps.pool.outputs.runs_on }}
+      retry_runs_on: ${{ steps.pool.outputs.retry_runs_on }}
+      label: ${{ steps.pool.outputs.label }}
+      persistent: ${{ steps.pool.outputs.persistent }}
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          sparse-checkout: |
+            scripts/ci/ios_runner_pool.py
+            scripts/ci/e2e_runner_pool.py
+            scripts/ci/pr_runner_pool.py
+            scripts/ci/simple_pool_picker.py
+          sparse-checkout-cone-mode: false
+          persist-credentials: false
+      - name: Mint the owned-pool routing token
+        id: route-token
+        if: (github.event_name == 'workflow_dispatch' || github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository)) && vars.CI_PR_POOL_OWNED == '1' && vars.CI_IOS_OWNED == '1' && vars.GLAEDA_ROUTE_APP_ID != ''
+        continue-on-error: true
+        timeout-minutes: 1
+        uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
+        with:
+          app-id: ${{ vars.GLAEDA_ROUTE_APP_ID }}
+          private-key: ${{ secrets.GLAEDA_ROUTE_APP_KEY }}
+          permission-administration: read
+          # Queue, job and capacity-snapshot reads use this App's quota.
+          permission-actions: read
+          permission-organization-self-hosted-runners: read
+      - name: Mint the repository routing token
+        id: route-token-repo
+        if: steps.route-token.outcome == 'failure'
+        continue-on-error: true
+        timeout-minutes: 1
+        uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
+        with:
+          app-id: ${{ vars.GLAEDA_ROUTE_APP_ID }}
+          private-key: ${{ secrets.GLAEDA_ROUTE_APP_KEY }}
+          permission-administration: read
+          permission-actions: read
+      - name: Pick the macOS pool
+        id: pool
+        env:
+          GH_TOKEN: ${{ github.token }}
+          GH_REPO: ${{ github.repository }}
+          ROUTE_TOKEN: ${{ steps.route-token.outputs.token || steps.route-token-repo.outputs.token }}
+          FORK_PULL_REQUEST: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }}
+          RUNNER_VARIABLE: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && 'blacksmith-6vcpu-macos-26' || vars.MACOS_RUNNER_TESTS || vars.MACOS_RUNNER_IOS }}
+          IOS_OWNED: ${{ vars.CI_IOS_OWNED }}
+          POOL_OWNED: ${{ vars.CI_PR_POOL_OWNED }}
+          OWNED_SLOTS: ${{ vars.CI_OWNED_POOL_SLOTS }}
+          PR_XCODE_APP: ${{ (github.event_name == 'workflow_dispatch' || github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) && vars.CMUX_CI_XCODE_APP_PR || '' }}
+          POOL_ORDER: ${{ vars.CI_PR_POOL_ORDER }}
+          POOL_MAX_QUEUED: ${{ vars.CI_PR_POOL_MAX_QUEUED }}
+        run: |
+          set -euo pipefail
+          python3 scripts/ci/ios_runner_pool.py \
+            --lane kit --fork "$FORK_PULL_REQUEST" --requested auto \
+            --variable "$RUNNER_VARIABLE" --ios-owned "$IOS_OWNED" --owned "$POOL_OWNED" \
+            --owned-slots "$OWNED_SLOTS" --pr-xcode-app "$PR_XCODE_APP" --order "$POOL_ORDER" \
+            --max-queued "$POOL_MAX_QUEUED" --queue-rounds 0 \
+            --device-family iphone | tee -a "$GITHUB_OUTPUT"
+      - name: Create the owned-pool marker
+        id: marker
+        if: steps.pool.outputs.persistent == 'true' && github.run_attempt == 1
+        continue-on-error: true
+        run: |
+          set -euo pipefail
+          marker="$RUNNER_TEMP/macos-pool-persistent.json"
+          echo '{}' > "$marker"
+          echo "path=$marker" >> "$GITHUB_OUTPUT"
+      - name: Upload the persistent pool marker
+        if: steps.marker.outputs.path != ''
+        continue-on-error: true
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: macos-pool-persistent-${{ github.run_id }}-${{ github.run_attempt }}-1-${{ steps.pool.outputs.label }}
+          path: ${{ steps.marker.outputs.path }}
+          retention-days: 1
+      - name: Upload the simulator capacity marker
+        if: steps.marker.outputs.path != ''
+        continue-on-error: true
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: macos-pool-persistent-${{ github.run_id }}-${{ github.run_attempt }}-1-glaeda-ios-sim
+          path: ${{ steps.marker.outputs.path }}
+          retention-days: 1
+      - name: Upload the owned-pool watch marker
+        if: steps.marker.outputs.path != ''
+        continue-on-error: true
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: owned-pool-watch
+          path: ${{ steps.marker.outputs.path }}
+          retention-days: 1
+
   kit-tests:
+    needs: [runner]
     name: CmuxMobileKit tests (macOS swift test, iOS simulator)
-    runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && 'blacksmith-6vcpu-macos-26' || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_26 || 'blacksmith-6vcpu-macos-26' }}
+    runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && 'blacksmith-6vcpu-macos-26' || fromJSON(github.run_attempt > 1 && needs.runner.outputs.retry_runs_on || needs.runner.outputs.runs_on) }}
     timeout-minutes: 60
+    env:
+      CMUX_CI_XCODE_APP: ${{ github.run_attempt == 1 && startsWith(needs.runner.outputs.label, 'glaeda-') && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && vars.CMUX_CI_XCODE_APP_PR || '' }}
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:

scripts/ci/ios_runner_pool.py

@@ -53,7 +53,9 @@
 for: SIM_LABEL must have the run's simulator jobs free now.
 
 Live capacity. test-ios.yml mints the org's glaeda-route App token (as ci.yml
-does) for same-repository runs and passes it as ROUTE_TOKEN. With it, "free"
+does) for same-repository runs and passes it as ROUTE_TOKEN. The same token
+reads the queue and snapshots when it has Actions read permission, with one
+fallback to the job token for older callers. With it, "free"
 is read from live state, not estimated from run titles. Each in-flight
 test-ios.yml and ios-screenshots.yml run of the last SIM_WINDOW_MINUTES (only
 those two workflows hold simulators) is charged what its jobs show it holds
@@ -191,7 +193,7 @@
 OWNED_CHOICE = "owned"
 IOS_OWNED_VARIABLE = "CI_IOS_OWNED"
 # The workflows whose in-flight runs since the snapshot hold simulators.
-IOS_WORKFLOWS = ("test-ios.yml", "ios-screenshots.yml")
+IOS_WORKFLOWS = ("test-ios.yml", "ios-screenshots.yml", "cmux-next-ios-kit.yml")
 # How far back an in-flight iOS run is charged its simulator jobs on the live
 # path: longer than any iOS run lives (ios-screenshots.yml's 300 minute
 # capture; see Live capacity).
@@ -209,7 +211,7 @@
 # The jobs that hold a mini's simulator: test-ios.yml's ios-simulator matrix
 # and ios-screenshots.yml's capture. ios-simulator-build carries SIM_LABEL only
 # to land on a mini with the runtime; glaeda runs it as an isolated build job.
-SIM_JOB = re.compile(r"ios-simulator(?: \(.*\))?|screenshots")
+SIM_JOB = re.compile(r"ios-simulator(?: \(.*\))?|screenshots|CmuxMobileKit tests \(macOS swift test, iOS simulator\)")
 # In-flight iOS runs whose jobs are read (one request each), newest first.
 MAX_JOB_READS = 20
 # Job statuses before a runner has taken the job.
@@ -234,8 +236,10 @@ class Lane:
     # mobile-core-package and ios-simulator-build run side by side; the
     # simulator matrix (iPhone, iPad) follows the build, two again.
     "test-ios": Lane(jobs=2, measured=True),
-    # One capture job.
+    # One capture or CmuxMobileKit job.
     "screenshots": Lane(jobs=1, measured=False),
+    # The kit job builds one iOS simulator target on one Mac.
+    "kit": Lane(jobs=1, measured=True),
 }
 # The most simulator jobs any one iOS run holds: test-ios.yml's two families.
 MAX_SIM_JOBS = 2
@@ -291,7 +295,7 @@ def retry_label(default: str) -> str:
 
 def sim_jobs(lane: str, device_family: str | None, swift_package: str | None = None) -> int:
     """The simulator jobs this run holds at once: one per device family, the one capture, or none."""
-    if lane == "screenshots":
+    if lane in ("screenshots", "kit"):
         return 1
     if (swift_package or "").strip():
         # mobile-core-package alone: SwiftPM tests on the host.
@@ -512,8 +516,11 @@ def charged_sim_jobs(run: Mapping[str, Any], placements: Placements | None = Non
     With `placements`, an `auto` run the picker sent to Blacksmith is charged nothing (see "Where an
     `auto` run went").
     """
-    if screenshots_run(run):
-        return LANES["screenshots"].jobs
+    kit_run = str(run.get("path") or "").split("@", 1)[0].endswith("/cmux-next-ios-kit.yml")
+    if screenshots_run(run) or kit_run:
+        if kit_run and placements is not None and placements.off_fleet(run, now or dt.datetime.now(dt.timezone.utc)):
+            return 0
+        return LANES["screenshots" if screenshots_run(run) else "kit"].jobs
     title = str(run.get("display_title") or "")
     fields = title.split(TITLE_SEPARATOR)
     if not title.startswith(TITLE_PREFIX) or len(fields) != 7 or not fields[6].startswith("on "):
@@ -531,8 +538,9 @@ def charged_sim_jobs(run: Mapping[str, Any], placements: Placements | None = Non
 
 def charged_jobs(run: Mapping[str, Any]) -> int:
     """The owned machines an in-flight iOS run may hold, read from its title; in full when unsure."""
-    if screenshots_run(run):
-        return LANES["screenshots"].jobs
+    kit_run = str(run.get("path") or "").split("@", 1)[0].endswith("/cmux-next-ios-kit.yml")
+    if screenshots_run(run) or kit_run:
+        return LANES["screenshots" if screenshots_run(run) else "kit"].jobs
     title = str(run.get("display_title") or "")
     fields = title.split(TITLE_SEPARATOR)
     if not title.startswith(TITLE_PREFIX) or len(fields) != 7 or not fields[6].startswith("on "):
@@ -786,29 +794,49 @@ def main(argv: Sequence[str] | None = None, env: Mapping[str, str] | None = None
 
     route_token = (env.get("ROUTE_TOKEN") or "").strip()
 
-    def measure() -> IOSLoad:
-        if not token or not repo:
-            raise RuntimeError("GH_TOKEN and GH_REPO are required")
-        client = pr_runner_pool.GitHub(token, repo)
+    def read_load(client: Any, runners: Sequence[Mapping[str, Any]] | None) -> IOSLoad:
         pools = pr_runner_pool.owned_pools(args.pr_xcode_app)
-        if route_token and pools:
+        if runners is not None and pools:
             try:
-                runners = pr_runner_pool.GitHub(route_token, repo).runners()
                 since = pr_runner_pool.iso(now - dt.timedelta(minutes=SIM_WINDOW_MINUTES))
                 recent = in_flight_ios_runs(client, since, exclude_run_id=exclude)
                 capacity = pr_runner_pool.capability_slots(args.owned_slots).get(SIM_LABEL, 0)
                 jobs, placements = live_placements(client, recent, now)
                 return IOSLoad(None, live=live_free(runners, pools[0], recent, now=now, capacity=capacity,
                                                     placements=placements, jobs=jobs))
             except Exception as error:  # noqa: BLE001 - the snapshot path still decides
-                print(f"::warning title=live owned capacity::could not list runners ({error}); using the snapshot",
+                print(f"::warning title=live owned capacity::could not read live capacity ({error}); using the snapshot",
                       file=sys.stderr)
         load = e2e_runner_pool.measure_load(client, now=now, exclude_run_id=exclude)
         if load is None:
             return IOSLoad(None)
         return IOSLoad(load, ios_runs_since(client, str(load.snapshot["generated_at"]), exclude_run_id=exclude,
                                             placements=owned_placements(client), now=now))
 
+    def measure() -> IOSLoad:
+        # The job token and routing App have separate quotas. Reuse the routing
+        # App for queue/artifact reads too; older callers that mint runner-only
+        # tokens retain one bounded fallback to the job token.
+        tokens = list(dict.fromkeys(value for value in (route_token, token) if value))
+        if not tokens or not repo:
+            raise RuntimeError("a GitHub token and GH_REPO are required")
+        runners = None
+        if route_token:
+            try:
+                runners = pr_runner_pool.GitHub(route_token, repo).runners()
+            except Exception as error:  # noqa: BLE001 - the snapshot still decides
+                print(f"::warning title=live owned capacity::could not list runners ({error}); using the snapshot",
+                      file=sys.stderr)
+        for index, read_token in enumerate(tokens):
+            try:
+                return read_load(pr_runner_pool.GitHub(read_token, repo), runners)
+            except Exception as error:  # noqa: BLE001 - no usable measurement keeps the default
+                if index + 1 == len(tokens):
+                    raise
+                print(f"::warning title=runner queue::routing-token read failed ({error}); trying the job token",
+                      file=sys.stderr)
+        raise AssertionError("unreachable")
+
     def log(message: str) -> None:
         print(message, file=sys.stderr)