ci: fix owned pool review items before the fleet is switched on

.github/workflows/ci-macos.yml

@@ -85,6 +85,13 @@ on:
         required: false
         default: ""
         type: string
+      # Set only when pr_runner is persistent (an owned Mac pool): the
+      # Blacksmith pool on the same Xcode that a re-run of failed jobs takes,
+      # because such a re-run reuses attempt 1's pick (pr_runner_pool.py).
+      pr_retry_runner:
+        required: false
+        default: ""
+        type: string
       pr_xcode_app:
         required: false
         default: ""
@@ -111,7 +118,7 @@ jobs:
     # compile on the pool and Xcode seed-derived-data.yml builds with, so both
     # can adopt its DerivedData seed below. Merge groups and dispatches on
     # other branches keep the macos-15 lane.
-    runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (startsWith(inputs.pr_runner, 'blacksmith-') && inputs.pr_runner || 'blacksmith-6vcpu-macos-15') || (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && (inputs.pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15') || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15') }}
+    runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (startsWith(inputs.pr_runner, 'blacksmith-') && inputs.pr_runner || 'blacksmith-6vcpu-macos-15') || (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && (github.run_attempt > 1 && inputs.pr_retry_runner || inputs.pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15') || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15') }}
     # A changed-suites run adds its tests after the compile: the same
     # 30-minute batch ceiling the separate worker had.
     timeout-minutes: ${{ inputs.unit_in_admission == 'true' && 105 || 75 }}
@@ -152,7 +159,7 @@ jobs:
       # keys on the toolchain and the build path instead, so a product built
       # here at the canonical root matches on any pool with the same Xcode.
       # The macOS runner guard still requires it to track runs-on.
-      CMUX_PRODUCT_RUNNER: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (startsWith(inputs.pr_runner, 'blacksmith-') && inputs.pr_runner || 'blacksmith-6vcpu-macos-15') || (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && (inputs.pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15') || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15') }}
+      CMUX_PRODUCT_RUNNER: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (startsWith(inputs.pr_runner, 'blacksmith-') && inputs.pr_runner || 'blacksmith-6vcpu-macos-15') || (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && (github.run_attempt > 1 && inputs.pr_retry_runner || inputs.pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15') || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15') }}
       # What the changed-suites steps at the end of this job read, with the
       # values `app-host unit tests` gives its changed-suites worker, shard 8.
       # The compile reads none of them: it runs plain xcodebuild and bakes the
@@ -1076,7 +1083,9 @@ jobs:
     # compile admission to macOS 26. To spread shards over providers again,
     # every pool involved must carry the admission's exact Xcode; the
     # restore step refuses an older one with both versions named.
-    runs-on: ${{ needs.macos-compile-admission.outputs.runner }}
+    # A re-run of failed shards after compile admission passed on an owned
+    # pool keeps its outputs; pr_retry_runner moves them to Blacksmith.
+    runs-on: ${{ github.run_attempt > 1 && inputs.pr_retry_runner || needs.macos-compile-admission.outputs.runner }}
     timeout-minutes: 75
     strategy:
       # A pull request wants every shard's failures in one run. A merge group
@@ -1158,7 +1167,7 @@ jobs:
       - name: Verify GitHub-hosted route
         env:
           RUNNER_ENVIRONMENT: ${{ runner.environment }}
-          REQUESTED_RUNNER: ${{ needs.macos-compile-admission.outputs.runner }}
+          REQUESTED_RUNNER: ${{ github.run_attempt > 1 && inputs.pr_retry_runner || needs.macos-compile-admission.outputs.runner }}
           RUNNER_CONTEXT_NAME: ${{ runner.name }}
         run: |
           set -euo pipefail
@@ -2050,7 +2059,7 @@ jobs:
     # reuses macos-compile-admission's artifact like the app-host shards do,
     # on the pool and Xcode that built it: the test bundle only loads under
     # the Xcode that linked it.
-    runs-on: ${{ needs.macos-compile-admission.outputs.runner }}
+    runs-on: ${{ github.run_attempt > 1 && inputs.pr_retry_runner || needs.macos-compile-admission.outputs.runner }}
     timeout-minutes: 40
     env:
       CMUX_NODE_PRODUCT_CACHE_ROOT: ${{ vars.CMUX_NODE_PRODUCT_CACHE_ROOT }}
@@ -2066,7 +2075,7 @@ jobs:
       - name: Verify GitHub-hosted route
         env:
           RUNNER_ENVIRONMENT: ${{ runner.environment }}
-          REQUESTED_RUNNER: ${{ needs.macos-compile-admission.outputs.runner }}
+          REQUESTED_RUNNER: ${{ github.run_attempt > 1 && inputs.pr_retry_runner || needs.macos-compile-admission.outputs.runner }}
           RUNNER_CONTEXT_NAME: ${{ runner.name }}
         run: |
           set -euo pipefail
@@ -2772,7 +2781,7 @@ jobs:
     # full-suite dispatch follows admission onto the pull-request pool and
     # Xcode. The product consumer guard in the CI change-area tests fails when
     # the two drift.
-    runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (startsWith(inputs.pr_runner, 'blacksmith-') && inputs.pr_runner || 'blacksmith-6vcpu-macos-15') || (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && (inputs.pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15') || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_DISPLAY || 'blacksmith-6vcpu-macos-15') }}
+    runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (startsWith(inputs.pr_runner, 'blacksmith-') && inputs.pr_runner || 'blacksmith-6vcpu-macos-15') || (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && (github.run_attempt > 1 && inputs.pr_retry_runner || inputs.pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15') || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_DISPLAY || 'blacksmith-6vcpu-macos-15') }}
     timeout-minutes: 75
     env:
       CMUX_NODE_PRODUCT_CACHE_ROOT: ${{ vars.CMUX_NODE_PRODUCT_CACHE_ROOT }}
@@ -2785,7 +2794,7 @@ jobs:
     steps:
       - name: Validate display runner identity
         env:
-          REQUESTED_RUNNER: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (startsWith(inputs.pr_runner, 'blacksmith-') && inputs.pr_runner || 'blacksmith-6vcpu-macos-15') || (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && (inputs.pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15') || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_DISPLAY || 'blacksmith-6vcpu-macos-15') }}
+          REQUESTED_RUNNER: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (startsWith(inputs.pr_runner, 'blacksmith-') && inputs.pr_runner || 'blacksmith-6vcpu-macos-15') || (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && (github.run_attempt > 1 && inputs.pr_retry_runner || inputs.pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15') || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_DISPLAY || 'blacksmith-6vcpu-macos-15') }}
           RUNNER_CONTEXT_NAME: ${{ runner.name }}
         run: |
           set -euo pipefail

.github/workflows/ci-owned-pool-rescue.yml

@@ -6,7 +6,10 @@ run-name: owned-pool-rescue-${{ github.event.workflow_run.id }}-${{ github.event
 # busy the run would wait for it indefinitely. This workflow watches such a run
 # and, when one of its jobs has waited past CI_OWNED_POOL_RESCUE_SECONDS,
 # cancels the run and re-runs it; the retry attempt never takes a persistent
-# pool, so it lands on Blacksmith. scripts/ci/owned_pool_rescue.py has the rule.
+# pool, so it lands on Blacksmith. A job an owned runner refused at job start
+# (it failed in seconds, before any step succeeded) gets its failed jobs re-run
+# instead, which moves them to the picker's retry_runner on Blacksmith.
+# scripts/ci/owned_pool_rescue.py has the rules.
 #
 # It needs actions: write, so it is triggered by workflow_run and its code comes
 # from main: a pull request cannot change it. Only attempt 1 is watched, in the

.github/workflows/ci-queue-janitor.yml

@@ -74,6 +74,9 @@ jobs:
           PR_POOL_OVERFLOW: ${{ vars.CI_PR_POOL_OVERFLOW }}
           PR_POOL_ORDER: ${{ vars.CI_PR_POOL_ORDER }}
           PR_POOL_MAX_QUEUED: ${{ vars.CI_PR_POOL_MAX_QUEUED }}
+          # Owned pools on: read each candidate run's owned-pool marker for the
+          # peak it declared. Not copied into the snapshot; forks never use them.
+          PR_POOL_OWNED: ${{ vars.CI_PR_POOL_OWNED }}
         run: python3 scripts/ci/queue_janitor.py
 
       # What this sweep saw on each macOS pool. ci.yml's `changes` job reads

.github/workflows/ci.yml

@@ -86,6 +86,10 @@ jobs:
       # pull request, and any uncertainty.
       macos_pr_runner: ${{ steps.macos-pool.outputs.runner }}
       macos_pr_xcode_app: ${{ steps.macos-pool.outputs.xcode_app }}
+      # Set only when the pool is persistent (owned Macs): the Blacksmith pool
+      # a re-run of failed jobs takes, since it reuses these outputs and must
+      # not queue on the owned pool unwatched.
+      macos_pr_retry_runner: ${{ steps.macos-pool.outputs.retry_runner }}
     permissions:
       actions: read
       contents: read
@@ -96,58 +100,6 @@ jobs:
         with:
           fetch-depth: 2
 
-      - name: Choose the pull request macOS pool
-        id: macos-pool
-        # Fail-safe: any error leaves the outputs empty, which is today's route.
-        continue-on-error: true
-        timeout-minutes: 1
-        env:
-          GH_TOKEN: ${{ github.token }}
-          EVENT_NAME: ${{ github.event_name }}
-          HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }}
-          DEFAULT_RUNNER: ${{ vars.MACOS_RUNNER_PR }}
-          POOL_OVERFLOW: ${{ vars.CI_PR_POOL_OVERFLOW }}
-          POOL_ORDER: ${{ vars.CI_PR_POOL_ORDER }}
-          POOL_MAX_QUEUED: ${{ vars.CI_PR_POOL_MAX_QUEUED }}
-          # Owned Mac pools (off unless 1), their machine counts, and the lane
-          # Xcode whose version names their label (glaeda-std-xcode-26.6).
-          POOL_OWNED: ${{ vars.CI_PR_POOL_OWNED }}
-          OWNED_SLOTS: ${{ vars.CI_OWNED_POOL_SLOTS }}
-          OWNED_JOBS_PER_RUN: ${{ vars.CI_OWNED_POOL_JOBS_PER_RUN }}
-          CMUX_CI_XCODE_APP_PR: ${{ github.event.pull_request.head.repo.full_name == github.repository && vars.CMUX_CI_XCODE_APP_PR || '' }}
-          # Handed to the macOS 15 pool's jobs only when the run lands there.
-          CMUX_CI_XCODE_APP_MACOS_15: ${{ vars.CMUX_CI_XCODE_APP_MACOS_15 }}
-        run: python3 scripts/ci/pr_runner_pool.py
-
-      # A job queued on a persistent pool (owned Macs) waits for it however long
-      # it stays busy. This marker tells ci-owned-pool-rescue.yml, running from
-      # main, to watch the run and re-run it on Blacksmith when a job waits past
-      # CI_OWNED_POOL_RESCUE_SECONDS. No persistent pool, no marker, no watching.
-      # Fail-safe like the picker: a missing marker only means the run is not
-      # watched, which is how every run behaved before the rescue existed.
-      - name: Mark a run on a persistent macOS pool
-        id: macos-pool-marker
-        if: ${{ steps.macos-pool.outputs.persistent == 'true' }}
-        continue-on-error: true
-        env:
-          POOL: ${{ steps.macos-pool.outputs.runner }}
-        run: |
-          set -euo pipefail
-          marker="$RUNNER_TEMP/macos-pool-persistent.json"
-          POOL="$POOL" MARKER="$marker" python3 -c 'import json,os; json.dump({"pool": os.environ["POOL"]}, open(os.environ["MARKER"], "w"))'
-          echo "path=$marker" >> "$GITHUB_OUTPUT"
-
-      - name: Upload the persistent pool marker
-        if: ${{ steps.macos-pool-marker.outputs.path != '' }}
-        continue-on-error: true
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: macos-pool-persistent-${{ github.run_id }}-${{ github.run_attempt }}
-          path: ${{ steps.macos-pool-marker.outputs.path }}
-          if-no-files-found: error
-          retention-days: 1
-          compression-level: 0
-
       - name: Record GitHub-selected source identity
         id: source-identity
         run: |
@@ -621,6 +573,67 @@ jobs:
             ${files_args[@]+"${files_args[@]}"} \
             --github-output "$GITHUB_OUTPUT"
 
+      - name: Choose the pull request macOS pool
+        id: macos-pool
+        # Fail-safe: any error leaves the outputs empty, which is today's route.
+        continue-on-error: true
+        timeout-minutes: 1
+        env:
+          GH_TOKEN: ${{ github.token }}
+          EVENT_NAME: ${{ github.event_name }}
+          HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }}
+          DEFAULT_RUNNER: ${{ vars.MACOS_RUNNER_PR }}
+          POOL_OVERFLOW: ${{ vars.CI_PR_POOL_OVERFLOW }}
+          POOL_ORDER: ${{ vars.CI_PR_POOL_ORDER }}
+          POOL_MAX_QUEUED: ${{ vars.CI_PR_POOL_MAX_QUEUED }}
+          # Owned Mac pools (off unless 1), their machine counts, and the lane
+          # Xcode whose version names their label (glaeda-std-xcode-26.6).
+          POOL_OWNED: ${{ vars.CI_PR_POOL_OWNED }}
+          OWNED_SLOTS: ${{ vars.CI_OWNED_POOL_SLOTS }}
+          CMUX_CI_XCODE_APP_PR: ${{ github.event.pull_request.head.repo.full_name == github.repository && vars.CMUX_CI_XCODE_APP_PR || '' }}
+          # Handed to the macOS 15 pool's jobs only when the run lands there.
+          CMUX_CI_XCODE_APP_MACOS_15: ${{ vars.CMUX_CI_XCODE_APP_MACOS_15 }}
+          # This run's macOS jobs, from the routing above, so an owned pool is
+          # taken only when every job of the run gets a machine at once.
+          RUN_MACOS: ${{ steps.detect.outputs.macos }}
+          RUN_FULL_SUITE: ${{ steps.suite.outputs.full_suite }}
+          RUN_UNIT_SUITE: ${{ steps.suite.outputs.unit_suite }}
+          RUN_UNIT_IN_ADMISSION: ${{ steps.suite.outputs.unit_in_admission }}
+          RUN_CLAUDE_WRAPPER: ${{ steps.standalone.outputs.claude_wrapper }}
+          RUN_CLI: ${{ steps.detect.outputs.cli }}
+          RUN_REMOTE_DAEMON: ${{ steps.standalone.outputs.remote_daemon }}
+        run: python3 scripts/ci/pr_runner_pool.py
+
+      # A job queued on a persistent pool (owned Macs) waits for it however long
+      # it stays busy. This marker tells ci-owned-pool-rescue.yml, running from
+      # main, to watch the run and re-run it on Blacksmith when a job waits past
+      # CI_OWNED_POOL_RESCUE_SECONDS. No persistent pool, no marker, no watching.
+      # Fail-safe like the picker: a missing marker only means the run is not
+      # watched, which is how every run behaved before the rescue existed.
+      - name: Mark a run on a persistent macOS pool
+        id: macos-pool-marker
+        if: ${{ steps.macos-pool.outputs.persistent == 'true' }}
+        continue-on-error: true
+        env:
+          POOL: ${{ steps.macos-pool.outputs.runner }}
+        run: |
+          set -euo pipefail
+          marker="$RUNNER_TEMP/macos-pool-persistent.json"
+          POOL="$POOL" MARKER="$marker" python3 -c 'import json,os; json.dump({"pool": os.environ["POOL"]}, open(os.environ["MARKER"], "w"))'
+          echo "path=$marker" >> "$GITHUB_OUTPUT"
+
+      - name: Upload the persistent pool marker
+        if: ${{ steps.macos-pool-marker.outputs.path != '' }}
+        continue-on-error: true
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          # The janitor reads the run's peak and pool from the name.
+          name: macos-pool-persistent-${{ github.run_id }}-${{ github.run_attempt }}-${{ steps.macos-pool.outputs.jobs }}-${{ steps.macos-pool.outputs.runner }}
+          path: ${{ steps.macos-pool-marker.outputs.path }}
+          if-no-files-found: error
+          retention-days: 1
+          compression-level: 0
+
       - name: Route Linux guard suites
         id: linux_guards
         env:
@@ -897,6 +910,7 @@ jobs:
     uses: ./.github/workflows/remote-daemon.yml
     with:
       pr_runner: ${{ needs.changes.outputs.macos_pr_runner }}
+      pr_retry_runner: ${{ needs.changes.outputs.macos_pr_retry_runner }}
       native_tests: ${{ needs.changes.outputs.remote_daemon_native == 'true' || contains(github.event.pull_request.labels.*.name, 'full-ci') }}
 
   cli:
@@ -905,6 +919,7 @@ jobs:
     uses: ./.github/workflows/cli-pipe-regressions.yml
     with:
       pr_runner: ${{ needs.changes.outputs.macos_pr_runner }}
+      pr_retry_runner: ${{ needs.changes.outputs.macos_pr_retry_runner }}
       pr_xcode_app: ${{ needs.changes.outputs.macos_pr_xcode_app }}
 
   web:
@@ -921,7 +936,7 @@ jobs:
     name: Claude wrapper regressions
     needs: [changes, static-preflight]
     if: ${{ !cancelled() && needs.changes.result == 'success' && needs.static-preflight.result == 'success' && (needs.changes.outputs.claude_wrapper == 'true' || (needs.changes.outputs.macos == 'true' && needs.changes.outputs.full_suite == 'true')) }}
-    runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (startsWith(needs.changes.outputs.macos_pr_runner, 'blacksmith-') && needs.changes.outputs.macos_pr_runner || 'blacksmith-6vcpu-macos-15') || github.event_name == 'pull_request' && (needs.changes.outputs.macos_pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15') || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15') }}
+    runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (startsWith(needs.changes.outputs.macos_pr_runner, 'blacksmith-') && needs.changes.outputs.macos_pr_runner || 'blacksmith-6vcpu-macos-15') || github.event_name == 'pull_request' && github.run_attempt > 1 && needs.changes.outputs.macos_pr_retry_runner || github.event_name == 'pull_request' && (needs.changes.outputs.macos_pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15') || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15') }}
     timeout-minutes: 10
     steps:
       - name: Checkout wrapper and test inputs
@@ -1052,6 +1067,7 @@ jobs:
       cache_backend: ${{ inputs.cache_backend }}
       release_archs: ${{ inputs.release_archs }}
       pr_runner: ${{ needs.changes.outputs.macos_pr_runner }}
+      pr_retry_runner: ${{ needs.changes.outputs.macos_pr_retry_runner }}
       pr_xcode_app: ${{ needs.changes.outputs.macos_pr_xcode_app }}
 
   tests: