ci: send compile admission to the owned Mac that kept a build of its merge base

.github/workflows/ci-guards.yml

@@ -372,6 +372,10 @@ jobs:
         if: ${{ matrix.group == 'ci' }}
         run: python3 tests/test_ci_owned_pool_rescue.py
 
+      - name: Validate the owned Mac warm labels
+        if: ${{ matrix.group == 'ci' }}
+        run: python3 tests/test_ci_owned_warm_labels.py
+
       - name: Validate CI health report
         if: ${{ matrix.group == 'ci' }}
         run: python3 tests/test_ci_health_report.py

.github/workflows/ci-macos.yml

@@ -119,6 +119,14 @@ on:
         required: false
         default: ""
         type: string
+      # Set only beside pr_root_runner, when an idle root runner kept a build
+      # of this run's merge base: the JSON array ["<root label>",
+      # "glaeda-warm-<sha12>"] (pr_runner_pool.py, warm affinity). Compile
+      # admission's attempt 1 takes it as its runs-on; retries do not.
+      pr_admission_runner:
+        required: false
+        default: ""
+        type: string
       # Set only when pr_runner is persistent: the jobs of attempt 1 that take
       # it, as " <key> <key> ". Every other job takes pr_retry_runner, so a
       # run can use the owned machines that are free and overflow the rest
@@ -168,8 +176,10 @@ jobs:
     # Pull requests and main's full-suite dispatch (ci-main-full-suite.yml)
     # compile on the pool and Xcode seed-derived-data.yml builds with, so both
     # can adopt its DerivedData seed below. Merge groups and dispatches on
-    # other branches keep the macos-15 lane.
-    runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (startsWith(inputs.pr_runner, 'blacksmith-') && inputs.pr_runner || 'blacksmith-6vcpu-macos-15') || (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && (github.run_attempt == 2 && github.triggering_actor == 'github-actions[bot]' && contains(inputs.pr_owned_jobs, ' admission ') && (inputs.pr_root_runner || inputs.pr_refused_retry_runner) || (github.run_attempt > 1 || !contains(inputs.pr_owned_jobs, ' admission ')) && inputs.pr_retry_runner || inputs.pr_root_runner || inputs.pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15') || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15') }}
+    # other branches keep the macos-15 lane. On an owned pool, attempt 1 may
+    # also require the warm label of a root runner that kept a build of the
+    # run's merge base (pr_admission_runner).
+    runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (startsWith(inputs.pr_runner, 'blacksmith-') && inputs.pr_runner || 'blacksmith-6vcpu-macos-15') || (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && (github.run_attempt == 2 && github.triggering_actor == 'github-actions[bot]' && contains(inputs.pr_owned_jobs, ' admission ') && (inputs.pr_root_runner || inputs.pr_refused_retry_runner) || (github.run_attempt > 1 || !contains(inputs.pr_owned_jobs, ' admission ')) && inputs.pr_retry_runner || github.run_attempt == 1 && inputs.pr_admission_runner && fromJSON(inputs.pr_admission_runner) || inputs.pr_root_runner || inputs.pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15') || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15') }}
     # A changed-suites run adds its tests after the compile: the same
     # 30-minute batch ceiling the separate worker had.
     timeout-minutes: ${{ inputs.unit_in_admission == 'true' && 105 || 75 }}
@@ -211,8 +221,10 @@ jobs:
       # Names the pool this job actually ran on. The app-host product contract
       # keys on the toolchain and the build path instead, so a product built
       # here at the canonical root matches on any pool with the same Xcode.
-      # The macOS runner guard still requires it to track runs-on.
-      CMUX_PRODUCT_RUNNER: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (startsWith(inputs.pr_runner, 'blacksmith-') && inputs.pr_runner || 'blacksmith-6vcpu-macos-15') || (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && (github.run_attempt == 2 && github.triggering_actor == 'github-actions[bot]' && contains(inputs.pr_owned_jobs, ' admission ') && (inputs.pr_root_runner || inputs.pr_refused_retry_runner) || (github.run_attempt > 1 || !contains(inputs.pr_owned_jobs, ' admission ')) && inputs.pr_retry_runner || inputs.pr_root_runner || inputs.pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15') || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15') }}
+      # The macOS runner guard still requires it to track runs-on. A warm
+      # admission (pr_admission_runner) names its first label, the root label:
+      # the consumers need a root runner, not this one.
+      CMUX_PRODUCT_RUNNER: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (startsWith(inputs.pr_runner, 'blacksmith-') && inputs.pr_runner || 'blacksmith-6vcpu-macos-15') || (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && (github.run_attempt == 2 && github.triggering_actor == 'github-actions[bot]' && contains(inputs.pr_owned_jobs, ' admission ') && (inputs.pr_root_runner || inputs.pr_refused_retry_runner) || (github.run_attempt > 1 || !contains(inputs.pr_owned_jobs, ' admission ')) && inputs.pr_retry_runner || github.run_attempt == 1 && inputs.pr_admission_runner && fromJSON(inputs.pr_admission_runner)[0] || inputs.pr_root_runner || inputs.pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15') || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15') }}
       # What the changed-suites steps at the end of this job read, with the
       # values `app-host unit tests` gives its changed-suites worker, shard 8.
       # The compile reads none of them: it runs plain xcodebuild and bakes the
@@ -734,6 +746,36 @@ jobs:
           FINGERPRINT: ${{ steps.owned-state.outputs.fingerprint }}
         run: python3 scripts/ci/owned_build_state.py keep "$CMUX_OWNED_STATE_ROOT" "$CMUX_COMPILE_ADMISSION_DERIVED_DATA" "$FINGERPRINT"
 
+      # The main commits this Mac's kept DerivedData starts from cheaply, for
+      # ci-owned-warm-labels.yml to label its root runner with, so a later run
+      # merging onto one of them can ask for this Mac (pr_runner_pool.py, warm
+      # affinity). Only once owned_build_state.py has `warm-keys`; before
+      # that, and on any failure, nothing is uploaded and nothing changes.
+      - name: List the commits this owned Mac starts from warm
+        id: owned-warm-keys
+        if: ${{ !cancelled() && steps.owned-state.outputs.fingerprint != '' }}
+        continue-on-error: true
+        run: |
+          set -euo pipefail
+          usage="$(python3 scripts/ci/owned_build_state.py --help 2>&1 || true)"
+          case "$usage" in
+            *"owned_build_state.py warm-keys"*) ;;
+            *) echo "owned_build_state.py has no warm-keys yet"; exit 0 ;;
+          esac
+          python3 scripts/ci/owned_build_state.py warm-keys "$CMUX_OWNED_STATE_ROOT" "$RUNNER_NAME" "$CMUX_PRODUCT_RUNNER" \
+            > "$RUNNER_TEMP/warm-keys.json"
+          echo "path=$RUNNER_TEMP/warm-keys.json" >> "$GITHUB_OUTPUT"
+
+      - name: Upload the owned Mac's warm keys
+        if: ${{ !cancelled() && steps.owned-warm-keys.outputs.path != '' }}
+        continue-on-error: true
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: owned-warm-keys-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ steps.owned-warm-keys.outputs.path }}
+          retention-days: 1
+          if-no-files-found: ignore
+
       - name: Generate Xcode build metrics receipt
         id: build-metrics
         if: always() && !cancelled() && steps.hosted-compile.outcome != 'skipped'

.github/workflows/ci-owned-warm-labels.yml

@@ -0,0 +1,84 @@
+name: CI owned warm labels
+run-name: owned-warm-labels-${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }}
+
+# A compile admission on an owned Mac keeps its DerivedData, and uploads the
+# main commits that build starts from cheaply (owned-warm-keys-<run>-<attempt>,
+# ci-macos.yml). When the CI run completes, this labels the runner that ran it
+# glaeda-warm-<sha12> for each, and takes those labels off the other runners of
+# its pool, so pr_runner_pool.py can send a later admission merging onto one of
+# them to that Mac (warm affinity). scripts/ci/owned_warm_labels.py has the
+# rules.
+#
+# Changing runner labels needs the org route App with administration: write,
+# so it is triggered by workflow_run and its code comes from main: a pull
+# request cannot change it. It trusts only the keys from the artifact; the
+# runner comes from the jobs API. Off unless CI_OWNED_WARM_LABELS is 1. It only
+# makes API requests, so it runs on a GitHub-hosted runner rather than holding
+# a slot in CI's Linux pool.
+on:
+  workflow_run:
+    workflows: [CI]
+    types: [completed]
+
+permissions: {}
+
+env:
+  # Renaming the source workflow silently stops every `workflow_run` consumer
+  # of it; tests/test_ci_workflow_run_sources.py derives the expected name from
+  # this path. The script confirms the run's own `path` again before acting.
+  SOURCE_WORKFLOW_PATHS: .github/workflows/ci.yml
+
+concurrency:
+  group: owned-warm-labels-${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }}
+  cancel-in-progress: true
+
+jobs:
+  label:
+    name: Label the owned Mac that kept this run's build
+    if: ${{ vars.CI_OWNED_WARM_LABELS == '1' && vars.GLAEDA_ROUTE_APP_ID != '' && github.event.workflow_run.head_repository.full_name == github.repository }}
+    runs-on: ubuntu-24.04 # github-hosted-required: only calls the Actions API; keeps CI's Linux pool free
+    timeout-minutes: 5
+    permissions:
+      actions: read
+      contents: read
+    steps:
+      # Most runs have no warm keys (no owned Mac, or no warm-keys yet); the
+      # download then fails and every step after it is skipped.
+      - name: Download the warm keys
+        id: keys
+        continue-on-error: true
+        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
+        with:
+          name: owned-warm-keys-${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }}
+          run-id: ${{ github.event.workflow_run.id }}
+          github-token: ${{ github.token }}
+          path: ${{ runner.temp }}/owned-warm-keys
+
+      - name: Checkout trusted labeler
+        if: steps.keys.outcome == 'success'
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          ref: main
+          persist-credentials: false
+
+      - name: Mint the runner labeling token
+        id: route-token
+        if: steps.keys.outcome == 'success'
+        continue-on-error: true
+        timeout-minutes: 1
+        uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
+        with:
+          app-id: ${{ vars.GLAEDA_ROUTE_APP_ID }}
+          private-key: ${{ secrets.GLAEDA_ROUTE_APP_KEY }}
+          permission-administration: write
+
+      - name: Label the runner
+        if: steps.keys.outcome == 'success' && steps.route-token.outputs.token != ''
+        env:
+          GH_TOKEN: ${{ github.token }}
+          ROUTE_TOKEN: ${{ steps.route-token.outputs.token }}
+          RUN_ID: ${{ github.event.workflow_run.id }}
+          RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }}
+          RUN_PATH: ${{ github.event.workflow_run.path }}
+          KEYS_FILE: ${{ runner.temp }}/owned-warm-keys/warm-keys.json
+        run: python3 scripts/ci/owned_warm_labels.py

.github/workflows/ci.yml

@@ -111,6 +111,10 @@ jobs:
       # macos_pr_owned_jobs (admission, shards, lag, cli-product) take it in
       # place of the pool label, on attempt 1 and after a refusal.
       macos_pr_root_runner: ${{ steps.macos-pool.outputs.root_runner }}
+      # Set only beside a root runner, when an idle one carries the warm label
+      # of this run's merge base: ["<root label>", "glaeda-warm-<sha12>"],
+      # the labels compile admission's attempt 1 takes instead.
+      macos_pr_admission_runner: ${{ steps.macos-pool.outputs.admission_runner }}
     permissions:
       actions: read
       contents: read
@@ -649,6 +653,10 @@ jobs:
           RUN_CLAUDE_WRAPPER: ${{ steps.standalone.outputs.claude_wrapper }}
           RUN_CLI: ${{ steps.detect.outputs.cli }}
           RUN_REMOTE_DAEMON: ${{ steps.standalone.outputs.remote_daemon }}
+          # The main commit this run merges onto, whose warm label (a root
+          # runner that kept a build of it) compile admission may take.
+          MERGED_ONTO: ${{ steps.source-identity.outputs.parent1 || github.event.pull_request.base.sha }}
+          WARM_LABELS: ${{ vars.CI_OWNED_WARM_LABELS }}
         run: python3 scripts/ci/pr_runner_pool.py
 
       # A job queued on a persistent pool (owned Macs) waits for it however long
@@ -1197,6 +1205,7 @@ jobs:
       pr_owned_jobs: ${{ needs.changes.outputs.macos_pr_owned_jobs }}
       pr_refused_retry_runner: ${{ needs.changes.outputs.macos_pr_refused_retry_runner }}
       pr_root_runner: ${{ needs.changes.outputs.macos_pr_root_runner }}
+      pr_admission_runner: ${{ needs.changes.outputs.macos_pr_admission_runner }}
       pr_xcode_app: ${{ needs.changes.outputs.macos_pr_xcode_app }}
 
   tests:

docs/ci-runners.md

@@ -230,6 +230,22 @@ Claude wrapper lanes always do. A root count above its pool's is an error.
 With 8 std minis and 2 light ones:
 `{"std": 32, "light": 4, "root-std": 8, "root-light": 2}`.
 
+Warm affinity (`CI_OWNED_WARM_LABELS=1`, off by default): an owned Mac keeps
+compile admission's DerivedData, and admission uploads the main commits that
+build starts from cheaply (`owned_build_state.py warm-keys`). When the CI run
+completes, `ci-owned-warm-labels.yml` (from main, with the route App's
+administration: write) labels the runner that ran admission
+`glaeda-warm-<sha12>` for each, at most 4, and removes those labels from the
+other runners of its root pool, so one runner per pool carries each commit.
+With live runners, the picker sends a run's admission to
+`["<root label>", "glaeda-warm-<merge base sha12>"]` when an idle root runner
+carries that label (the `admission_runner` output, attempt 1 only); otherwise
+admission takes the root label as before. The picker also reads the variable, so
+turning it off ignores labels already set. v1 matches the merge base exactly;
+it does not rank runners by commit distance. A warm runner taken between the
+pick and the queue leaves admission waiting, and the rescue moves it to
+Blacksmith like any other stuck owned job.
+
 An owned pool is persistent, which needs one more rule because GitHub never
 re-routes a queued job: one queued there waits for that pool however long it
 stays busy. An offline mini still counts as a slot, and the snapshot can be