ci: overflow E2E to the 12vcpu macOS 26 pool only when the 6vcpu pool is backed up

.github/workflows/test-e2e.yml

@@ -29,7 +29,7 @@ on:
         default: true
         type: boolean
       runner:
-        description: "Runner OS (auto follows MACOS_RUNNER_TESTS; tart-* choices use isolated VMs)"
+        description: "Runner OS (auto follows MACOS_RUNNER_TESTS and overflows to the 12vcpu macOS 26 pool only when the 6vcpu pool is backed up and 12vcpu is idle; tart-* choices use isolated VMs)"
         required: false
         default: "auto"
         type: choice
@@ -43,6 +43,11 @@ on:
           - tart-dual
           - tart-small
 
+# run-name and the concurrency group cannot read job outputs, so they spell
+# `auto` as the 6vcpu default even when the runner job overflows the run to
+# the 12vcpu pool. Every auto dispatch of one ref and filter still shares a
+# group, so a newer one still cancels the older wherever it landed. run-e2e.sh
+# names the pool it chose explicitly, so its titles are exact.
 concurrency:
   group: e2e-${{ (!inputs.runner || inputs.runner == 'auto') && (vars.MACOS_RUNNER_TESTS || 'blacksmith-6vcpu-macos-26') || inputs.runner }}-${{ inputs.ref || github.ref_name }}-${{ inputs.test_filter }}
   cancel-in-progress: true
@@ -60,6 +65,60 @@ jobs:
     with:
       ref: ${{ inputs.ref }}
 
+  runner:
+    # The macOS pool build and test run on. `auto` follows MACOS_RUNNER_TESTS.
+    # The 12vcpu macOS 26 pool is reserved first for release and nightly
+    # builds, so on the 6vcpu default an E2E run overflows to 12vcpu only when
+    # the 6vcpu pool is backed up (at least CI_E2E_OVERFLOW_MIN_QUEUED other
+    # E2E runs in flight on it, default 4) and 12vcpu has spare room (no
+    # release or nightly run in flight, nothing queued, and fewer than
+    # CI_E2E_OVERFLOW_MAX_LARGE_RUNNING E2E runs on it, default 2). Any error
+    # reading the queue stays on 6vcpu, and CI_E2E_LARGE_POOL_OVERFLOW=0 turns
+    # overflow off. run-e2e.sh applies the same rule from the same script.
+    runs-on: ${{ vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }}
+    timeout-minutes: 5
+    # Reads one page each of in-progress and queued runs (two API calls at
+    # most; see e2e_runner_pool.py for the budget) to estimate pool demand.
+    permissions:
+      contents: read
+      actions: read
+    outputs:
+      label: ${{ steps.pool.outputs.label }}
+    steps:
+      # The helper comes from this workflow's revision, not the tested one,
+      # which may predate it. The workflow files tell it which release and
+      # nightly workflows never use macOS.
+      - name: Checkout pool helper
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          sparse-checkout: |
+            scripts/ci/e2e_runner_pool.py
+            scripts/ci/queue_janitor.py
+            .github/workflows/
+          sparse-checkout-cone-mode: false
+          persist-credentials: false
+
+      - name: Pick the macOS pool
+        id: pool
+        env:
+          GH_TOKEN: ${{ github.token }}
+          GH_REPO: ${{ github.repository }}
+          REQUESTED_RUNNER: ${{ inputs.runner }}
+          RUNNER_VARIABLE: ${{ vars.MACOS_RUNNER_TESTS }}
+          LARGE_POOL_OVERFLOW: ${{ vars.CI_E2E_LARGE_POOL_OVERFLOW }}
+          OVERFLOW_MIN_QUEUED: ${{ vars.CI_E2E_OVERFLOW_MIN_QUEUED }}
+          OVERFLOW_MAX_LARGE_RUNNING: ${{ vars.CI_E2E_OVERFLOW_MAX_LARGE_RUNNING }}
+        run: |
+          set -euo pipefail
+          label="$(python3 scripts/ci/e2e_runner_pool.py \
+            --requested "$REQUESTED_RUNNER" \
+            --variable "$RUNNER_VARIABLE" \
+            --overflow "$LARGE_POOL_OVERFLOW" \
+            --min-queued "$OVERFLOW_MIN_QUEUED" \
+            --max-large-running "$OVERFLOW_MAX_LARGE_RUNNING")"
+          echo "label=$label" >> "$GITHUB_OUTPUT"
+          echo "Runner: $label (requested ${REQUESTED_RUNNER:-auto})"
+
   filter:
     # Seconds on Linux, and it rejects a malformed selector before either
     # macOS job is scheduled. After the build/test split a bad filter would
@@ -190,8 +249,8 @@ jobs:
           echo "Resolved selectors ($count): $selectors"
 
   build:
-    needs: [resolve-ref, filter]
-    runs-on: ${{ (!inputs.runner || inputs.runner == 'auto') && (vars.MACOS_RUNNER_TESTS || 'blacksmith-6vcpu-macos-26') || inputs.runner }}
+    needs: [resolve-ref, filter, runner]
+    runs-on: ${{ needs.runner.outputs.label }}
     # Reuse lists this contract's artifacts and downloads one from an earlier
     # run. Nothing else in this lane reads the Actions API, and nothing writes.
     permissions:
@@ -214,13 +273,13 @@ jobs:
       # same Xcode build. The intra-run handoff uses the artifact id, not the
       # name, so nothing here depends on it.
       CMUX_SKIP_ZIG_BUILD: "1"
-      CMUX_PRODUCT_RUNNER: ${{ (!inputs.runner || inputs.runner == 'auto') && (vars.MACOS_RUNNER_TESTS || 'blacksmith-6vcpu-macos-26') || inputs.runner }}
+      CMUX_PRODUCT_RUNNER: ${{ needs.runner.outputs.label }}
     steps:
 
       - name: Validate Tart canary identity
-        if: ${{ startsWith((!inputs.runner || inputs.runner == 'auto') && (vars.MACOS_RUNNER_TESTS || 'blacksmith-6vcpu-macos-26') || inputs.runner, 'tart-') }}
+        if: ${{ startsWith(needs.runner.outputs.label, 'tart-') }}
         env:
-          REQUESTED_RUNNER: ${{ (!inputs.runner || inputs.runner == 'auto') && (vars.MACOS_RUNNER_TESTS || 'blacksmith-6vcpu-macos-26') || inputs.runner }}
+          REQUESTED_RUNNER: ${{ needs.runner.outputs.label }}
           RUNNER_CONTEXT_NAME: ${{ runner.name }}
         run: |
           set -euo pipefail
@@ -613,15 +672,15 @@ jobs:
           esac
 
   test:
-    needs: [resolve-ref, filter, build]
+    needs: [resolve-ref, filter, runner, build]
     # scripts/ci/parallel_artifact_download.py reads this run's artifact
     # metadata through the Actions API; the workflow-level token is
     # contents-only. Narrow and additive, mirroring ci-macos.yml's admission
     # job rather than widening the whole workflow.
     permissions:
       contents: read
       actions: read
-    runs-on: ${{ (!inputs.runner || inputs.runner == 'auto') && (vars.MACOS_RUNNER_TESTS || 'blacksmith-6vcpu-macos-26') || inputs.runner }}
+    runs-on: ${{ needs.runner.outputs.label }}
     timeout-minutes: ${{ fromJSON(inputs.job_timeout || '45') }}
     env:
       CMUX_CI_MAX_MACOS_SDK_MAJOR: "26"
@@ -639,9 +698,9 @@ jobs:
     steps:
 
       - name: Validate Tart canary identity
-        if: ${{ startsWith((!inputs.runner || inputs.runner == 'auto') && (vars.MACOS_RUNNER_TESTS || 'blacksmith-6vcpu-macos-26') || inputs.runner, 'tart-') }}
+        if: ${{ startsWith(needs.runner.outputs.label, 'tart-') }}
         env:
-          REQUESTED_RUNNER: ${{ (!inputs.runner || inputs.runner == 'auto') && (vars.MACOS_RUNNER_TESTS || 'blacksmith-6vcpu-macos-26') || inputs.runner }}
+          REQUESTED_RUNNER: ${{ needs.runner.outputs.label }}
           RUNNER_CONTEXT_NAME: ${{ runner.name }}
         run: |
           set -euo pipefail

.github/workflows/test-macos-suite.yml

@@ -92,6 +92,9 @@ jobs:
           # The job token cannot list repository variables, and without the
           # runner the wrapper cannot see an identical run already in flight.
           CMUX_MACOS_RUNNER_TESTS: ${{ vars.MACOS_RUNNER_TESTS }}
+          CMUX_CI_E2E_LARGE_POOL_OVERFLOW: ${{ vars.CI_E2E_LARGE_POOL_OVERFLOW }}
+          CMUX_CI_E2E_OVERFLOW_MIN_QUEUED: ${{ vars.CI_E2E_OVERFLOW_MIN_QUEUED }}
+          CMUX_CI_E2E_OVERFLOW_MAX_LARGE_RUNNING: ${{ vars.CI_E2E_OVERFLOW_MAX_LARGE_RUNNING }}
           UNIT_TEST_SUITES: ${{ inputs.unit_test_suites }}
           TEST_REF: ${{ inputs.ref || github.sha }}
           TEST_TIMEOUT: ${{ inputs.test_timeout }}