diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 0b516bcd35e0..8a0adc8037ca 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -1,5 +1,5 @@ name: E2E test with video recording -run-name: ${{ inputs.test_filter }} on ${{ (!inputs.runner || inputs.runner == 'auto') && (vars.MACOS_RUNNER_TESTS || 'blacksmith-6vcpu-macos-15') || inputs.runner }} @ ${{ inputs.ref || github.ref_name }}${{ inputs.dispatch_id != '' && format(' [{0}]', inputs.dispatch_id) || '' }} +run-name: ${{ inputs.test_filter }} on ${{ (!inputs.runner || inputs.runner == 'auto') && (vars.MACOS_RUNNER_TESTS || 'blacksmith-6vcpu-macos-26') || inputs.runner }} @ ${{ inputs.ref || github.ref_name }}${{ inputs.dispatch_id != '' && format(' [{0}]', inputs.dispatch_id) || '' }} on: workflow_dispatch: @@ -43,7 +43,7 @@ on: - tart-small concurrency: - group: e2e-${{ (!inputs.runner || inputs.runner == 'auto') && (vars.MACOS_RUNNER_TESTS || 'blacksmith-6vcpu-macos-15') || inputs.runner }}-${{ inputs.ref || github.ref_name }}-${{ inputs.test_filter }} + group: e2e-${{ (!inputs.runner || inputs.runner == 'auto') && (vars.MACOS_RUNNER_TESTS || 'blacksmith-6vcpu-macos-26') || inputs.runner }}-${{ inputs.ref || github.ref_name }}-${{ inputs.test_filter }} cancel-in-progress: true # This lane checks out and compiles a dispatcher-chosen ref, so it should hold @@ -61,16 +61,16 @@ jobs: e2e: needs: resolve-ref - runs-on: ${{ (!inputs.runner || inputs.runner == 'auto') && (vars.MACOS_RUNNER_TESTS || 'blacksmith-6vcpu-macos-15') || inputs.runner }} + runs-on: ${{ (!inputs.runner || inputs.runner == 'auto') && (vars.MACOS_RUNNER_TESTS || 'blacksmith-6vcpu-macos-26') || inputs.runner }} timeout-minutes: ${{ fromJSON(inputs.job_timeout || '20') }} env: TEST_REF: ${{ needs.resolve-ref.outputs.sha }} CMUX_CI_MAX_MACOS_SDK_MAJOR: "26" steps: - name: Validate Tart canary identity - if: ${{ startsWith((!inputs.runner || inputs.runner == 'auto') && (vars.MACOS_RUNNER_TESTS || 'blacksmith-6vcpu-macos-15') || inputs.runner, 'tart-') }} + if: ${{ startsWith((!inputs.runner || inputs.runner == 'auto') && (vars.MACOS_RUNNER_TESTS || 'blacksmith-6vcpu-macos-26') || inputs.runner, 'tart-') }} env: - REQUESTED_RUNNER: ${{ (!inputs.runner || inputs.runner == 'auto') && (vars.MACOS_RUNNER_TESTS || 'blacksmith-6vcpu-macos-15') || inputs.runner }} + REQUESTED_RUNNER: ${{ (!inputs.runner || inputs.runner == 'auto') && (vars.MACOS_RUNNER_TESTS || 'blacksmith-6vcpu-macos-26') || inputs.runner }} RUNNER_CONTEXT_NAME: ${{ runner.name }} run: | set -euo pipefail @@ -393,8 +393,8 @@ jobs: uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: .ci-source-packages - key: spm-${{ (!inputs.runner || inputs.runner == 'auto') && (vars.MACOS_RUNNER_TESTS || 'blacksmith-6vcpu-macos-15') || inputs.runner }}-${{ hashFiles('cmux.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved') }} - restore-keys: spm-${{ (!inputs.runner || inputs.runner == 'auto') && (vars.MACOS_RUNNER_TESTS || 'blacksmith-6vcpu-macos-15') || inputs.runner }}- + key: spm-${{ (!inputs.runner || inputs.runner == 'auto') && (vars.MACOS_RUNNER_TESTS || 'blacksmith-6vcpu-macos-26') || inputs.runner }}-${{ hashFiles('cmux.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved') }} + restore-keys: spm-${{ (!inputs.runner || inputs.runner == 'auto') && (vars.MACOS_RUNNER_TESTS || 'blacksmith-6vcpu-macos-26') || inputs.runner }}- - name: Sanitize Swift package cache run: python3 scripts/ci/sanitize-xcode-source-packages-cache.py .ci-source-packages diff --git a/docs/ci-runners.md b/docs/ci-runners.md index d2db4d1859ef..91a83b2ef178 100644 --- a/docs/ci-runners.md +++ b/docs/ci-runners.md @@ -24,7 +24,7 @@ gh variable list --repo manaflow-ai/cmux | `LINUX_ARM64_RUNNER` | native ARM64 package entrypoint verification | `ubuntu-24.04-arm` | `ubuntu-24.04-arm` | | `MACOS_RUNNER_15` | the macOS 15 default: `macos-compile-admission`, `app-host-unit-tests`, nightly helper and test-cache jobs | `blacksmith-6vcpu-macos-15` | `blacksmith-6vcpu-macos-15` | | `MACOS_RUNNER_PR` | **pull-request** macOS jobs only, in `ci-macos.yml`, `cli-pipe-regressions.yml` and `terminal-hang-diagnostics.yml` | unset (see "Lanes" below) | `blacksmith-6vcpu-macos-15` | -| `MACOS_RUNNER_TESTS` | the manual test-debugging lanes: `test-e2e.yml` and `test-depot.yml` | unset (see "Lanes" below) | `blacksmith-6vcpu-macos-15` | +| `MACOS_RUNNER_TESTS` | the manual test-debugging lanes: `test-e2e.yml` and `test-depot.yml` | unset (see "Lanes" below) | `blacksmith-6vcpu-macos-26` for `test-e2e.yml`, `blacksmith-6vcpu-macos-15` for `test-depot.yml` | | `MACOS_RUNNER_DUAL_XCODE` | `swift-package-tests` (SDK 15 release helper, then SDK 26 package tests) on non-pull-request events; pull requests take `MACOS_RUNNER_PR` | `blacksmith-6vcpu-macos-15` | `blacksmith-6vcpu-macos-15` | | `MACOS_RUNNER_26` | macOS 26 compatibility jobs and nightly sign/notarize | `blacksmith-6vcpu-macos-26` | `blacksmith-6vcpu-macos-26` | | `MACOS_RUNNER_26_NIGHTLY_BUILD` | changed-revision universal Nightly app builds | `blacksmith-12vcpu-macos-26` | `blacksmith-6vcpu-macos-26` | @@ -49,6 +49,9 @@ the same cost profile or the same urgency. - **Manual test debugging** (`test-e2e.yml`, `test-depot.yml`) resolves through `MACOS_RUNNER_TESTS`, and deliberately does **not** follow `MACOS_RUNNER_15`. Re-running one test to chase a flake should never reach for paid capacity. + Both fallbacks stay on Blacksmith for that reason; `test-e2e.yml` falls back + to macOS 26 because the macOS 15 pool's queue-to-start p90 was 83 min against + 1.0 min on 26, measured over 60 dispatches on 2026-09-22/23. `MACOS_RUNNER_PR` and `MACOS_RUNNER_TESTS` are escape hatches: leaving them unset is the intended state, and setting one overrides just that lane without @@ -73,7 +76,9 @@ also asserts that no workflow names a Warp label as a literal anywhere. Because forks cannot see repository variables, a fork pull request resolves `MACOS_RUNNER_PR` and `MACOS_RUNNER_TESTS` to empty and lands on the Blacksmith -fallback. That is the same runner it used before those variables existed. +fallback, never on paid capacity. `test-e2e.yml` is `workflow_dispatch`-only, +so a fork never reaches its fallback at all; for the lanes a fork does reach, +the fallback is still the runner they used before these variables existed. ## Background lane diff --git a/skills/cmux-testing/references/local-vs-ci-validation.md b/skills/cmux-testing/references/local-vs-ci-validation.md index 7ee09b0be000..9729dbab14b6 100644 --- a/skills/cmux-testing/references/local-vs-ci-validation.md +++ b/skills/cmux-testing/references/local-vs-ci-validation.md @@ -31,6 +31,8 @@ Dispatch through the wrapper. It pins the exact pushed commit, carries a `dispat **Compile the test target locally before dispatching.** One focused run costs 10-20 macOS runner-minutes, and it compiles the whole tree before it runs anything, so the most common red result on a feature branch is a Swift compile error rather than a test failure. The `cmux-unit` command above catches those in a fraction of the time and without a runner. +**The default runner is `blacksmith-6vcpu-macos-26`.** `--runner` overrides it. Over 60 consecutive dispatches (2026-09-22/23) the macOS 15 pool queued for a median 2.4 min but a p90 of 83 min and a worst case of 178 min, while macOS 26 queued 0.3 min median / 1.0 min p90; execution time on 26 ran about 4 min longer. Pick `blacksmith-6vcpu-macos-15` explicitly only when the question is specifically about macOS 15 behavior, and expect to wait for it. + **Do not re-dispatch the same selector at the same commit.** A focused run's result is a property of the commit; repeating it reprints the same failure at full cost. The wrapper now refuses a selector that already failed at that commit and points at the earlier run; read that run, fix the branch, push, and dispatch the new commit. `--force` exists for the rare case where you know the failure was infrastructure. If a run fails with `selected test filter matched zero tests`, the selector is wrong or the test file is not wired into `project.pbxproj` (see the test wiring section of SKILL.md). Fix the selector; retrying an unmatched filter costs another full build and matches nothing again. diff --git a/tests/test_ci_self_hosted_guard.sh b/tests/test_ci_self_hosted_guard.sh index 8b7d8608a97f..1d82daf0b36d 100755 --- a/tests/test_ci_self_hosted_guard.sh +++ b/tests/test_ci_self_hosted_guard.sh @@ -159,7 +159,7 @@ check_e2e_runner_fallbacks() { if ! awk ' /^[[:space:]]*- name: Validate Tart canary identity$/ { in_tart_step=1; next } in_tart_step && /^ - / { in_tart_step=0; in_runner_reject=0; in_marker_reject=0 } - in_tart_step && /startsWith\(\(!inputs\.runner \|\| inputs\.runner == '\''auto'\''\) && \(vars\.MACOS_RUNNER_[A-Z0-9_]+ \|\| '\''blacksmith-6vcpu-macos-15'\''\) \|\| inputs\.runner, '\''tart-'\''\)/ { saw_effective_runner=1 } + in_tart_step && /startsWith\(\(!inputs\.runner \|\| inputs\.runner == '\''auto'\''\) && \(vars\.MACOS_RUNNER_[A-Z0-9_]+ \|\| '\''blacksmith-6vcpu-macos-[0-9]+'\''\) \|\| inputs\.runner, '\''tart-'\''\)/ { saw_effective_runner=1 } in_tart_step && /REQUESTED_RUNNER:.*inputs\.runner/ { saw_requested_runner=1 } in_tart_step && /RUNNER_CONTEXT_NAME: \$\{\{ runner\.name \}\}/ { saw_runner_context=1 } in_tart_step && /tart-cmux-\*/ { saw_runner_pattern=1 }