From ae4a312623f27e069e1deb91e30773740435f846 Mon Sep 17 00:00:00 2001 From: Aziz Albahar Date: Thu, 18 Jun 2026 20:40:17 -0700 Subject: [PATCH 1/2] ci: run all macOS jobs on paid managed runners; retire self-hosted minis We're no longer using the self-hosted Mac minis for CI. Remove the vars.MACOS_RUNNER_* indirection from every workflow so macOS jobs target paid managed runners directly, with no self-hosted option: - vars.MACOS_RUNNER_15 / _26 / _26_RELEASE / _DISPLAY / _IOS references removed; jobs now pin warp (warp-macos-15/26-arm64-6x), depot (perf-activation PRs, unchanged), or hosted macos-26 (iOS). - perf-activation and test-e2e keep their manual `inputs.runner` override but default to warp instead of the self-hosted variable. - Make the tests-job pin permanent (it was a temporary 2026-06-18 workaround for the austin minis' missing GUI session) and update stale comments/descriptions that still advertised the self-hosted fleet. No workflow still references MACOS_RUNNER or the cmux-aws-macos-15 / cmux-macos-26 labels. Follow-up (settings, not code): the now-unused vars.MACOS_RUNNER_* repo variables can be deleted and the self-hosted runners deregistered. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/build-ghosttykit.yml | 2 +- .github/workflows/ci-macos-compat.yml | 4 ++-- .github/workflows/ci.yml | 24 ++++++++++-------------- .github/workflows/ios-testflight.yml | 2 +- .github/workflows/nightly.yml | 2 +- .github/workflows/perf-activation.yml | 12 ++++++------ .github/workflows/release.yml | 4 ++-- .github/workflows/reload-build.yml | 2 +- .github/workflows/test-depot.yml | 2 +- .github/workflows/test-e2e.yml | 16 ++++++++-------- .github/workflows/test-ios.yml | 4 ++-- .github/workflows/tmux-corpus.yml | 2 +- 12 files changed, 36 insertions(+), 40 deletions(-) diff --git a/.github/workflows/build-ghosttykit.yml b/.github/workflows/build-ghosttykit.yml index 7d8df6861489..2c35fb93b7bf 100644 --- a/.github/workflows/build-ghosttykit.yml +++ b/.github/workflows/build-ghosttykit.yml @@ -9,7 +9,7 @@ concurrency: jobs: build-ghosttykit: - runs-on: ${{ vars.MACOS_RUNNER_15 || 'warp-macos-15-arm64-6x' }} + runs-on: warp-macos-15-arm64-6x timeout-minutes: 20 env: GHOSTTYKIT_CRASH_REPORT_SUBDIR: cmux/crash diff --git a/.github/workflows/ci-macos-compat.yml b/.github/workflows/ci-macos-compat.yml index 7ee2341137ea..3a2777ad7fbe 100644 --- a/.github/workflows/ci-macos-compat.yml +++ b/.github/workflows/ci-macos-compat.yml @@ -9,12 +9,12 @@ jobs: fail-fast: false matrix: include: - - os: ${{ vars.MACOS_RUNNER_15 || 'warp-macos-15-arm64-6x' }} + - os: warp-macos-15-arm64-6x timeout: 30 startup_smoke: true virtual_display: true skip_zig: false - - os: ${{ vars.MACOS_RUNNER_26 || 'warp-macos-26-arm64-6x' }} + - os: warp-macos-26-arm64-6x timeout: 30 startup_smoke: true virtual_display: false diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 62146b114114..c03b1a20ece3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -253,14 +253,10 @@ jobs: run: bun run test:db:behavior tests: - # TEMPORARY (2026-06-18): pinned to hosted runners. The self-hosted austin - # mac-minis in the MACOS_RUNNER_15 pool cannot broker the XCTest control - # session with testmanagerd ("Timed out 120s initiating control session - # with daemon" -> Executed 0 tests -> idle-timeout), so this required check - # can never go green there. Revert to - # `${{ vars.MACOS_RUNNER_15 || 'warp-macos-15-arm64-6x' }}` once the austin - # runners are repaired (GUI login session + automation mode + unwedged - # testmanagerd). Tracked by the CI-flakiness handoff. + # macOS CI runs on paid managed runners only. The self-hosted Mac minis + # were retired (2026-06-18): with no logged-in GUI session, testmanagerd + # could not broker the XCTest control session and tests never ran there. + # All macOS jobs now use warp / depot / hosted runners. runs-on: warp-macos-15-arm64-6x timeout-minutes: 75 env: @@ -667,7 +663,7 @@ jobs: # Keep lag validation separate from UI regressions so functional UI failures # and performance regressions stay isolated. Broader interactive UI suites # still run via test-e2e.yml on GitHub-hosted runners. - runs-on: ${{ vars.MACOS_RUNNER_DISPLAY || 'warp-macos-15-arm64-6x' }} + runs-on: warp-macos-15-arm64-6x # A cold DerivedData cache (any project.pbxproj or Package.resolved change # mints a new cache key with no restore-keys fallback) forces a full # cmux build whose Swift codegen alone can run 20+ min. Project/package @@ -677,7 +673,7 @@ jobs: steps: - name: Validate display runner identity env: - REQUESTED_RUNNER: ${{ vars.MACOS_RUNNER_DISPLAY || 'warp-macos-15-arm64-6x' }} + REQUESTED_RUNNER: warp-macos-15-arm64-6x RUNNER_CONTEXT_NAME: ${{ runner.name }} run: | set -euo pipefail @@ -992,7 +988,7 @@ jobs: rm -f "${VDISPLAY_HELPER_PATH:-}" "${VDISPLAY_READY:-}" "${VDISPLAY_ID_PATH:-}" "${VDISPLAY_LOG:-}" release-ghostty-cli-helper: - runs-on: ${{ vars.MACOS_RUNNER_15 || 'warp-macos-15-arm64-6x' }} + runs-on: warp-macos-15-arm64-6x timeout-minutes: 20 steps: - name: Clear stale git locks (self-hosted reused workspace) @@ -1057,7 +1053,7 @@ jobs: # Release builds need enough disk for a universal Release build plus the # restored SwiftPM cache. Default to a clean paid macOS 26 runner instead # of the generic persistent macOS 26 pool. - runs-on: ${{ vars.MACOS_RUNNER_26_RELEASE || 'warp-macos-26-arm64-6x' }} + runs-on: warp-macos-26-arm64-6x timeout-minutes: 45 steps: - name: Clear stale git locks (self-hosted reused workspace) @@ -1190,14 +1186,14 @@ jobs: [[ "$SDK_VERSION" == 26.* ]] ui-regressions: - runs-on: ${{ vars.MACOS_RUNNER_DISPLAY || 'warp-macos-15-arm64-6x' }} + runs-on: warp-macos-15-arm64-6x # Cold builds after project/package changes can spend more than 25 minutes # in build-for-testing before the UI regression script starts. timeout-minutes: 45 steps: - name: Validate display runner identity env: - REQUESTED_RUNNER: ${{ vars.MACOS_RUNNER_DISPLAY || 'warp-macos-15-arm64-6x' }} + REQUESTED_RUNNER: warp-macos-15-arm64-6x RUNNER_CONTEXT_NAME: ${{ runner.name }} run: | set -euo pipefail diff --git a/.github/workflows/ios-testflight.yml b/.github/workflows/ios-testflight.yml index 5d2b58d79771..baa4e7edc464 100644 --- a/.github/workflows/ios-testflight.yml +++ b/.github/workflows/ios-testflight.yml @@ -104,7 +104,7 @@ jobs: # blocks publishing arbitrary code by dispatching the workflow against a # feature branch (the ASC secrets are only meant to ship reviewed main). if: needs.decide.outputs.should_build == 'true' && github.ref == 'refs/heads/main' - runs-on: ${{ vars.MACOS_RUNNER_IOS || 'macos-26' }} + runs-on: macos-26 timeout-minutes: 60 env: ASC_API_KEY_ID: ${{ secrets.ASC_API_KEY_ID }} diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 150e57878deb..4dc65b7f7eba 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -108,7 +108,7 @@ jobs: # injected before signing, preferring a pre-26 SDK when the runner image has # one but falling back to the selected app Xcode when the image only ships # Xcode 26. The helper build remains required and lipo-verified below. - runs-on: ${{ vars.MACOS_RUNNER_15 || 'warp-macos-15-arm64-6x' }} + runs-on: warp-macos-15-arm64-6x timeout-minutes: 30 steps: - name: Clear stale git locks (self-hosted reused workspace) diff --git a/.github/workflows/perf-activation.yml b/.github/workflows/perf-activation.yml index 5b1b9b73e914..677041c33c02 100644 --- a/.github/workflows/perf-activation.yml +++ b/.github/workflows/perf-activation.yml @@ -9,7 +9,7 @@ on: required: false default: "" runner: - description: macOS runner (PRs use Depot for GUI activation; manual auto follows MACOS_RUNNER_15, then warp) + description: macOS runner (PRs use Depot for GUI activation; manual auto uses warp) required: false default: auto type: choice @@ -45,15 +45,15 @@ concurrency: jobs: activation-session: - runs-on: ${{ github.event_name == 'pull_request' && 'depot-macos-latest' || ((!inputs.runner || inputs.runner == 'auto') && (vars.MACOS_RUNNER_15 || 'warp-macos-15-arm64-6x') || inputs.runner) }} + runs-on: ${{ github.event_name == 'pull_request' && 'depot-macos-latest' || ((!inputs.runner || inputs.runner == 'auto') && 'warp-macos-15-arm64-6x' || inputs.runner) }} timeout-minutes: 45 env: PERF_TAG: perf-${{ github.run_id }}-${{ github.run_attempt }} steps: - name: Validate Depot runner identity - if: ${{ startsWith(github.event_name == 'pull_request' && 'depot-macos-latest' || ((!inputs.runner || inputs.runner == 'auto') && (vars.MACOS_RUNNER_15 || 'warp-macos-15-arm64-6x') || inputs.runner), 'depot-macos-') }} + if: ${{ startsWith(github.event_name == 'pull_request' && 'depot-macos-latest' || ((!inputs.runner || inputs.runner == 'auto') && 'warp-macos-15-arm64-6x' || inputs.runner), 'depot-macos-') }} env: - REQUESTED_RUNNER: ${{ github.event_name == 'pull_request' && 'depot-macos-latest' || ((!inputs.runner || inputs.runner == 'auto') && (vars.MACOS_RUNNER_15 || 'warp-macos-15-arm64-6x') || inputs.runner) }} + REQUESTED_RUNNER: ${{ github.event_name == 'pull_request' && 'depot-macos-latest' || ((!inputs.runner || inputs.runner == 'auto') && 'warp-macos-15-arm64-6x' || inputs.runner) }} RUNNER_CONTEXT_NAME: ${{ runner.name }} run: | set -euo pipefail @@ -131,8 +131,8 @@ jobs: uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: .ci-source-packages - key: spm-${{ github.event_name == 'pull_request' && 'depot-macos-latest' || ((!inputs.runner || inputs.runner == 'auto') && (vars.MACOS_RUNNER_15 || 'warp-macos-15-arm64-6x') || inputs.runner) }}-${{ hashFiles('cmux.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved') }} - restore-keys: spm-${{ github.event_name == 'pull_request' && 'depot-macos-latest' || ((!inputs.runner || inputs.runner == 'auto') && (vars.MACOS_RUNNER_15 || 'warp-macos-15-arm64-6x') || inputs.runner) }}- + key: spm-${{ github.event_name == 'pull_request' && 'depot-macos-latest' || ((!inputs.runner || inputs.runner == 'auto') && 'warp-macos-15-arm64-6x' || inputs.runner) }}-${{ hashFiles('cmux.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved') }} + restore-keys: spm-${{ github.event_name == 'pull_request' && 'depot-macos-latest' || ((!inputs.runner || inputs.runner == 'auto') && 'warp-macos-15-arm64-6x' || inputs.runner) }}- - name: Sanitize Swift package cache run: python3 scripts/ci/sanitize-xcode-source-packages-cache.py .ci-source-packages diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ac39b8a2316b..6bb9452e2cdf 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -16,7 +16,7 @@ env: jobs: build-ghostty-cli-helper: - runs-on: ${{ vars.MACOS_RUNNER_15 || 'warp-macos-15-arm64-6x' }} + runs-on: warp-macos-15-arm64-6x timeout-minutes: 20 steps: - name: Clear stale git locks (self-hosted reused workspace) @@ -67,7 +67,7 @@ jobs: # macOS 15 above because Zig 0.15.2 cannot link it on macOS 26. # Import the Apple Developer ID intermediate chain into the build keychain # below so signing does not depend on mutable runner login-keychain state. - runs-on: ${{ vars.MACOS_RUNNER_26 || 'warp-macos-26-arm64-6x' }} + runs-on: warp-macos-26-arm64-6x # Notarization wait times vary on Apple's side; v0.64.14 finished at 19m16s # and v0.64.15 attempt 1 was killed by a 20-minute budget mid-notarization. timeout-minutes: 40 diff --git a/.github/workflows/reload-build.yml b/.github/workflows/reload-build.yml index d610c7a6feee..caffbaea8f54 100644 --- a/.github/workflows/reload-build.yml +++ b/.github/workflows/reload-build.yml @@ -32,7 +32,7 @@ on: runner: description: >- macOS runner label to build on. Blacksmith (blacksmith-6vcpu-macos-26), - our self-hosted fleet (cmux-macos-26 / cmux-aws-macos-15), warp, or depot. + warp, or depot. Paid managed runners only; self-hosted minis retired. required: false default: blacksmith-6vcpu-macos-26 type: string diff --git a/.github/workflows/test-depot.yml b/.github/workflows/test-depot.yml index b4da4269688f..cc2ce2445f50 100644 --- a/.github/workflows/test-depot.yml +++ b/.github/workflows/test-depot.yml @@ -28,7 +28,7 @@ on: jobs: tests: - runs-on: ${{ vars.MACOS_RUNNER_15 || 'warp-macos-15-arm64-6x' }} + runs-on: warp-macos-15-arm64-6x timeout-minutes: 20 steps: - name: Clear stale git locks (self-hosted reused workspace) diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index e9a070ef6af8..9beae1b4ab7f 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -1,5 +1,5 @@ name: E2E test with video recording -run-name: ${{ inputs.test_filter }} on ${{ (!inputs.runner || inputs.runner == 'auto') && (vars.MACOS_RUNNER_15 || 'warp-macos-15-arm64-6x') || inputs.runner }} @ ${{ inputs.ref || github.ref_name }} +run-name: ${{ inputs.test_filter }} on ${{ (!inputs.runner || inputs.runner == 'auto') && 'warp-macos-15-arm64-6x' || inputs.runner }} @ ${{ inputs.ref || github.ref_name }} on: workflow_dispatch: @@ -25,7 +25,7 @@ on: default: true type: boolean runner: - description: "Runner OS (auto follows the MACOS_RUNNER_15 repo variable, then warp; pick depot-macos-* for GUI activation)" + description: "Runner OS (auto uses warp; pick depot-macos-* for GUI activation)" required: false default: "auto" type: choice @@ -40,20 +40,20 @@ on: - depot-macos-14 concurrency: - group: e2e-${{ (!inputs.runner || inputs.runner == 'auto') && (vars.MACOS_RUNNER_15 || 'warp-macos-15-arm64-6x') || inputs.runner }}-${{ inputs.ref || github.ref_name }}-${{ inputs.test_filter }} + group: e2e-${{ (!inputs.runner || inputs.runner == 'auto') && 'warp-macos-15-arm64-6x' || inputs.runner }}-${{ inputs.ref || github.ref_name }}-${{ inputs.test_filter }} cancel-in-progress: true jobs: e2e: - runs-on: ${{ (!inputs.runner || inputs.runner == 'auto') && (vars.MACOS_RUNNER_15 || 'warp-macos-15-arm64-6x') || inputs.runner }} + runs-on: ${{ (!inputs.runner || inputs.runner == 'auto') && 'warp-macos-15-arm64-6x' || inputs.runner }} timeout-minutes: ${{ fromJSON(inputs.job_timeout || '20') }} env: TEST_REF: ${{ inputs.ref || github.ref }} steps: - name: Validate Depot runner identity - if: ${{ startsWith((!inputs.runner || inputs.runner == 'auto') && (vars.MACOS_RUNNER_15 || 'warp-macos-15-arm64-6x') || inputs.runner, 'depot-macos-') }} + if: ${{ startsWith((!inputs.runner || inputs.runner == 'auto') && 'warp-macos-15-arm64-6x' || inputs.runner, 'depot-macos-') }} env: - REQUESTED_RUNNER: ${{ (!inputs.runner || inputs.runner == 'auto') && (vars.MACOS_RUNNER_15 || 'warp-macos-15-arm64-6x') || inputs.runner }} + REQUESTED_RUNNER: ${{ (!inputs.runner || inputs.runner == 'auto') && 'warp-macos-15-arm64-6x' || inputs.runner }} RUNNER_CONTEXT_NAME: ${{ runner.name }} run: | set -euo pipefail @@ -248,8 +248,8 @@ jobs: uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: .ci-source-packages - key: spm-${{ (!inputs.runner || inputs.runner == 'auto') && (vars.MACOS_RUNNER_15 || 'warp-macos-15-arm64-6x') || inputs.runner }}-${{ hashFiles('cmux.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved') }} - restore-keys: spm-${{ (!inputs.runner || inputs.runner == 'auto') && (vars.MACOS_RUNNER_15 || 'warp-macos-15-arm64-6x') || inputs.runner }}- + key: spm-${{ (!inputs.runner || inputs.runner == 'auto') && 'warp-macos-15-arm64-6x' || inputs.runner }}-${{ hashFiles('cmux.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved') }} + restore-keys: spm-${{ (!inputs.runner || inputs.runner == 'auto') && 'warp-macos-15-arm64-6x' || inputs.runner }}- - name: Sanitize Swift package cache run: python3 scripts/ci/sanitize-xcode-source-packages-cache.py .ci-source-packages diff --git a/.github/workflows/test-ios.yml b/.github/workflows/test-ios.yml index c4dd6f6012ab..a636ab5bb9ed 100644 --- a/.github/workflows/test-ios.yml +++ b/.github/workflows/test-ios.yml @@ -110,7 +110,7 @@ jobs: mobile-core-package: needs: detect-ios-changes if: ${{ needs.detect-ios-changes.outputs.should_run == 'true' }} - runs-on: ${{ vars.MACOS_RUNNER_IOS || 'macos-26' }} + runs-on: macos-26 timeout-minutes: 10 steps: - name: Checkout @@ -150,7 +150,7 @@ jobs: ios-simulator: needs: detect-ios-changes if: ${{ needs.detect-ios-changes.outputs.should_run == 'true' }} - runs-on: ${{ vars.MACOS_RUNNER_IOS || 'macos-26' }} + runs-on: macos-26 timeout-minutes: 35 strategy: fail-fast: false diff --git a/.github/workflows/tmux-corpus.yml b/.github/workflows/tmux-corpus.yml index 5bbb2918dee2..dd6928bb5883 100644 --- a/.github/workflows/tmux-corpus.yml +++ b/.github/workflows/tmux-corpus.yml @@ -61,7 +61,7 @@ jobs: terminal-nightly: if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' - runs-on: ${{ vars.MACOS_RUNNER_15 || 'warp-macos-15-arm64-6x' }} + runs-on: warp-macos-15-arm64-6x timeout-minutes: 30 env: # XCTest app-host crashes can leave xcodebuild waiting in Swift's crash From 71b8ecb1af84c2f053a7485b130a207252b94488 Mon Sep 17 00:00:00 2001 From: Aziz Albahar Date: Thu, 18 Jun 2026 20:48:46 -0700 Subject: [PATCH 2/2] ci: update runner guards + docs for paid-only macOS runners Match test_ci_self_hosted_guard.sh and test_ci_release_sdk_lane.sh to the paid-only workflow config (literal warp/depot labels instead of the removed vars.MACOS_RUNNER_*), preserving their intent: paid runners only, depot identity validation, macOS-15-helper/macOS-26-app SDK split, and a new explicit check that retired self-hosted labels never reappear. Update docs/ci-runners.md to the paid-only model. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/ci-runners.md | 132 ++++++++++++----------------- tests/test_ci_release_sdk_lane.sh | 20 +++-- tests/test_ci_self_hosted_guard.sh | 96 ++++++++++++++++----- 3 files changed, 139 insertions(+), 109 deletions(-) diff --git a/docs/ci-runners.md b/docs/ci-runners.md index 5a355a6dd01b..325761f2e527 100644 --- a/docs/ci-runners.md +++ b/docs/ci-runners.md @@ -1,93 +1,65 @@ # CI runners -Every CI/CD job picks its runner from a repository variable instead of a -hardcoded label. We run **Blacksmith only** for every runner type and accept -the occasional sub-minute Blacksmith queue rather than overflowing elsewhere. -There is no automatic Warp overflow. WarpBuild stays wired in only as a manual -break-glass fallback (and as the home of the macOS XCTest/GUI jobs Blacksmith -can't run; see exceptions below). Switching a runner type to the break-glass -fallback is a single repo-variable change that takes effect on the next -workflow run, with no PR or commit. - -| Variable | Used by | Blacksmith (primary) | Fallback baked into the workflow | -| ------------------- | ---------------------------------------------------------- | --------------------------- | -------------------------------- | -| `LINUX_RUNNER` | every Linux job (`ci.yml` web/typecheck/db, presence, cloud-vm, nightly/ios decide jobs, claude, homebrew, tmux fuzz) | `blacksmith-4vcpu-ubuntu-2404` | `warp-ubuntu-latest-x64-4x` | -| `MACOS_RUNNER_15` | universal Release app builds: nightly, stable release, `release-ghostty-cli-helper`, most macOS defaults | `blacksmith-6vcpu-macos-15` | `warp-macos-15-arm64-6x` | -| `MACOS_RUNNER_26` | macOS 26 compat + jobs that do not need Zig | `blacksmith-6vcpu-macos-26` | `warp-macos-26-arm64-6x` | -| `MACOS_RUNNER_26_RELEASE` | disk-heavy `release-build` universal app | `blacksmith-6vcpu-macos-26` | `warp-macos-26-arm64-6x` | -| `MACOS_RUNNER_IOS` | iOS simulator tests + TestFlight upload (`test-ios.yml`, `ios-testflight.yml`) | `blacksmith-6vcpu-macos-26` | `macos-26` (free GitHub-hosted) | - -Workflows reference them as `runs-on: ${{ vars.LINUX_RUNNER || 'warp-ubuntu-latest-x64-4x' }}`. -If a variable is unset the job uses the fallback, so CI is never broken by a -missing variable. - -## Deliberate exceptions (not on Blacksmith) +macOS CI runs on **paid managed runners only**. Every macOS job names a paid +label directly (WarpBuild for the always-on lanes, Depot for GUI-activation +runs); the old `vars.MACOS_RUNNER_*` indirection and the self-hosted Mac minis +(labels `cmux-aws-macos-15` / `cmux-macos-26`) were retired. Linux jobs still +pick their runner from the `LINUX_RUNNER` repo variable so the +Blacksmith<->Warp overflow switch for Linux stays a single repo-variable flip +with no PR. + +| Runner | Used by | Notes | +| ------ | ------- | ----- | +| `vars.LINUX_RUNNER` (fallback `warp-ubuntu-latest-x64-4x`) | every Linux job (`ci.yml` web/typecheck/db, presence, cloud-vm, nightly/ios decide jobs, claude, homebrew, tmux fuzz) | Blacksmith<->Warp flip via the repo variable | +| `warp-macos-15-arm64-6x` | universal Release app builds: nightly, stable release, `release-ghostty-cli-helper`, `tests`, `tests-build-and-lag`, `ui-regressions`, `build-ghosttykit`, tmux corpus, e2e/perf `auto` default | paid macOS 15 | +| `warp-macos-26-arm64-6x` | macOS 26 app build/sign jobs: `release-build`, release `build-sign-notarize`, macOS 26 compat | paid macOS 26 (disk-heavy universal builds) | +| `depot-macos-*` | `perf-activation.yml` PR runs (`depot-macos-latest`) and explicit `depot-macos-*` choices in `perf-activation.yml` / `test-e2e.yml` | paid Depot, GUI activation; identity-guarded | +| `macos-26` (free GitHub-hosted) | iOS simulator tests + TestFlight upload (`test-ios.yml`, `ios-testflight.yml`) | only sanctioned bare hosted runner; iOS sim path does not need a paid GUI runner | + +Linux jobs reference the variable as +`runs-on: ${{ vars.LINUX_RUNNER || 'warp-ubuntu-latest-x64-4x' }}`; if the +variable is unset the job uses the baked-in Warp fallback, so CI is never broken +by a missing variable. + +## GUI-activation runners (Depot) Blacksmith macOS runners cannot initiate a testmanagerd control session (no GUI -login session / automation mode), so XCTest-driven and virtual-display jobs -hang at "Timed out 120s initiating control session with daemon" and never go -green there. These stay on Warp or Depot on purpose: - -- `ci.yml` `tests` is hard-pinned to `warp-macos-15-arm64-6x` (see the comment at - the job). Revert to `vars.MACOS_RUNNER_15` once Blacksmith macOS testmanagerd - is repaired. -- `ci.yml` `tests-build-and-lag` and `ui-regressions`, `perf-activation.yml` - PR runs, and the `virtual_display` compat row use `MACOS_RUNNER_DISPLAY` - (Depot/Warp) because Cmd-Tab timing, virtual displays, and XCTest automation - need a GUI-capable runner. A Depot identity guard validates these. - -`MACOS_RUNNER_IOS` defaults to Blacksmith but keeps a free GitHub-hosted -`macos-26` fallback because iOS simulator XCUITests may hit the same -testmanagerd limitation. If an iOS job wedges on Blacksmith, flip -`MACOS_RUNNER_IOS` back to `macos-26`. - -## Break-glass: switch a runner type off Blacksmith - -We do not auto-overflow. If Blacksmith is genuinely down or queuing for minutes -(not the sub-minute queue we accept by default), manually flip the affected -variable to its fallback; revert it once Blacksmith recovers. Use Blacksmith -(default): - -```bash -gh variable set LINUX_RUNNER --repo manaflow-ai/cmux -b blacksmith-4vcpu-ubuntu-2404 -gh variable set MACOS_RUNNER_15 --repo manaflow-ai/cmux -b blacksmith-6vcpu-macos-15 -gh variable set MACOS_RUNNER_26 --repo manaflow-ai/cmux -b blacksmith-6vcpu-macos-26 -gh variable set MACOS_RUNNER_26_RELEASE --repo manaflow-ai/cmux -b blacksmith-6vcpu-macos-26 -gh variable set MACOS_RUNNER_IOS --repo manaflow-ai/cmux -b blacksmith-6vcpu-macos-26 -``` - -Break-glass a type to WarpBuild only when Blacksmith is down or queuing for -minutes (as happened for macOS in https://github.com/manaflow-ai/cmux/pull/4926). -Either delete the variable to use the baked-in fallback, or set it explicitly: - -```bash -gh variable set LINUX_RUNNER --repo manaflow-ai/cmux -b warp-ubuntu-latest-x64-4x -gh variable set MACOS_RUNNER_15 --repo manaflow-ai/cmux -b warp-macos-15-arm64-6x -gh variable delete MACOS_RUNNER_26 --repo manaflow-ai/cmux # reverts to the Warp fallback -``` - -Check current values: - -```bash -gh variable list --repo manaflow-ai/cmux -``` +login session / automation mode), so XCTest-driven and virtual-display jobs hang +at "Timed out 120s initiating control session with daemon" there. macOS jobs +therefore run on Warp or Depot: + +- `ci.yml` `tests`, `tests-build-and-lag`, and `ui-regressions` pin + `warp-macos-15-arm64-6x`. `tests-build-and-lag` and `ui-regressions` keep a + "Validate display runner identity" step: if the pinned label is ever changed + to a `depot-macos-*` value, the step asserts the resolved `runner.name` is + actually Depot and fails otherwise. +- `perf-activation.yml` PR runs use `depot-macos-latest`; manual runs default to + `warp-macos-15-arm64-6x` (`auto`) and expose `depot-macos-*` choices. +- `test-e2e.yml` defaults to `warp-macos-15-arm64-6x` (`auto`) and exposes + `depot-macos-*` choices. Any Depot choice is identity-guarded. +- `macOS 26` compat (`ci-macos-compat.yml`) runs on `warp-macos-26-arm64-6x`. + +The iOS jobs run on free GitHub-hosted `macos-26` because iOS simulator +XCUITests and the TestFlight upload do not need a paid GUI-activation runner. ## Manual runs `perf-activation.yml` and `test-e2e.yml` keep a `runner` choice input that -defaults to `auto`. Manual `auto` runs follow `MACOS_RUNNER_15` then the Warp -fallback, so flipping the repo variable redirects those workflows. An explicit -manual choice wins over the variable; both dropdowns expose Blacksmith, Warp, -and `depot-macos-*` choices, with a Depot identity guard for GUI-activation -runs. +defaults to `auto`. `auto` resolves to `warp-macos-15-arm64-6x`; an explicit +manual choice (Blacksmith, Warp, or `depot-macos-*`) wins over the default. A +Depot identity guard validates GUI-activation runs. ## Guard `tests/test_ci_self_hosted_guard.sh` (run by the `workflow-guard-tests` job) -asserts that no job pins a bare GitHub-hosted runner (`ubuntu-*` / `macos-NN`): -every job must route through a runner repo variable so the overflow switch stays -a single variable flip. It also asserts every paid macOS job references -`vars.MACOS_RUNNER_*` or a Blacksmith/Warp/Depot label so it can never silently -fall back to a free runner. Bare paid-provider labels (`blacksmith-*`, `warp-*`, -`depot-*`) stay allowed for deliberate single-runner pins. Keep new labels in +asserts that every guarded macOS job runs on a paid managed label +(`warp-*`/`depot-*`/`blacksmith-*`) and never on a free GitHub-hosted runner, +that Linux jobs still route through `vars.LINUX_RUNNER`, and that the retired +self-hosted Mac labels (`cmux-aws-macos-15` / `cmux-macos-26`, or any +`self-hosted` label) never reappear in a `runs-on`. The only sanctioned bare +GitHub-hosted runner is `macos-26` for the iOS jobs in `test-ios.yml` and +`ios-testflight.yml`; the guard fails if any other workflow pins a bare hosted +runner or if an iOS workflow pins a hosted macOS label other than `macos-26`. +`tests/test_ci_release_sdk_lane.sh` additionally asserts the macOS 15 helper / +macOS 26 app SDK split for the release lane. Keep new labels in `.github/actionlint.yaml`. diff --git a/tests/test_ci_release_sdk_lane.sh b/tests/test_ci_release_sdk_lane.sh index a1a8a8a21bb0..11703d2f6822 100755 --- a/tests/test_ci_release_sdk_lane.sh +++ b/tests/test_ci_release_sdk_lane.sh @@ -29,29 +29,33 @@ require_job_contains() { fi } +# macOS CI now routes straight to PAID managed runners (warp/depot); the +# vars.MACOS_RUNNER_* indirection was removed intentionally. These checks still +# assert the SDK lane split (helper built on macOS 15, app signed/built on +# macOS 26) but key off the literal paid Warp labels instead of the old vars. require_job_contains \ "$RELEASE_FILE" \ "build-ghostty-cli-helper" \ - 'runs-on: ${{ vars.MACOS_RUNNER_15 || '\''warp-macos-15-arm64-6x'\'' }}' \ - "release must build the real Ghostty CLI helper on macOS 15" + 'runs-on: warp-macos-15-arm64-6x' \ + "release must build the real Ghostty CLI helper on the paid macOS 15 runner" require_job_contains \ "$RELEASE_FILE" \ "build-sign-notarize" \ - 'runs-on: ${{ vars.MACOS_RUNNER_26 || '\''warp-macos-26-arm64-6x'\'' }}' \ - "release must sign+notarize on the macOS 26 runner variable after importing the Developer ID intermediate chain" + 'runs-on: warp-macos-26-arm64-6x' \ + "release must sign+notarize on the paid macOS 26 runner after importing the Developer ID intermediate chain" require_job_contains \ "$CI_FILE" \ "release-ghostty-cli-helper" \ - 'runs-on: ${{ vars.MACOS_RUNNER_15 || '\''warp-macos-15-arm64-6x'\'' }}' \ - "CI must build the real Ghostty CLI helper on macOS 15" + 'runs-on: warp-macos-15-arm64-6x' \ + "CI must build the real Ghostty CLI helper on the paid macOS 15 runner" require_job_contains \ "$CI_FILE" \ "release-build" \ - 'runs-on: ${{ vars.MACOS_RUNNER_26_RELEASE || '\''warp-macos-26-arm64-6x'\'' }}' \ - "CI release-build must compile the app on macOS 26 using the release-specific runner variable" + 'runs-on: warp-macos-26-arm64-6x' \ + "CI release-build must compile the app on the paid macOS 26 runner" for workflow in "$CI_FILE" "$RELEASE_FILE"; do if ! grep -Fq "CMUX_SKIP_ZIG_BUILD=1 xcodebuild" "$workflow"; then diff --git a/tests/test_ci_self_hosted_guard.sh b/tests/test_ci_self_hosted_guard.sh index 96b577f99052..a16741ff650e 100755 --- a/tests/test_ci_self_hosted_guard.sh +++ b/tests/test_ci_self_hosted_guard.sh @@ -1,9 +1,13 @@ #!/usr/bin/env bash # Regression test for https://github.com/manaflow-ai/cmux/issues/385. -# Ensures paid CI jobs use a paid macOS runner (Blacksmith or WarpBuild, routed -# through the MACOS_RUNNER_15 / MACOS_RUNNER_26 repo variables), never a free -# GitHub-hosted runner. Flip Blacksmith<->Warp by editing those repo variables; -# see docs/ci-runners.md. +# Ensures guarded macOS CI jobs run on a PAID managed runner (WarpBuild or +# Depot, pinned by literal label such as warp-macos-15-arm64-6x / +# warp-macos-26-arm64-6x / depot-macos-*), never a free GitHub-hosted runner. +# The vars.MACOS_RUNNER_* indirection was removed; macOS jobs now name the paid +# label directly. Linux jobs still route through vars.LINUX_RUNNER. The retired +# self-hosted Mac minis (labels cmux-aws-macos-15 / cmux-macos-26) must never +# return. The only sanctioned bare GitHub-hosted runner is hosted macos-26 for +# the iOS jobs in test-ios.yml / ios-testflight.yml. See docs/ci-runners.md. # Fork PRs are gated by GitHub's built-in "Require approval for outside # collaborators" setting, so workflow-level fork guards are not needed. set -euo pipefail @@ -31,11 +35,18 @@ check_macos_runner() { } check_display_runner_identity_guard() { + # The display jobs (tests-build-and-lag, ui-regressions) no longer route + # through vars.MACOS_RUNNER_DISPLAY; they pin the paid Warp label + # warp-macos-15-arm64-6x directly. The Depot identity guard STEP is still + # present and still meaningful: if someone repoints REQUESTED_RUNNER at a + # depot-* label, the step verifies the resolved runner.name actually is Depot + # and fails otherwise. We keep validating that guard, keyed off the literal + # paid Warp label instead of the removed var. local file="$1" job="$2" if ! awk -v job="$job" ' $0 ~ "^ "job":" { in_job=1; next } in_job && /^ [^[:space:]#][^:]*:[[:space:]]*(#.*)?$/ { in_job=0 } - in_job && /REQUESTED_RUNNER:.*vars\.MACOS_RUNNER_DISPLAY/ { saw_requested=1 } + in_job && /REQUESTED_RUNNER:[[:space:]]*warp-macos-15-arm64-6x/ { saw_requested=1 } in_job && /RUNNER_CONTEXT_NAME:[[:space:]]*\$\{\{ runner\.name \}\}/ { saw_runner_name=1 } in_job && /case "\$REQUESTED_RUNNER" in/ { saw_requested_case=1 } in_job && /depot-\*\)/ { saw_depot_case=1 } @@ -43,25 +54,29 @@ check_display_runner_identity_guard() { in_job && /resolved outside Depot/ { saw_error=1 } END { exit !(saw_requested && saw_runner_name && saw_requested_case && saw_depot_case && saw_non_depot_skip && saw_error) } ' "$file"; then - echo "FAIL: $job in $(basename "$file") must validate actual Depot identity when MACOS_RUNNER_DISPLAY resolves to a depot-* runner" + echo "FAIL: $job in $(basename "$file") must pin the paid Warp display runner and still validate actual Depot identity when REQUESTED_RUNNER resolves to a depot-* runner" exit 1 fi - echo "PASS: $job in $(basename "$file") validates display runner identity" + echo "PASS: $job in $(basename "$file") pins the paid display runner and validates Depot identity" } check_release_build_runner_disk_capacity() { + # release-build now pins the paid macOS 26 Warp runner directly + # (vars.MACOS_RUNNER_26_RELEASE was removed with the rest of the macOS runner + # vars). Assert the literal paid label so a disk-heavy universal Release build + # can never silently land on a free hosted macOS runner. if ! awk ' /^ release-build:/ { in_job=1; next } in_job && /^ [^[:space:]#][^:]*:[[:space:]]*(#.*)?$/ { in_job=0 } - in_job && /runs-on:/ && /vars\.MACOS_RUNNER_26_RELEASE/ && /warp-macos-26-arm64-6x/ { saw_release_runner=1 } + in_job && /runs-on:[[:space:]]*warp-macos-26-arm64-6x[[:space:]]*$/ { saw_release_runner=1 } END { exit !saw_release_runner } ' "$CI_FILE"; then - echo "FAIL: release-build must use the release-specific macOS 26 runner var with clean Warp fallback for disk-heavy universal builds" + echo "FAIL: release-build must run on the paid macOS 26 runner warp-macos-26-arm64-6x for disk-heavy universal builds" exit 1 fi - echo "PASS: release-build uses release-specific macOS 26 runner fallback" + echo "PASS: release-build pins the paid macOS 26 runner" } check_e2e_runner_fallbacks() { @@ -96,7 +111,9 @@ check_e2e_runner_fallbacks() { exit 1 fi - if ! grep -Fq "startsWith((!inputs.runner || inputs.runner == 'auto') && (vars.MACOS_RUNNER_15 || 'warp-macos-15-arm64-6x') || inputs.runner, 'depot-macos-')" "$E2E_FILE"; then + # `auto` now resolves straight to the paid Warp label (no vars.MACOS_RUNNER_15 + # indirection). The Depot identity guard still gates every depot-macos-* choice. + if ! grep -Fq "startsWith((!inputs.runner || inputs.runner == 'auto') && 'warp-macos-15-arm64-6x' || inputs.runner, 'depot-macos-')" "$E2E_FILE"; then echo "FAIL: test-e2e.yml must validate all Depot macOS runner choices" exit 1 fi @@ -678,20 +695,57 @@ check_tmux_terminal_nightly_isolation() { } check_no_bare_github_hosted_runners() { - # Every job must route its runner through a repo variable (LINUX_RUNNER, - # MACOS_RUNNER_*) so the Blacksmith<->Warp / Blacksmith<->macos-26 overflow - # switch is a single repo-variable flip with no PR. A bare GitHub-hosted - # label (ubuntu-*, macos-NN) cannot be redirected, so it is forbidden. - # Bare paid-provider labels (blacksmith-*, warp-*, depot-*) stay allowed for - # deliberate single-runner pins such as the testmanagerd-wedged `tests` job. + # Core purpose (issue #385): no GUARDED macOS CI job may run on a free + # GitHub-hosted runner. Linux jobs still route through vars.LINUX_RUNNER so + # the Blacksmith<->Warp overflow switch is a single repo-variable flip; a bare + # ubuntu-* label is therefore still forbidden. Paid macOS jobs run directly on + # paid managed labels (warp-*, depot-*, blacksmith-*), which stay allowed. + # + # The ONLY sanctioned bare GitHub-hosted macOS pins are the iOS jobs in + # test-ios.yml and ios-testflight.yml: iOS simulator XCUITests and the + # TestFlight upload deliberately run on hosted `macos-26` (the paid Mac minis + # were retired and the iOS sim path does not need a paid GUI-activation + # runner). Those, and only those, are excluded below; every other bare hosted + # runner still fails this guard. local hits - hits="$(grep -rnE "runs-on:[[:space:]]*(ubuntu-[a-z0-9.]+|macos-[a-z0-9]+)[[:space:]]*$" "$ROOT_DIR/.github/workflows" || true)" + hits="$(grep -rnE "runs-on:[[:space:]]*(ubuntu-[a-z0-9.]+|macos-[a-z0-9]+)[[:space:]]*$" "$ROOT_DIR/.github/workflows" \ + | grep -vE "/(test-ios|ios-testflight)\.yml:[0-9]+:[[:space:]]*runs-on:[[:space:]]*macos-26[[:space:]]*$" \ + || true)" if [[ -n "$hits" ]]; then - echo "FAIL: these jobs use a bare GitHub-hosted runner; route them through vars.LINUX_RUNNER / vars.MACOS_RUNNER_IOS so Blacksmith<->overflow stays a repo-variable flip:" + echo "FAIL: these jobs use a bare free GitHub-hosted runner; Linux must route through vars.LINUX_RUNNER and macOS jobs must use a paid managed label (warp-*/depot-*/blacksmith-*). Only the iOS macos-26 jobs in test-ios.yml/ios-testflight.yml are exempt:" echo "$hits" exit 1 fi - echo "PASS: no workflow pins a bare GitHub-hosted runner; all route through runner repo variables" + + # Defense in depth: the iOS exemption must stay narrow. If an iOS workflow + # ever pins a different bare hosted macOS label (e.g. macos-15, macos-latest), + # that is NOT covered by the exclusion above and must be caught here so the + # exemption can't silently widen. + local ios_bad + ios_bad="$(grep -rnE "runs-on:[[:space:]]*macos-[a-z0-9]+[[:space:]]*$" \ + "$ROOT_DIR/.github/workflows/test-ios.yml" \ + "$ROOT_DIR/.github/workflows/ios-testflight.yml" 2>/dev/null \ + | grep -vE "runs-on:[[:space:]]*macos-26[[:space:]]*$" || true)" + if [[ -n "$ios_bad" ]]; then + echo "FAIL: iOS workflows may only use the sanctioned bare hosted runner macos-26; this pins a different free hosted label:" + echo "$ios_bad" + exit 1 + fi + + # The retired self-hosted Mac minis (labels cmux-aws-macos-15 / cmux-macos-26) + # were intentionally removed. They must never come back as a runs-on label, + # since a self-hosted runner is exactly the unpaid/unmanaged surface this + # guard exists to keep macOS CI off of. (They would not match the hosted + # ubuntu-*/macos-* pattern above, so they need their own check.) + local retired + retired="$(grep -rnE "runs-on:.*(self-hosted|cmux-aws-macos|cmux-macos-26)" "$ROOT_DIR/.github/workflows" || true)" + if [[ -n "$retired" ]]; then + echo "FAIL: a job pins a retired self-hosted Mac runner; macOS CI must stay on paid managed runners (warp-*/depot-*/blacksmith-*):" + echo "$retired" + exit 1 + fi + + echo "PASS: no guarded job pins a free GitHub-hosted or retired self-hosted runner (iOS macos-26 jobs are the only sanctioned exemption)" } # ci.yml jobs @@ -708,7 +762,7 @@ check_display_runner_identity_guard "$CI_FILE" "ui-regressions" # build-ghosttykit.yml check_macos_runner "$GHOSTTYKIT_FILE" "build-ghosttykit" -# ci-macos-compat.yml (matrix.os routed through the MACOS_RUNNER_* repo vars) +# ci-macos-compat.yml (matrix.os pins paid warp-macos-* labels directly) check_macos_runner "$COMPAT_FILE" "compat-tests" # test-e2e.yml is manual, so keep the Depot GUI runner choices but cancel