From 8bd775919181df9f3b03b64b251a4cf81d811231 Mon Sep 17 00:00:00 2001 From: Lawrence Chen Date: Wed, 27 May 2026 15:42:14 -0700 Subject: [PATCH 1/5] Migrate macOS CI/CD runners from WarpBuild/Depot to Blacksmith MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Switch all macOS jobs to blacksmith-6vcpu-macos-{15,26,latest}: - ci.yml (tests, tests-build-and-lag, release-build, ui-regressions) - build-ghosttykit, nightly, release, test-depot, perf-activation, tmux-corpus, ci-macos-compat - test-e2e default → blacksmith-6vcpu-macos-latest; existing workflow_dispatch input keeps depot-macos-* as fallback options Update tests/test_ci_self_hosted_guard.sh to assert Blacksmith runners on the paid jobs from issue #385. --- .github/workflows/build-ghosttykit.yml | 2 +- .github/workflows/ci-macos-compat.yml | 4 ++-- .github/workflows/ci.yml | 8 ++++---- .github/workflows/nightly.yml | 2 +- .github/workflows/perf-activation.yml | 9 +++++---- .github/workflows/release.yml | 2 +- .github/workflows/test-depot.yml | 2 +- .github/workflows/test-e2e.yml | 13 +++++++----- .github/workflows/tmux-corpus.yml | 2 +- tests/test_ci_self_hosted_guard.sh | 28 +++++++++++++------------- 10 files changed, 38 insertions(+), 34 deletions(-) diff --git a/.github/workflows/build-ghosttykit.yml b/.github/workflows/build-ghosttykit.yml index 109ed364690b..8a9ffd697c44 100644 --- a/.github/workflows/build-ghosttykit.yml +++ b/.github/workflows/build-ghosttykit.yml @@ -9,7 +9,7 @@ concurrency: jobs: build-ghosttykit: - runs-on: warp-macos-15-arm64-6x + runs-on: blacksmith-6vcpu-macos-15 timeout-minutes: 20 env: GHOSTTYKIT_CRASH_REPORT_SUBDIR: cmux/crash diff --git a/.github/workflows/ci-macos-compat.yml b/.github/workflows/ci-macos-compat.yml index 61f98c5e3d83..0e451b6c98e6 100644 --- a/.github/workflows/ci-macos-compat.yml +++ b/.github/workflows/ci-macos-compat.yml @@ -9,12 +9,12 @@ jobs: fail-fast: false matrix: include: - - os: warp-macos-15-arm64-6x + - os: blacksmith-6vcpu-macos-15 timeout: 30 startup_smoke: true virtual_display: true skip_zig: false - - os: warp-macos-26-arm64-6x + - os: blacksmith-6vcpu-macos-26 timeout: 30 startup_smoke: true virtual_display: false diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a75b9b0a42d2..a61d61723ba8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -158,7 +158,7 @@ jobs: bun test tests/vm-db-read-model.test.ts tests: - runs-on: warp-macos-15-arm64-6x + runs-on: blacksmith-6vcpu-macos-15 timeout-minutes: 75 env: CMUX_SKIP_ZIG_BUILD: "1" @@ -380,7 +380,7 @@ jobs: # Keep lag validation separate from UI regressions so functional UI failures # and performance regressions stay isolated. Broader interactive UI suites # still run via test-e2e.yml on GitHub-hosted runners. - runs-on: warp-macos-15-arm64-6x + runs-on: blacksmith-6vcpu-macos-15 timeout-minutes: 20 steps: - name: Checkout @@ -568,7 +568,7 @@ jobs: # Compile the same unsigned universal Release app that nightly builds before # signing, notarization, and publishing. This catches DEBUG/Release boundary # mistakes before they reach main. - runs-on: warp-macos-26-arm64-6x + runs-on: blacksmith-6vcpu-macos-26 timeout-minutes: 20 steps: - name: Checkout @@ -653,7 +653,7 @@ jobs: CODE_SIGNING_ALLOWED=NO ASSETCATALOG_COMPILER_APPICON_NAME=AppIcon-Nightly build ui-regressions: - runs-on: warp-macos-15-arm64-6x + runs-on: blacksmith-6vcpu-macos-15 timeout-minutes: 25 steps: - name: Checkout diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 700825a43565..8bba32fecafb 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -100,7 +100,7 @@ jobs: build-sign-notarize-nightly: needs: decide if: needs.decide.outputs.should_build == 'true' - runs-on: warp-macos-26-arm64-6x + runs-on: blacksmith-6vcpu-macos-26 timeout-minutes: 20 steps: - name: Checkout build ref diff --git a/.github/workflows/perf-activation.yml b/.github/workflows/perf-activation.yml index fd2967833239..61a05999d5d0 100644 --- a/.github/workflows/perf-activation.yml +++ b/.github/workflows/perf-activation.yml @@ -11,11 +11,12 @@ on: runner: description: macOS runner required: false - default: warp-macos-15-arm64-6x + default: blacksmith-6vcpu-macos-15 type: choice options: - - warp-macos-15-arm64-6x - - depot-macos-latest + - blacksmith-6vcpu-macos-15 + - blacksmith-6vcpu-macos-26 + - blacksmith-6vcpu-macos-latest workspace_count: description: Fixture workspace count required: false @@ -35,7 +36,7 @@ concurrency: jobs: activation-session: - runs-on: ${{ inputs.runner || 'warp-macos-15-arm64-6x' }} + runs-on: ${{ inputs.runner || 'blacksmith-6vcpu-macos-15' }} timeout-minutes: 45 env: PERF_TAG: perfci diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c842a3496972..3adfc6d47b8b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -16,7 +16,7 @@ env: jobs: build-sign-notarize: - runs-on: warp-macos-26-arm64-6x + runs-on: blacksmith-6vcpu-macos-26 timeout-minutes: 20 steps: - name: Checkout diff --git a/.github/workflows/test-depot.yml b/.github/workflows/test-depot.yml index d81dd94bcb9e..18b1873f709f 100644 --- a/.github/workflows/test-depot.yml +++ b/.github/workflows/test-depot.yml @@ -28,7 +28,7 @@ on: jobs: tests: - runs-on: warp-macos-15-arm64-6x + runs-on: blacksmith-6vcpu-macos-15 timeout-minutes: 20 steps: - name: Checkout diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index bbf8aee9c0e4..92f4dd53b271 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -20,17 +20,20 @@ on: default: true type: boolean runner: - description: "Runner OS (Depot runners for GUI activation support)" + description: "Runner OS" required: false - default: "depot-macos-latest" + default: "blacksmith-6vcpu-macos-latest" type: choice options: + - blacksmith-6vcpu-macos-latest + - blacksmith-6vcpu-macos-26 + - blacksmith-6vcpu-macos-15 - depot-macos-latest - depot-macos-14 jobs: e2e: - runs-on: ${{ inputs.runner || 'depot-macos-latest' }} + runs-on: ${{ inputs.runner || 'blacksmith-6vcpu-macos-latest' }} timeout-minutes: 20 env: TEST_REF: ${{ inputs.ref || github.ref }} @@ -172,8 +175,8 @@ jobs: uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: .ci-source-packages - key: spm-${{ inputs.runner || 'depot-macos-latest' }}-${{ hashFiles('cmux.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved') }} - restore-keys: spm-${{ inputs.runner || 'depot-macos-latest' }}- + key: spm-${{ inputs.runner || 'blacksmith-6vcpu-macos-latest' }}-${{ hashFiles('cmux.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved') }} + restore-keys: spm-${{ inputs.runner || 'blacksmith-6vcpu-macos-latest' }}- - name: Resolve Swift packages run: | diff --git a/.github/workflows/tmux-corpus.yml b/.github/workflows/tmux-corpus.yml index 4f193acdaf7c..b7cc29224310 100644 --- a/.github/workflows/tmux-corpus.yml +++ b/.github/workflows/tmux-corpus.yml @@ -57,7 +57,7 @@ jobs: terminal-nightly: if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' - runs-on: [self-hosted, warp-macos-15-arm64-6x] + runs-on: blacksmith-6vcpu-macos-15 timeout-minutes: 30 steps: - name: Checkout diff --git a/tests/test_ci_self_hosted_guard.sh b/tests/test_ci_self_hosted_guard.sh index ab06f4e01f4b..052469e45dcf 100755 --- a/tests/test_ci_self_hosted_guard.sh +++ b/tests/test_ci_self_hosted_guard.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Regression test for https://github.com/manaflow-ai/cmux/issues/385. -# Ensures paid CI jobs use WarpBuild runners. +# Ensures paid CI jobs use Blacksmith macOS runners. # Fork PRs are gated by GitHub's built-in "Require approval for outside # collaborators" setting, so workflow-level fork guards are not needed. set -euo pipefail @@ -10,29 +10,29 @@ CI_FILE="$ROOT_DIR/.github/workflows/ci.yml" GHOSTTYKIT_FILE="$ROOT_DIR/.github/workflows/build-ghosttykit.yml" COMPAT_FILE="$ROOT_DIR/.github/workflows/ci-macos-compat.yml" -check_warp_runner() { +check_blacksmith_runner() { local file="$1" job="$2" if ! awk -v job="$job" ' $0 ~ "^ "job":" { in_job=1; next } in_job && /^ [^[:space:]]/ { in_job=0 } - in_job && /runs-on:.*warp-macos-.*-arm64/ { saw_warp=1 } - in_job && /os: warp-macos-.*-arm64/ { saw_warp=1 } - END { exit !(saw_warp) } + in_job && /runs-on:.*blacksmith-[0-9]+vcpu-macos-/ { saw=1 } + in_job && /os: blacksmith-[0-9]+vcpu-macos-/ { saw=1 } + END { exit !(saw) } ' "$file"; then - echo "FAIL: $job in $(basename "$file") must use a WarpBuild runner" + echo "FAIL: $job in $(basename "$file") must use a Blacksmith macOS runner" exit 1 fi - echo "PASS: $job WarpBuild runner is present" + echo "PASS: $job Blacksmith runner is present" } # ci.yml jobs -check_warp_runner "$CI_FILE" "tests" -check_warp_runner "$CI_FILE" "tests-build-and-lag" -check_warp_runner "$CI_FILE" "release-build" -check_warp_runner "$CI_FILE" "ui-regressions" +check_blacksmith_runner "$CI_FILE" "tests" +check_blacksmith_runner "$CI_FILE" "tests-build-and-lag" +check_blacksmith_runner "$CI_FILE" "release-build" +check_blacksmith_runner "$CI_FILE" "ui-regressions" # build-ghosttykit.yml -check_warp_runner "$GHOSTTYKIT_FILE" "build-ghosttykit" +check_blacksmith_runner "$GHOSTTYKIT_FILE" "build-ghosttykit" -# ci-macos-compat.yml (uses matrix.os with WarpBuild runners) -check_warp_runner "$COMPAT_FILE" "compat-tests" +# ci-macos-compat.yml (uses matrix.os with Blacksmith runners) +check_blacksmith_runner "$COMPAT_FILE" "compat-tests" From e7f187068d563911e8dd49510a4a27256e92fb53 Mon Sep 17 00:00:00 2001 From: Lawrence Chen Date: Wed, 27 May 2026 16:07:13 -0700 Subject: [PATCH 2/5] Diagnose GUI session on Blacksmith for Cmd-Tab perf bench Cmd-Tab activation bench times out waiting for a visible CGWindow on Blacksmith macOS runners. perf-activation-session.py works because it talks to the debug socket, so the app process launches fine, but the Cmd-Tab bench uses CGWindowListCopyWindowInfo([.optionOnScreenOnly,...]) which only returns windows the WindowServer is rendering. Print enough runner state to decide whether to launchctl asuser the bench step or move it back to a Warp/Depot GUI-enabled runner. --- .github/workflows/perf-activation.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/workflows/perf-activation.yml b/.github/workflows/perf-activation.yml index 61a05999d5d0..e961432e2d23 100644 --- a/.github/workflows/perf-activation.yml +++ b/.github/workflows/perf-activation.yml @@ -130,6 +130,26 @@ jobs: --output perf-results/activation-session.json \ --junit perf-results/activation-session.junit.xml + - name: Diagnose GUI session + run: | + set -uo pipefail + echo "job user: $(id -un) (uid $(id -u))" + CONSOLE_USER="$(stat -f %Su /dev/console 2>/dev/null || echo '?')" + echo "console user: $CONSOLE_USER" + if [ "$CONSOLE_USER" != "?" ] && [ "$CONSOLE_USER" != "root" ]; then + echo "console uid: $(id -u "$CONSOLE_USER" 2>/dev/null || echo '?')" + fi + echo "--- launchctl list (filtered for WindowServer/loginwindow) ---" + launchctl list 2>/dev/null | grep -iE 'windowserver|loginwindow|securityagent|coreaudio' || echo "(no matches)" + echo "--- WindowServer process ---" + pgrep -lf WindowServer || echo "(no WindowServer running)" + echo "--- loginwindow process ---" + pgrep -lf loginwindow || echo "(no loginwindow running)" + echo "--- ioreg display ---" + ioreg -lw0 | grep -i 'IODisplayConnect\|IOGraphics' | head -5 || true + echo "--- screencapture probe ---" + screencapture -x /tmp/probe.png 2>&1 && ls -la /tmp/probe.png || echo "(screencapture failed)" + - name: Run Cmd-Tab activation benchmark run: | set -euo pipefail From 47741c7dd2d3f7dc9983ce342f6874c0c47f92db Mon Sep 17 00:00:00 2001 From: Lawrence Chen Date: Wed, 27 May 2026 16:23:07 -0700 Subject: [PATCH 3/5] Run Cmd-Tab perf bench inside the console Aqua session On Blacksmith macOS runners (and likely any future GitHub-image-based hosted runner), the runner job runs in launchd's system bootstrap even though it is uid 501. Apps launched via NSWorkspace.openApplication still attach to the WindowServer, but CGWindowListCopyWindowInfo([.optionOnScreenOnly,...]) from a system-bootstrap process does not see those windows in the console user's Aqua session, so the Cmd-Tab benchmark times out waiting for an initial visible window. Re-enter the console user's launchd domain with launchctl asuser before invoking the swift bench so visibility polling sees the real on-screen windows. perf-activation-session.py was unaffected because it talks to the debug socket and does not need WindowServer visibility. --- .github/workflows/perf-activation.yml | 37 ++++++++------------------- 1 file changed, 11 insertions(+), 26 deletions(-) diff --git a/.github/workflows/perf-activation.yml b/.github/workflows/perf-activation.yml index e961432e2d23..ee9de0efaffb 100644 --- a/.github/workflows/perf-activation.yml +++ b/.github/workflows/perf-activation.yml @@ -130,36 +130,21 @@ jobs: --output perf-results/activation-session.json \ --junit perf-results/activation-session.junit.xml - - name: Diagnose GUI session - run: | - set -uo pipefail - echo "job user: $(id -un) (uid $(id -u))" - CONSOLE_USER="$(stat -f %Su /dev/console 2>/dev/null || echo '?')" - echo "console user: $CONSOLE_USER" - if [ "$CONSOLE_USER" != "?" ] && [ "$CONSOLE_USER" != "root" ]; then - echo "console uid: $(id -u "$CONSOLE_USER" 2>/dev/null || echo '?')" - fi - echo "--- launchctl list (filtered for WindowServer/loginwindow) ---" - launchctl list 2>/dev/null | grep -iE 'windowserver|loginwindow|securityagent|coreaudio' || echo "(no matches)" - echo "--- WindowServer process ---" - pgrep -lf WindowServer || echo "(no WindowServer running)" - echo "--- loginwindow process ---" - pgrep -lf loginwindow || echo "(no loginwindow running)" - echo "--- ioreg display ---" - ioreg -lw0 | grep -i 'IODisplayConnect\|IOGraphics' | head -5 || true - echo "--- screencapture probe ---" - screencapture -x /tmp/probe.png 2>&1 && ls -la /tmp/probe.png || echo "(screencapture failed)" - - name: Run Cmd-Tab activation benchmark run: | set -euo pipefail APP_PATH="$HOME/Library/Developer/Xcode/DerivedData/cmux-$PERF_TAG/Build/Products/Debug/cmux DEV $PERF_TAG.app" - swift scripts/bench-window-visibility.swift \ - "$APP_PATH" \ - "com.cmuxterm.app.debug.$PERF_TAG" \ - 30 \ - --cmd-tab-activation \ - --cg-visibility \ + # The GitHub Actions runner process runs in launchd's system + # bootstrap, not the console user's Aqua session, so windows + # created by apps launched via NSWorkspace.openApplication do + # not show up in CGWindowListCopyWindowInfo([.optionOnScreenOnly]). + # Re-enter the Aqua session via launchctl asuser before running + # the bench so visibility polling sees real on-screen windows. + CONSOLE_USER="$(stat -f %Su /dev/console)" + CONSOLE_UID="$(id -u "$CONSOLE_USER")" + sudo -n launchctl asuser "$CONSOLE_UID" sudo -n -u "$CONSOLE_USER" -E \ + env PATH="$PATH" DEVELOPER_DIR="$DEVELOPER_DIR" \ + bash -c "cd '$PWD' && swift scripts/bench-window-visibility.swift '$APP_PATH' 'com.cmuxterm.app.debug.$PERF_TAG' 30 --cmd-tab-activation --cg-visibility" \ > perf-results/cmd-tab-activation.txt cat perf-results/cmd-tab-activation.txt From 7405c279571b2c12ea3a0010145c2980b906996f Mon Sep 17 00:00:00 2001 From: Lawrence Chen Date: Wed, 27 May 2026 18:02:42 -0700 Subject: [PATCH 4/5] Harden Blacksmith CI runner config --- .github/actionlint.yaml | 7 +++++++ .github/workflows/perf-activation.yml | 4 ++-- .github/workflows/test-e2e.yml | 14 +++++++------- 3 files changed, 16 insertions(+), 9 deletions(-) create mode 100644 .github/actionlint.yaml diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 000000000000..c4d5692a9d9a --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,7 @@ +self-hosted-runner: + labels: + - blacksmith-6vcpu-macos-15 + - blacksmith-6vcpu-macos-26 + - blacksmith-6vcpu-macos-latest + +config-variables: null diff --git a/.github/workflows/perf-activation.yml b/.github/workflows/perf-activation.yml index ee9de0efaffb..00ddd8410b35 100644 --- a/.github/workflows/perf-activation.yml +++ b/.github/workflows/perf-activation.yml @@ -86,8 +86,8 @@ jobs: uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: .ci-source-packages - key: spm-${{ hashFiles('cmux.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved') }} - restore-keys: spm- + key: spm-${{ inputs.runner || 'blacksmith-6vcpu-macos-15' }}-${{ hashFiles('cmux.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved') }} + restore-keys: spm-${{ inputs.runner || 'blacksmith-6vcpu-macos-15' }}- - name: Resolve Swift packages run: | diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 92f4dd53b271..f5eb617268e4 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -20,20 +20,20 @@ on: default: true type: boolean runner: - description: "Runner OS" + description: "Runner OS (macOS 15 default for GUI activation support)" required: false - default: "blacksmith-6vcpu-macos-latest" + default: "blacksmith-6vcpu-macos-15" type: choice options: - - blacksmith-6vcpu-macos-latest - - blacksmith-6vcpu-macos-26 - blacksmith-6vcpu-macos-15 + - blacksmith-6vcpu-macos-26 + - blacksmith-6vcpu-macos-latest - depot-macos-latest - depot-macos-14 jobs: e2e: - runs-on: ${{ inputs.runner || 'blacksmith-6vcpu-macos-latest' }} + runs-on: ${{ inputs.runner || 'blacksmith-6vcpu-macos-15' }} timeout-minutes: 20 env: TEST_REF: ${{ inputs.ref || github.ref }} @@ -175,8 +175,8 @@ jobs: uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: .ci-source-packages - key: spm-${{ inputs.runner || 'blacksmith-6vcpu-macos-latest' }}-${{ hashFiles('cmux.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved') }} - restore-keys: spm-${{ inputs.runner || 'blacksmith-6vcpu-macos-latest' }}- + key: spm-${{ inputs.runner || 'blacksmith-6vcpu-macos-15' }}-${{ hashFiles('cmux.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved') }} + restore-keys: spm-${{ inputs.runner || 'blacksmith-6vcpu-macos-15' }}- - name: Resolve Swift packages run: | From a61c666ae2a7ee1f6c1eca6ae51eeaeae0701395 Mon Sep 17 00:00:00 2001 From: Lawrence Chen Date: Wed, 27 May 2026 18:15:17 -0700 Subject: [PATCH 5/5] Harden macOS runner guard diagnostics --- tests/test_ci_self_hosted_guard.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_ci_self_hosted_guard.sh b/tests/test_ci_self_hosted_guard.sh index 052469e45dcf..d1ee974f0a09 100755 --- a/tests/test_ci_self_hosted_guard.sh +++ b/tests/test_ci_self_hosted_guard.sh @@ -14,15 +14,15 @@ check_blacksmith_runner() { local file="$1" job="$2" if ! awk -v job="$job" ' $0 ~ "^ "job":" { in_job=1; next } - in_job && /^ [^[:space:]]/ { in_job=0 } + in_job && /^ [^[:space:]#][^:]*:[[:space:]]*(#.*)?$/ { in_job=0 } in_job && /runs-on:.*blacksmith-[0-9]+vcpu-macos-/ { saw=1 } in_job && /os: blacksmith-[0-9]+vcpu-macos-/ { saw=1 } END { exit !(saw) } ' "$file"; then - echo "FAIL: $job in $(basename "$file") must use a Blacksmith macOS runner" + echo "FAIL: $job in $(basename "$file") must use the expected macOS runner" exit 1 fi - echo "PASS: $job Blacksmith runner is present" + echo "PASS: $job in $(basename "$file") uses the expected macOS runner" } # ci.yml jobs