From ff8c378f719c4dde1d51f9f77110a09fe569576f Mon Sep 17 00:00:00 2001 From: Aziz Albahar Date: Tue, 7 Jul 2026 21:17:14 -0700 Subject: [PATCH 01/13] Stage macOS CI behind linux preflight --- .github/workflows/ci.yml | 691 ++++----------------- scripts/ci/run-display-ui-regressions.sh | 364 +++++++++++ scripts/select-ci-xcode.sh | 85 ++- tests/test_ci_change_areas.py | 63 +- tests/test_ci_release_sdk_lane.sh | 38 +- tests/test_ci_self_hosted_guard.sh | 63 +- tests/test_ci_xcode_selection_fast_path.sh | 103 +++ 7 files changed, 765 insertions(+), 642 deletions(-) create mode 100755 scripts/ci/run-display-ui-regressions.sh create mode 100755 tests/test_ci_xcode_selection_fast_path.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 73038ae44e69..1185f3e7ace1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -159,6 +159,9 @@ jobs: - name: Validate nightly Xcode selection run: ./tests/test_ci_nightly_xcode_selection.sh + - name: Validate CI Xcode selection fast path + run: ./tests/test_ci_xcode_selection_fast_path.sh + - name: Validate universal nightly workflow run: bash ./tests/test_nightly_universal_build.sh @@ -379,7 +382,9 @@ jobs: run: bun run test:db:behavior app-host-unit-tests: - needs: changes + needs: + - changes + - linux-preflight if: ${{ needs.changes.outputs.macos == 'true' }} name: app-host unit tests (${{ matrix.shard }}/4) # App-host XCTest needs a runner that can broker testmanagerd control @@ -393,6 +398,7 @@ jobs: matrix: shard: [1, 2, 3, 4] env: + CMUX_CI_XCODE_APP: ${{ vars.CMUX_CI_XCODE_APP_MACOS_15 }} CMUX_SKIP_ZIG_BUILD: "1" # Keep one-off focused gates on the lightest measured shard so shard 1 no # longer carries the full shard plus every extra regression guard. @@ -702,6 +708,7 @@ jobs: # that stranded the old "tests" check). Add new ci.yml test/build jobs here. needs: - changes + - linux-preflight - app-host-unit-tests - swift-package-tests - agent-session-web-resources @@ -727,6 +734,11 @@ jobs: print(f"changes: {changes['result']}", file=sys.stderr) sys.exit(1) + preflight = needs["linux-preflight"] + if preflight["result"] != "success": + print(f"linux preflight did not pass: {preflight['result']}", file=sys.stderr) + sys.exit(1) + if macos == "true" and tests["result"] != "success": print(f"app-host unit tests were required but did not pass: {tests['result']}", file=sys.stderr) sys.exit(1) @@ -746,16 +758,21 @@ jobs: sys.exit(1) print(f"changes.macos={macos}") + print(f"linux-preflight={preflight['result']}") print(f"app-host unit tests={tests['result']}") for name in ("swift-package-tests", "agent-session-web-resources"): print(f"{name}={needs[name]['result']}") PY swift-package-tests: - needs: changes + needs: + - changes + - linux-preflight if: ${{ needs.changes.outputs.macos == 'true' }} runs-on: ${{ vars.MACOS_RUNNER_15 || 'warp-macos-15-arm64-6x' }} timeout-minutes: 20 + env: + CMUX_CI_XCODE_APP: ${{ vars.CMUX_CI_XCODE_APP_MACOS_15 }} steps: - name: Clear stale git locks (self-hosted reused workspace) shell: bash @@ -928,20 +945,84 @@ jobs: bun run agent-session-web:test git diff --exit-code -- Resources/agent-session-react Resources/agent-session-solid + linux-preflight: + name: linux-preflight + needs: + - changes + - workflow-guard-tests + - remote-daemon-tests + - web-typecheck + - react-apps-check + - web-db-migrations + - agent-session-web-resources + if: ${{ always() }} + runs-on: ${{ vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }} + timeout-minutes: 5 + steps: + - name: Check cheap CI layer before macOS runners + env: + PREFLIGHT_NEEDS: ${{ toJSON(needs) }} + run: | + python3 - <<'PY' + import json + import os + import sys + + needs = json.loads(os.environ["PREFLIGHT_NEEDS"]) + required = ("changes", "workflow-guard-tests") + allowed_routed = { + "remote-daemon-tests", + "web-typecheck", + "react-apps-check", + "web-db-migrations", + "agent-session-web-resources", + } + + bad = {} + for name in required: + result = needs[name]["result"] + if result != "success": + bad[name] = result + + for name in sorted(allowed_routed): + result = needs[name]["result"] + if result not in {"success", "skipped"}: + bad[name] = result + + if bad: + for name, result in bad.items(): + print(f"{name}: {result}", file=sys.stderr) + sys.exit(1) + + outputs = needs["changes"].get("outputs", {}) + print( + "routes: " + f"macos={outputs.get('macos')} " + f"web={outputs.get('web')} " + f"go={outputs.get('go')} " + f"agent_session_web={outputs.get('agent_session_web')}" + ) + for name, data in sorted(needs.items()): + print(f"{name}: {data['result']}") + PY + tests-build-and-lag: - needs: changes + needs: + - changes + - linux-preflight if: ${{ needs.changes.outputs.macos == 'true' }} - # Build the full cmux scheme and run the lag regression on macOS CI. - # Keep lag validation separate from UI regressions so functional UI failures - # and performance regressions stay isolated. Broader interactive UI suites - # still run via test-e2e.yml on GitHub-hosted runners. + # Build the full cmux scheme once, then run the required display/runtime + # regressions from the same DerivedData instead of queuing a second display + # runner for UI-only checks. runs-on: ${{ vars.MACOS_RUNNER_DISPLAY || 'warp-macos-15-arm64-6x' }} # A cold DerivedData cache (any project.pbxproj or Package.resolved change # mints a new cache key with no restore-keys fallback) forces a full # cmux build whose Swift codegen alone can run 20+ min. Project/package # changes from the sidebar extension kit pushed this full build plus the - # CA and lag regressions beyond 35 min before the cache could repopulate. - timeout-minutes: 55 + # CA, lag, and UI regressions beyond 35 min before the cache could repopulate. + timeout-minutes: 75 + env: + CMUX_CI_XCODE_APP: ${{ vars.CMUX_CI_XCODE_APP_MACOS_15 }} steps: - name: Validate display runner identity env: @@ -1076,7 +1157,7 @@ jobs: sleep $((attempt * 5)) done - - name: Build app + - name: Build for runtime regressions run: | set -euo pipefail SOURCE_PACKAGES_DIR="$PWD/.ci-source-packages" @@ -1084,7 +1165,7 @@ jobs: -derivedDataPath "$CMUX_DERIVED_DATA_PATH" \ -clonedSourcePackagesDirPath "$SOURCE_PACKAGES_DIR" \ -disableAutomaticPackageResolution \ - -destination "platform=macOS" build 2>&1 | tee /tmp/cmux-build-output.txt + -destination "platform=macOS" build-for-testing 2>&1 | tee /tmp/cmux-build-output.txt - name: Validate Swift warning budget run: python3 scripts/swift_warning_budget.py --log /tmp/cmux-build-output.txt @@ -1246,68 +1327,13 @@ jobs: scripts/ci/virtual-display-lock.sh release || true rm -f "${VDISPLAY_HELPER_PATH:-}" "${VDISPLAY_READY:-}" "${VDISPLAY_ID_PATH:-}" "${VDISPLAY_LOG:-}" - release-ghostty-cli-helper: - needs: changes - if: ${{ needs.changes.outputs.macos == 'true' }} - runs-on: ${{ vars.MACOS_RUNNER_15 || 'warp-macos-15-arm64-6x' }} - timeout-minutes: 20 - steps: - - name: Clear stale git locks (self-hosted reused workspace) - shell: bash - run: | - # Self-hosted macOS runners reuse the workspace. A job cancelled or - # killed mid-checkout can leave a stale .git/modules/*/index.lock that - # fails every later submodule checkout (e.g. ghostty). Clear them first. - ws="${GITHUB_WORKSPACE:-$PWD}" - rm -f "$ws/.git/index.lock" 2>/dev/null || true - if [ -d "$ws/.git/modules" ]; then - find "$ws/.git/modules" -type f -name "*.lock" -delete 2>/dev/null || true - fi - - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - submodules: recursive - - - name: Install zig - run: ./scripts/install-zig-ci.sh - - - name: Cache Zig packages - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 - with: - path: ~/.cache/zig - key: zig-packages-${{ hashFiles('ghostty/build.zig.zon', 'ghostty/build.zig.zon.json') }} - restore-keys: zig-packages- - - - name: Build universal Ghostty CLI helper - run: | - set -euo pipefail - mkdir -p ghostty-cli-helper - ./scripts/build-ghostty-cli-helper.sh --universal --output ghostty-cli-helper/ghostty - lipo ghostty-cli-helper/ghostty -verify_arch arm64 x86_64 - - - name: Upload universal Ghostty CLI helper - id: upload-ghostty-cli-helper - continue-on-error: true - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: cmux-ghostty-cli-helper - path: ghostty-cli-helper/ghostty - if-no-files-found: error - - - name: Retry universal Ghostty CLI helper upload - if: steps.upload-ghostty-cli-helper.outcome == 'failure' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: cmux-ghostty-cli-helper - path: ghostty-cli-helper/ghostty - if-no-files-found: error - overwrite: true + - name: Run display UI regressions + run: scripts/ci/run-display-ui-regressions.sh release-build: needs: - changes - - release-ghostty-cli-helper + - linux-preflight if: ${{ needs.changes.outputs.macos == 'true' }} # Compile the same unsigned universal Release app that nightly builds before # signing, notarization, and publishing. This catches DEBUG/Release boundary @@ -1318,7 +1344,9 @@ jobs: # restored SwiftPM cache. Default to a clean paid macOS 26 runner instead # of the generic persistent macOS 26 pool. runs-on: ${{ vars.MACOS_RUNNER_26_RELEASE || 'blacksmith-6vcpu-macos-26' }} - timeout-minutes: 45 + timeout-minutes: 60 + env: + CMUX_CI_XCODE_APP: ${{ vars.CMUX_CI_XCODE_APP_MACOS_26 }} steps: - name: Clear stale git locks (self-hosted reused workspace) shell: bash @@ -1372,6 +1400,24 @@ jobs: run: | ./scripts/install-rust-ci.sh + - name: Install zig + run: | + ./scripts/install-zig-ci.sh + + - name: Cache Zig packages + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: ~/.cache/zig + key: zig-packages-${{ hashFiles('ghostty/build.zig.zon', 'ghostty/build.zig.zon.json') }} + restore-keys: zig-packages- + + - name: Build universal Ghostty CLI helper + run: | + set -euo pipefail + mkdir -p ghostty-cli-helper + ./scripts/build-ghostty-cli-helper.sh --universal --output ghostty-cli-helper/ghostty + lipo ghostty-cli-helper/ghostty -verify_arch arm64 x86_64 + - name: Cache DerivedData uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: @@ -1400,12 +1446,6 @@ jobs: ONLY_ACTIVE_ARCH=NO \ CODE_SIGNING_ALLOWED=NO ASSETCATALOG_COMPILER_APPICON_NAME=AppIcon-Nightly build - - name: Download universal Ghostty CLI helper - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 - with: - name: cmux-ghostty-cli-helper - path: ghostty-cli-helper - - name: Install universal Ghostty CLI helper run: | set -euo pipefail @@ -1430,502 +1470,6 @@ jobs: lipo "$HELPER_BINARY" -verify_arch arm64 x86_64 [[ "$SDK_VERSION" == 26.* ]] - ui-regressions: - needs: changes - if: ${{ needs.changes.outputs.macos == 'true' }} - runs-on: ${{ vars.MACOS_RUNNER_DISPLAY || 'warp-macos-15-arm64-6x' }} - # Cold builds after project/package changes can spend more than 25 minutes - # in build-for-testing before the UI regression script starts. - timeout-minutes: 45 - steps: - - name: Validate display runner identity - env: - REQUESTED_RUNNER: ${{ vars.MACOS_RUNNER_DISPLAY || 'warp-macos-15-arm64-6x' }} - RUNNER_CONTEXT_NAME: ${{ runner.name }} - run: | - set -euo pipefail - echo "Requested runner: $REQUESTED_RUNNER" - case "$REQUESTED_RUNNER" in - depot-*) - case "$RUNNER_CONTEXT_NAME" in - depot-*) - echo "Resolved runner matches depot-*? yes" - ;; - *) - echo "Resolved runner matches depot-*? no" - echo "::error::$REQUESTED_RUNNER resolved outside Depot. Remove $REQUESTED_RUNNER from non-Depot self-hosted runners or choose a different runner." - exit 1 - ;; - esac - ;; - *) - echo "Display runner is not Depot; skipping Depot identity guard" - ;; - esac - - - name: Clear stale git locks (self-hosted reused workspace) - shell: bash - run: | - # Self-hosted macOS runners reuse the workspace. A job cancelled or - # killed mid-checkout can leave a stale .git/modules/*/index.lock that - # fails every later submodule checkout (e.g. ghostty). Clear them first. - ws="${GITHUB_WORKSPACE:-$PWD}" - rm -f "$ws/.git/index.lock" 2>/dev/null || true - if [ -d "$ws/.git/modules" ]; then - find "$ws/.git/modules" -type f -name "*.lock" -delete 2>/dev/null || true - fi - - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - submodules: recursive - - - name: Select Xcode - run: | - set -euo pipefail - ./scripts/select-ci-xcode.sh - - - name: Prepare isolated DerivedData - run: | - set -euo pipefail - DERIVED_DATA_PATH="$RUNNER_TEMP/cmux-deriveddata-ui-regressions-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" - rm -rf "$DERIVED_DATA_PATH" - echo "CMUX_DERIVED_DATA_PATH=$DERIVED_DATA_PATH" >> "$GITHUB_ENV" - - - name: Download pre-built GhosttyKit.xcframework - run: ./scripts/download-prebuilt-ghosttykit.sh - - - name: Install zig - run: | - ./scripts/install-zig-ci.sh - - - name: Install Rust - run: | - ./scripts/install-rust-ci.sh - - - name: Cache Zig packages - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 - with: - path: ~/.cache/zig - key: zig-packages-${{ hashFiles('ghostty/build.zig.zon', 'ghostty/build.zig.zon.json') }} - restore-keys: zig-packages- - - - name: Cache Swift packages - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 - with: - path: .ci-source-packages - key: spm-ui-regressions-${{ hashFiles('cmux.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved') }} - restore-keys: spm-ui-regressions- - - - name: Sanitize Swift package cache - run: python3 scripts/ci/sanitize-xcode-source-packages-cache.py .ci-source-packages - - - name: Resolve Swift packages - run: | - set -euo pipefail - SOURCE_PACKAGES_DIR="$PWD/.ci-source-packages" - rm -rf "$CMUX_DERIVED_DATA_PATH" - mkdir -p "$SOURCE_PACKAGES_DIR" - - for attempt in 1 2 3; do - if xcodebuild -project cmux.xcodeproj -scheme cmux -configuration Debug \ - -derivedDataPath "$CMUX_DERIVED_DATA_PATH" \ - -clonedSourcePackagesDirPath "$SOURCE_PACKAGES_DIR" \ - -resolvePackageDependencies; then - # Guard against a stale/poisoned Swift-package cache: a restored - # .ci-source-packages can make resolve report success without the - # binary artifacts (Sparkle/Sentry) actually present, which then - # fails the build. Verify they materialized; if not, clear and retry. - if [ -d "$SOURCE_PACKAGES_DIR/artifacts/sparkle/Sparkle/Sparkle.xcframework" ] && [ -d "$SOURCE_PACKAGES_DIR/artifacts/sentry-cocoa/Sentry/Sentry.xcframework" ]; then - exit 0 - fi - echo "Resolve succeeded but binary artifacts are missing (stale cache); clearing and retrying" >&2 - rm -rf "$SOURCE_PACKAGES_DIR" # whole dir — resolve will not re-materialize artifacts into a partial tree - fi - if [ "$attempt" -eq 3 ]; then - echo "Failed to resolve Swift packages after 3 attempts" >&2 - exit 1 - fi - echo "Package resolution failed on attempt $attempt, retrying..." - sleep $((attempt * 5)) - done - - - name: Build for testing (display resolution) - run: | - set -euo pipefail - SOURCE_PACKAGES_DIR="$PWD/.ci-source-packages" - xcodebuild -project cmux.xcodeproj -scheme cmux -configuration Debug \ - -derivedDataPath "$CMUX_DERIVED_DATA_PATH" \ - -clonedSourcePackagesDirPath "$SOURCE_PACKAGES_DIR" \ - -disableAutomaticPackageResolution \ - -destination "platform=macOS" \ - build-for-testing - - - name: Enable XCTest automation mode - run: | - set -euo pipefail - if ! command -v automationmodetool >/dev/null 2>&1; then - echo "::warning::automationmodetool is unavailable; XCTest will use its default automation-mode setup" - exit 0 - fi - - if sudo -n true 2>/dev/null; then - sudo -n automationmodetool enable-automationmode-without-authentication - else - echo "::warning::Passwordless sudo unavailable; XCTest will use its default automation-mode setup" - fi - - - name: Run display resolution churn UI regression - run: | - set -euo pipefail - SOURCE_PACKAGES_DIR="$PWD/.ci-source-packages" - TOKEN="$(uuidgen)" - HELPER_PATH="$RUNNER_TEMP/create-virtual-display-display-churn-${TOKEN}" - DIAG_PATH="/tmp/cmux-ui-test-display-churn-${TOKEN}.json" - DISPLAY_READY="/tmp/cmux-ui-test-display-${TOKEN}.ready" - DISPLAY_ID_PATH="/tmp/cmux-ui-test-display-${TOKEN}.id" - DISPLAY_START="/tmp/cmux-ui-test-display-${TOKEN}.start" - DISPLAY_DONE="/tmp/cmux-ui-test-display-${TOKEN}.done" - HELPER_LOG="/tmp/cmux-ui-test-display-${TOKEN}-helper.log" - XCODEBUILD_LOG="/tmp/cmux-ui-test-display-${TOKEN}-xcodebuild.log" - BASELINE_READY_MARKER="CMUX_DISPLAY_CHURN_BASELINE_READY_${TOKEN}" - HELPER_PID="" - START_SIGNAL_PID="" - DISPLAY_LOCK_DIR="" - DISPLAY_LOCK_TOKEN="" - - acquire_display_lock() { - LOCK_ENV="$(scripts/ci/virtual-display-lock.sh acquire)" - eval "$LOCK_ENV" - export CMUX_VDISPLAY_LOCK_DIR CMUX_VDISPLAY_LOCK_TOKEN - DISPLAY_LOCK_DIR="$CMUX_VDISPLAY_LOCK_DIR" - DISPLAY_LOCK_TOKEN="$CMUX_VDISPLAY_LOCK_TOKEN" - } - - release_display_lock() { - if [ -n "${DISPLAY_LOCK_DIR:-}" ]; then - CMUX_VDISPLAY_LOCK_DIR="$DISPLAY_LOCK_DIR" \ - CMUX_VDISPLAY_LOCK_TOKEN="$DISPLAY_LOCK_TOKEN" \ - scripts/ci/virtual-display-lock.sh release || true - DISPLAY_LOCK_DIR="" - DISPLAY_LOCK_TOKEN="" - fi - } - - cleanup_attempt() { - if [ -n "${START_SIGNAL_PID:-}" ]; then - kill "$START_SIGNAL_PID" 2>/dev/null || true - wait "$START_SIGNAL_PID" 2>/dev/null || true - START_SIGNAL_PID="" - fi - if [ -n "${HELPER_PID:-}" ]; then - kill "$HELPER_PID" 2>/dev/null || true - wait "$HELPER_PID" 2>/dev/null || true - HELPER_PID="" - fi - release_display_lock - pkill -x "cmux DEV" 2>/dev/null || true - rm -f "$DIAG_PATH" "$DISPLAY_READY" "$DISPLAY_ID_PATH" "$DISPLAY_START" "$DISPLAY_DONE" "$HELPER_LOG" "$XCODEBUILD_LOG" - rm -f /tmp/cmux-ui-test-prelaunch.json /tmp/cmux-ui-test-display-harness.json - } - - cleanup() { - cleanup_attempt - rm -f "$HELPER_PATH" - } - trap cleanup EXIT - - # Build display helper - clang -framework Foundation -framework CoreGraphics \ - -o "$HELPER_PATH" scripts/create-virtual-display.m - - # Find the app binary - APP_BINARY=$(find "$CMUX_DERIVED_DATA_PATH" -path "*/Build/Products/Debug/cmux DEV.app/Contents/MacOS/cmux DEV" -print -quit 2>/dev/null || true) - if [ -z "$APP_BINARY" ]; then - echo "ERROR: App binary not found in DerivedData" >&2 - exit 1 - fi - echo "App binary: $APP_BINARY" - - for attempt in 1 2; do - cleanup_attempt 2>/dev/null || true - - acquire_display_lock - - # Reap any leaked display helper now that we hold the lock, so a - # CGVirtualDisplay orphaned by a crashed/cancelled job cannot block - # this create on persistent self-hosted runners. - scripts/ci/virtual-display-lock.sh reap-strays || true - - # Launch display helper from shell (non-sandboxed). The helper waits - # on a shell-owned /tmp start file; XCTest prints a baseline-ready - # marker after sampling render stats, then the shell writes startPath. - "$HELPER_PATH" \ - --modes "1920x1080,1728x1117,1600x900,1440x810" \ - --ready-path "$DISPLAY_READY" \ - --display-id-path "$DISPLAY_ID_PATH" \ - --start-path "$DISPLAY_START" \ - --done-path "$DISPLAY_DONE" \ - --iterations 40 \ - --interval-ms 40 \ - > "$HELPER_LOG" 2>&1 & - HELPER_PID=$! - scripts/ci/virtual-display-lock.sh set-owner "$HELPER_PID" - - # Wait for display ready - echo "Waiting for virtual display..." - DISPLAY_READY_OK=false - for _ in $(seq 1 100); do - if [ -s "$DISPLAY_READY" ] && [ -s "$DISPLAY_ID_PATH" ]; then - DISPLAY_READY_OK=true - break - fi - if ! kill -0 "$HELPER_PID" 2>/dev/null; then - echo "ERROR: Virtual display helper exited before readiness" >&2 - cat "$HELPER_LOG" 2>/dev/null || true - break - fi - sleep 0.1 - done - if [ "$DISPLAY_READY_OK" != "true" ]; then - echo "ERROR: Virtual display not ready after 10s" >&2 - cat "$HELPER_LOG" 2>/dev/null || true - cleanup_attempt - if [ "$attempt" -eq 2 ]; then - echo "Display resolution UI regression failed after 2 virtual display setup attempts" >&2 - exit 1 - fi - sleep 3 - continue - fi - DISPLAY_ID=$(tr -d '\n' < "$DISPLAY_ID_PATH") - echo "Virtual display ready: ID=$DISPLAY_ID" - - # Launch app from shell (non-sandboxed, outside XCTest sandbox) - CMUX_UI_TEST_MODE=1 \ - CMUX_UI_TEST_DIAGNOSTICS_PATH="$DIAG_PATH" \ - CMUX_UI_TEST_DISPLAY_RENDER_STATS=1 \ - CMUX_UI_TEST_TARGET_DISPLAY_ID="$DISPLAY_ID" \ - CMUX_TAG="ui-tests-display-resolution" \ - "$APP_BINARY" > /tmp/cmux-ui-test-app.log 2>&1 & - APP_PID=$! - echo "App launched: PID=$APP_PID" - - # Wait for app diagnostics - echo "Waiting for app diagnostics..." - APP_READY=false - for i in $(seq 1 30); do - if [ -f "$DIAG_PATH" ]; then - if python3 -c "import json; d=json.load(open('$DIAG_PATH')); assert d.get('pid')" 2>/dev/null; then - APP_READY=true - break - fi - fi - if ! kill -0 "$APP_PID" 2>/dev/null; then - echo "ERROR: App crashed during startup" - cat /tmp/cmux-ui-test-app.log 2>/dev/null | tail -30 || true - break - fi - sleep 0.5 - done - - if [ "$APP_READY" != "true" ]; then - echo "Attempt $attempt: App not ready after 15s" - pkill -x "cmux DEV" 2>/dev/null || true - kill "$HELPER_PID" 2>/dev/null || true - if [ "$attempt" -eq 2 ]; then - echo "Display resolution UI regression failed after 2 attempts" >&2 - echo "--- App log ---" - cat /tmp/cmux-ui-test-app.log 2>/dev/null | tail -50 || true - echo "--- Helper log ---" - cat "$HELPER_LOG" 2>/dev/null | tail -20 || true - echo "--- Diagnostics ---" - cat "$DIAG_PATH" 2>/dev/null || echo "(not found)" - exit 1 - fi - sleep 3 - continue - fi - - echo "App started. Diagnostics:" - cat "$DIAG_PATH" - - # Wait for render stats (terminal surface initialization) - echo "Waiting for render stats..." - RENDER_READY=false - for i in $(seq 1 40); do - if python3 -c "import json; d=json.load(open('$DIAG_PATH')); assert d.get('renderStatsAvailable') == '1'" 2>/dev/null; then - RENDER_READY=true - echo "Render stats available after $((i / 2))s" - break - fi - sleep 0.5 - done - if [ "$RENDER_READY" != "true" ]; then - echo "WARNING: Render stats not available after 20s. Diagnostics:" - cat "$DIAG_PATH" 2>/dev/null || true - echo "--- App log ---" - cat /tmp/cmux-ui-test-app.log 2>/dev/null | tail -30 || true - fi - - # Write manifests so test can find the pre-launched state - MANIFEST_PATH="/tmp/cmux-ui-test-display-harness.json" - cat >"$MANIFEST_PATH" <"$PRELAUNCH_PATH" </dev/null; then - echo "XCTest baseline marker observed; starting display churn" - printf 'start\n' > "$DISPLAY_START" - exit 0 - fi - sleep 0.25 - done - echo "ERROR: XCTest baseline marker not observed before display-churn start timeout" >&2 - echo "--- xcodebuild log tail ---" >&2 - tail -80 "$XCODEBUILD_LOG" >&2 2>/dev/null || true - echo "--- diagnostics ---" >&2 - cat "$DIAG_PATH" >&2 2>/dev/null || true - exit 1 - ) & - START_SIGNAL_PID=$! - - XCODEBUILD_OK=false - if xcodebuild -project cmux.xcodeproj -scheme cmux -configuration Debug \ - -derivedDataPath "$CMUX_DERIVED_DATA_PATH" \ - -clonedSourcePackagesDirPath "$SOURCE_PACKAGES_DIR" \ - -disableAutomaticPackageResolution \ - -destination "platform=macOS" \ - -only-testing:cmuxUITests/DisplayResolutionRegressionUITests \ - test-without-building 2>&1 | tee "$XCODEBUILD_LOG"; then - XCODEBUILD_OK=true - fi - - if [ -n "${START_SIGNAL_PID:-}" ]; then - kill "$START_SIGNAL_PID" 2>/dev/null || true - wait "$START_SIGNAL_PID" 2>/dev/null || true - START_SIGNAL_PID="" - fi - - if [ "$XCODEBUILD_OK" = "true" ]; then - cleanup_attempt - exit 0 - fi - - pkill -x "cmux DEV" 2>/dev/null || true - cleanup_attempt - - if [ "$attempt" -eq 2 ]; then - echo "Display resolution UI regression failed after 2 attempts" >&2 - exit 1 - fi - echo "Attempt $attempt failed, retrying..." - sleep 3 - done - - - name: Create persistent virtual display - run: | - set -euo pipefail - LOCK_ENV="$(scripts/ci/virtual-display-lock.sh acquire)" - eval "$LOCK_ENV" - export CMUX_VDISPLAY_LOCK_DIR CMUX_VDISPLAY_LOCK_TOKEN - { - echo "CMUX_VDISPLAY_LOCK_DIR=$CMUX_VDISPLAY_LOCK_DIR" - echo "CMUX_VDISPLAY_LOCK_TOKEN=$CMUX_VDISPLAY_LOCK_TOKEN" - } >> "$GITHUB_ENV" - - HELPER_PATH="$RUNNER_TEMP/create-virtual-display-persistent" - clang -framework Foundation -framework CoreGraphics \ - -o "$HELPER_PATH" scripts/create-virtual-display.m - - VDISPLAY_READY="$RUNNER_TEMP/cmux-vdisplay-persistent.ready" - VDISPLAY_ID_PATH="$RUNNER_TEMP/cmux-vdisplay-persistent.id" - VDISPLAY_LOG="$RUNNER_TEMP/cmux-vdisplay-persistent.log" - rm -f "$VDISPLAY_READY" "$VDISPLAY_ID_PATH" "$VDISPLAY_LOG" - - # Now that we hold the lock, reap any leaked display helper so a - # CGVirtualDisplay orphaned by a crashed/cancelled job cannot block - # this create on persistent self-hosted runners. - scripts/ci/virtual-display-lock.sh reap-strays || true - - "$HELPER_PATH" \ - --modes "1920x1080" \ - --ready-path "$VDISPLAY_READY" \ - --display-id-path "$VDISPLAY_ID_PATH" \ - >"$VDISPLAY_LOG" 2>&1 & - VDISPLAY_PERSISTENT_PID=$! - scripts/ci/virtual-display-lock.sh set-owner "$VDISPLAY_PERSISTENT_PID" - - { - echo "VDISPLAY_PERSISTENT_PID=$VDISPLAY_PERSISTENT_PID" - echo "VDISPLAY_PERSISTENT_HELPER_PATH=$HELPER_PATH" - echo "VDISPLAY_PERSISTENT_READY=$VDISPLAY_READY" - echo "VDISPLAY_PERSISTENT_ID_PATH=$VDISPLAY_ID_PATH" - echo "VDISPLAY_PERSISTENT_LOG=$VDISPLAY_LOG" - } >> "$GITHUB_ENV" - - echo "Waiting for persistent virtual display..." - for _ in $(seq 1 100); do - if [ -s "$VDISPLAY_READY" ] && [ -s "$VDISPLAY_ID_PATH" ]; then - break - fi - if ! kill -0 "$VDISPLAY_PERSISTENT_PID" 2>/dev/null; then - echo "Persistent virtual display helper exited before readiness" >&2 - cat "$VDISPLAY_LOG" >&2 || true - exit 1 - fi - sleep 0.1 - done - - if [ ! -s "$VDISPLAY_READY" ] || [ ! -s "$VDISPLAY_ID_PATH" ]; then - echo "ERROR: Persistent virtual display not ready after 10s" >&2 - cat "$VDISPLAY_LOG" >&2 || true - exit 1 - fi - - echo "Persistent virtual display ready: ID=$(tr -d '\n' < "$VDISPLAY_ID_PATH")" - cat "$VDISPLAY_LOG" - - - name: Run browser find focus UI regression - run: | - set -euo pipefail - SOURCE_PACKAGES_DIR="$PWD/.ci-source-packages" - if [ -n "${VDISPLAY_PERSISTENT_PID:-}" ] && ! kill -0 "$VDISPLAY_PERSISTENT_PID" 2>/dev/null; then - echo "Persistent virtual display exited before browser find UI regression" >&2 - cat "${VDISPLAY_PERSISTENT_LOG:-/dev/null}" >&2 || true - exit 1 - fi - - xcodebuild -project cmux.xcodeproj -scheme cmux -configuration Debug \ - -derivedDataPath "$CMUX_DERIVED_DATA_PATH" \ - -clonedSourcePackagesDirPath "$SOURCE_PACKAGES_DIR" \ - -disableAutomaticPackageResolution \ - -destination "platform=macOS" \ - -maximum-test-execution-time-allowance 180 \ - -only-testing:cmuxUITests/BrowserPaneNavigationKeybindUITests/testCmdFOpensBrowserFindAfterCmdDCmdLNavigation \ - test-without-building - - - name: Cleanup persistent virtual display - if: always() - run: | - if [ -n "${VDISPLAY_PERSISTENT_PID:-}" ]; then - kill "$VDISPLAY_PERSISTENT_PID" >/dev/null 2>&1 || true - for _ in $(seq 1 50); do - kill -0 "$VDISPLAY_PERSISTENT_PID" >/dev/null 2>&1 || break - sleep 0.1 - done - fi - scripts/ci/virtual-display-lock.sh release || true - rm -f "${VDISPLAY_PERSISTENT_HELPER_PATH:-}" "${VDISPLAY_PERSISTENT_READY:-}" "${VDISPLAY_PERSISTENT_ID_PATH:-}" "${VDISPLAY_PERSISTENT_LOG:-}" - ci-status: needs: - changes @@ -1934,14 +1478,13 @@ jobs: - web-typecheck - react-apps-check - web-db-migrations + - linux-preflight - app-host-unit-tests - tests - swift-package-tests - agent-session-web-resources - tests-build-and-lag - - release-ghostty-cli-helper - release-build - - ui-regressions if: ${{ always() }} runs-on: ${{ vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }} steps: diff --git a/scripts/ci/run-display-ui-regressions.sh b/scripts/ci/run-display-ui-regressions.sh new file mode 100755 index 000000000000..3f4135ac0a54 --- /dev/null +++ b/scripts/ci/run-display-ui-regressions.sh @@ -0,0 +1,364 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${CMUX_DERIVED_DATA_PATH:?CMUX_DERIVED_DATA_PATH is required}" +SOURCE_PACKAGES_DIR="${CMUX_SOURCE_PACKAGES_DIR:-$PWD/.ci-source-packages}" + +DRC_HELPER_PATH="" +DRC_DIAG_PATH="" +DRC_DISPLAY_READY="" +DRC_DISPLAY_ID_PATH="" +DRC_DISPLAY_START="" +DRC_DISPLAY_DONE="" +DRC_HELPER_LOG="" +DRC_XCODEBUILD_LOG="" +DRC_HELPER_PID="" +DRC_START_SIGNAL_PID="" +DRC_DISPLAY_LOCK_DIR="" +DRC_DISPLAY_LOCK_TOKEN="" + +PERSISTENT_HELPER_PATH="" +PERSISTENT_READY="" +PERSISTENT_ID_PATH="" +PERSISTENT_LOG="" +PERSISTENT_PID="" +PERSISTENT_LOCK_DIR="" +PERSISTENT_LOCK_TOKEN="" + +release_lock() { + local lock_dir="$1" + local lock_token="$2" + if [ -n "$lock_dir" ]; then + CMUX_VDISPLAY_LOCK_DIR="$lock_dir" \ + CMUX_VDISPLAY_LOCK_TOKEN="$lock_token" \ + scripts/ci/virtual-display-lock.sh release || true + fi +} + +cleanup_display_churn() { + if [ -n "${DRC_START_SIGNAL_PID:-}" ]; then + kill "$DRC_START_SIGNAL_PID" 2>/dev/null || true + wait "$DRC_START_SIGNAL_PID" 2>/dev/null || true + DRC_START_SIGNAL_PID="" + fi + if [ -n "${DRC_HELPER_PID:-}" ]; then + kill "$DRC_HELPER_PID" 2>/dev/null || true + wait "$DRC_HELPER_PID" 2>/dev/null || true + DRC_HELPER_PID="" + fi + release_lock "$DRC_DISPLAY_LOCK_DIR" "$DRC_DISPLAY_LOCK_TOKEN" + DRC_DISPLAY_LOCK_DIR="" + DRC_DISPLAY_LOCK_TOKEN="" + pkill -x "cmux DEV" 2>/dev/null || true + rm -f "$DRC_DIAG_PATH" "$DRC_DISPLAY_READY" "$DRC_DISPLAY_ID_PATH" "$DRC_DISPLAY_START" "$DRC_DISPLAY_DONE" "$DRC_HELPER_LOG" "$DRC_XCODEBUILD_LOG" + rm -f /tmp/cmux-ui-test-prelaunch.json /tmp/cmux-ui-test-display-harness.json +} + +cleanup_persistent_display() { + if [ -n "${PERSISTENT_PID:-}" ]; then + kill "$PERSISTENT_PID" >/dev/null 2>&1 || true + for _ in $(seq 1 50); do + kill -0 "$PERSISTENT_PID" >/dev/null 2>&1 || break + sleep 0.1 + done + PERSISTENT_PID="" + fi + release_lock "$PERSISTENT_LOCK_DIR" "$PERSISTENT_LOCK_TOKEN" + PERSISTENT_LOCK_DIR="" + PERSISTENT_LOCK_TOKEN="" + rm -f "$PERSISTENT_HELPER_PATH" "$PERSISTENT_READY" "$PERSISTENT_ID_PATH" "$PERSISTENT_LOG" +} + +cleanup_all() { + cleanup_display_churn + cleanup_persistent_display +} +trap cleanup_all EXIT + +enable_xctest_automation_mode() { + if ! command -v automationmodetool >/dev/null 2>&1; then + echo "::warning::automationmodetool is unavailable; XCTest will use its default automation-mode setup" + return 0 + fi + + if sudo -n true 2>/dev/null; then + sudo -n automationmodetool enable-automationmode-without-authentication + else + echo "::warning::Passwordless sudo unavailable; XCTest will use its default automation-mode setup" + fi +} + +find_app_binary() { + find "$CMUX_DERIVED_DATA_PATH" -path "*/Build/Products/Debug/cmux DEV.app/Contents/MacOS/cmux DEV" -print -quit 2>/dev/null || true +} + +run_display_resolution_churn() { + local token app_binary display_id app_pid app_ready render_ready xcodebuild_ok + token="$(uuidgen)" + DRC_HELPER_PATH="$RUNNER_TEMP/create-virtual-display-display-churn-${token}" + DRC_DIAG_PATH="/tmp/cmux-ui-test-display-churn-${token}.json" + DRC_DISPLAY_READY="/tmp/cmux-ui-test-display-${token}.ready" + DRC_DISPLAY_ID_PATH="/tmp/cmux-ui-test-display-${token}.id" + DRC_DISPLAY_START="/tmp/cmux-ui-test-display-${token}.start" + DRC_DISPLAY_DONE="/tmp/cmux-ui-test-display-${token}.done" + DRC_HELPER_LOG="/tmp/cmux-ui-test-display-${token}-helper.log" + DRC_XCODEBUILD_LOG="/tmp/cmux-ui-test-display-${token}-xcodebuild.log" + local baseline_ready_marker="CMUX_DISPLAY_CHURN_BASELINE_READY_${token}" + + clang -framework Foundation -framework CoreGraphics \ + -o "$DRC_HELPER_PATH" scripts/create-virtual-display.m + + app_binary="$(find_app_binary)" + if [ -z "$app_binary" ]; then + echo "ERROR: App binary not found in DerivedData" >&2 + exit 1 + fi + echo "App binary: $app_binary" + + for attempt in 1 2; do + cleanup_display_churn 2>/dev/null || true + + local lock_env + lock_env="$(scripts/ci/virtual-display-lock.sh acquire)" + eval "$lock_env" + export CMUX_VDISPLAY_LOCK_DIR CMUX_VDISPLAY_LOCK_TOKEN + DRC_DISPLAY_LOCK_DIR="$CMUX_VDISPLAY_LOCK_DIR" + DRC_DISPLAY_LOCK_TOKEN="$CMUX_VDISPLAY_LOCK_TOKEN" + + scripts/ci/virtual-display-lock.sh reap-strays || true + + "$DRC_HELPER_PATH" \ + --modes "1920x1080,1728x1117,1600x900,1440x810" \ + --ready-path "$DRC_DISPLAY_READY" \ + --display-id-path "$DRC_DISPLAY_ID_PATH" \ + --start-path "$DRC_DISPLAY_START" \ + --done-path "$DRC_DISPLAY_DONE" \ + --iterations 40 \ + --interval-ms 40 \ + > "$DRC_HELPER_LOG" 2>&1 & + DRC_HELPER_PID=$! + scripts/ci/virtual-display-lock.sh set-owner "$DRC_HELPER_PID" + + echo "Waiting for virtual display..." + local display_ready_ok=false + for _ in $(seq 1 100); do + if [ -s "$DRC_DISPLAY_READY" ] && [ -s "$DRC_DISPLAY_ID_PATH" ]; then + display_ready_ok=true + break + fi + if ! kill -0 "$DRC_HELPER_PID" 2>/dev/null; then + echo "ERROR: Virtual display helper exited before readiness" >&2 + cat "$DRC_HELPER_LOG" 2>/dev/null || true + break + fi + sleep 0.1 + done + if [ "$display_ready_ok" != "true" ]; then + echo "ERROR: Virtual display not ready after 10s" >&2 + cat "$DRC_HELPER_LOG" 2>/dev/null || true + cleanup_display_churn + if [ "$attempt" -eq 2 ]; then + echo "Display resolution UI regression failed after 2 virtual display setup attempts" >&2 + exit 1 + fi + sleep 3 + continue + fi + + display_id="$(tr -d '\n' < "$DRC_DISPLAY_ID_PATH")" + echo "Virtual display ready: ID=$display_id" + + CMUX_UI_TEST_MODE=1 \ + CMUX_UI_TEST_DIAGNOSTICS_PATH="$DRC_DIAG_PATH" \ + CMUX_UI_TEST_DISPLAY_RENDER_STATS=1 \ + CMUX_UI_TEST_TARGET_DISPLAY_ID="$display_id" \ + CMUX_TAG="ui-tests-display-resolution" \ + "$app_binary" > /tmp/cmux-ui-test-app.log 2>&1 & + app_pid=$! + echo "App launched: PID=$app_pid" + + echo "Waiting for app diagnostics..." + app_ready=false + for _ in $(seq 1 30); do + if [ -f "$DRC_DIAG_PATH" ]; then + if python3 -c "import json; d=json.load(open('$DRC_DIAG_PATH')); assert d.get('pid')" 2>/dev/null; then + app_ready=true + break + fi + fi + if ! kill -0 "$app_pid" 2>/dev/null; then + echo "ERROR: App crashed during startup" + cat /tmp/cmux-ui-test-app.log 2>/dev/null | tail -30 || true + break + fi + sleep 0.5 + done + + if [ "$app_ready" != "true" ]; then + echo "Attempt $attempt: App not ready after 15s" + pkill -x "cmux DEV" 2>/dev/null || true + kill "$DRC_HELPER_PID" 2>/dev/null || true + if [ "$attempt" -eq 2 ]; then + echo "Display resolution UI regression failed after 2 attempts" >&2 + echo "--- App log ---" + cat /tmp/cmux-ui-test-app.log 2>/dev/null | tail -50 || true + echo "--- Helper log ---" + cat "$DRC_HELPER_LOG" 2>/dev/null | tail -20 || true + echo "--- Diagnostics ---" + cat "$DRC_DIAG_PATH" 2>/dev/null || echo "(not found)" + exit 1 + fi + sleep 3 + continue + fi + + echo "App started. Diagnostics:" + cat "$DRC_DIAG_PATH" + + echo "Waiting for render stats..." + render_ready=false + for i in $(seq 1 40); do + if python3 -c "import json; d=json.load(open('$DRC_DIAG_PATH')); assert d.get('renderStatsAvailable') == '1'" 2>/dev/null; then + render_ready=true + echo "Render stats available after $((i / 2))s" + break + fi + sleep 0.5 + done + if [ "$render_ready" != "true" ]; then + echo "WARNING: Render stats not available after 20s. Diagnostics:" + cat "$DRC_DIAG_PATH" 2>/dev/null || true + echo "--- App log ---" + cat /tmp/cmux-ui-test-app.log 2>/dev/null | tail -30 || true + fi + + cat >"/tmp/cmux-ui-test-display-harness.json" <"/tmp/cmux-ui-test-prelaunch.json" </dev/null; then + echo "XCTest baseline marker observed; starting display churn" + printf 'start\n' > "$DRC_DISPLAY_START" + exit 0 + fi + sleep 0.25 + done + echo "ERROR: XCTest baseline marker not observed before display-churn start timeout" >&2 + echo "--- xcodebuild log tail ---" >&2 + tail -80 "$DRC_XCODEBUILD_LOG" >&2 2>/dev/null || true + echo "--- diagnostics ---" >&2 + cat "$DRC_DIAG_PATH" >&2 2>/dev/null || true + exit 1 + ) & + DRC_START_SIGNAL_PID=$! + + xcodebuild_ok=false + if xcodebuild -project cmux.xcodeproj -scheme cmux -configuration Debug \ + -derivedDataPath "$CMUX_DERIVED_DATA_PATH" \ + -clonedSourcePackagesDirPath "$SOURCE_PACKAGES_DIR" \ + -disableAutomaticPackageResolution \ + -destination "platform=macOS" \ + -only-testing:cmuxUITests/DisplayResolutionRegressionUITests \ + test-without-building 2>&1 | tee "$DRC_XCODEBUILD_LOG"; then + xcodebuild_ok=true + fi + + if [ -n "${DRC_START_SIGNAL_PID:-}" ]; then + kill "$DRC_START_SIGNAL_PID" 2>/dev/null || true + wait "$DRC_START_SIGNAL_PID" 2>/dev/null || true + DRC_START_SIGNAL_PID="" + fi + + if [ "$xcodebuild_ok" = "true" ]; then + cleanup_display_churn + return 0 + fi + + cleanup_display_churn + + if [ "$attempt" -eq 2 ]; then + echo "Display resolution UI regression failed after 2 attempts" >&2 + exit 1 + fi + echo "Attempt $attempt failed, retrying..." + sleep 3 + done +} + +create_persistent_display() { + local lock_env + lock_env="$(scripts/ci/virtual-display-lock.sh acquire)" + eval "$lock_env" + export CMUX_VDISPLAY_LOCK_DIR CMUX_VDISPLAY_LOCK_TOKEN + PERSISTENT_LOCK_DIR="$CMUX_VDISPLAY_LOCK_DIR" + PERSISTENT_LOCK_TOKEN="$CMUX_VDISPLAY_LOCK_TOKEN" + + PERSISTENT_HELPER_PATH="$RUNNER_TEMP/create-virtual-display-persistent" + clang -framework Foundation -framework CoreGraphics \ + -o "$PERSISTENT_HELPER_PATH" scripts/create-virtual-display.m + + PERSISTENT_READY="$RUNNER_TEMP/cmux-vdisplay-persistent.ready" + PERSISTENT_ID_PATH="$RUNNER_TEMP/cmux-vdisplay-persistent.id" + PERSISTENT_LOG="$RUNNER_TEMP/cmux-vdisplay-persistent.log" + rm -f "$PERSISTENT_READY" "$PERSISTENT_ID_PATH" "$PERSISTENT_LOG" + + scripts/ci/virtual-display-lock.sh reap-strays || true + + "$PERSISTENT_HELPER_PATH" \ + --modes "1920x1080" \ + --ready-path "$PERSISTENT_READY" \ + --display-id-path "$PERSISTENT_ID_PATH" \ + >"$PERSISTENT_LOG" 2>&1 & + PERSISTENT_PID=$! + scripts/ci/virtual-display-lock.sh set-owner "$PERSISTENT_PID" + + echo "Waiting for persistent virtual display..." + for _ in $(seq 1 100); do + if [ -s "$PERSISTENT_READY" ] && [ -s "$PERSISTENT_ID_PATH" ]; then + break + fi + if ! kill -0 "$PERSISTENT_PID" 2>/dev/null; then + echo "Persistent virtual display helper exited before readiness" >&2 + cat "$PERSISTENT_LOG" >&2 || true + exit 1 + fi + sleep 0.1 + done + + if [ ! -s "$PERSISTENT_READY" ] || [ ! -s "$PERSISTENT_ID_PATH" ]; then + echo "ERROR: Persistent virtual display not ready after 10s" >&2 + cat "$PERSISTENT_LOG" >&2 || true + exit 1 + fi + + echo "Persistent virtual display ready: ID=$(tr -d '\n' < "$PERSISTENT_ID_PATH")" + cat "$PERSISTENT_LOG" +} + +run_browser_find_focus() { + if [ -n "${PERSISTENT_PID:-}" ] && ! kill -0 "$PERSISTENT_PID" 2>/dev/null; then + echo "Persistent virtual display exited before browser find UI regression" >&2 + cat "${PERSISTENT_LOG:-/dev/null}" >&2 || true + exit 1 + fi + + xcodebuild -project cmux.xcodeproj -scheme cmux -configuration Debug \ + -derivedDataPath "$CMUX_DERIVED_DATA_PATH" \ + -clonedSourcePackagesDirPath "$SOURCE_PACKAGES_DIR" \ + -disableAutomaticPackageResolution \ + -destination "platform=macOS" \ + -maximum-test-execution-time-allowance 180 \ + -only-testing:cmuxUITests/BrowserPaneNavigationKeybindUITests/testCmdFOpensBrowserFindAfterCmdDCmdLNavigation \ + test-without-building +} + +enable_xctest_automation_mode +run_display_resolution_churn +create_persistent_display +run_browser_find_focus diff --git a/scripts/select-ci-xcode.sh b/scripts/select-ci-xcode.sh index d82b5f2c9729..80475a2cf727 100755 --- a/scripts/select-ci-xcode.sh +++ b/scripts/select-ci-xcode.sh @@ -16,6 +16,60 @@ set -euo pipefail APPLICATIONS_DIR="${CMUX_XCODE_APPLICATIONS_DIR:-/Applications}" +select_developer_dir() { + local selected_dir="$1" sdk_version="$2" label="$3" + + echo "$label (DEVELOPER_DIR): $selected_dir (macOS SDK $sdk_version)" + if [ -n "${GITHUB_ENV:-}" ]; then + echo "DEVELOPER_DIR=$selected_dir" >> "$GITHUB_ENV" + fi + export DEVELOPER_DIR="$selected_dir" + + # Also point the *system* xcode-select default at the selected toolchain. Tools + # that ignore DEVELOPER_DIR resolve `xcodebuild` via the xcode-select default, + # notably Apple's `/usr/bin/git` shim (`xcodebuild -find git`). The xctest host + # spawns git subprocesses that do NOT inherit our DEVELOPER_DIR, so on runner VMs + # whose default is the old Xcode symlink, `git` runs the old `xcodebuild`, which + # dlopen()s a libxcodebuildLoader ABI-incompatible with the newer-Xcode-built test + # host and crashes ("Symbol not found"), failing git-shell-out tests + # (e.g. ExtensionWorktreePrototypeTests) before they can assert - nondeterministic + # per which VM a shard lands on. Aligning the default removes that divergence. + # Best-effort: never hard-fail a runner that disallows the switch. + if xcode-select -s "$selected_dir" 2>/dev/null; then + echo "xcode-select default -> $selected_dir" + elif command -v sudo >/dev/null 2>&1 && sudo -n xcode-select -s "$selected_dir" 2>/dev/null; then + echo "xcode-select default (via sudo) -> $selected_dir" + else + echo "WARN: could not switch xcode-select default to $selected_dir (continuing; DEVELOPER_DIR is still set for steps that honor it)" >&2 + fi + + xcodebuild -version + # Diagnostic: resolve the SDK with DEVELOPER_DIR set in-process. The workflow + # step that calls this script gets DEVELOPER_DIR only via GITHUB_ENV, which + # applies to *later* steps, not the current shell, so a bare `xcrun` on the + # next line of the same step would still resolve the old xcode-select default. + xcrun --sdk macosx --show-sdk-path +} + +PINNED_DEVELOPER_DIR="${CMUX_CI_DEVELOPER_DIR:-}" +if [ -z "$PINNED_DEVELOPER_DIR" ] && [ -n "${CMUX_CI_XCODE_APP:-}" ]; then + PINNED_DEVELOPER_DIR="${CMUX_CI_XCODE_APP%/}/Contents/Developer" +fi + +if [ -n "$PINNED_DEVELOPER_DIR" ]; then + if [ ! -d "$PINNED_DEVELOPER_DIR" ]; then + echo "Pinned Xcode developer dir does not exist: $PINNED_DEVELOPER_DIR" >&2 + exit 1 + fi + PINNED_SDK_VER="$(DEVELOPER_DIR="$PINNED_DEVELOPER_DIR" xcrun --sdk macosx --show-sdk-version 2>/dev/null || true)" + if [ -z "$PINNED_SDK_VER" ]; then + echo "Pinned Xcode developer dir has no usable macOS SDK: $PINNED_DEVELOPER_DIR" >&2 + exit 1 + fi + select_developer_dir "$PINNED_DEVELOPER_DIR" "$PINNED_SDK_VER" "Selected pinned Xcode" + exit 0 +fi + # Rank by macOS SDK as maj*1000+min so 26.2 (26002) outranks 15.5 (15005). sdk_rank() { local v="$1" maj min @@ -80,33 +134,4 @@ if [ -z "$BEST_DIR" ]; then exit 1 fi -echo "Selected Xcode (DEVELOPER_DIR): $BEST_DIR (macOS SDK $BEST_VER)" -if [ -n "${GITHUB_ENV:-}" ]; then - echo "DEVELOPER_DIR=$BEST_DIR" >> "$GITHUB_ENV" -fi -export DEVELOPER_DIR="$BEST_DIR" - -# Also point the *system* xcode-select default at the selected toolchain. Tools -# that ignore DEVELOPER_DIR resolve `xcodebuild` via the xcode-select default, -# notably Apple's `/usr/bin/git` shim (`xcodebuild -find git`). The xctest host -# spawns git subprocesses that do NOT inherit our DEVELOPER_DIR, so on runner VMs -# whose default is the old Xcode symlink, `git` runs the old `xcodebuild`, which -# dlopen()s a libxcodebuildLoader ABI-incompatible with the newer-Xcode-built test -# host and crashes ("Symbol not found"), failing git-shell-out tests -# (e.g. ExtensionWorktreePrototypeTests) before they can assert — nondeterministic -# per which VM a shard lands on. Aligning the default removes that divergence. -# Best-effort: never hard-fail a runner that disallows the switch. -if xcode-select -s "$BEST_DIR" 2>/dev/null; then - echo "xcode-select default -> $BEST_DIR" -elif command -v sudo >/dev/null 2>&1 && sudo -n xcode-select -s "$BEST_DIR" 2>/dev/null; then - echo "xcode-select default (via sudo) -> $BEST_DIR" -else - echo "WARN: could not switch xcode-select default to $BEST_DIR (continuing; DEVELOPER_DIR is still set for steps that honor it)" >&2 -fi - -xcodebuild -version -# Diagnostic: resolve the SDK with DEVELOPER_DIR set in-process. The workflow -# step that calls this script gets DEVELOPER_DIR only via GITHUB_ENV, which -# applies to *later* steps, not the current shell, so a bare `xcrun` on the -# next line of the same step would still resolve the old xcode-select default. -xcrun --sdk macosx --show-sdk-path +select_developer_dir "$BEST_DIR" "$BEST_VER" "Selected Xcode" diff --git a/tests/test_ci_change_areas.py b/tests/test_ci_change_areas.py index e03934e3fd01..ceea56009e48 100644 --- a/tests/test_ci_change_areas.py +++ b/tests/test_ci_change_areas.py @@ -393,12 +393,11 @@ def test_ci_status_job_accepts_skipped_routed_jobs() -> None: "web-typecheck", "react-apps-check", "web-db-migrations", + "linux-preflight", "app-host-unit-tests", "tests", "tests-build-and-lag", - "release-ghostty-cli-helper", "release-build", - "ui-regressions", ]: assert f" - {job_name}" in block @@ -411,12 +410,72 @@ def test_required_tests_status_waits_for_app_host_matrix() -> None: assert "name: tests" in block assert " - changes" in block + assert " - linux-preflight" in block assert " - app-host-unit-tests" in block assert "if: ${{ always() }}" in block + assert 'preflight["result"] != "success"' in block assert 'macos == "true" and tests["result"] != "success"' in block assert 'tests["result"] not in {"success", "skipped"}' in block +def test_macos_jobs_wait_for_linux_preflight() -> None: + for job_name in [ + "app-host-unit-tests", + "swift-package-tests", + "tests-build-and-lag", + "release-build", + ]: + block = workflow_job_block(job_name) + assert " - changes" in block + assert " - linux-preflight" in block + assert "if: ${{ needs.changes.outputs.macos == 'true' }}" in block + + +def test_linux_preflight_blocks_macos_on_cheap_layer_failure() -> None: + block = workflow_job_block("linux-preflight") + + assert "name: linux-preflight" in block + assert " - changes" in block + assert " - workflow-guard-tests" in block + assert " - remote-daemon-tests" in block + assert " - web-typecheck" in block + assert " - react-apps-check" in block + assert " - web-db-migrations" in block + assert " - agent-session-web-resources" in block + assert "if: ${{ always() }}" in block + assert 'required = ("changes", "workflow-guard-tests")' in block + assert 'allowed_routed = {' in block + assert 'result not in {"success", "skipped"}' in block + + +def test_macos_jobs_use_lane_specific_xcode_pin_vars() -> None: + for job_name in [ + "app-host-unit-tests", + "swift-package-tests", + "tests-build-and-lag", + ]: + block = workflow_job_block(job_name) + assert "CMUX_CI_XCODE_APP: ${{ vars.CMUX_CI_XCODE_APP_MACOS_15 }}" in block + + release_block = workflow_job_block("release-build") + assert "CMUX_CI_XCODE_APP: ${{ vars.CMUX_CI_XCODE_APP_MACOS_26 }}" in release_block + + +def test_required_macos_topology_collapses_display_and_release_helper_jobs() -> None: + workflow = CI_WORKFLOW.read_text(encoding="utf-8") + runtime_block = workflow_job_block("tests-build-and-lag") + release_block = workflow_job_block("release-build") + + assert "\n ui-regressions:" not in workflow + assert "\n release-ghostty-cli-helper:" not in workflow + assert "build-for-testing" in runtime_block + assert "Run display UI regressions" in runtime_block + assert "scripts/ci/run-display-ui-regressions.sh" in runtime_block + assert "Build universal Ghostty CLI helper" in release_block + assert "./scripts/build-ghostty-cli-helper.sh --universal --output ghostty-cli-helper/ghostty" in release_block + assert "actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131" not in release_block + + def test_agent_session_web_resources_runs_only_for_agent_session_web_area() -> None: block = workflow_job_block("agent-session-web-resources") diff --git a/tests/test_ci_release_sdk_lane.sh b/tests/test_ci_release_sdk_lane.sh index c6a05fd8d53d..5e4677619e24 100755 --- a/tests/test_ci_release_sdk_lane.sh +++ b/tests/test_ci_release_sdk_lane.sh @@ -7,8 +7,8 @@ RELEASE_FILE="$ROOT_DIR/.github/workflows/release.yml" # nightly.yml is intentionally not covered here. It has its own helper-build # model and guards via test_ci_nightly_xcode_selection.sh plus -# test_nightly_universal_build.sh. This lane guards the release/CI -# artifact-download model. +# test_nightly_universal_build.sh. This lane guards the release artifact-download +# model and the CI inline-helper release-build model. job_section() { local file="$1" job="$2" @@ -41,12 +41,6 @@ require_job_contains \ 'runs-on: ${{ vars.MACOS_RUNNER_26 || '\''blacksmith-6vcpu-macos-26'\'' }}' \ "release must sign+notarize on the macOS 26 runner variable after importing the Developer ID intermediate chain" -require_job_contains \ - "$CI_FILE" \ - "release-ghostty-cli-helper" \ - 'runs-on: ${{ vars.MACOS_RUNNER_15 || '\''warp-macos-15-arm64-6x'\'' }}' \ - "CI must build the real Ghostty CLI helper on macOS 15" - require_job_contains \ "$CI_FILE" \ "release-build" \ @@ -59,11 +53,6 @@ for workflow in "$CI_FILE" "$RELEASE_FILE"; do exit 1 fi - if ! grep -Fq "actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0" "$workflow"; then - echo "FAIL: $(basename "$workflow") must download the macOS 15-built helper artifact" >&2 - exit 1 - fi - if ! grep -Fq "./scripts/install-prebuilt-ghostty-cli-helper.sh" "$workflow"; then echo "FAIL: $(basename "$workflow") must install the prebuilt Ghostty CLI helper into the app" >&2 exit 1 @@ -75,4 +64,25 @@ for workflow in "$CI_FILE" "$RELEASE_FILE"; do fi done -echo "PASS: release and CI app builds use macOS 26 SDK with a macOS 15-built Ghostty CLI helper" +if ! grep -Fq "actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0" "$RELEASE_FILE"; then + echo "FAIL: release.yml must download the macOS 15-built helper artifact" >&2 + exit 1 +fi + +release_build_section="$(job_section "$CI_FILE" "release-build")" +if [[ "$release_build_section" != *"./scripts/build-ghostty-cli-helper.sh --universal --output ghostty-cli-helper/ghostty"* ]]; then + echo "FAIL: CI release-build must build the universal Ghostty CLI helper inline" >&2 + exit 1 +fi + +if [[ "$release_build_section" == *"actions/download-artifact@"* ]]; then + echo "FAIL: CI release-build must not wait on a separate Ghostty helper artifact job" >&2 + exit 1 +fi + +if grep -Fq "release-ghostty-cli-helper:" "$CI_FILE"; then + echo "FAIL: CI must not define a separate release-ghostty-cli-helper job" >&2 + exit 1 +fi + +echo "PASS: release uses artifact helper handoff; CI release-build builds the helper inline on the macOS 26 lane" diff --git a/tests/test_ci_self_hosted_guard.sh b/tests/test_ci_self_hosted_guard.sh index 63e9b02786cc..1bb13386a867 100755 --- a/tests/test_ci_self_hosted_guard.sh +++ b/tests/test_ci_self_hosted_guard.sh @@ -207,34 +207,55 @@ check_release_build_disk_cleanup() { echo "PASS: release-build reclaims runner disk before large cache restores" } -check_release_helper_upload_retry() { +check_release_helper_built_inline() { if ! awk ' - /^ release-ghostty-cli-helper:/ { in_job=1; next } + /^ release-build:/ { in_job=1; next } in_job && /^ [^[:space:]#][^:]*:[[:space:]]*(#.*)?$/ { in_job=0 } - in_job && /- name: Upload universal Ghostty CLI helper/ { in_upload=1; next } - in_upload && /^[[:space:]]*- name:/ { in_upload=0 } - in_upload && /id:[[:space:]]*upload-ghostty-cli-helper/ { upload_id=1 } - in_upload && /continue-on-error:[[:space:]]*true/ { upload_continue=1 } - in_upload && /uses: actions\/upload-artifact@/ { upload_action=1 } - in_upload && /if-no-files-found:[[:space:]]*error/ { upload_required=1 } - - in_job && /- name: Retry universal Ghostty CLI helper upload/ { in_retry=1; retry_step=1; next } - in_retry && /^[[:space:]]*- name:/ { in_retry=0 } - in_retry && index($0, "steps.upload-ghostty-cli-helper.outcome == '\''failure'\''") { retry_if=1 } - in_retry && /uses: actions\/upload-artifact@/ { retry_action=1 } - in_retry && /if-no-files-found:[[:space:]]*error/ { retry_required=1 } - in_retry && /overwrite:[[:space:]]*true/ { retry_overwrite=1 } + in_job && /- name: Build universal Ghostty CLI helper/ { saw_build_step=1; next } + in_job && /\.\/scripts\/build-ghostty-cli-helper\.sh --universal --output ghostty-cli-helper\/ghostty/ { saw_build=1 } + in_job && /lipo ghostty-cli-helper\/ghostty -verify_arch arm64 x86_64/ { saw_lipo=1 } + in_job && /- name: Install universal Ghostty CLI helper/ { saw_install_step=1; next } + in_job && /\.\/scripts\/install-prebuilt-ghostty-cli-helper\.sh/ { saw_install=1 } + in_job && /actions\/download-artifact@/ { saw_download=1 } END { - exit !(upload_id && upload_continue && upload_action && upload_required && retry_step && retry_if && retry_action && retry_required && retry_overwrite) + exit !(saw_build_step && saw_build && saw_lipo && saw_install_step && saw_install && !saw_download) } ' "$CI_FILE"; then - echo "FAIL: release-ghostty-cli-helper must retry required Ghostty helper artifact uploads instead of failing on a single transient upload error" + echo "FAIL: release-build must build and install the universal Ghostty helper inline without a separate CI artifact job" + exit 1 + fi + + if grep -Fq "release-ghostty-cli-helper:" "$CI_FILE"; then + echo "FAIL: CI must not queue a separate release-ghostty-cli-helper job" + exit 1 + fi + + echo "PASS: release-build builds and installs the universal Ghostty helper inline" +} + +check_runtime_regressions_collapsed() { + if grep -Fq "ui-regressions:" "$CI_FILE"; then + echo "FAIL: CI must not queue a separate ui-regressions job" + exit 1 + fi + + if ! awk ' + /^ tests-build-and-lag:/ { in_job=1; next } + in_job && /^ [^[:space:]#][^:]*:[[:space:]]*(#.*)?$/ { in_job=0 } + + in_job && /build-for-testing/ { saw_build_for_testing=1 } + in_job && /scripts\/ci\/run-display-ui-regressions\.sh/ { saw_ui_script=1 } + in_job && /timeout-minutes:[[:space:]]*75/ { saw_timeout=1 } + + END { exit !(saw_build_for_testing && saw_ui_script && saw_timeout) } + ' "$CI_FILE"; then + echo "FAIL: tests-build-and-lag must build once for testing and run the display UI regressions from that DerivedData" exit 1 fi - echo "PASS: release-ghostty-cli-helper retries required Ghostty helper artifact uploads" + echo "PASS: runtime display regressions are collapsed into tests-build-and-lag" } check_signing_intermediate_imports() { @@ -796,12 +817,9 @@ check_no_bare_github_hosted_runners check_no_self_hosted_fleet_runners check_macos_runner "$CI_FILE" "app-host-unit-tests" check_macos_runner "$CI_FILE" "tests-build-and-lag" -check_macos_runner "$CI_FILE" "release-ghostty-cli-helper" check_macos_runner "$CI_FILE" "release-build" -check_macos_runner "$CI_FILE" "ui-regressions" check_release_build_runner_disk_capacity check_display_runner_identity_guard "$CI_FILE" "tests-build-and-lag" -check_display_runner_identity_guard "$CI_FILE" "ui-regressions" check_build_lag_deriveddata_cache_path # build-ghosttykit.yml @@ -817,7 +835,8 @@ check_e2e_runner_fallbacks check_xcode_selection check_release_build_signal check_release_build_disk_cleanup -check_release_helper_upload_retry +check_release_helper_built_inline +check_runtime_regressions_collapsed check_signing_intermediate_imports check_signing_intermediate_helper_behavior check_sentry_cli_install_portability diff --git a/tests/test_ci_xcode_selection_fast_path.sh b/tests/test_ci_xcode_selection_fast_path.sh new file mode 100755 index 000000000000..bce7729d8756 --- /dev/null +++ b/tests/test_ci_xcode_selection_fast_path.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)" +SCRIPT="$ROOT_DIR/scripts/select-ci-xcode.sh" + +tmp_dir="$(mktemp -d)" +trap 'rm -rf "$tmp_dir"' EXIT + +bin_dir="$tmp_dir/bin" +env_file="$tmp_dir/github-env" +xcode_select_log="$tmp_dir/xcode-select.log" +mkdir -p "$bin_dir" +touch "$env_file" "$xcode_select_log" + +cat > "$bin_dir/xcrun" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +case "$*" in + "--sdk macosx --show-sdk-version") + cat "$DEVELOPER_DIR/sdk-version" + ;; + "--sdk macosx --show-sdk-path") + printf '%s\n' "$DEVELOPER_DIR/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk" + ;; + *) + echo "unexpected xcrun args: $*" >&2 + exit 64 + ;; +esac +EOF +chmod +x "$bin_dir/xcrun" + +cat > "$bin_dir/xcode-select" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' "$*" >> "$CMUX_TEST_XCODE_SELECT_LOG" +EOF +chmod +x "$bin_dir/xcode-select" + +cat > "$bin_dir/xcodebuild" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' "Xcode 26.2" "Build version 17C52" +EOF +chmod +x "$bin_dir/xcodebuild" + +pinned_app="$tmp_dir/Xcode_26.2.app" +pinned_developer="$pinned_app/Contents/Developer" +mkdir -p "$pinned_developer" +printf '%s\n' "26.2" > "$pinned_developer/sdk-version" + +output="$( + PATH="$bin_dir:/usr/bin:/bin" \ + GITHUB_ENV="$env_file" \ + CMUX_TEST_XCODE_SELECT_LOG="$xcode_select_log" \ + CMUX_CI_DEVELOPER_DIR="$pinned_developer" \ + CMUX_XCODE_APPLICATIONS_DIR="$tmp_dir/no-apps" \ + "$SCRIPT" +)" + +if ! grep -Fq "Selected pinned Xcode (DEVELOPER_DIR): $pinned_developer (macOS SDK 26.2)" <<< "$output"; then + echo "FAIL: pinned developer dir was not selected" + printf '%s\n' "$output" >&2 + exit 1 +fi + +if grep -Fq "Found " <<< "$output"; then + echo "FAIL: pinned developer dir path should skip scanning Xcode apps" + printf '%s\n' "$output" >&2 + exit 1 +fi + +if [[ "$(cat "$env_file")" != "DEVELOPER_DIR=$pinned_developer" ]]; then + echo "FAIL: select-ci-xcode.sh did not export the pinned developer dir" + cat "$env_file" >&2 + exit 1 +fi + +if [[ "$(cat "$xcode_select_log")" != "-s $pinned_developer" ]]; then + echo "FAIL: select-ci-xcode.sh did not point xcode-select at the pinned developer dir" + cat "$xcode_select_log" >&2 + exit 1 +fi + +missing_output="$( + PATH="$bin_dir:/usr/bin:/bin" \ + GITHUB_ENV="$env_file" \ + CMUX_TEST_XCODE_SELECT_LOG="$xcode_select_log" \ + CMUX_CI_DEVELOPER_DIR="$tmp_dir/missing/Contents/Developer" \ + "$SCRIPT" 2>&1 >/dev/null +)" && { + echo "FAIL: missing pinned developer dir should fail" + exit 1 +} + +if ! grep -Fq "Pinned Xcode developer dir does not exist" <<< "$missing_output"; then + echo "FAIL: missing pinned developer dir failure was not explained" + printf '%s\n' "$missing_output" >&2 + exit 1 +fi + +echo "PASS: CI Xcode selection fast path" From 074e87a055a44f4b9a088454cf712bf1ce9c5f1d Mon Sep 17 00:00:00 2001 From: Aziz Albahar Date: Tue, 7 Jul 2026 21:52:11 -0700 Subject: [PATCH 02/13] Validate pinned CI Xcode SDK lanes --- .github/workflows/ci.yml | 4 ++++ scripts/select-ci-xcode.sh | 27 ++++++++++++++++++++++ tests/test_ci_change_areas.py | 2 ++ tests/test_ci_xcode_selection_fast_path.sh | 24 +++++++++++++++++++ 4 files changed, 57 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1185f3e7ace1..5beffa5006a1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -399,6 +399,7 @@ jobs: shard: [1, 2, 3, 4] env: CMUX_CI_XCODE_APP: ${{ vars.CMUX_CI_XCODE_APP_MACOS_15 }} + CMUX_CI_REQUIRED_MACOS_SDK_MAJOR: "26" CMUX_SKIP_ZIG_BUILD: "1" # Keep one-off focused gates on the lightest measured shard so shard 1 no # longer carries the full shard plus every extra regression guard. @@ -773,6 +774,7 @@ jobs: timeout-minutes: 20 env: CMUX_CI_XCODE_APP: ${{ vars.CMUX_CI_XCODE_APP_MACOS_15 }} + CMUX_CI_REQUIRED_MACOS_SDK_MAJOR: "26" steps: - name: Clear stale git locks (self-hosted reused workspace) shell: bash @@ -1023,6 +1025,7 @@ jobs: timeout-minutes: 75 env: CMUX_CI_XCODE_APP: ${{ vars.CMUX_CI_XCODE_APP_MACOS_15 }} + CMUX_CI_REQUIRED_MACOS_SDK_MAJOR: "26" steps: - name: Validate display runner identity env: @@ -1347,6 +1350,7 @@ jobs: timeout-minutes: 60 env: CMUX_CI_XCODE_APP: ${{ vars.CMUX_CI_XCODE_APP_MACOS_26 }} + CMUX_CI_REQUIRED_MACOS_SDK_MAJOR: "26" steps: - name: Clear stale git locks (self-hosted reused workspace) shell: bash diff --git a/scripts/select-ci-xcode.sh b/scripts/select-ci-xcode.sh index 80475a2cf727..a50105e38ad5 100755 --- a/scripts/select-ci-xcode.sh +++ b/scripts/select-ci-xcode.sh @@ -15,10 +15,37 @@ set -euo pipefail APPLICATIONS_DIR="${CMUX_XCODE_APPLICATIONS_DIR:-/Applications}" +REQUIRED_SDK_MAJOR="${CMUX_CI_REQUIRED_MACOS_SDK_MAJOR:-}" + +sdk_major() { + local v="$1" maj + maj="${v%%.*}" + case "$maj" in ''|*[!0-9]*) return 1 ;; esac + printf '%s' "$maj" +} + +validate_required_sdk() { + local selected_dir="$1" sdk_version="$2" actual_major + [ -n "$REQUIRED_SDK_MAJOR" ] || return 0 + case "$REQUIRED_SDK_MAJOR" in ''|*[!0-9]*) + echo "CMUX_CI_REQUIRED_MACOS_SDK_MAJOR must be numeric, got: $REQUIRED_SDK_MAJOR" >&2 + exit 1 + ;; + esac + if ! actual_major="$(sdk_major "$sdk_version")"; then + echo "Could not parse macOS SDK version for $selected_dir: $sdk_version" >&2 + exit 1 + fi + if [ "$actual_major" != "$REQUIRED_SDK_MAJOR" ]; then + echo "Selected Xcode at $selected_dir has macOS SDK $sdk_version; required major is $REQUIRED_SDK_MAJOR" >&2 + exit 1 + fi +} select_developer_dir() { local selected_dir="$1" sdk_version="$2" label="$3" + validate_required_sdk "$selected_dir" "$sdk_version" echo "$label (DEVELOPER_DIR): $selected_dir (macOS SDK $sdk_version)" if [ -n "${GITHUB_ENV:-}" ]; then echo "DEVELOPER_DIR=$selected_dir" >> "$GITHUB_ENV" diff --git a/tests/test_ci_change_areas.py b/tests/test_ci_change_areas.py index ceea56009e48..4d36f55eb8fd 100644 --- a/tests/test_ci_change_areas.py +++ b/tests/test_ci_change_areas.py @@ -456,9 +456,11 @@ def test_macos_jobs_use_lane_specific_xcode_pin_vars() -> None: ]: block = workflow_job_block(job_name) assert "CMUX_CI_XCODE_APP: ${{ vars.CMUX_CI_XCODE_APP_MACOS_15 }}" in block + assert 'CMUX_CI_REQUIRED_MACOS_SDK_MAJOR: "26"' in block release_block = workflow_job_block("release-build") assert "CMUX_CI_XCODE_APP: ${{ vars.CMUX_CI_XCODE_APP_MACOS_26 }}" in release_block + assert 'CMUX_CI_REQUIRED_MACOS_SDK_MAJOR: "26"' in release_block def test_required_macos_topology_collapses_display_and_release_helper_jobs() -> None: diff --git a/tests/test_ci_xcode_selection_fast_path.sh b/tests/test_ci_xcode_selection_fast_path.sh index bce7729d8756..eebcdf3efc60 100755 --- a/tests/test_ci_xcode_selection_fast_path.sh +++ b/tests/test_ci_xcode_selection_fast_path.sh @@ -55,6 +55,7 @@ output="$( GITHUB_ENV="$env_file" \ CMUX_TEST_XCODE_SELECT_LOG="$xcode_select_log" \ CMUX_CI_DEVELOPER_DIR="$pinned_developer" \ + CMUX_CI_REQUIRED_MACOS_SDK_MAJOR=26 \ CMUX_XCODE_APPLICATIONS_DIR="$tmp_dir/no-apps" \ "$SCRIPT" )" @@ -83,6 +84,29 @@ if [[ "$(cat "$xcode_select_log")" != "-s $pinned_developer" ]]; then exit 1 fi +old_app="$tmp_dir/Xcode_16.4.app" +old_developer="$old_app/Contents/Developer" +mkdir -p "$old_developer" +printf '%s\n' "15.5" > "$old_developer/sdk-version" + +wrong_sdk_output="$( + PATH="$bin_dir:/usr/bin:/bin" \ + GITHUB_ENV="$env_file" \ + CMUX_TEST_XCODE_SELECT_LOG="$xcode_select_log" \ + CMUX_CI_DEVELOPER_DIR="$old_developer" \ + CMUX_CI_REQUIRED_MACOS_SDK_MAJOR=26 \ + "$SCRIPT" 2>&1 >/dev/null +)" && { + echo "FAIL: pinned Xcode with the wrong SDK major should fail" + exit 1 +} + +if ! grep -Fq "required major is 26" <<< "$wrong_sdk_output"; then + echo "FAIL: wrong pinned SDK major failure was not explained" + printf '%s\n' "$wrong_sdk_output" >&2 + exit 1 +fi + missing_output="$( PATH="$bin_dir:/usr/bin:/bin" \ GITHUB_ENV="$env_file" \ From e923d84d87956e3429c2c9fdd18a0c2e9d842e29 Mon Sep 17 00:00:00 2001 From: Aziz Albahar Date: Tue, 7 Jul 2026 22:02:54 -0700 Subject: [PATCH 03/13] Move CI Ghostty helper handoff to package lane --- .github/workflows/ci.yml | 60 +++++++++++++++++++++--------- tests/test_ci_change_areas.py | 12 ++++-- tests/test_ci_release_sdk_lane.sh | 28 +++++++++++--- tests/test_ci_self_hosted_guard.sh | 33 ++++++++++++---- 4 files changed, 99 insertions(+), 34 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5beffa5006a1..58a33d2b6337 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -831,6 +831,41 @@ jobs: run: | ./scripts/install-rust-ci.sh + - name: Install zig + run: ./scripts/install-zig-ci.sh + + - name: Cache Zig packages + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: ~/.cache/zig + key: zig-packages-${{ hashFiles('ghostty/build.zig.zon', 'ghostty/build.zig.zon.json') }} + restore-keys: zig-packages- + + - name: Build universal Ghostty CLI helper + run: | + set -euo pipefail + mkdir -p ghostty-cli-helper + ./scripts/build-ghostty-cli-helper.sh --universal --output ghostty-cli-helper/ghostty + lipo ghostty-cli-helper/ghostty -verify_arch arm64 x86_64 + + - name: Upload universal Ghostty CLI helper + id: upload-ghostty-cli-helper + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: cmux-ghostty-cli-helper + path: ghostty-cli-helper/ghostty + if-no-files-found: error + + - name: Retry universal Ghostty CLI helper upload + if: steps.upload-ghostty-cli-helper.outcome == 'failure' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: cmux-ghostty-cli-helper + path: ghostty-cli-helper/ghostty + if-no-files-found: error + overwrite: true + - name: Run Swift package unit tests run: | set -euo pipefail @@ -1337,6 +1372,7 @@ jobs: needs: - changes - linux-preflight + - swift-package-tests if: ${{ needs.changes.outputs.macos == 'true' }} # Compile the same unsigned universal Release app that nightly builds before # signing, notarization, and publishing. This catches DEBUG/Release boundary @@ -1404,24 +1440,6 @@ jobs: run: | ./scripts/install-rust-ci.sh - - name: Install zig - run: | - ./scripts/install-zig-ci.sh - - - name: Cache Zig packages - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 - with: - path: ~/.cache/zig - key: zig-packages-${{ hashFiles('ghostty/build.zig.zon', 'ghostty/build.zig.zon.json') }} - restore-keys: zig-packages- - - - name: Build universal Ghostty CLI helper - run: | - set -euo pipefail - mkdir -p ghostty-cli-helper - ./scripts/build-ghostty-cli-helper.sh --universal --output ghostty-cli-helper/ghostty - lipo ghostty-cli-helper/ghostty -verify_arch arm64 x86_64 - - name: Cache DerivedData uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: @@ -1450,6 +1468,12 @@ jobs: ONLY_ACTIVE_ARCH=NO \ CODE_SIGNING_ALLOWED=NO ASSETCATALOG_COMPILER_APPICON_NAME=AppIcon-Nightly build + - name: Download universal Ghostty CLI helper + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: cmux-ghostty-cli-helper + path: ghostty-cli-helper + - name: Install universal Ghostty CLI helper run: | set -euo pipefail diff --git a/tests/test_ci_change_areas.py b/tests/test_ci_change_areas.py index 4d36f55eb8fd..539f396246fc 100644 --- a/tests/test_ci_change_areas.py +++ b/tests/test_ci_change_areas.py @@ -466,6 +466,7 @@ def test_macos_jobs_use_lane_specific_xcode_pin_vars() -> None: def test_required_macos_topology_collapses_display_and_release_helper_jobs() -> None: workflow = CI_WORKFLOW.read_text(encoding="utf-8") runtime_block = workflow_job_block("tests-build-and-lag") + package_block = workflow_job_block("swift-package-tests") release_block = workflow_job_block("release-build") assert "\n ui-regressions:" not in workflow @@ -473,9 +474,14 @@ def test_required_macos_topology_collapses_display_and_release_helper_jobs() -> assert "build-for-testing" in runtime_block assert "Run display UI regressions" in runtime_block assert "scripts/ci/run-display-ui-regressions.sh" in runtime_block - assert "Build universal Ghostty CLI helper" in release_block - assert "./scripts/build-ghostty-cli-helper.sh --universal --output ghostty-cli-helper/ghostty" in release_block - assert "actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131" not in release_block + assert "Build universal Ghostty CLI helper" in package_block + assert "./scripts/build-ghostty-cli-helper.sh --universal --output ghostty-cli-helper/ghostty" in package_block + assert "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a" in package_block + assert " - swift-package-tests" in release_block + assert "Download universal Ghostty CLI helper" in release_block + assert "actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131" in release_block + assert "Install universal Ghostty CLI helper" in release_block + assert "./scripts/build-ghostty-cli-helper.sh --universal --output ghostty-cli-helper/ghostty" not in release_block def test_agent_session_web_resources_runs_only_for_agent_session_web_area() -> None: diff --git a/tests/test_ci_release_sdk_lane.sh b/tests/test_ci_release_sdk_lane.sh index 5e4677619e24..7b7bf47b8a0c 100755 --- a/tests/test_ci_release_sdk_lane.sh +++ b/tests/test_ci_release_sdk_lane.sh @@ -8,7 +8,7 @@ RELEASE_FILE="$ROOT_DIR/.github/workflows/release.yml" # nightly.yml is intentionally not covered here. It has its own helper-build # model and guards via test_ci_nightly_xcode_selection.sh plus # test_nightly_universal_build.sh. This lane guards the release artifact-download -# model and the CI inline-helper release-build model. +# model and the CI package-lane helper handoff model. job_section() { local file="$1" job="$2" @@ -69,14 +69,30 @@ if ! grep -Fq "actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef35013 exit 1 fi +swift_package_section="$(job_section "$CI_FILE" "swift-package-tests")" +if [[ "$swift_package_section" != *"./scripts/build-ghostty-cli-helper.sh --universal --output ghostty-cli-helper/ghostty"* ]]; then + echo "FAIL: CI swift-package-tests must build the universal Ghostty CLI helper on the macOS 15 lane" >&2 + exit 1 +fi + +if [[ "$swift_package_section" != *"actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1"* ]]; then + echo "FAIL: CI swift-package-tests must upload the macOS 15-built Ghostty helper artifact" >&2 + exit 1 +fi + release_build_section="$(job_section "$CI_FILE" "release-build")" -if [[ "$release_build_section" != *"./scripts/build-ghostty-cli-helper.sh --universal --output ghostty-cli-helper/ghostty"* ]]; then - echo "FAIL: CI release-build must build the universal Ghostty CLI helper inline" >&2 +if [[ "$release_build_section" != *"actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0"* ]]; then + echo "FAIL: CI release-build must download the macOS 15-built Ghostty helper artifact" >&2 + exit 1 +fi + +if [[ "$release_build_section" != *"- swift-package-tests"* ]]; then + echo "FAIL: CI release-build must wait for the helper-producing swift-package-tests lane" >&2 exit 1 fi -if [[ "$release_build_section" == *"actions/download-artifact@"* ]]; then - echo "FAIL: CI release-build must not wait on a separate Ghostty helper artifact job" >&2 +if [[ "$release_build_section" == *"./scripts/build-ghostty-cli-helper.sh --universal --output ghostty-cli-helper/ghostty"* ]]; then + echo "FAIL: CI release-build must not build the Ghostty helper on macOS 26" >&2 exit 1 fi @@ -85,4 +101,4 @@ if grep -Fq "release-ghostty-cli-helper:" "$CI_FILE"; then exit 1 fi -echo "PASS: release uses artifact helper handoff; CI release-build builds the helper inline on the macOS 26 lane" +echo "PASS: release uses artifact helper handoff; CI release-build downloads the helper from the existing macOS 15 package lane" diff --git a/tests/test_ci_self_hosted_guard.sh b/tests/test_ci_self_hosted_guard.sh index 1bb13386a867..0f9933871241 100755 --- a/tests/test_ci_self_hosted_guard.sh +++ b/tests/test_ci_self_hosted_guard.sh @@ -207,23 +207,42 @@ check_release_build_disk_cleanup() { echo "PASS: release-build reclaims runner disk before large cache restores" } -check_release_helper_built_inline() { +check_release_helper_artifact_from_package_lane() { if ! awk ' - /^ release-build:/ { in_job=1; next } + /^ swift-package-tests:/ { in_job=1; next } in_job && /^ [^[:space:]#][^:]*:[[:space:]]*(#.*)?$/ { in_job=0 } in_job && /- name: Build universal Ghostty CLI helper/ { saw_build_step=1; next } in_job && /\.\/scripts\/build-ghostty-cli-helper\.sh --universal --output ghostty-cli-helper\/ghostty/ { saw_build=1 } in_job && /lipo ghostty-cli-helper\/ghostty -verify_arch arm64 x86_64/ { saw_lipo=1 } + in_job && /- name: Upload universal Ghostty CLI helper/ { saw_upload_step=1; next } + in_job && /uses: actions\/upload-artifact@/ { saw_upload=1 } + in_job && /name:[[:space:]]*cmux-ghostty-cli-helper/ { saw_artifact_name=1 } + + END { + exit !(saw_build_step && saw_build && saw_lipo && saw_upload_step && saw_upload && saw_artifact_name) + } + ' "$CI_FILE"; then + echo "FAIL: swift-package-tests must build and upload the universal Ghostty helper from an existing macOS 15 lane" + exit 1 + fi + + if ! awk ' + /^ release-build:/ { in_job=1; next } + in_job && /^ [^[:space:]#][^:]*:[[:space:]]*(#.*)?$/ { in_job=0 } + + in_job && /- swift-package-tests/ { saw_need=1 } + in_job && /- name: Download universal Ghostty CLI helper/ { saw_download_step=1; next } + in_job && /uses: actions\/download-artifact@/ { saw_download=1 } + in_job && /name:[[:space:]]*cmux-ghostty-cli-helper/ { saw_artifact_name=1 } in_job && /- name: Install universal Ghostty CLI helper/ { saw_install_step=1; next } in_job && /\.\/scripts\/install-prebuilt-ghostty-cli-helper\.sh/ { saw_install=1 } - in_job && /actions\/download-artifact@/ { saw_download=1 } END { - exit !(saw_build_step && saw_build && saw_lipo && saw_install_step && saw_install && !saw_download) + exit !(saw_need && saw_download_step && saw_download && saw_artifact_name && saw_install_step && saw_install) } ' "$CI_FILE"; then - echo "FAIL: release-build must build and install the universal Ghostty helper inline without a separate CI artifact job" + echo "FAIL: release-build must depend on swift-package-tests, download the helper artifact, and install it into the app" exit 1 fi @@ -232,7 +251,7 @@ check_release_helper_built_inline() { exit 1 fi - echo "PASS: release-build builds and installs the universal Ghostty helper inline" + echo "PASS: release-build consumes the Ghostty helper artifact built by swift-package-tests" } check_runtime_regressions_collapsed() { @@ -835,7 +854,7 @@ check_e2e_runner_fallbacks check_xcode_selection check_release_build_signal check_release_build_disk_cleanup -check_release_helper_built_inline +check_release_helper_artifact_from_package_lane check_runtime_regressions_collapsed check_signing_intermediate_imports check_signing_intermediate_helper_behavior From 85c1807e9499ed19624f263671fd93082cb121da Mon Sep 17 00:00:00 2001 From: Aziz Albahar Date: Tue, 7 Jul 2026 22:17:17 -0700 Subject: [PATCH 04/13] Build CI Ghostty helper before Xcode selection --- .github/workflows/ci.yml | 70 +++++++++++++++--------------- tests/test_ci_change_areas.py | 2 + tests/test_ci_release_sdk_lane.sh | 11 +++++ tests/test_ci_self_hosted_guard.sh | 21 +++++++-- 4 files changed, 65 insertions(+), 39 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 58a33d2b6337..13633ab36e39 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -793,6 +793,41 @@ jobs: with: submodules: recursive + - name: Install zig + run: ./scripts/install-zig-ci.sh + + - name: Cache Zig packages + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: ~/.cache/zig + key: zig-packages-${{ hashFiles('ghostty/build.zig.zon', 'ghostty/build.zig.zon.json') }} + restore-keys: zig-packages- + + - name: Build universal Ghostty CLI helper + run: | + set -euo pipefail + mkdir -p ghostty-cli-helper + ./scripts/build-ghostty-cli-helper.sh --universal --output ghostty-cli-helper/ghostty + lipo ghostty-cli-helper/ghostty -verify_arch arm64 x86_64 + + - name: Upload universal Ghostty CLI helper + id: upload-ghostty-cli-helper + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: cmux-ghostty-cli-helper + path: ghostty-cli-helper/ghostty + if-no-files-found: error + + - name: Retry universal Ghostty CLI helper upload + if: steps.upload-ghostty-cli-helper.outcome == 'failure' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: cmux-ghostty-cli-helper + path: ghostty-cli-helper/ghostty + if-no-files-found: error + overwrite: true + - name: Select Xcode run: | set -euo pipefail @@ -831,41 +866,6 @@ jobs: run: | ./scripts/install-rust-ci.sh - - name: Install zig - run: ./scripts/install-zig-ci.sh - - - name: Cache Zig packages - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 - with: - path: ~/.cache/zig - key: zig-packages-${{ hashFiles('ghostty/build.zig.zon', 'ghostty/build.zig.zon.json') }} - restore-keys: zig-packages- - - - name: Build universal Ghostty CLI helper - run: | - set -euo pipefail - mkdir -p ghostty-cli-helper - ./scripts/build-ghostty-cli-helper.sh --universal --output ghostty-cli-helper/ghostty - lipo ghostty-cli-helper/ghostty -verify_arch arm64 x86_64 - - - name: Upload universal Ghostty CLI helper - id: upload-ghostty-cli-helper - continue-on-error: true - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: cmux-ghostty-cli-helper - path: ghostty-cli-helper/ghostty - if-no-files-found: error - - - name: Retry universal Ghostty CLI helper upload - if: steps.upload-ghostty-cli-helper.outcome == 'failure' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: cmux-ghostty-cli-helper - path: ghostty-cli-helper/ghostty - if-no-files-found: error - overwrite: true - - name: Run Swift package unit tests run: | set -euo pipefail diff --git a/tests/test_ci_change_areas.py b/tests/test_ci_change_areas.py index 539f396246fc..90f17165a78f 100644 --- a/tests/test_ci_change_areas.py +++ b/tests/test_ci_change_areas.py @@ -477,6 +477,8 @@ def test_required_macos_topology_collapses_display_and_release_helper_jobs() -> assert "Build universal Ghostty CLI helper" in package_block assert "./scripts/build-ghostty-cli-helper.sh --universal --output ghostty-cli-helper/ghostty" in package_block assert "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a" in package_block + assert package_block.index("Build universal Ghostty CLI helper") < package_block.index("Select Xcode") + assert package_block.index("Upload universal Ghostty CLI helper") < package_block.index("Select Xcode") assert " - swift-package-tests" in release_block assert "Download universal Ghostty CLI helper" in release_block assert "actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131" in release_block diff --git a/tests/test_ci_release_sdk_lane.sh b/tests/test_ci_release_sdk_lane.sh index 7b7bf47b8a0c..1504474748a1 100755 --- a/tests/test_ci_release_sdk_lane.sh +++ b/tests/test_ci_release_sdk_lane.sh @@ -80,6 +80,17 @@ if [[ "$swift_package_section" != *"actions/upload-artifact@043fb46d1a93c77aae65 exit 1 fi +swift_package_before_xcode="${swift_package_section%%- name: Select Xcode*}" +if [[ "$swift_package_before_xcode" != *"./scripts/build-ghostty-cli-helper.sh --universal --output ghostty-cli-helper/ghostty"* ]]; then + echo "FAIL: CI swift-package-tests must build the Ghostty helper before selecting the Xcode 26 SDK" >&2 + exit 1 +fi + +if [[ "$swift_package_before_xcode" != *"actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1"* ]]; then + echo "FAIL: CI swift-package-tests must upload the Ghostty helper before selecting the Xcode 26 SDK" >&2 + exit 1 +fi + release_build_section="$(job_section "$CI_FILE" "release-build")" if [[ "$release_build_section" != *"actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0"* ]]; then echo "FAIL: CI release-build must download the macOS 15-built Ghostty helper artifact" >&2 diff --git a/tests/test_ci_self_hosted_guard.sh b/tests/test_ci_self_hosted_guard.sh index 0f9933871241..a2f6d663bda0 100755 --- a/tests/test_ci_self_hosted_guard.sh +++ b/tests/test_ci_self_hosted_guard.sh @@ -212,18 +212,31 @@ check_release_helper_artifact_from_package_lane() { /^ swift-package-tests:/ { in_job=1; next } in_job && /^ [^[:space:]#][^:]*:[[:space:]]*(#.*)?$/ { in_job=0 } - in_job && /- name: Build universal Ghostty CLI helper/ { saw_build_step=1; next } + in_job && /- name: Select Xcode/ { saw_select=1; after_select=1; next } + in_job && /- name: Build universal Ghostty CLI helper/ { + saw_build_step=1 + if (after_select) { + saw_build_after_select=1 + } + next + } in_job && /\.\/scripts\/build-ghostty-cli-helper\.sh --universal --output ghostty-cli-helper\/ghostty/ { saw_build=1 } in_job && /lipo ghostty-cli-helper\/ghostty -verify_arch arm64 x86_64/ { saw_lipo=1 } - in_job && /- name: Upload universal Ghostty CLI helper/ { saw_upload_step=1; next } + in_job && /- name: Upload universal Ghostty CLI helper/ { + saw_upload_step=1 + if (after_select) { + saw_upload_after_select=1 + } + next + } in_job && /uses: actions\/upload-artifact@/ { saw_upload=1 } in_job && /name:[[:space:]]*cmux-ghostty-cli-helper/ { saw_artifact_name=1 } END { - exit !(saw_build_step && saw_build && saw_lipo && saw_upload_step && saw_upload && saw_artifact_name) + exit !(saw_build_step && saw_build && saw_lipo && saw_upload_step && saw_upload && saw_artifact_name && saw_select && !saw_build_after_select && !saw_upload_after_select) } ' "$CI_FILE"; then - echo "FAIL: swift-package-tests must build and upload the universal Ghostty helper from an existing macOS 15 lane" + echo "FAIL: swift-package-tests must build and upload the universal Ghostty helper before selecting Xcode 26" exit 1 fi From 609e259f5338d2da1203b8320bd50832d34cf98f Mon Sep 17 00:00:00 2001 From: Aziz Albahar Date: Tue, 7 Jul 2026 22:28:58 -0700 Subject: [PATCH 05/13] Pin CI Ghostty helper to macOS 15 SDK --- .github/workflows/ci.yml | 17 ++++++++++++++++- tests/test_ci_change_areas.py | 6 ++++++ tests/test_ci_release_sdk_lane.sh | 20 ++++++++++++++++++++ tests/test_ci_self_hosted_guard.sh | 9 +++++++-- 4 files changed, 49 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 13633ab36e39..a2ad81c75059 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -771,9 +771,10 @@ jobs: - linux-preflight if: ${{ needs.changes.outputs.macos == 'true' }} runs-on: ${{ vars.MACOS_RUNNER_15 || 'warp-macos-15-arm64-6x' }} - timeout-minutes: 20 + timeout-minutes: 40 env: CMUX_CI_XCODE_APP: ${{ vars.CMUX_CI_XCODE_APP_MACOS_15 }} + CMUX_CI_HELPER_XCODE_APP: ${{ vars.CMUX_CI_HELPER_XCODE_APP_MACOS_15 || '/Applications/Xcode_16.4.app' }} CMUX_CI_REQUIRED_MACOS_SDK_MAJOR: "26" steps: - name: Clear stale git locks (self-hosted reused workspace) @@ -793,6 +794,13 @@ jobs: with: submodules: recursive + - name: Select helper Xcode + run: | + set -euo pipefail + CMUX_CI_XCODE_APP="$CMUX_CI_HELPER_XCODE_APP" \ + CMUX_CI_REQUIRED_MACOS_SDK_MAJOR=15 \ + ./scripts/select-ci-xcode.sh + - name: Install zig run: ./scripts/install-zig-ci.sh @@ -809,6 +817,13 @@ jobs: mkdir -p ghostty-cli-helper ./scripts/build-ghostty-cli-helper.sh --universal --output ghostty-cli-helper/ghostty lipo ghostty-cli-helper/ghostty -verify_arch arm64 x86_64 + for arch in arm64 x86_64; do + thin="ghostty-cli-helper/ghostty-$arch" + lipo ghostty-cli-helper/ghostty -thin "$arch" -output "$thin" + HELPER_SDK_VERSION="$(otool -l "$thin" | awk '/LC_BUILD_VERSION/ { in_version=1; next } in_version && /sdk / { print $2; exit }')" + echo "Ghostty helper $arch SDK version: $HELPER_SDK_VERSION" + [[ "$HELPER_SDK_VERSION" == 15.* ]] + done - name: Upload universal Ghostty CLI helper id: upload-ghostty-cli-helper diff --git a/tests/test_ci_change_areas.py b/tests/test_ci_change_areas.py index 90f17165a78f..78daac62532e 100644 --- a/tests/test_ci_change_areas.py +++ b/tests/test_ci_change_areas.py @@ -474,9 +474,15 @@ def test_required_macos_topology_collapses_display_and_release_helper_jobs() -> assert "build-for-testing" in runtime_block assert "Run display UI regressions" in runtime_block assert "scripts/ci/run-display-ui-regressions.sh" in runtime_block + assert "timeout-minutes: 40" in package_block + assert "CMUX_CI_HELPER_XCODE_APP" in package_block + assert "Select helper Xcode" in package_block + assert "CMUX_CI_REQUIRED_MACOS_SDK_MAJOR=15" in package_block assert "Build universal Ghostty CLI helper" in package_block assert "./scripts/build-ghostty-cli-helper.sh --universal --output ghostty-cli-helper/ghostty" in package_block + assert '[[ "$HELPER_SDK_VERSION" == 15.* ]]' in package_block assert "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a" in package_block + assert package_block.index("Select helper Xcode") < package_block.index("Build universal Ghostty CLI helper") assert package_block.index("Build universal Ghostty CLI helper") < package_block.index("Select Xcode") assert package_block.index("Upload universal Ghostty CLI helper") < package_block.index("Select Xcode") assert " - swift-package-tests" in release_block diff --git a/tests/test_ci_release_sdk_lane.sh b/tests/test_ci_release_sdk_lane.sh index 1504474748a1..eeca4068781a 100755 --- a/tests/test_ci_release_sdk_lane.sh +++ b/tests/test_ci_release_sdk_lane.sh @@ -70,6 +70,16 @@ if ! grep -Fq "actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef35013 fi swift_package_section="$(job_section "$CI_FILE" "swift-package-tests")" +if [[ "$swift_package_section" != *"timeout-minutes: 40"* ]]; then + echo "FAIL: CI swift-package-tests must have enough timeout budget for helper build plus package tests" >&2 + exit 1 +fi + +if [[ "$swift_package_section" != *"CMUX_CI_HELPER_XCODE_APP"* ]]; then + echo "FAIL: CI swift-package-tests must use a helper-specific Xcode pin" >&2 + exit 1 +fi + if [[ "$swift_package_section" != *"./scripts/build-ghostty-cli-helper.sh --universal --output ghostty-cli-helper/ghostty"* ]]; then echo "FAIL: CI swift-package-tests must build the universal Ghostty CLI helper on the macOS 15 lane" >&2 exit 1 @@ -81,6 +91,11 @@ if [[ "$swift_package_section" != *"actions/upload-artifact@043fb46d1a93c77aae65 fi swift_package_before_xcode="${swift_package_section%%- name: Select Xcode*}" +if [[ "$swift_package_before_xcode" != *"CMUX_CI_REQUIRED_MACOS_SDK_MAJOR=15"* ]]; then + echo "FAIL: CI swift-package-tests must require a macOS 15 SDK for the helper build" >&2 + exit 1 +fi + if [[ "$swift_package_before_xcode" != *"./scripts/build-ghostty-cli-helper.sh --universal --output ghostty-cli-helper/ghostty"* ]]; then echo "FAIL: CI swift-package-tests must build the Ghostty helper before selecting the Xcode 26 SDK" >&2 exit 1 @@ -91,6 +106,11 @@ if [[ "$swift_package_before_xcode" != *"actions/upload-artifact@043fb46d1a93c77 exit 1 fi +if [[ "$swift_package_section" != *'[[ "$HELPER_SDK_VERSION" == 15.* ]]'* ]]; then + echo "FAIL: CI swift-package-tests must validate the uploaded Ghostty helper was built with a macOS 15 SDK" >&2 + exit 1 +fi + release_build_section="$(job_section "$CI_FILE" "release-build")" if [[ "$release_build_section" != *"actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0"* ]]; then echo "FAIL: CI release-build must download the macOS 15-built Ghostty helper artifact" >&2 diff --git a/tests/test_ci_self_hosted_guard.sh b/tests/test_ci_self_hosted_guard.sh index a2f6d663bda0..dcd54910b169 100755 --- a/tests/test_ci_self_hosted_guard.sh +++ b/tests/test_ci_self_hosted_guard.sh @@ -212,6 +212,10 @@ check_release_helper_artifact_from_package_lane() { /^ swift-package-tests:/ { in_job=1; next } in_job && /^ [^[:space:]#][^:]*:[[:space:]]*(#.*)?$/ { in_job=0 } + in_job && /timeout-minutes:[[:space:]]*40/ { saw_timeout=1 } + in_job && /CMUX_CI_HELPER_XCODE_APP:/ { saw_helper_xcode_env=1 } + in_job && /- name: Select helper Xcode/ { saw_helper_select=1; next } + in_job && /CMUX_CI_REQUIRED_MACOS_SDK_MAJOR=15/ { saw_helper_sdk_pin=1 } in_job && /- name: Select Xcode/ { saw_select=1; after_select=1; next } in_job && /- name: Build universal Ghostty CLI helper/ { saw_build_step=1 @@ -231,12 +235,13 @@ check_release_helper_artifact_from_package_lane() { } in_job && /uses: actions\/upload-artifact@/ { saw_upload=1 } in_job && /name:[[:space:]]*cmux-ghostty-cli-helper/ { saw_artifact_name=1 } + in_job && /\[\[ "\$HELPER_SDK_VERSION" == 15\.\* \]\]/ { saw_helper_sdk_validation=1 } END { - exit !(saw_build_step && saw_build && saw_lipo && saw_upload_step && saw_upload && saw_artifact_name && saw_select && !saw_build_after_select && !saw_upload_after_select) + exit !(saw_timeout && saw_helper_xcode_env && saw_helper_select && saw_helper_sdk_pin && saw_build_step && saw_build && saw_lipo && saw_helper_sdk_validation && saw_upload_step && saw_upload && saw_artifact_name && saw_select && !saw_build_after_select && !saw_upload_after_select) } ' "$CI_FILE"; then - echo "FAIL: swift-package-tests must build and upload the universal Ghostty helper before selecting Xcode 26" + echo "FAIL: swift-package-tests must pin and validate the macOS 15 Ghostty helper before selecting Xcode 26" exit 1 fi From 89c945625e133f8d38a3d5acd30f82a9fa0cca08 Mon Sep 17 00:00:00 2001 From: Aziz Albahar Date: Tue, 7 Jul 2026 22:46:38 -0700 Subject: [PATCH 06/13] Clean virtual displays before releasing CI lock --- .github/workflows/ci.yml | 10 ++++++++++ tests/test_ci_change_areas.py | 3 +++ tests/test_ci_self_hosted_guard.sh | 6 ++++-- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a2ad81c75059..d213681eba2e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1242,8 +1242,13 @@ jobs: kill -0 "$VDISPLAY_PID" >/dev/null 2>&1 || break sleep 0.1 done + if kill -0 "$VDISPLAY_PID" >/dev/null 2>&1; then + kill -9 "$VDISPLAY_PID" >/dev/null 2>&1 || true + wait "$VDISPLAY_PID" >/dev/null 2>&1 || true + fi VDISPLAY_PID="" fi + scripts/ci/virtual-display-lock.sh reap-strays || true scripts/ci/virtual-display-lock.sh release || true } @@ -1376,7 +1381,12 @@ jobs: kill -0 "$VDISPLAY_PID" >/dev/null 2>&1 || break sleep 0.1 done + if kill -0 "$VDISPLAY_PID" >/dev/null 2>&1; then + kill -9 "$VDISPLAY_PID" >/dev/null 2>&1 || true + wait "$VDISPLAY_PID" >/dev/null 2>&1 || true + fi fi + scripts/ci/virtual-display-lock.sh reap-strays || true scripts/ci/virtual-display-lock.sh release || true rm -f "${VDISPLAY_HELPER_PATH:-}" "${VDISPLAY_READY:-}" "${VDISPLAY_ID_PATH:-}" "${VDISPLAY_LOG:-}" diff --git a/tests/test_ci_change_areas.py b/tests/test_ci_change_areas.py index 78daac62532e..ea3fdce32232 100644 --- a/tests/test_ci_change_areas.py +++ b/tests/test_ci_change_areas.py @@ -474,6 +474,9 @@ def test_required_macos_topology_collapses_display_and_release_helper_jobs() -> assert "build-for-testing" in runtime_block assert "Run display UI regressions" in runtime_block assert "scripts/ci/run-display-ui-regressions.sh" in runtime_block + assert 'kill -9 "$VDISPLAY_PID"' in runtime_block + assert "scripts/ci/virtual-display-lock.sh reap-strays" in runtime_block + assert runtime_block.rfind("scripts/ci/virtual-display-lock.sh reap-strays") < runtime_block.rfind("scripts/ci/virtual-display-lock.sh release") assert "timeout-minutes: 40" in package_block assert "CMUX_CI_HELPER_XCODE_APP" in package_block assert "Select helper Xcode" in package_block diff --git a/tests/test_ci_self_hosted_guard.sh b/tests/test_ci_self_hosted_guard.sh index dcd54910b169..e047d441f8dc 100755 --- a/tests/test_ci_self_hosted_guard.sh +++ b/tests/test_ci_self_hosted_guard.sh @@ -284,11 +284,13 @@ check_runtime_regressions_collapsed() { in_job && /build-for-testing/ { saw_build_for_testing=1 } in_job && /scripts\/ci\/run-display-ui-regressions\.sh/ { saw_ui_script=1 } + in_job && /kill -9 "\$VDISPLAY_PID"/ { saw_force_kill=1 } + in_job && /scripts\/ci\/virtual-display-lock\.sh reap-strays/ { saw_reap_strays=1 } in_job && /timeout-minutes:[[:space:]]*75/ { saw_timeout=1 } - END { exit !(saw_build_for_testing && saw_ui_script && saw_timeout) } + END { exit !(saw_build_for_testing && saw_ui_script && saw_force_kill && saw_reap_strays && saw_timeout) } ' "$CI_FILE"; then - echo "FAIL: tests-build-and-lag must build once for testing and run the display UI regressions from that DerivedData" + echo "FAIL: tests-build-and-lag must build once, run display UI regressions from that DerivedData, and clean virtual displays before releasing the lock" exit 1 fi From aee12e7710d34156e20ea6215fe97790070d9d2b Mon Sep 17 00:00:00 2001 From: Aziz Albahar Date: Tue, 7 Jul 2026 22:59:42 -0700 Subject: [PATCH 07/13] Filter CI Xcode scan by required SDK --- scripts/select-ci-xcode.sh | 22 ++++++++++++++++ tests/test_ci_xcode_selection_fast_path.sh | 30 ++++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/scripts/select-ci-xcode.sh b/scripts/select-ci-xcode.sh index a50105e38ad5..f22a7e66f534 100755 --- a/scripts/select-ci-xcode.sh +++ b/scripts/select-ci-xcode.sh @@ -97,6 +97,14 @@ if [ -n "$PINNED_DEVELOPER_DIR" ]; then exit 0 fi +if [ -n "$REQUIRED_SDK_MAJOR" ]; then + case "$REQUIRED_SDK_MAJOR" in ''|*[!0-9]*) + echo "CMUX_CI_REQUIRED_MACOS_SDK_MAJOR must be numeric, got: $REQUIRED_SDK_MAJOR" >&2 + exit 1 + ;; + esac +fi + # Rank by macOS SDK as maj*1000+min so 26.2 (26002) outranks 15.5 (15005). sdk_rank() { local v="$1" maj min @@ -121,6 +129,16 @@ while IFS= read -r app; do [ -d "$dev" ] || continue sdk_ver="$(DEVELOPER_DIR="$dev" xcrun --sdk macosx --show-sdk-version 2>/dev/null || true)" [ -n "$sdk_ver" ] || continue + if [ -n "$REQUIRED_SDK_MAJOR" ]; then + if ! actual_major="$(sdk_major "$sdk_ver")"; then + echo "Ignoring $app with unparsable macOS SDK version: $sdk_ver" >&2 + continue + fi + if [ "$actual_major" != "$REQUIRED_SDK_MAJOR" ]; then + echo "Skipping $app -> macOS SDK $sdk_ver; required major is $REQUIRED_SDK_MAJOR" + continue + fi + fi if ! rank="$(sdk_rank "$sdk_ver")"; then echo "Ignoring $app with unparsable macOS SDK version: $sdk_ver" >&2 continue @@ -157,6 +175,10 @@ if [ -z "$BEST_DIR" ] && [ -n "$BETA_DIR" ]; then fi if [ -z "$BEST_DIR" ]; then + if [ -n "$REQUIRED_SDK_MAJOR" ]; then + echo "No Xcode.app found under $APPLICATIONS_DIR with macOS SDK major $REQUIRED_SDK_MAJOR" >&2 + exit 1 + fi echo "No Xcode.app found under $APPLICATIONS_DIR" >&2 exit 1 fi diff --git a/tests/test_ci_xcode_selection_fast_path.sh b/tests/test_ci_xcode_selection_fast_path.sh index eebcdf3efc60..c69df5af719f 100755 --- a/tests/test_ci_xcode_selection_fast_path.sh +++ b/tests/test_ci_xcode_selection_fast_path.sh @@ -107,6 +107,36 @@ if ! grep -Fq "required major is 26" <<< "$wrong_sdk_output"; then exit 1 fi +: > "$env_file" +: > "$xcode_select_log" + +scan_output="$( + PATH="$bin_dir:/usr/bin:/bin" \ + GITHUB_ENV="$env_file" \ + CMUX_TEST_XCODE_SELECT_LOG="$xcode_select_log" \ + CMUX_XCODE_APPLICATIONS_DIR="$tmp_dir" \ + CMUX_CI_REQUIRED_MACOS_SDK_MAJOR=15 \ + "$SCRIPT" +)" + +if ! grep -Fq "Selected Xcode (DEVELOPER_DIR): $old_developer (macOS SDK 15.5)" <<< "$scan_output"; then + echo "FAIL: unpinned required-SDK scan did not select the matching SDK 15 Xcode" + printf '%s\n' "$scan_output" >&2 + exit 1 +fi + +if ! grep -Fq "Skipping $pinned_app -> macOS SDK 26.2; required major is 15" <<< "$scan_output"; then + echo "FAIL: unpinned required-SDK scan did not report skipping the non-matching SDK 26 Xcode" + printf '%s\n' "$scan_output" >&2 + exit 1 +fi + +if [[ "$(cat "$env_file")" != "DEVELOPER_DIR=$old_developer" ]]; then + echo "FAIL: unpinned required-SDK scan did not export the matching developer dir" + cat "$env_file" >&2 + exit 1 +fi + missing_output="$( PATH="$bin_dir:/usr/bin:/bin" \ GITHUB_ENV="$env_file" \ From f6ac4e67073e3db253f52aee557e7bab26683e36 Mon Sep 17 00:00:00 2001 From: Aziz Albahar Date: Tue, 7 Jul 2026 23:11:25 -0700 Subject: [PATCH 08/13] Let helper Xcode selection scan by SDK --- .github/workflows/ci.yml | 2 +- tests/test_ci_change_areas.py | 1 + tests/test_ci_release_sdk_lane.sh | 5 +++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d213681eba2e..73a5dc3aa4aa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -774,7 +774,7 @@ jobs: timeout-minutes: 40 env: CMUX_CI_XCODE_APP: ${{ vars.CMUX_CI_XCODE_APP_MACOS_15 }} - CMUX_CI_HELPER_XCODE_APP: ${{ vars.CMUX_CI_HELPER_XCODE_APP_MACOS_15 || '/Applications/Xcode_16.4.app' }} + CMUX_CI_HELPER_XCODE_APP: ${{ vars.CMUX_CI_HELPER_XCODE_APP_MACOS_15 }} CMUX_CI_REQUIRED_MACOS_SDK_MAJOR: "26" steps: - name: Clear stale git locks (self-hosted reused workspace) diff --git a/tests/test_ci_change_areas.py b/tests/test_ci_change_areas.py index ea3fdce32232..87ae0dc169a3 100644 --- a/tests/test_ci_change_areas.py +++ b/tests/test_ci_change_areas.py @@ -479,6 +479,7 @@ def test_required_macos_topology_collapses_display_and_release_helper_jobs() -> assert runtime_block.rfind("scripts/ci/virtual-display-lock.sh reap-strays") < runtime_block.rfind("scripts/ci/virtual-display-lock.sh release") assert "timeout-minutes: 40" in package_block assert "CMUX_CI_HELPER_XCODE_APP" in package_block + assert "/Applications/Xcode_16.4.app" not in package_block assert "Select helper Xcode" in package_block assert "CMUX_CI_REQUIRED_MACOS_SDK_MAJOR=15" in package_block assert "Build universal Ghostty CLI helper" in package_block diff --git a/tests/test_ci_release_sdk_lane.sh b/tests/test_ci_release_sdk_lane.sh index eeca4068781a..99be8b32e0fa 100755 --- a/tests/test_ci_release_sdk_lane.sh +++ b/tests/test_ci_release_sdk_lane.sh @@ -80,6 +80,11 @@ if [[ "$swift_package_section" != *"CMUX_CI_HELPER_XCODE_APP"* ]]; then exit 1 fi +if [[ "$swift_package_section" == *"/Applications/Xcode_16.4.app"* ]]; then + echo "FAIL: CI swift-package-tests must scan for a macOS 15 SDK when the helper Xcode override is unset" >&2 + exit 1 +fi + if [[ "$swift_package_section" != *"./scripts/build-ghostty-cli-helper.sh --universal --output ghostty-cli-helper/ghostty"* ]]; then echo "FAIL: CI swift-package-tests must build the universal Ghostty CLI helper on the macOS 15 lane" >&2 exit 1 From c0e240c1bc63d4709f5298fef9ce55f192b611b2 Mon Sep 17 00:00:00 2001 From: Aziz Albahar Date: Tue, 7 Jul 2026 23:34:32 -0700 Subject: [PATCH 09/13] Run display UI regressions before lag display --- .github/workflows/ci.yml | 21 ++++++-- tests/test_ci_change_areas.py | 97 ++++++++++++++++++++++++++++++++++- 2 files changed, 112 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 73a5dc3aa4aa..ef54a12cfb1d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1036,9 +1036,21 @@ jobs: if result != "success": bad[name] = result + outputs = needs["changes"].get("outputs", {}) + routed_outputs = { + "remote-daemon-tests": "go", + "web-typecheck": "web", + "react-apps-check": "web", + "web-db-migrations": "web", + "agent-session-web-resources": "agent_session_web", + } for name in sorted(allowed_routed): result = needs[name]["result"] - if result not in {"success", "skipped"}: + route = routed_outputs[name] + if outputs.get(route) == "true": + if result != "success": + bad[name] = f"{result} (route {route}=true)" + elif result not in {"success", "skipped"}: bad[name] = result if bad: @@ -1046,7 +1058,6 @@ jobs: print(f"{name}: {result}", file=sys.stderr) sys.exit(1) - outputs = needs["changes"].get("outputs", {}) print( "routes: " f"macos={outputs.get('macos')} " @@ -1223,6 +1234,9 @@ jobs: - name: Validate Swift warning budget run: python3 scripts/swift_warning_budget.py --log /tmp/cmux-build-output.txt + - name: Run display UI regressions + run: scripts/ci/run-display-ui-regressions.sh + - name: Create virtual display run: | set -euo pipefail @@ -1390,9 +1404,6 @@ jobs: scripts/ci/virtual-display-lock.sh release || true rm -f "${VDISPLAY_HELPER_PATH:-}" "${VDISPLAY_READY:-}" "${VDISPLAY_ID_PATH:-}" "${VDISPLAY_LOG:-}" - - name: Run display UI regressions - run: scripts/ci/run-display-ui-regressions.sh - release-build: needs: - changes diff --git a/tests/test_ci_change_areas.py b/tests/test_ci_change_areas.py index 87ae0dc169a3..b11e518bc847 100644 --- a/tests/test_ci_change_areas.py +++ b/tests/test_ci_change_areas.py @@ -4,6 +4,7 @@ from __future__ import annotations import importlib.util +import json import os import subprocess import sys @@ -176,6 +177,77 @@ def workflow_job_block(job_name: str, workflow_path: Path = CI_WORKFLOW) -> str: raise AssertionError(f"{job_name} job not found") +def workflow_job_step_script(job_name: str, step_name: str, workflow_path: Path = CI_WORKFLOW) -> str: + lines = workflow_path.read_text(encoding="utf-8").splitlines() + job_marker = f" {job_name}:" + step_marker = f" - name: {step_name}" + in_job = False + for index, line in enumerate(lines): + if line == job_marker: + in_job = True + continue + if in_job and line.startswith(" ") and not line.startswith(" ") and line.strip(): + break + if in_job and line == step_marker: + for run_index in range(index + 1, len(lines)): + if lines[run_index] == " run: |": + body: list[str] = [] + for body_line in lines[run_index + 1 :]: + if body_line.startswith(" "): + body.append(body_line[10:]) + continue + if not body_line.strip(): + body.append("") + continue + break + return "\n".join(body) + break + raise AssertionError(f"{step_name} run block not found in {job_name}") + + +def run_linux_preflight(needs: dict[str, object]) -> subprocess.CompletedProcess[str]: + script = workflow_job_step_script("linux-preflight", "Check cheap CI layer before macOS runners") + env = {**os.environ, "PREFLIGHT_NEEDS": json.dumps(needs)} + return subprocess.run( + ["bash", "-c", script], + cwd=ROOT, + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + +def linux_preflight_needs( + *, + outputs: dict[str, str] | None = None, + results: dict[str, str] | None = None, +) -> dict[str, object]: + route_outputs = { + "macos": "true", + "web": "true", + "go": "true", + "agent_session_web": "true", + } + if outputs: + route_outputs.update(outputs) + job_results = { + "changes": "success", + "workflow-guard-tests": "success", + "remote-daemon-tests": "success", + "web-typecheck": "success", + "react-apps-check": "success", + "web-db-migrations": "success", + "agent-session-web-resources": "success", + } + if results: + job_results.update(results) + return { + name: {"result": result, "outputs": route_outputs if name == "changes" else {}} + for name, result in job_results.items() + } + + def run_detect_step_for_paths( paths: list[str], workflow_path: Path = CI_WORKFLOW, @@ -445,7 +517,29 @@ def test_linux_preflight_blocks_macos_on_cheap_layer_failure() -> None: assert "if: ${{ always() }}" in block assert 'required = ("changes", "workflow-guard-tests")' in block assert 'allowed_routed = {' in block - assert 'result not in {"success", "skipped"}' in block + assert 'routed_outputs = {' in block + assert 'bad[name] = f"{result} (route {route}=true)"' in block + + +def test_linux_preflight_fails_when_routed_job_skips() -> None: + result = run_linux_preflight( + linux_preflight_needs(results={"remote-daemon-tests": "skipped"}) + ) + + assert result.returncode != 0 + assert "remote-daemon-tests: skipped (route go=true)" in result.stderr + + +def test_linux_preflight_allows_unrouted_job_skip() -> None: + result = run_linux_preflight( + linux_preflight_needs( + outputs={"go": "false"}, + results={"remote-daemon-tests": "skipped"}, + ) + ) + + assert result.returncode == 0, result.stderr + assert "remote-daemon-tests: skipped" in result.stdout def test_macos_jobs_use_lane_specific_xcode_pin_vars() -> None: @@ -474,6 +568,7 @@ def test_required_macos_topology_collapses_display_and_release_helper_jobs() -> assert "build-for-testing" in runtime_block assert "Run display UI regressions" in runtime_block assert "scripts/ci/run-display-ui-regressions.sh" in runtime_block + assert runtime_block.index("Run display UI regressions") < runtime_block.index("Create virtual display") assert 'kill -9 "$VDISPLAY_PID"' in runtime_block assert "scripts/ci/virtual-display-lock.sh reap-strays" in runtime_block assert runtime_block.rfind("scripts/ci/virtual-display-lock.sh reap-strays") < runtime_block.rfind("scripts/ci/virtual-display-lock.sh release") From f02dda98bcae452100561106d606a1d4c7049f02 Mon Sep 17 00:00:00 2001 From: Aziz Albahar Date: Wed, 8 Jul 2026 00:07:45 -0700 Subject: [PATCH 10/13] Target persistent display in browser UI regression --- scripts/ci/run-display-ui-regressions.sh | 9 ++++++++- tests/test_ci_self_hosted_guard.sh | 11 +++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/scripts/ci/run-display-ui-regressions.sh b/scripts/ci/run-display-ui-regressions.sh index 3f4135ac0a54..e36a895a0ff3 100755 --- a/scripts/ci/run-display-ui-regressions.sh +++ b/scripts/ci/run-display-ui-regressions.sh @@ -342,13 +342,20 @@ create_persistent_display() { } run_browser_find_focus() { + local persistent_display_id if [ -n "${PERSISTENT_PID:-}" ] && ! kill -0 "$PERSISTENT_PID" 2>/dev/null; then echo "Persistent virtual display exited before browser find UI regression" >&2 cat "${PERSISTENT_LOG:-/dev/null}" >&2 || true exit 1 fi + if [ ! -s "${PERSISTENT_ID_PATH:-}" ]; then + echo "Persistent virtual display ID missing before browser find UI regression" >&2 + exit 1 + fi + persistent_display_id="$(tr -d '\n' < "$PERSISTENT_ID_PATH")" - xcodebuild -project cmux.xcodeproj -scheme cmux -configuration Debug \ + CMUX_UI_TEST_TARGET_DISPLAY_ID="$persistent_display_id" \ + xcodebuild -project cmux.xcodeproj -scheme cmux -configuration Debug \ -derivedDataPath "$CMUX_DERIVED_DATA_PATH" \ -clonedSourcePackagesDirPath "$SOURCE_PACKAGES_DIR" \ -disableAutomaticPackageResolution \ diff --git a/tests/test_ci_self_hosted_guard.sh b/tests/test_ci_self_hosted_guard.sh index e047d441f8dc..ab618fedb186 100755 --- a/tests/test_ci_self_hosted_guard.sh +++ b/tests/test_ci_self_hosted_guard.sh @@ -294,6 +294,17 @@ check_runtime_regressions_collapsed() { exit 1 fi + if ! awk ' + /^run_browser_find_focus\(\) \{/ { in_func=1; next } + in_func && /^}/ { in_func=0 } + in_func && /persistent_display_id="\$\(tr -d/ { saw_display_id_read=1 } + in_func && /CMUX_UI_TEST_TARGET_DISPLAY_ID="\$persistent_display_id"/ { saw_display_env=1 } + END { exit !(saw_display_id_read && saw_display_env) } + ' "$ROOT_DIR/scripts/ci/run-display-ui-regressions.sh"; then + echo "FAIL: browser-find UI regression must target the persistent virtual display" + exit 1 + fi + echo "PASS: runtime display regressions are collapsed into tests-build-and-lag" } From cc69679cf77650d152a5ec0fb5006d271e7fe0cb Mon Sep 17 00:00:00 2001 From: Aziz Albahar Date: Wed, 8 Jul 2026 00:26:50 -0700 Subject: [PATCH 11/13] Forward browser UI test display target --- cmuxUITests/BrowserPaneNavigationKeybindUITests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmuxUITests/BrowserPaneNavigationKeybindUITests.swift b/cmuxUITests/BrowserPaneNavigationKeybindUITests.swift index 1ce337ed62b7..3f0b13b786e6 100644 --- a/cmuxUITests/BrowserPaneNavigationKeybindUITests.swift +++ b/cmuxUITests/BrowserPaneNavigationKeybindUITests.swift @@ -1316,7 +1316,6 @@ final class BrowserPaneNavigationKeybindUITests: XCTestCase { "Expected command palette to dismiss after creating a workspace" ) } - private func focusLeftPaneForFindScenario(_ app: XCUIApplication, route: FindFocusRoute) { switch route { case .cmdOptionArrows: @@ -1370,6 +1369,7 @@ final class BrowserPaneNavigationKeybindUITests: XCTestCase { app.launchEnvironment["CMUX_UI_TEST_SOCKET_SANITY"] = "1" app.launchEnvironment["CMUX_UI_TEST_DIAGNOSTICS_PATH"] = diagnosticsPath app.launchEnvironment["CMUX_TAG"] = launchTag + app.launchEnvironment["CMUX_UI_TEST_TARGET_DISPLAY_ID"] = ProcessInfo.processInfo.environment["CMUX_UI_TEST_TARGET_DISPLAY_ID"] } private func socketReadinessFailureMessage() -> String { From 6f9d8f43e492bf0a1014f1069f79063e49546678 Mon Sep 17 00:00:00 2001 From: Aziz Albahar Date: Wed, 8 Jul 2026 00:39:19 -0700 Subject: [PATCH 12/13] Clean display churn helper binary --- scripts/ci/run-display-ui-regressions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci/run-display-ui-regressions.sh b/scripts/ci/run-display-ui-regressions.sh index e36a895a0ff3..a46e80ddf8b3 100755 --- a/scripts/ci/run-display-ui-regressions.sh +++ b/scripts/ci/run-display-ui-regressions.sh @@ -50,7 +50,7 @@ cleanup_display_churn() { DRC_DISPLAY_LOCK_DIR="" DRC_DISPLAY_LOCK_TOKEN="" pkill -x "cmux DEV" 2>/dev/null || true - rm -f "$DRC_DIAG_PATH" "$DRC_DISPLAY_READY" "$DRC_DISPLAY_ID_PATH" "$DRC_DISPLAY_START" "$DRC_DISPLAY_DONE" "$DRC_HELPER_LOG" "$DRC_XCODEBUILD_LOG" + rm -f "$DRC_HELPER_PATH" "$DRC_DIAG_PATH" "$DRC_DISPLAY_READY" "$DRC_DISPLAY_ID_PATH" "$DRC_DISPLAY_START" "$DRC_DISPLAY_DONE" "$DRC_HELPER_LOG" "$DRC_XCODEBUILD_LOG" rm -f /tmp/cmux-ui-test-prelaunch.json /tmp/cmux-ui-test-display-harness.json } From 652eb8465a31d9a67804e38bdca1cfc7cdbe8d9f Mon Sep 17 00:00:00 2001 From: Aziz Albahar Date: Wed, 8 Jul 2026 00:48:08 -0700 Subject: [PATCH 13/13] Clean display helper on final trap --- scripts/ci/run-display-ui-regressions.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/ci/run-display-ui-regressions.sh b/scripts/ci/run-display-ui-regressions.sh index a46e80ddf8b3..85d9812ff718 100755 --- a/scripts/ci/run-display-ui-regressions.sh +++ b/scripts/ci/run-display-ui-regressions.sh @@ -50,7 +50,7 @@ cleanup_display_churn() { DRC_DISPLAY_LOCK_DIR="" DRC_DISPLAY_LOCK_TOKEN="" pkill -x "cmux DEV" 2>/dev/null || true - rm -f "$DRC_HELPER_PATH" "$DRC_DIAG_PATH" "$DRC_DISPLAY_READY" "$DRC_DISPLAY_ID_PATH" "$DRC_DISPLAY_START" "$DRC_DISPLAY_DONE" "$DRC_HELPER_LOG" "$DRC_XCODEBUILD_LOG" + rm -f "$DRC_DIAG_PATH" "$DRC_DISPLAY_READY" "$DRC_DISPLAY_ID_PATH" "$DRC_DISPLAY_START" "$DRC_DISPLAY_DONE" "$DRC_HELPER_LOG" "$DRC_XCODEBUILD_LOG" rm -f /tmp/cmux-ui-test-prelaunch.json /tmp/cmux-ui-test-display-harness.json } @@ -72,6 +72,7 @@ cleanup_persistent_display() { cleanup_all() { cleanup_display_churn cleanup_persistent_display + rm -f "$DRC_HELPER_PATH" } trap cleanup_all EXIT