diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 73038ae44e69..ef54a12cfb1d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -159,6 +159,9 @@ jobs: - name: Validate nightly Xcode selection run: ./tests/test_ci_nightly_xcode_selection.sh + - name: Validate CI Xcode selection fast path + run: ./tests/test_ci_xcode_selection_fast_path.sh + - name: Validate universal nightly workflow run: bash ./tests/test_nightly_universal_build.sh @@ -379,7 +382,9 @@ jobs: run: bun run test:db:behavior app-host-unit-tests: - needs: changes + needs: + - changes + - linux-preflight if: ${{ needs.changes.outputs.macos == 'true' }} name: app-host unit tests (${{ matrix.shard }}/4) # App-host XCTest needs a runner that can broker testmanagerd control @@ -393,6 +398,8 @@ jobs: matrix: shard: [1, 2, 3, 4] env: + CMUX_CI_XCODE_APP: ${{ vars.CMUX_CI_XCODE_APP_MACOS_15 }} + CMUX_CI_REQUIRED_MACOS_SDK_MAJOR: "26" CMUX_SKIP_ZIG_BUILD: "1" # Keep one-off focused gates on the lightest measured shard so shard 1 no # longer carries the full shard plus every extra regression guard. @@ -702,6 +709,7 @@ jobs: # that stranded the old "tests" check). Add new ci.yml test/build jobs here. needs: - changes + - linux-preflight - app-host-unit-tests - swift-package-tests - agent-session-web-resources @@ -727,6 +735,11 @@ jobs: print(f"changes: {changes['result']}", file=sys.stderr) sys.exit(1) + preflight = needs["linux-preflight"] + if preflight["result"] != "success": + print(f"linux preflight did not pass: {preflight['result']}", file=sys.stderr) + sys.exit(1) + if macos == "true" and tests["result"] != "success": print(f"app-host unit tests were required but did not pass: {tests['result']}", file=sys.stderr) sys.exit(1) @@ -746,16 +759,23 @@ jobs: sys.exit(1) print(f"changes.macos={macos}") + print(f"linux-preflight={preflight['result']}") print(f"app-host unit tests={tests['result']}") for name in ("swift-package-tests", "agent-session-web-resources"): print(f"{name}={needs[name]['result']}") PY swift-package-tests: - needs: changes + needs: + - changes + - linux-preflight if: ${{ needs.changes.outputs.macos == 'true' }} runs-on: ${{ vars.MACOS_RUNNER_15 || 'warp-macos-15-arm64-6x' }} - timeout-minutes: 20 + timeout-minutes: 40 + env: + CMUX_CI_XCODE_APP: ${{ vars.CMUX_CI_XCODE_APP_MACOS_15 }} + CMUX_CI_HELPER_XCODE_APP: ${{ vars.CMUX_CI_HELPER_XCODE_APP_MACOS_15 }} + CMUX_CI_REQUIRED_MACOS_SDK_MAJOR: "26" steps: - name: Clear stale git locks (self-hosted reused workspace) shell: bash @@ -774,6 +794,55 @@ jobs: with: submodules: recursive + - name: Select helper Xcode + run: | + set -euo pipefail + CMUX_CI_XCODE_APP="$CMUX_CI_HELPER_XCODE_APP" \ + CMUX_CI_REQUIRED_MACOS_SDK_MAJOR=15 \ + ./scripts/select-ci-xcode.sh + + - name: Install zig + run: ./scripts/install-zig-ci.sh + + - name: Cache Zig packages + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: ~/.cache/zig + key: zig-packages-${{ hashFiles('ghostty/build.zig.zon', 'ghostty/build.zig.zon.json') }} + restore-keys: zig-packages- + + - name: Build universal Ghostty CLI helper + run: | + set -euo pipefail + mkdir -p ghostty-cli-helper + ./scripts/build-ghostty-cli-helper.sh --universal --output ghostty-cli-helper/ghostty + lipo ghostty-cli-helper/ghostty -verify_arch arm64 x86_64 + for arch in arm64 x86_64; do + thin="ghostty-cli-helper/ghostty-$arch" + lipo ghostty-cli-helper/ghostty -thin "$arch" -output "$thin" + HELPER_SDK_VERSION="$(otool -l "$thin" | awk '/LC_BUILD_VERSION/ { in_version=1; next } in_version && /sdk / { print $2; exit }')" + echo "Ghostty helper $arch SDK version: $HELPER_SDK_VERSION" + [[ "$HELPER_SDK_VERSION" == 15.* ]] + done + + - name: Upload universal Ghostty CLI helper + id: upload-ghostty-cli-helper + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: cmux-ghostty-cli-helper + path: ghostty-cli-helper/ghostty + if-no-files-found: error + + - name: Retry universal Ghostty CLI helper upload + if: steps.upload-ghostty-cli-helper.outcome == 'failure' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: cmux-ghostty-cli-helper + path: ghostty-cli-helper/ghostty + if-no-files-found: error + overwrite: true + - name: Select Xcode run: | set -euo pipefail @@ -928,20 +997,96 @@ jobs: bun run agent-session-web:test git diff --exit-code -- Resources/agent-session-react Resources/agent-session-solid + linux-preflight: + name: linux-preflight + needs: + - changes + - workflow-guard-tests + - remote-daemon-tests + - web-typecheck + - react-apps-check + - web-db-migrations + - agent-session-web-resources + if: ${{ always() }} + runs-on: ${{ vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }} + timeout-minutes: 5 + steps: + - name: Check cheap CI layer before macOS runners + env: + PREFLIGHT_NEEDS: ${{ toJSON(needs) }} + run: | + python3 - <<'PY' + import json + import os + import sys + + needs = json.loads(os.environ["PREFLIGHT_NEEDS"]) + required = ("changes", "workflow-guard-tests") + allowed_routed = { + "remote-daemon-tests", + "web-typecheck", + "react-apps-check", + "web-db-migrations", + "agent-session-web-resources", + } + + bad = {} + for name in required: + result = needs[name]["result"] + if result != "success": + bad[name] = result + + outputs = needs["changes"].get("outputs", {}) + routed_outputs = { + "remote-daemon-tests": "go", + "web-typecheck": "web", + "react-apps-check": "web", + "web-db-migrations": "web", + "agent-session-web-resources": "agent_session_web", + } + for name in sorted(allowed_routed): + result = needs[name]["result"] + route = routed_outputs[name] + if outputs.get(route) == "true": + if result != "success": + bad[name] = f"{result} (route {route}=true)" + elif result not in {"success", "skipped"}: + bad[name] = result + + if bad: + for name, result in bad.items(): + print(f"{name}: {result}", file=sys.stderr) + sys.exit(1) + + print( + "routes: " + f"macos={outputs.get('macos')} " + f"web={outputs.get('web')} " + f"go={outputs.get('go')} " + f"agent_session_web={outputs.get('agent_session_web')}" + ) + for name, data in sorted(needs.items()): + print(f"{name}: {data['result']}") + PY + tests-build-and-lag: - needs: changes + needs: + - changes + - linux-preflight if: ${{ needs.changes.outputs.macos == 'true' }} - # Build the full cmux scheme and run the lag regression on macOS CI. - # Keep lag validation separate from UI regressions so functional UI failures - # and performance regressions stay isolated. Broader interactive UI suites - # still run via test-e2e.yml on GitHub-hosted runners. + # Build the full cmux scheme once, then run the required display/runtime + # regressions from the same DerivedData instead of queuing a second display + # runner for UI-only checks. runs-on: ${{ vars.MACOS_RUNNER_DISPLAY || 'warp-macos-15-arm64-6x' }} # A cold DerivedData cache (any project.pbxproj or Package.resolved change # mints a new cache key with no restore-keys fallback) forces a full # cmux build whose Swift codegen alone can run 20+ min. Project/package # changes from the sidebar extension kit pushed this full build plus the - # CA and lag regressions beyond 35 min before the cache could repopulate. - timeout-minutes: 55 + # CA, lag, and UI regressions beyond 35 min before the cache could repopulate. + timeout-minutes: 75 + env: + CMUX_CI_XCODE_APP: ${{ vars.CMUX_CI_XCODE_APP_MACOS_15 }} + CMUX_CI_REQUIRED_MACOS_SDK_MAJOR: "26" steps: - name: Validate display runner identity env: @@ -1076,7 +1221,7 @@ jobs: sleep $((attempt * 5)) done - - name: Build app + - name: Build for runtime regressions run: | set -euo pipefail SOURCE_PACKAGES_DIR="$PWD/.ci-source-packages" @@ -1084,11 +1229,14 @@ jobs: -derivedDataPath "$CMUX_DERIVED_DATA_PATH" \ -clonedSourcePackagesDirPath "$SOURCE_PACKAGES_DIR" \ -disableAutomaticPackageResolution \ - -destination "platform=macOS" build 2>&1 | tee /tmp/cmux-build-output.txt + -destination "platform=macOS" build-for-testing 2>&1 | tee /tmp/cmux-build-output.txt - name: Validate Swift warning budget run: python3 scripts/swift_warning_budget.py --log /tmp/cmux-build-output.txt + - name: Run display UI regressions + run: scripts/ci/run-display-ui-regressions.sh + - name: Create virtual display run: | set -euo pipefail @@ -1108,8 +1256,13 @@ jobs: kill -0 "$VDISPLAY_PID" >/dev/null 2>&1 || break sleep 0.1 done + if kill -0 "$VDISPLAY_PID" >/dev/null 2>&1; then + kill -9 "$VDISPLAY_PID" >/dev/null 2>&1 || true + wait "$VDISPLAY_PID" >/dev/null 2>&1 || true + fi VDISPLAY_PID="" fi + scripts/ci/virtual-display-lock.sh reap-strays || true scripts/ci/virtual-display-lock.sh release || true } @@ -1242,72 +1395,20 @@ jobs: kill -0 "$VDISPLAY_PID" >/dev/null 2>&1 || break sleep 0.1 done + if kill -0 "$VDISPLAY_PID" >/dev/null 2>&1; then + kill -9 "$VDISPLAY_PID" >/dev/null 2>&1 || true + wait "$VDISPLAY_PID" >/dev/null 2>&1 || true + fi fi + scripts/ci/virtual-display-lock.sh reap-strays || true scripts/ci/virtual-display-lock.sh release || true rm -f "${VDISPLAY_HELPER_PATH:-}" "${VDISPLAY_READY:-}" "${VDISPLAY_ID_PATH:-}" "${VDISPLAY_LOG:-}" - release-ghostty-cli-helper: - needs: changes - if: ${{ needs.changes.outputs.macos == 'true' }} - runs-on: ${{ vars.MACOS_RUNNER_15 || 'warp-macos-15-arm64-6x' }} - timeout-minutes: 20 - steps: - - name: Clear stale git locks (self-hosted reused workspace) - shell: bash - run: | - # Self-hosted macOS runners reuse the workspace. A job cancelled or - # killed mid-checkout can leave a stale .git/modules/*/index.lock that - # fails every later submodule checkout (e.g. ghostty). Clear them first. - ws="${GITHUB_WORKSPACE:-$PWD}" - rm -f "$ws/.git/index.lock" 2>/dev/null || true - if [ -d "$ws/.git/modules" ]; then - find "$ws/.git/modules" -type f -name "*.lock" -delete 2>/dev/null || true - fi - - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - submodules: recursive - - - name: Install zig - run: ./scripts/install-zig-ci.sh - - - name: Cache Zig packages - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 - with: - path: ~/.cache/zig - key: zig-packages-${{ hashFiles('ghostty/build.zig.zon', 'ghostty/build.zig.zon.json') }} - restore-keys: zig-packages- - - - name: Build universal Ghostty CLI helper - run: | - set -euo pipefail - mkdir -p ghostty-cli-helper - ./scripts/build-ghostty-cli-helper.sh --universal --output ghostty-cli-helper/ghostty - lipo ghostty-cli-helper/ghostty -verify_arch arm64 x86_64 - - - name: Upload universal Ghostty CLI helper - id: upload-ghostty-cli-helper - continue-on-error: true - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: cmux-ghostty-cli-helper - path: ghostty-cli-helper/ghostty - if-no-files-found: error - - - name: Retry universal Ghostty CLI helper upload - if: steps.upload-ghostty-cli-helper.outcome == 'failure' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: cmux-ghostty-cli-helper - path: ghostty-cli-helper/ghostty - if-no-files-found: error - overwrite: true - release-build: needs: - changes - - release-ghostty-cli-helper + - linux-preflight + - swift-package-tests if: ${{ needs.changes.outputs.macos == 'true' }} # Compile the same unsigned universal Release app that nightly builds before # signing, notarization, and publishing. This catches DEBUG/Release boundary @@ -1318,7 +1419,10 @@ jobs: # restored SwiftPM cache. Default to a clean paid macOS 26 runner instead # of the generic persistent macOS 26 pool. runs-on: ${{ vars.MACOS_RUNNER_26_RELEASE || 'blacksmith-6vcpu-macos-26' }} - timeout-minutes: 45 + timeout-minutes: 60 + env: + CMUX_CI_XCODE_APP: ${{ vars.CMUX_CI_XCODE_APP_MACOS_26 }} + CMUX_CI_REQUIRED_MACOS_SDK_MAJOR: "26" steps: - name: Clear stale git locks (self-hosted reused workspace) shell: bash @@ -1430,502 +1534,6 @@ jobs: lipo "$HELPER_BINARY" -verify_arch arm64 x86_64 [[ "$SDK_VERSION" == 26.* ]] - ui-regressions: - needs: changes - if: ${{ needs.changes.outputs.macos == 'true' }} - runs-on: ${{ vars.MACOS_RUNNER_DISPLAY || 'warp-macos-15-arm64-6x' }} - # Cold builds after project/package changes can spend more than 25 minutes - # in build-for-testing before the UI regression script starts. - timeout-minutes: 45 - steps: - - name: Validate display runner identity - env: - REQUESTED_RUNNER: ${{ vars.MACOS_RUNNER_DISPLAY || 'warp-macos-15-arm64-6x' }} - RUNNER_CONTEXT_NAME: ${{ runner.name }} - run: | - set -euo pipefail - echo "Requested runner: $REQUESTED_RUNNER" - case "$REQUESTED_RUNNER" in - depot-*) - case "$RUNNER_CONTEXT_NAME" in - depot-*) - echo "Resolved runner matches depot-*? yes" - ;; - *) - echo "Resolved runner matches depot-*? no" - echo "::error::$REQUESTED_RUNNER resolved outside Depot. Remove $REQUESTED_RUNNER from non-Depot self-hosted runners or choose a different runner." - exit 1 - ;; - esac - ;; - *) - echo "Display runner is not Depot; skipping Depot identity guard" - ;; - esac - - - name: Clear stale git locks (self-hosted reused workspace) - shell: bash - run: | - # Self-hosted macOS runners reuse the workspace. A job cancelled or - # killed mid-checkout can leave a stale .git/modules/*/index.lock that - # fails every later submodule checkout (e.g. ghostty). Clear them first. - ws="${GITHUB_WORKSPACE:-$PWD}" - rm -f "$ws/.git/index.lock" 2>/dev/null || true - if [ -d "$ws/.git/modules" ]; then - find "$ws/.git/modules" -type f -name "*.lock" -delete 2>/dev/null || true - fi - - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - submodules: recursive - - - name: Select Xcode - run: | - set -euo pipefail - ./scripts/select-ci-xcode.sh - - - name: Prepare isolated DerivedData - run: | - set -euo pipefail - DERIVED_DATA_PATH="$RUNNER_TEMP/cmux-deriveddata-ui-regressions-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" - rm -rf "$DERIVED_DATA_PATH" - echo "CMUX_DERIVED_DATA_PATH=$DERIVED_DATA_PATH" >> "$GITHUB_ENV" - - - name: Download pre-built GhosttyKit.xcframework - run: ./scripts/download-prebuilt-ghosttykit.sh - - - name: Install zig - run: | - ./scripts/install-zig-ci.sh - - - name: Install Rust - run: | - ./scripts/install-rust-ci.sh - - - name: Cache Zig packages - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 - with: - path: ~/.cache/zig - key: zig-packages-${{ hashFiles('ghostty/build.zig.zon', 'ghostty/build.zig.zon.json') }} - restore-keys: zig-packages- - - - name: Cache Swift packages - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 - with: - path: .ci-source-packages - key: spm-ui-regressions-${{ hashFiles('cmux.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved') }} - restore-keys: spm-ui-regressions- - - - name: Sanitize Swift package cache - run: python3 scripts/ci/sanitize-xcode-source-packages-cache.py .ci-source-packages - - - name: Resolve Swift packages - run: | - set -euo pipefail - SOURCE_PACKAGES_DIR="$PWD/.ci-source-packages" - rm -rf "$CMUX_DERIVED_DATA_PATH" - mkdir -p "$SOURCE_PACKAGES_DIR" - - for attempt in 1 2 3; do - if xcodebuild -project cmux.xcodeproj -scheme cmux -configuration Debug \ - -derivedDataPath "$CMUX_DERIVED_DATA_PATH" \ - -clonedSourcePackagesDirPath "$SOURCE_PACKAGES_DIR" \ - -resolvePackageDependencies; then - # Guard against a stale/poisoned Swift-package cache: a restored - # .ci-source-packages can make resolve report success without the - # binary artifacts (Sparkle/Sentry) actually present, which then - # fails the build. Verify they materialized; if not, clear and retry. - if [ -d "$SOURCE_PACKAGES_DIR/artifacts/sparkle/Sparkle/Sparkle.xcframework" ] && [ -d "$SOURCE_PACKAGES_DIR/artifacts/sentry-cocoa/Sentry/Sentry.xcframework" ]; then - exit 0 - fi - echo "Resolve succeeded but binary artifacts are missing (stale cache); clearing and retrying" >&2 - rm -rf "$SOURCE_PACKAGES_DIR" # whole dir — resolve will not re-materialize artifacts into a partial tree - fi - if [ "$attempt" -eq 3 ]; then - echo "Failed to resolve Swift packages after 3 attempts" >&2 - exit 1 - fi - echo "Package resolution failed on attempt $attempt, retrying..." - sleep $((attempt * 5)) - done - - - name: Build for testing (display resolution) - run: | - set -euo pipefail - SOURCE_PACKAGES_DIR="$PWD/.ci-source-packages" - xcodebuild -project cmux.xcodeproj -scheme cmux -configuration Debug \ - -derivedDataPath "$CMUX_DERIVED_DATA_PATH" \ - -clonedSourcePackagesDirPath "$SOURCE_PACKAGES_DIR" \ - -disableAutomaticPackageResolution \ - -destination "platform=macOS" \ - build-for-testing - - - name: Enable XCTest automation mode - run: | - set -euo pipefail - if ! command -v automationmodetool >/dev/null 2>&1; then - echo "::warning::automationmodetool is unavailable; XCTest will use its default automation-mode setup" - exit 0 - fi - - if sudo -n true 2>/dev/null; then - sudo -n automationmodetool enable-automationmode-without-authentication - else - echo "::warning::Passwordless sudo unavailable; XCTest will use its default automation-mode setup" - fi - - - name: Run display resolution churn UI regression - run: | - set -euo pipefail - SOURCE_PACKAGES_DIR="$PWD/.ci-source-packages" - TOKEN="$(uuidgen)" - HELPER_PATH="$RUNNER_TEMP/create-virtual-display-display-churn-${TOKEN}" - DIAG_PATH="/tmp/cmux-ui-test-display-churn-${TOKEN}.json" - DISPLAY_READY="/tmp/cmux-ui-test-display-${TOKEN}.ready" - DISPLAY_ID_PATH="/tmp/cmux-ui-test-display-${TOKEN}.id" - DISPLAY_START="/tmp/cmux-ui-test-display-${TOKEN}.start" - DISPLAY_DONE="/tmp/cmux-ui-test-display-${TOKEN}.done" - HELPER_LOG="/tmp/cmux-ui-test-display-${TOKEN}-helper.log" - XCODEBUILD_LOG="/tmp/cmux-ui-test-display-${TOKEN}-xcodebuild.log" - BASELINE_READY_MARKER="CMUX_DISPLAY_CHURN_BASELINE_READY_${TOKEN}" - HELPER_PID="" - START_SIGNAL_PID="" - DISPLAY_LOCK_DIR="" - DISPLAY_LOCK_TOKEN="" - - acquire_display_lock() { - LOCK_ENV="$(scripts/ci/virtual-display-lock.sh acquire)" - eval "$LOCK_ENV" - export CMUX_VDISPLAY_LOCK_DIR CMUX_VDISPLAY_LOCK_TOKEN - DISPLAY_LOCK_DIR="$CMUX_VDISPLAY_LOCK_DIR" - DISPLAY_LOCK_TOKEN="$CMUX_VDISPLAY_LOCK_TOKEN" - } - - release_display_lock() { - if [ -n "${DISPLAY_LOCK_DIR:-}" ]; then - CMUX_VDISPLAY_LOCK_DIR="$DISPLAY_LOCK_DIR" \ - CMUX_VDISPLAY_LOCK_TOKEN="$DISPLAY_LOCK_TOKEN" \ - scripts/ci/virtual-display-lock.sh release || true - DISPLAY_LOCK_DIR="" - DISPLAY_LOCK_TOKEN="" - fi - } - - cleanup_attempt() { - if [ -n "${START_SIGNAL_PID:-}" ]; then - kill "$START_SIGNAL_PID" 2>/dev/null || true - wait "$START_SIGNAL_PID" 2>/dev/null || true - START_SIGNAL_PID="" - fi - if [ -n "${HELPER_PID:-}" ]; then - kill "$HELPER_PID" 2>/dev/null || true - wait "$HELPER_PID" 2>/dev/null || true - HELPER_PID="" - fi - release_display_lock - pkill -x "cmux DEV" 2>/dev/null || true - rm -f "$DIAG_PATH" "$DISPLAY_READY" "$DISPLAY_ID_PATH" "$DISPLAY_START" "$DISPLAY_DONE" "$HELPER_LOG" "$XCODEBUILD_LOG" - rm -f /tmp/cmux-ui-test-prelaunch.json /tmp/cmux-ui-test-display-harness.json - } - - cleanup() { - cleanup_attempt - rm -f "$HELPER_PATH" - } - trap cleanup EXIT - - # Build display helper - clang -framework Foundation -framework CoreGraphics \ - -o "$HELPER_PATH" scripts/create-virtual-display.m - - # Find the app binary - APP_BINARY=$(find "$CMUX_DERIVED_DATA_PATH" -path "*/Build/Products/Debug/cmux DEV.app/Contents/MacOS/cmux DEV" -print -quit 2>/dev/null || true) - if [ -z "$APP_BINARY" ]; then - echo "ERROR: App binary not found in DerivedData" >&2 - exit 1 - fi - echo "App binary: $APP_BINARY" - - for attempt in 1 2; do - cleanup_attempt 2>/dev/null || true - - acquire_display_lock - - # Reap any leaked display helper now that we hold the lock, so a - # CGVirtualDisplay orphaned by a crashed/cancelled job cannot block - # this create on persistent self-hosted runners. - scripts/ci/virtual-display-lock.sh reap-strays || true - - # Launch display helper from shell (non-sandboxed). The helper waits - # on a shell-owned /tmp start file; XCTest prints a baseline-ready - # marker after sampling render stats, then the shell writes startPath. - "$HELPER_PATH" \ - --modes "1920x1080,1728x1117,1600x900,1440x810" \ - --ready-path "$DISPLAY_READY" \ - --display-id-path "$DISPLAY_ID_PATH" \ - --start-path "$DISPLAY_START" \ - --done-path "$DISPLAY_DONE" \ - --iterations 40 \ - --interval-ms 40 \ - > "$HELPER_LOG" 2>&1 & - HELPER_PID=$! - scripts/ci/virtual-display-lock.sh set-owner "$HELPER_PID" - - # Wait for display ready - echo "Waiting for virtual display..." - DISPLAY_READY_OK=false - for _ in $(seq 1 100); do - if [ -s "$DISPLAY_READY" ] && [ -s "$DISPLAY_ID_PATH" ]; then - DISPLAY_READY_OK=true - break - fi - if ! kill -0 "$HELPER_PID" 2>/dev/null; then - echo "ERROR: Virtual display helper exited before readiness" >&2 - cat "$HELPER_LOG" 2>/dev/null || true - break - fi - sleep 0.1 - done - if [ "$DISPLAY_READY_OK" != "true" ]; then - echo "ERROR: Virtual display not ready after 10s" >&2 - cat "$HELPER_LOG" 2>/dev/null || true - cleanup_attempt - if [ "$attempt" -eq 2 ]; then - echo "Display resolution UI regression failed after 2 virtual display setup attempts" >&2 - exit 1 - fi - sleep 3 - continue - fi - DISPLAY_ID=$(tr -d '\n' < "$DISPLAY_ID_PATH") - echo "Virtual display ready: ID=$DISPLAY_ID" - - # Launch app from shell (non-sandboxed, outside XCTest sandbox) - CMUX_UI_TEST_MODE=1 \ - CMUX_UI_TEST_DIAGNOSTICS_PATH="$DIAG_PATH" \ - CMUX_UI_TEST_DISPLAY_RENDER_STATS=1 \ - CMUX_UI_TEST_TARGET_DISPLAY_ID="$DISPLAY_ID" \ - CMUX_TAG="ui-tests-display-resolution" \ - "$APP_BINARY" > /tmp/cmux-ui-test-app.log 2>&1 & - APP_PID=$! - echo "App launched: PID=$APP_PID" - - # Wait for app diagnostics - echo "Waiting for app diagnostics..." - APP_READY=false - for i in $(seq 1 30); do - if [ -f "$DIAG_PATH" ]; then - if python3 -c "import json; d=json.load(open('$DIAG_PATH')); assert d.get('pid')" 2>/dev/null; then - APP_READY=true - break - fi - fi - if ! kill -0 "$APP_PID" 2>/dev/null; then - echo "ERROR: App crashed during startup" - cat /tmp/cmux-ui-test-app.log 2>/dev/null | tail -30 || true - break - fi - sleep 0.5 - done - - if [ "$APP_READY" != "true" ]; then - echo "Attempt $attempt: App not ready after 15s" - pkill -x "cmux DEV" 2>/dev/null || true - kill "$HELPER_PID" 2>/dev/null || true - if [ "$attempt" -eq 2 ]; then - echo "Display resolution UI regression failed after 2 attempts" >&2 - echo "--- App log ---" - cat /tmp/cmux-ui-test-app.log 2>/dev/null | tail -50 || true - echo "--- Helper log ---" - cat "$HELPER_LOG" 2>/dev/null | tail -20 || true - echo "--- Diagnostics ---" - cat "$DIAG_PATH" 2>/dev/null || echo "(not found)" - exit 1 - fi - sleep 3 - continue - fi - - echo "App started. Diagnostics:" - cat "$DIAG_PATH" - - # Wait for render stats (terminal surface initialization) - echo "Waiting for render stats..." - RENDER_READY=false - for i in $(seq 1 40); do - if python3 -c "import json; d=json.load(open('$DIAG_PATH')); assert d.get('renderStatsAvailable') == '1'" 2>/dev/null; then - RENDER_READY=true - echo "Render stats available after $((i / 2))s" - break - fi - sleep 0.5 - done - if [ "$RENDER_READY" != "true" ]; then - echo "WARNING: Render stats not available after 20s. Diagnostics:" - cat "$DIAG_PATH" 2>/dev/null || true - echo "--- App log ---" - cat /tmp/cmux-ui-test-app.log 2>/dev/null | tail -30 || true - fi - - # Write manifests so test can find the pre-launched state - MANIFEST_PATH="/tmp/cmux-ui-test-display-harness.json" - cat >"$MANIFEST_PATH" <"$PRELAUNCH_PATH" </dev/null; then - echo "XCTest baseline marker observed; starting display churn" - printf 'start\n' > "$DISPLAY_START" - exit 0 - fi - sleep 0.25 - done - echo "ERROR: XCTest baseline marker not observed before display-churn start timeout" >&2 - echo "--- xcodebuild log tail ---" >&2 - tail -80 "$XCODEBUILD_LOG" >&2 2>/dev/null || true - echo "--- diagnostics ---" >&2 - cat "$DIAG_PATH" >&2 2>/dev/null || true - exit 1 - ) & - START_SIGNAL_PID=$! - - XCODEBUILD_OK=false - if xcodebuild -project cmux.xcodeproj -scheme cmux -configuration Debug \ - -derivedDataPath "$CMUX_DERIVED_DATA_PATH" \ - -clonedSourcePackagesDirPath "$SOURCE_PACKAGES_DIR" \ - -disableAutomaticPackageResolution \ - -destination "platform=macOS" \ - -only-testing:cmuxUITests/DisplayResolutionRegressionUITests \ - test-without-building 2>&1 | tee "$XCODEBUILD_LOG"; then - XCODEBUILD_OK=true - fi - - if [ -n "${START_SIGNAL_PID:-}" ]; then - kill "$START_SIGNAL_PID" 2>/dev/null || true - wait "$START_SIGNAL_PID" 2>/dev/null || true - START_SIGNAL_PID="" - fi - - if [ "$XCODEBUILD_OK" = "true" ]; then - cleanup_attempt - exit 0 - fi - - pkill -x "cmux DEV" 2>/dev/null || true - cleanup_attempt - - if [ "$attempt" -eq 2 ]; then - echo "Display resolution UI regression failed after 2 attempts" >&2 - exit 1 - fi - echo "Attempt $attempt failed, retrying..." - sleep 3 - done - - - name: Create persistent virtual display - run: | - set -euo pipefail - LOCK_ENV="$(scripts/ci/virtual-display-lock.sh acquire)" - eval "$LOCK_ENV" - export CMUX_VDISPLAY_LOCK_DIR CMUX_VDISPLAY_LOCK_TOKEN - { - echo "CMUX_VDISPLAY_LOCK_DIR=$CMUX_VDISPLAY_LOCK_DIR" - echo "CMUX_VDISPLAY_LOCK_TOKEN=$CMUX_VDISPLAY_LOCK_TOKEN" - } >> "$GITHUB_ENV" - - HELPER_PATH="$RUNNER_TEMP/create-virtual-display-persistent" - clang -framework Foundation -framework CoreGraphics \ - -o "$HELPER_PATH" scripts/create-virtual-display.m - - VDISPLAY_READY="$RUNNER_TEMP/cmux-vdisplay-persistent.ready" - VDISPLAY_ID_PATH="$RUNNER_TEMP/cmux-vdisplay-persistent.id" - VDISPLAY_LOG="$RUNNER_TEMP/cmux-vdisplay-persistent.log" - rm -f "$VDISPLAY_READY" "$VDISPLAY_ID_PATH" "$VDISPLAY_LOG" - - # Now that we hold the lock, reap any leaked display helper so a - # CGVirtualDisplay orphaned by a crashed/cancelled job cannot block - # this create on persistent self-hosted runners. - scripts/ci/virtual-display-lock.sh reap-strays || true - - "$HELPER_PATH" \ - --modes "1920x1080" \ - --ready-path "$VDISPLAY_READY" \ - --display-id-path "$VDISPLAY_ID_PATH" \ - >"$VDISPLAY_LOG" 2>&1 & - VDISPLAY_PERSISTENT_PID=$! - scripts/ci/virtual-display-lock.sh set-owner "$VDISPLAY_PERSISTENT_PID" - - { - echo "VDISPLAY_PERSISTENT_PID=$VDISPLAY_PERSISTENT_PID" - echo "VDISPLAY_PERSISTENT_HELPER_PATH=$HELPER_PATH" - echo "VDISPLAY_PERSISTENT_READY=$VDISPLAY_READY" - echo "VDISPLAY_PERSISTENT_ID_PATH=$VDISPLAY_ID_PATH" - echo "VDISPLAY_PERSISTENT_LOG=$VDISPLAY_LOG" - } >> "$GITHUB_ENV" - - echo "Waiting for persistent virtual display..." - for _ in $(seq 1 100); do - if [ -s "$VDISPLAY_READY" ] && [ -s "$VDISPLAY_ID_PATH" ]; then - break - fi - if ! kill -0 "$VDISPLAY_PERSISTENT_PID" 2>/dev/null; then - echo "Persistent virtual display helper exited before readiness" >&2 - cat "$VDISPLAY_LOG" >&2 || true - exit 1 - fi - sleep 0.1 - done - - if [ ! -s "$VDISPLAY_READY" ] || [ ! -s "$VDISPLAY_ID_PATH" ]; then - echo "ERROR: Persistent virtual display not ready after 10s" >&2 - cat "$VDISPLAY_LOG" >&2 || true - exit 1 - fi - - echo "Persistent virtual display ready: ID=$(tr -d '\n' < "$VDISPLAY_ID_PATH")" - cat "$VDISPLAY_LOG" - - - name: Run browser find focus UI regression - run: | - set -euo pipefail - SOURCE_PACKAGES_DIR="$PWD/.ci-source-packages" - if [ -n "${VDISPLAY_PERSISTENT_PID:-}" ] && ! kill -0 "$VDISPLAY_PERSISTENT_PID" 2>/dev/null; then - echo "Persistent virtual display exited before browser find UI regression" >&2 - cat "${VDISPLAY_PERSISTENT_LOG:-/dev/null}" >&2 || true - exit 1 - fi - - xcodebuild -project cmux.xcodeproj -scheme cmux -configuration Debug \ - -derivedDataPath "$CMUX_DERIVED_DATA_PATH" \ - -clonedSourcePackagesDirPath "$SOURCE_PACKAGES_DIR" \ - -disableAutomaticPackageResolution \ - -destination "platform=macOS" \ - -maximum-test-execution-time-allowance 180 \ - -only-testing:cmuxUITests/BrowserPaneNavigationKeybindUITests/testCmdFOpensBrowserFindAfterCmdDCmdLNavigation \ - test-without-building - - - name: Cleanup persistent virtual display - if: always() - run: | - if [ -n "${VDISPLAY_PERSISTENT_PID:-}" ]; then - kill "$VDISPLAY_PERSISTENT_PID" >/dev/null 2>&1 || true - for _ in $(seq 1 50); do - kill -0 "$VDISPLAY_PERSISTENT_PID" >/dev/null 2>&1 || break - sleep 0.1 - done - fi - scripts/ci/virtual-display-lock.sh release || true - rm -f "${VDISPLAY_PERSISTENT_HELPER_PATH:-}" "${VDISPLAY_PERSISTENT_READY:-}" "${VDISPLAY_PERSISTENT_ID_PATH:-}" "${VDISPLAY_PERSISTENT_LOG:-}" - ci-status: needs: - changes @@ -1934,14 +1542,13 @@ jobs: - web-typecheck - react-apps-check - web-db-migrations + - linux-preflight - app-host-unit-tests - tests - swift-package-tests - agent-session-web-resources - tests-build-and-lag - - release-ghostty-cli-helper - release-build - - ui-regressions if: ${{ always() }} runs-on: ${{ vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }} steps: diff --git a/cmuxUITests/BrowserPaneNavigationKeybindUITests.swift b/cmuxUITests/BrowserPaneNavigationKeybindUITests.swift index 1ce337ed62b7..3f0b13b786e6 100644 --- a/cmuxUITests/BrowserPaneNavigationKeybindUITests.swift +++ b/cmuxUITests/BrowserPaneNavigationKeybindUITests.swift @@ -1316,7 +1316,6 @@ final class BrowserPaneNavigationKeybindUITests: XCTestCase { "Expected command palette to dismiss after creating a workspace" ) } - private func focusLeftPaneForFindScenario(_ app: XCUIApplication, route: FindFocusRoute) { switch route { case .cmdOptionArrows: @@ -1370,6 +1369,7 @@ final class BrowserPaneNavigationKeybindUITests: XCTestCase { app.launchEnvironment["CMUX_UI_TEST_SOCKET_SANITY"] = "1" app.launchEnvironment["CMUX_UI_TEST_DIAGNOSTICS_PATH"] = diagnosticsPath app.launchEnvironment["CMUX_TAG"] = launchTag + app.launchEnvironment["CMUX_UI_TEST_TARGET_DISPLAY_ID"] = ProcessInfo.processInfo.environment["CMUX_UI_TEST_TARGET_DISPLAY_ID"] } private func socketReadinessFailureMessage() -> String { diff --git a/scripts/ci/run-display-ui-regressions.sh b/scripts/ci/run-display-ui-regressions.sh new file mode 100755 index 000000000000..85d9812ff718 --- /dev/null +++ b/scripts/ci/run-display-ui-regressions.sh @@ -0,0 +1,372 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${CMUX_DERIVED_DATA_PATH:?CMUX_DERIVED_DATA_PATH is required}" +SOURCE_PACKAGES_DIR="${CMUX_SOURCE_PACKAGES_DIR:-$PWD/.ci-source-packages}" + +DRC_HELPER_PATH="" +DRC_DIAG_PATH="" +DRC_DISPLAY_READY="" +DRC_DISPLAY_ID_PATH="" +DRC_DISPLAY_START="" +DRC_DISPLAY_DONE="" +DRC_HELPER_LOG="" +DRC_XCODEBUILD_LOG="" +DRC_HELPER_PID="" +DRC_START_SIGNAL_PID="" +DRC_DISPLAY_LOCK_DIR="" +DRC_DISPLAY_LOCK_TOKEN="" + +PERSISTENT_HELPER_PATH="" +PERSISTENT_READY="" +PERSISTENT_ID_PATH="" +PERSISTENT_LOG="" +PERSISTENT_PID="" +PERSISTENT_LOCK_DIR="" +PERSISTENT_LOCK_TOKEN="" + +release_lock() { + local lock_dir="$1" + local lock_token="$2" + if [ -n "$lock_dir" ]; then + CMUX_VDISPLAY_LOCK_DIR="$lock_dir" \ + CMUX_VDISPLAY_LOCK_TOKEN="$lock_token" \ + scripts/ci/virtual-display-lock.sh release || true + fi +} + +cleanup_display_churn() { + if [ -n "${DRC_START_SIGNAL_PID:-}" ]; then + kill "$DRC_START_SIGNAL_PID" 2>/dev/null || true + wait "$DRC_START_SIGNAL_PID" 2>/dev/null || true + DRC_START_SIGNAL_PID="" + fi + if [ -n "${DRC_HELPER_PID:-}" ]; then + kill "$DRC_HELPER_PID" 2>/dev/null || true + wait "$DRC_HELPER_PID" 2>/dev/null || true + DRC_HELPER_PID="" + fi + release_lock "$DRC_DISPLAY_LOCK_DIR" "$DRC_DISPLAY_LOCK_TOKEN" + DRC_DISPLAY_LOCK_DIR="" + DRC_DISPLAY_LOCK_TOKEN="" + pkill -x "cmux DEV" 2>/dev/null || true + rm -f "$DRC_DIAG_PATH" "$DRC_DISPLAY_READY" "$DRC_DISPLAY_ID_PATH" "$DRC_DISPLAY_START" "$DRC_DISPLAY_DONE" "$DRC_HELPER_LOG" "$DRC_XCODEBUILD_LOG" + rm -f /tmp/cmux-ui-test-prelaunch.json /tmp/cmux-ui-test-display-harness.json +} + +cleanup_persistent_display() { + if [ -n "${PERSISTENT_PID:-}" ]; then + kill "$PERSISTENT_PID" >/dev/null 2>&1 || true + for _ in $(seq 1 50); do + kill -0 "$PERSISTENT_PID" >/dev/null 2>&1 || break + sleep 0.1 + done + PERSISTENT_PID="" + fi + release_lock "$PERSISTENT_LOCK_DIR" "$PERSISTENT_LOCK_TOKEN" + PERSISTENT_LOCK_DIR="" + PERSISTENT_LOCK_TOKEN="" + rm -f "$PERSISTENT_HELPER_PATH" "$PERSISTENT_READY" "$PERSISTENT_ID_PATH" "$PERSISTENT_LOG" +} + +cleanup_all() { + cleanup_display_churn + cleanup_persistent_display + rm -f "$DRC_HELPER_PATH" +} +trap cleanup_all EXIT + +enable_xctest_automation_mode() { + if ! command -v automationmodetool >/dev/null 2>&1; then + echo "::warning::automationmodetool is unavailable; XCTest will use its default automation-mode setup" + return 0 + fi + + if sudo -n true 2>/dev/null; then + sudo -n automationmodetool enable-automationmode-without-authentication + else + echo "::warning::Passwordless sudo unavailable; XCTest will use its default automation-mode setup" + fi +} + +find_app_binary() { + find "$CMUX_DERIVED_DATA_PATH" -path "*/Build/Products/Debug/cmux DEV.app/Contents/MacOS/cmux DEV" -print -quit 2>/dev/null || true +} + +run_display_resolution_churn() { + local token app_binary display_id app_pid app_ready render_ready xcodebuild_ok + token="$(uuidgen)" + DRC_HELPER_PATH="$RUNNER_TEMP/create-virtual-display-display-churn-${token}" + DRC_DIAG_PATH="/tmp/cmux-ui-test-display-churn-${token}.json" + DRC_DISPLAY_READY="/tmp/cmux-ui-test-display-${token}.ready" + DRC_DISPLAY_ID_PATH="/tmp/cmux-ui-test-display-${token}.id" + DRC_DISPLAY_START="/tmp/cmux-ui-test-display-${token}.start" + DRC_DISPLAY_DONE="/tmp/cmux-ui-test-display-${token}.done" + DRC_HELPER_LOG="/tmp/cmux-ui-test-display-${token}-helper.log" + DRC_XCODEBUILD_LOG="/tmp/cmux-ui-test-display-${token}-xcodebuild.log" + local baseline_ready_marker="CMUX_DISPLAY_CHURN_BASELINE_READY_${token}" + + clang -framework Foundation -framework CoreGraphics \ + -o "$DRC_HELPER_PATH" scripts/create-virtual-display.m + + app_binary="$(find_app_binary)" + if [ -z "$app_binary" ]; then + echo "ERROR: App binary not found in DerivedData" >&2 + exit 1 + fi + echo "App binary: $app_binary" + + for attempt in 1 2; do + cleanup_display_churn 2>/dev/null || true + + local lock_env + lock_env="$(scripts/ci/virtual-display-lock.sh acquire)" + eval "$lock_env" + export CMUX_VDISPLAY_LOCK_DIR CMUX_VDISPLAY_LOCK_TOKEN + DRC_DISPLAY_LOCK_DIR="$CMUX_VDISPLAY_LOCK_DIR" + DRC_DISPLAY_LOCK_TOKEN="$CMUX_VDISPLAY_LOCK_TOKEN" + + scripts/ci/virtual-display-lock.sh reap-strays || true + + "$DRC_HELPER_PATH" \ + --modes "1920x1080,1728x1117,1600x900,1440x810" \ + --ready-path "$DRC_DISPLAY_READY" \ + --display-id-path "$DRC_DISPLAY_ID_PATH" \ + --start-path "$DRC_DISPLAY_START" \ + --done-path "$DRC_DISPLAY_DONE" \ + --iterations 40 \ + --interval-ms 40 \ + > "$DRC_HELPER_LOG" 2>&1 & + DRC_HELPER_PID=$! + scripts/ci/virtual-display-lock.sh set-owner "$DRC_HELPER_PID" + + echo "Waiting for virtual display..." + local display_ready_ok=false + for _ in $(seq 1 100); do + if [ -s "$DRC_DISPLAY_READY" ] && [ -s "$DRC_DISPLAY_ID_PATH" ]; then + display_ready_ok=true + break + fi + if ! kill -0 "$DRC_HELPER_PID" 2>/dev/null; then + echo "ERROR: Virtual display helper exited before readiness" >&2 + cat "$DRC_HELPER_LOG" 2>/dev/null || true + break + fi + sleep 0.1 + done + if [ "$display_ready_ok" != "true" ]; then + echo "ERROR: Virtual display not ready after 10s" >&2 + cat "$DRC_HELPER_LOG" 2>/dev/null || true + cleanup_display_churn + if [ "$attempt" -eq 2 ]; then + echo "Display resolution UI regression failed after 2 virtual display setup attempts" >&2 + exit 1 + fi + sleep 3 + continue + fi + + display_id="$(tr -d '\n' < "$DRC_DISPLAY_ID_PATH")" + echo "Virtual display ready: ID=$display_id" + + CMUX_UI_TEST_MODE=1 \ + CMUX_UI_TEST_DIAGNOSTICS_PATH="$DRC_DIAG_PATH" \ + CMUX_UI_TEST_DISPLAY_RENDER_STATS=1 \ + CMUX_UI_TEST_TARGET_DISPLAY_ID="$display_id" \ + CMUX_TAG="ui-tests-display-resolution" \ + "$app_binary" > /tmp/cmux-ui-test-app.log 2>&1 & + app_pid=$! + echo "App launched: PID=$app_pid" + + echo "Waiting for app diagnostics..." + app_ready=false + for _ in $(seq 1 30); do + if [ -f "$DRC_DIAG_PATH" ]; then + if python3 -c "import json; d=json.load(open('$DRC_DIAG_PATH')); assert d.get('pid')" 2>/dev/null; then + app_ready=true + break + fi + fi + if ! kill -0 "$app_pid" 2>/dev/null; then + echo "ERROR: App crashed during startup" + cat /tmp/cmux-ui-test-app.log 2>/dev/null | tail -30 || true + break + fi + sleep 0.5 + done + + if [ "$app_ready" != "true" ]; then + echo "Attempt $attempt: App not ready after 15s" + pkill -x "cmux DEV" 2>/dev/null || true + kill "$DRC_HELPER_PID" 2>/dev/null || true + if [ "$attempt" -eq 2 ]; then + echo "Display resolution UI regression failed after 2 attempts" >&2 + echo "--- App log ---" + cat /tmp/cmux-ui-test-app.log 2>/dev/null | tail -50 || true + echo "--- Helper log ---" + cat "$DRC_HELPER_LOG" 2>/dev/null | tail -20 || true + echo "--- Diagnostics ---" + cat "$DRC_DIAG_PATH" 2>/dev/null || echo "(not found)" + exit 1 + fi + sleep 3 + continue + fi + + echo "App started. Diagnostics:" + cat "$DRC_DIAG_PATH" + + echo "Waiting for render stats..." + render_ready=false + for i in $(seq 1 40); do + if python3 -c "import json; d=json.load(open('$DRC_DIAG_PATH')); assert d.get('renderStatsAvailable') == '1'" 2>/dev/null; then + render_ready=true + echo "Render stats available after $((i / 2))s" + break + fi + sleep 0.5 + done + if [ "$render_ready" != "true" ]; then + echo "WARNING: Render stats not available after 20s. Diagnostics:" + cat "$DRC_DIAG_PATH" 2>/dev/null || true + echo "--- App log ---" + cat /tmp/cmux-ui-test-app.log 2>/dev/null | tail -30 || true + fi + + cat >"/tmp/cmux-ui-test-display-harness.json" <"/tmp/cmux-ui-test-prelaunch.json" </dev/null; then + echo "XCTest baseline marker observed; starting display churn" + printf 'start\n' > "$DRC_DISPLAY_START" + exit 0 + fi + sleep 0.25 + done + echo "ERROR: XCTest baseline marker not observed before display-churn start timeout" >&2 + echo "--- xcodebuild log tail ---" >&2 + tail -80 "$DRC_XCODEBUILD_LOG" >&2 2>/dev/null || true + echo "--- diagnostics ---" >&2 + cat "$DRC_DIAG_PATH" >&2 2>/dev/null || true + exit 1 + ) & + DRC_START_SIGNAL_PID=$! + + xcodebuild_ok=false + if xcodebuild -project cmux.xcodeproj -scheme cmux -configuration Debug \ + -derivedDataPath "$CMUX_DERIVED_DATA_PATH" \ + -clonedSourcePackagesDirPath "$SOURCE_PACKAGES_DIR" \ + -disableAutomaticPackageResolution \ + -destination "platform=macOS" \ + -only-testing:cmuxUITests/DisplayResolutionRegressionUITests \ + test-without-building 2>&1 | tee "$DRC_XCODEBUILD_LOG"; then + xcodebuild_ok=true + fi + + if [ -n "${DRC_START_SIGNAL_PID:-}" ]; then + kill "$DRC_START_SIGNAL_PID" 2>/dev/null || true + wait "$DRC_START_SIGNAL_PID" 2>/dev/null || true + DRC_START_SIGNAL_PID="" + fi + + if [ "$xcodebuild_ok" = "true" ]; then + cleanup_display_churn + return 0 + fi + + cleanup_display_churn + + if [ "$attempt" -eq 2 ]; then + echo "Display resolution UI regression failed after 2 attempts" >&2 + exit 1 + fi + echo "Attempt $attempt failed, retrying..." + sleep 3 + done +} + +create_persistent_display() { + local lock_env + lock_env="$(scripts/ci/virtual-display-lock.sh acquire)" + eval "$lock_env" + export CMUX_VDISPLAY_LOCK_DIR CMUX_VDISPLAY_LOCK_TOKEN + PERSISTENT_LOCK_DIR="$CMUX_VDISPLAY_LOCK_DIR" + PERSISTENT_LOCK_TOKEN="$CMUX_VDISPLAY_LOCK_TOKEN" + + PERSISTENT_HELPER_PATH="$RUNNER_TEMP/create-virtual-display-persistent" + clang -framework Foundation -framework CoreGraphics \ + -o "$PERSISTENT_HELPER_PATH" scripts/create-virtual-display.m + + PERSISTENT_READY="$RUNNER_TEMP/cmux-vdisplay-persistent.ready" + PERSISTENT_ID_PATH="$RUNNER_TEMP/cmux-vdisplay-persistent.id" + PERSISTENT_LOG="$RUNNER_TEMP/cmux-vdisplay-persistent.log" + rm -f "$PERSISTENT_READY" "$PERSISTENT_ID_PATH" "$PERSISTENT_LOG" + + scripts/ci/virtual-display-lock.sh reap-strays || true + + "$PERSISTENT_HELPER_PATH" \ + --modes "1920x1080" \ + --ready-path "$PERSISTENT_READY" \ + --display-id-path "$PERSISTENT_ID_PATH" \ + >"$PERSISTENT_LOG" 2>&1 & + PERSISTENT_PID=$! + scripts/ci/virtual-display-lock.sh set-owner "$PERSISTENT_PID" + + echo "Waiting for persistent virtual display..." + for _ in $(seq 1 100); do + if [ -s "$PERSISTENT_READY" ] && [ -s "$PERSISTENT_ID_PATH" ]; then + break + fi + if ! kill -0 "$PERSISTENT_PID" 2>/dev/null; then + echo "Persistent virtual display helper exited before readiness" >&2 + cat "$PERSISTENT_LOG" >&2 || true + exit 1 + fi + sleep 0.1 + done + + if [ ! -s "$PERSISTENT_READY" ] || [ ! -s "$PERSISTENT_ID_PATH" ]; then + echo "ERROR: Persistent virtual display not ready after 10s" >&2 + cat "$PERSISTENT_LOG" >&2 || true + exit 1 + fi + + echo "Persistent virtual display ready: ID=$(tr -d '\n' < "$PERSISTENT_ID_PATH")" + cat "$PERSISTENT_LOG" +} + +run_browser_find_focus() { + local persistent_display_id + if [ -n "${PERSISTENT_PID:-}" ] && ! kill -0 "$PERSISTENT_PID" 2>/dev/null; then + echo "Persistent virtual display exited before browser find UI regression" >&2 + cat "${PERSISTENT_LOG:-/dev/null}" >&2 || true + exit 1 + fi + if [ ! -s "${PERSISTENT_ID_PATH:-}" ]; then + echo "Persistent virtual display ID missing before browser find UI regression" >&2 + exit 1 + fi + persistent_display_id="$(tr -d '\n' < "$PERSISTENT_ID_PATH")" + + CMUX_UI_TEST_TARGET_DISPLAY_ID="$persistent_display_id" \ + xcodebuild -project cmux.xcodeproj -scheme cmux -configuration Debug \ + -derivedDataPath "$CMUX_DERIVED_DATA_PATH" \ + -clonedSourcePackagesDirPath "$SOURCE_PACKAGES_DIR" \ + -disableAutomaticPackageResolution \ + -destination "platform=macOS" \ + -maximum-test-execution-time-allowance 180 \ + -only-testing:cmuxUITests/BrowserPaneNavigationKeybindUITests/testCmdFOpensBrowserFindAfterCmdDCmdLNavigation \ + test-without-building +} + +enable_xctest_automation_mode +run_display_resolution_churn +create_persistent_display +run_browser_find_focus diff --git a/scripts/select-ci-xcode.sh b/scripts/select-ci-xcode.sh index d82b5f2c9729..f22a7e66f534 100755 --- a/scripts/select-ci-xcode.sh +++ b/scripts/select-ci-xcode.sh @@ -15,6 +15,95 @@ set -euo pipefail APPLICATIONS_DIR="${CMUX_XCODE_APPLICATIONS_DIR:-/Applications}" +REQUIRED_SDK_MAJOR="${CMUX_CI_REQUIRED_MACOS_SDK_MAJOR:-}" + +sdk_major() { + local v="$1" maj + maj="${v%%.*}" + case "$maj" in ''|*[!0-9]*) return 1 ;; esac + printf '%s' "$maj" +} + +validate_required_sdk() { + local selected_dir="$1" sdk_version="$2" actual_major + [ -n "$REQUIRED_SDK_MAJOR" ] || return 0 + case "$REQUIRED_SDK_MAJOR" in ''|*[!0-9]*) + echo "CMUX_CI_REQUIRED_MACOS_SDK_MAJOR must be numeric, got: $REQUIRED_SDK_MAJOR" >&2 + exit 1 + ;; + esac + if ! actual_major="$(sdk_major "$sdk_version")"; then + echo "Could not parse macOS SDK version for $selected_dir: $sdk_version" >&2 + exit 1 + fi + if [ "$actual_major" != "$REQUIRED_SDK_MAJOR" ]; then + echo "Selected Xcode at $selected_dir has macOS SDK $sdk_version; required major is $REQUIRED_SDK_MAJOR" >&2 + exit 1 + fi +} + +select_developer_dir() { + local selected_dir="$1" sdk_version="$2" label="$3" + + validate_required_sdk "$selected_dir" "$sdk_version" + echo "$label (DEVELOPER_DIR): $selected_dir (macOS SDK $sdk_version)" + if [ -n "${GITHUB_ENV:-}" ]; then + echo "DEVELOPER_DIR=$selected_dir" >> "$GITHUB_ENV" + fi + export DEVELOPER_DIR="$selected_dir" + + # Also point the *system* xcode-select default at the selected toolchain. Tools + # that ignore DEVELOPER_DIR resolve `xcodebuild` via the xcode-select default, + # notably Apple's `/usr/bin/git` shim (`xcodebuild -find git`). The xctest host + # spawns git subprocesses that do NOT inherit our DEVELOPER_DIR, so on runner VMs + # whose default is the old Xcode symlink, `git` runs the old `xcodebuild`, which + # dlopen()s a libxcodebuildLoader ABI-incompatible with the newer-Xcode-built test + # host and crashes ("Symbol not found"), failing git-shell-out tests + # (e.g. ExtensionWorktreePrototypeTests) before they can assert - nondeterministic + # per which VM a shard lands on. Aligning the default removes that divergence. + # Best-effort: never hard-fail a runner that disallows the switch. + if xcode-select -s "$selected_dir" 2>/dev/null; then + echo "xcode-select default -> $selected_dir" + elif command -v sudo >/dev/null 2>&1 && sudo -n xcode-select -s "$selected_dir" 2>/dev/null; then + echo "xcode-select default (via sudo) -> $selected_dir" + else + echo "WARN: could not switch xcode-select default to $selected_dir (continuing; DEVELOPER_DIR is still set for steps that honor it)" >&2 + fi + + xcodebuild -version + # Diagnostic: resolve the SDK with DEVELOPER_DIR set in-process. The workflow + # step that calls this script gets DEVELOPER_DIR only via GITHUB_ENV, which + # applies to *later* steps, not the current shell, so a bare `xcrun` on the + # next line of the same step would still resolve the old xcode-select default. + xcrun --sdk macosx --show-sdk-path +} + +PINNED_DEVELOPER_DIR="${CMUX_CI_DEVELOPER_DIR:-}" +if [ -z "$PINNED_DEVELOPER_DIR" ] && [ -n "${CMUX_CI_XCODE_APP:-}" ]; then + PINNED_DEVELOPER_DIR="${CMUX_CI_XCODE_APP%/}/Contents/Developer" +fi + +if [ -n "$PINNED_DEVELOPER_DIR" ]; then + if [ ! -d "$PINNED_DEVELOPER_DIR" ]; then + echo "Pinned Xcode developer dir does not exist: $PINNED_DEVELOPER_DIR" >&2 + exit 1 + fi + PINNED_SDK_VER="$(DEVELOPER_DIR="$PINNED_DEVELOPER_DIR" xcrun --sdk macosx --show-sdk-version 2>/dev/null || true)" + if [ -z "$PINNED_SDK_VER" ]; then + echo "Pinned Xcode developer dir has no usable macOS SDK: $PINNED_DEVELOPER_DIR" >&2 + exit 1 + fi + select_developer_dir "$PINNED_DEVELOPER_DIR" "$PINNED_SDK_VER" "Selected pinned Xcode" + exit 0 +fi + +if [ -n "$REQUIRED_SDK_MAJOR" ]; then + case "$REQUIRED_SDK_MAJOR" in ''|*[!0-9]*) + echo "CMUX_CI_REQUIRED_MACOS_SDK_MAJOR must be numeric, got: $REQUIRED_SDK_MAJOR" >&2 + exit 1 + ;; + esac +fi # Rank by macOS SDK as maj*1000+min so 26.2 (26002) outranks 15.5 (15005). sdk_rank() { @@ -40,6 +129,16 @@ while IFS= read -r app; do [ -d "$dev" ] || continue sdk_ver="$(DEVELOPER_DIR="$dev" xcrun --sdk macosx --show-sdk-version 2>/dev/null || true)" [ -n "$sdk_ver" ] || continue + if [ -n "$REQUIRED_SDK_MAJOR" ]; then + if ! actual_major="$(sdk_major "$sdk_ver")"; then + echo "Ignoring $app with unparsable macOS SDK version: $sdk_ver" >&2 + continue + fi + if [ "$actual_major" != "$REQUIRED_SDK_MAJOR" ]; then + echo "Skipping $app -> macOS SDK $sdk_ver; required major is $REQUIRED_SDK_MAJOR" + continue + fi + fi if ! rank="$(sdk_rank "$sdk_ver")"; then echo "Ignoring $app with unparsable macOS SDK version: $sdk_ver" >&2 continue @@ -76,37 +175,12 @@ if [ -z "$BEST_DIR" ] && [ -n "$BETA_DIR" ]; then fi if [ -z "$BEST_DIR" ]; then + if [ -n "$REQUIRED_SDK_MAJOR" ]; then + echo "No Xcode.app found under $APPLICATIONS_DIR with macOS SDK major $REQUIRED_SDK_MAJOR" >&2 + exit 1 + fi echo "No Xcode.app found under $APPLICATIONS_DIR" >&2 exit 1 fi -echo "Selected Xcode (DEVELOPER_DIR): $BEST_DIR (macOS SDK $BEST_VER)" -if [ -n "${GITHUB_ENV:-}" ]; then - echo "DEVELOPER_DIR=$BEST_DIR" >> "$GITHUB_ENV" -fi -export DEVELOPER_DIR="$BEST_DIR" - -# Also point the *system* xcode-select default at the selected toolchain. Tools -# that ignore DEVELOPER_DIR resolve `xcodebuild` via the xcode-select default, -# notably Apple's `/usr/bin/git` shim (`xcodebuild -find git`). The xctest host -# spawns git subprocesses that do NOT inherit our DEVELOPER_DIR, so on runner VMs -# whose default is the old Xcode symlink, `git` runs the old `xcodebuild`, which -# dlopen()s a libxcodebuildLoader ABI-incompatible with the newer-Xcode-built test -# host and crashes ("Symbol not found"), failing git-shell-out tests -# (e.g. ExtensionWorktreePrototypeTests) before they can assert — nondeterministic -# per which VM a shard lands on. Aligning the default removes that divergence. -# Best-effort: never hard-fail a runner that disallows the switch. -if xcode-select -s "$BEST_DIR" 2>/dev/null; then - echo "xcode-select default -> $BEST_DIR" -elif command -v sudo >/dev/null 2>&1 && sudo -n xcode-select -s "$BEST_DIR" 2>/dev/null; then - echo "xcode-select default (via sudo) -> $BEST_DIR" -else - echo "WARN: could not switch xcode-select default to $BEST_DIR (continuing; DEVELOPER_DIR is still set for steps that honor it)" >&2 -fi - -xcodebuild -version -# Diagnostic: resolve the SDK with DEVELOPER_DIR set in-process. The workflow -# step that calls this script gets DEVELOPER_DIR only via GITHUB_ENV, which -# applies to *later* steps, not the current shell, so a bare `xcrun` on the -# next line of the same step would still resolve the old xcode-select default. -xcrun --sdk macosx --show-sdk-path +select_developer_dir "$BEST_DIR" "$BEST_VER" "Selected Xcode" diff --git a/tests/test_ci_change_areas.py b/tests/test_ci_change_areas.py index e03934e3fd01..b11e518bc847 100644 --- a/tests/test_ci_change_areas.py +++ b/tests/test_ci_change_areas.py @@ -4,6 +4,7 @@ from __future__ import annotations import importlib.util +import json import os import subprocess import sys @@ -176,6 +177,77 @@ def workflow_job_block(job_name: str, workflow_path: Path = CI_WORKFLOW) -> str: raise AssertionError(f"{job_name} job not found") +def workflow_job_step_script(job_name: str, step_name: str, workflow_path: Path = CI_WORKFLOW) -> str: + lines = workflow_path.read_text(encoding="utf-8").splitlines() + job_marker = f" {job_name}:" + step_marker = f" - name: {step_name}" + in_job = False + for index, line in enumerate(lines): + if line == job_marker: + in_job = True + continue + if in_job and line.startswith(" ") and not line.startswith(" ") and line.strip(): + break + if in_job and line == step_marker: + for run_index in range(index + 1, len(lines)): + if lines[run_index] == " run: |": + body: list[str] = [] + for body_line in lines[run_index + 1 :]: + if body_line.startswith(" "): + body.append(body_line[10:]) + continue + if not body_line.strip(): + body.append("") + continue + break + return "\n".join(body) + break + raise AssertionError(f"{step_name} run block not found in {job_name}") + + +def run_linux_preflight(needs: dict[str, object]) -> subprocess.CompletedProcess[str]: + script = workflow_job_step_script("linux-preflight", "Check cheap CI layer before macOS runners") + env = {**os.environ, "PREFLIGHT_NEEDS": json.dumps(needs)} + return subprocess.run( + ["bash", "-c", script], + cwd=ROOT, + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + +def linux_preflight_needs( + *, + outputs: dict[str, str] | None = None, + results: dict[str, str] | None = None, +) -> dict[str, object]: + route_outputs = { + "macos": "true", + "web": "true", + "go": "true", + "agent_session_web": "true", + } + if outputs: + route_outputs.update(outputs) + job_results = { + "changes": "success", + "workflow-guard-tests": "success", + "remote-daemon-tests": "success", + "web-typecheck": "success", + "react-apps-check": "success", + "web-db-migrations": "success", + "agent-session-web-resources": "success", + } + if results: + job_results.update(results) + return { + name: {"result": result, "outputs": route_outputs if name == "changes" else {}} + for name, result in job_results.items() + } + + def run_detect_step_for_paths( paths: list[str], workflow_path: Path = CI_WORKFLOW, @@ -393,12 +465,11 @@ def test_ci_status_job_accepts_skipped_routed_jobs() -> None: "web-typecheck", "react-apps-check", "web-db-migrations", + "linux-preflight", "app-host-unit-tests", "tests", "tests-build-and-lag", - "release-ghostty-cli-helper", "release-build", - "ui-regressions", ]: assert f" - {job_name}" in block @@ -411,12 +482,115 @@ def test_required_tests_status_waits_for_app_host_matrix() -> None: assert "name: tests" in block assert " - changes" in block + assert " - linux-preflight" in block assert " - app-host-unit-tests" in block assert "if: ${{ always() }}" in block + assert 'preflight["result"] != "success"' in block assert 'macos == "true" and tests["result"] != "success"' in block assert 'tests["result"] not in {"success", "skipped"}' in block +def test_macos_jobs_wait_for_linux_preflight() -> None: + for job_name in [ + "app-host-unit-tests", + "swift-package-tests", + "tests-build-and-lag", + "release-build", + ]: + block = workflow_job_block(job_name) + assert " - changes" in block + assert " - linux-preflight" in block + assert "if: ${{ needs.changes.outputs.macos == 'true' }}" in block + + +def test_linux_preflight_blocks_macos_on_cheap_layer_failure() -> None: + block = workflow_job_block("linux-preflight") + + assert "name: linux-preflight" in block + assert " - changes" in block + assert " - workflow-guard-tests" in block + assert " - remote-daemon-tests" in block + assert " - web-typecheck" in block + assert " - react-apps-check" in block + assert " - web-db-migrations" in block + assert " - agent-session-web-resources" in block + assert "if: ${{ always() }}" in block + assert 'required = ("changes", "workflow-guard-tests")' in block + assert 'allowed_routed = {' in block + assert 'routed_outputs = {' in block + assert 'bad[name] = f"{result} (route {route}=true)"' in block + + +def test_linux_preflight_fails_when_routed_job_skips() -> None: + result = run_linux_preflight( + linux_preflight_needs(results={"remote-daemon-tests": "skipped"}) + ) + + assert result.returncode != 0 + assert "remote-daemon-tests: skipped (route go=true)" in result.stderr + + +def test_linux_preflight_allows_unrouted_job_skip() -> None: + result = run_linux_preflight( + linux_preflight_needs( + outputs={"go": "false"}, + results={"remote-daemon-tests": "skipped"}, + ) + ) + + assert result.returncode == 0, result.stderr + assert "remote-daemon-tests: skipped" in result.stdout + + +def test_macos_jobs_use_lane_specific_xcode_pin_vars() -> None: + for job_name in [ + "app-host-unit-tests", + "swift-package-tests", + "tests-build-and-lag", + ]: + block = workflow_job_block(job_name) + assert "CMUX_CI_XCODE_APP: ${{ vars.CMUX_CI_XCODE_APP_MACOS_15 }}" in block + assert 'CMUX_CI_REQUIRED_MACOS_SDK_MAJOR: "26"' in block + + release_block = workflow_job_block("release-build") + assert "CMUX_CI_XCODE_APP: ${{ vars.CMUX_CI_XCODE_APP_MACOS_26 }}" in release_block + assert 'CMUX_CI_REQUIRED_MACOS_SDK_MAJOR: "26"' in release_block + + +def test_required_macos_topology_collapses_display_and_release_helper_jobs() -> None: + workflow = CI_WORKFLOW.read_text(encoding="utf-8") + runtime_block = workflow_job_block("tests-build-and-lag") + package_block = workflow_job_block("swift-package-tests") + release_block = workflow_job_block("release-build") + + assert "\n ui-regressions:" not in workflow + assert "\n release-ghostty-cli-helper:" not in workflow + assert "build-for-testing" in runtime_block + assert "Run display UI regressions" in runtime_block + assert "scripts/ci/run-display-ui-regressions.sh" in runtime_block + assert runtime_block.index("Run display UI regressions") < runtime_block.index("Create virtual display") + assert 'kill -9 "$VDISPLAY_PID"' in runtime_block + assert "scripts/ci/virtual-display-lock.sh reap-strays" in runtime_block + assert runtime_block.rfind("scripts/ci/virtual-display-lock.sh reap-strays") < runtime_block.rfind("scripts/ci/virtual-display-lock.sh release") + assert "timeout-minutes: 40" in package_block + assert "CMUX_CI_HELPER_XCODE_APP" in package_block + assert "/Applications/Xcode_16.4.app" not in package_block + assert "Select helper Xcode" in package_block + assert "CMUX_CI_REQUIRED_MACOS_SDK_MAJOR=15" in package_block + assert "Build universal Ghostty CLI helper" in package_block + assert "./scripts/build-ghostty-cli-helper.sh --universal --output ghostty-cli-helper/ghostty" in package_block + assert '[[ "$HELPER_SDK_VERSION" == 15.* ]]' in package_block + assert "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a" in package_block + assert package_block.index("Select helper Xcode") < package_block.index("Build universal Ghostty CLI helper") + assert package_block.index("Build universal Ghostty CLI helper") < package_block.index("Select Xcode") + assert package_block.index("Upload universal Ghostty CLI helper") < package_block.index("Select Xcode") + assert " - swift-package-tests" in release_block + assert "Download universal Ghostty CLI helper" in release_block + assert "actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131" in release_block + assert "Install universal Ghostty CLI helper" in release_block + assert "./scripts/build-ghostty-cli-helper.sh --universal --output ghostty-cli-helper/ghostty" not in release_block + + def test_agent_session_web_resources_runs_only_for_agent_session_web_area() -> None: block = workflow_job_block("agent-session-web-resources") diff --git a/tests/test_ci_release_sdk_lane.sh b/tests/test_ci_release_sdk_lane.sh index c6a05fd8d53d..99be8b32e0fa 100755 --- a/tests/test_ci_release_sdk_lane.sh +++ b/tests/test_ci_release_sdk_lane.sh @@ -7,8 +7,8 @@ RELEASE_FILE="$ROOT_DIR/.github/workflows/release.yml" # nightly.yml is intentionally not covered here. It has its own helper-build # model and guards via test_ci_nightly_xcode_selection.sh plus -# test_nightly_universal_build.sh. This lane guards the release/CI -# artifact-download model. +# test_nightly_universal_build.sh. This lane guards the release artifact-download +# model and the CI package-lane helper handoff model. job_section() { local file="$1" job="$2" @@ -41,12 +41,6 @@ require_job_contains \ 'runs-on: ${{ vars.MACOS_RUNNER_26 || '\''blacksmith-6vcpu-macos-26'\'' }}' \ "release must sign+notarize on the macOS 26 runner variable after importing the Developer ID intermediate chain" -require_job_contains \ - "$CI_FILE" \ - "release-ghostty-cli-helper" \ - 'runs-on: ${{ vars.MACOS_RUNNER_15 || '\''warp-macos-15-arm64-6x'\'' }}' \ - "CI must build the real Ghostty CLI helper on macOS 15" - require_job_contains \ "$CI_FILE" \ "release-build" \ @@ -59,11 +53,6 @@ for workflow in "$CI_FILE" "$RELEASE_FILE"; do exit 1 fi - if ! grep -Fq "actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0" "$workflow"; then - echo "FAIL: $(basename "$workflow") must download the macOS 15-built helper artifact" >&2 - exit 1 - fi - if ! grep -Fq "./scripts/install-prebuilt-ghostty-cli-helper.sh" "$workflow"; then echo "FAIL: $(basename "$workflow") must install the prebuilt Ghostty CLI helper into the app" >&2 exit 1 @@ -75,4 +64,77 @@ for workflow in "$CI_FILE" "$RELEASE_FILE"; do fi done -echo "PASS: release and CI app builds use macOS 26 SDK with a macOS 15-built Ghostty CLI helper" +if ! grep -Fq "actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0" "$RELEASE_FILE"; then + echo "FAIL: release.yml must download the macOS 15-built helper artifact" >&2 + exit 1 +fi + +swift_package_section="$(job_section "$CI_FILE" "swift-package-tests")" +if [[ "$swift_package_section" != *"timeout-minutes: 40"* ]]; then + echo "FAIL: CI swift-package-tests must have enough timeout budget for helper build plus package tests" >&2 + exit 1 +fi + +if [[ "$swift_package_section" != *"CMUX_CI_HELPER_XCODE_APP"* ]]; then + echo "FAIL: CI swift-package-tests must use a helper-specific Xcode pin" >&2 + exit 1 +fi + +if [[ "$swift_package_section" == *"/Applications/Xcode_16.4.app"* ]]; then + echo "FAIL: CI swift-package-tests must scan for a macOS 15 SDK when the helper Xcode override is unset" >&2 + exit 1 +fi + +if [[ "$swift_package_section" != *"./scripts/build-ghostty-cli-helper.sh --universal --output ghostty-cli-helper/ghostty"* ]]; then + echo "FAIL: CI swift-package-tests must build the universal Ghostty CLI helper on the macOS 15 lane" >&2 + exit 1 +fi + +if [[ "$swift_package_section" != *"actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1"* ]]; then + echo "FAIL: CI swift-package-tests must upload the macOS 15-built Ghostty helper artifact" >&2 + exit 1 +fi + +swift_package_before_xcode="${swift_package_section%%- name: Select Xcode*}" +if [[ "$swift_package_before_xcode" != *"CMUX_CI_REQUIRED_MACOS_SDK_MAJOR=15"* ]]; then + echo "FAIL: CI swift-package-tests must require a macOS 15 SDK for the helper build" >&2 + exit 1 +fi + +if [[ "$swift_package_before_xcode" != *"./scripts/build-ghostty-cli-helper.sh --universal --output ghostty-cli-helper/ghostty"* ]]; then + echo "FAIL: CI swift-package-tests must build the Ghostty helper before selecting the Xcode 26 SDK" >&2 + exit 1 +fi + +if [[ "$swift_package_before_xcode" != *"actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1"* ]]; then + echo "FAIL: CI swift-package-tests must upload the Ghostty helper before selecting the Xcode 26 SDK" >&2 + exit 1 +fi + +if [[ "$swift_package_section" != *'[[ "$HELPER_SDK_VERSION" == 15.* ]]'* ]]; then + echo "FAIL: CI swift-package-tests must validate the uploaded Ghostty helper was built with a macOS 15 SDK" >&2 + exit 1 +fi + +release_build_section="$(job_section "$CI_FILE" "release-build")" +if [[ "$release_build_section" != *"actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0"* ]]; then + echo "FAIL: CI release-build must download the macOS 15-built Ghostty helper artifact" >&2 + exit 1 +fi + +if [[ "$release_build_section" != *"- swift-package-tests"* ]]; then + echo "FAIL: CI release-build must wait for the helper-producing swift-package-tests lane" >&2 + exit 1 +fi + +if [[ "$release_build_section" == *"./scripts/build-ghostty-cli-helper.sh --universal --output ghostty-cli-helper/ghostty"* ]]; then + echo "FAIL: CI release-build must not build the Ghostty helper on macOS 26" >&2 + exit 1 +fi + +if grep -Fq "release-ghostty-cli-helper:" "$CI_FILE"; then + echo "FAIL: CI must not define a separate release-ghostty-cli-helper job" >&2 + exit 1 +fi + +echo "PASS: release uses artifact helper handoff; CI release-build downloads the helper from the existing macOS 15 package lane" diff --git a/tests/test_ci_self_hosted_guard.sh b/tests/test_ci_self_hosted_guard.sh index 63e9b02786cc..ab618fedb186 100755 --- a/tests/test_ci_self_hosted_guard.sh +++ b/tests/test_ci_self_hosted_guard.sh @@ -207,34 +207,105 @@ check_release_build_disk_cleanup() { echo "PASS: release-build reclaims runner disk before large cache restores" } -check_release_helper_upload_retry() { +check_release_helper_artifact_from_package_lane() { if ! awk ' - /^ release-ghostty-cli-helper:/ { in_job=1; next } + /^ swift-package-tests:/ { in_job=1; next } in_job && /^ [^[:space:]#][^:]*:[[:space:]]*(#.*)?$/ { in_job=0 } - in_job && /- name: Upload universal Ghostty CLI helper/ { in_upload=1; next } - in_upload && /^[[:space:]]*- name:/ { in_upload=0 } - in_upload && /id:[[:space:]]*upload-ghostty-cli-helper/ { upload_id=1 } - in_upload && /continue-on-error:[[:space:]]*true/ { upload_continue=1 } - in_upload && /uses: actions\/upload-artifact@/ { upload_action=1 } - in_upload && /if-no-files-found:[[:space:]]*error/ { upload_required=1 } + in_job && /timeout-minutes:[[:space:]]*40/ { saw_timeout=1 } + in_job && /CMUX_CI_HELPER_XCODE_APP:/ { saw_helper_xcode_env=1 } + in_job && /- name: Select helper Xcode/ { saw_helper_select=1; next } + in_job && /CMUX_CI_REQUIRED_MACOS_SDK_MAJOR=15/ { saw_helper_sdk_pin=1 } + in_job && /- name: Select Xcode/ { saw_select=1; after_select=1; next } + in_job && /- name: Build universal Ghostty CLI helper/ { + saw_build_step=1 + if (after_select) { + saw_build_after_select=1 + } + next + } + in_job && /\.\/scripts\/build-ghostty-cli-helper\.sh --universal --output ghostty-cli-helper\/ghostty/ { saw_build=1 } + in_job && /lipo ghostty-cli-helper\/ghostty -verify_arch arm64 x86_64/ { saw_lipo=1 } + in_job && /- name: Upload universal Ghostty CLI helper/ { + saw_upload_step=1 + if (after_select) { + saw_upload_after_select=1 + } + next + } + in_job && /uses: actions\/upload-artifact@/ { saw_upload=1 } + in_job && /name:[[:space:]]*cmux-ghostty-cli-helper/ { saw_artifact_name=1 } + in_job && /\[\[ "\$HELPER_SDK_VERSION" == 15\.\* \]\]/ { saw_helper_sdk_validation=1 } - in_job && /- name: Retry universal Ghostty CLI helper upload/ { in_retry=1; retry_step=1; next } - in_retry && /^[[:space:]]*- name:/ { in_retry=0 } - in_retry && index($0, "steps.upload-ghostty-cli-helper.outcome == '\''failure'\''") { retry_if=1 } - in_retry && /uses: actions\/upload-artifact@/ { retry_action=1 } - in_retry && /if-no-files-found:[[:space:]]*error/ { retry_required=1 } - in_retry && /overwrite:[[:space:]]*true/ { retry_overwrite=1 } + END { + exit !(saw_timeout && saw_helper_xcode_env && saw_helper_select && saw_helper_sdk_pin && saw_build_step && saw_build && saw_lipo && saw_helper_sdk_validation && saw_upload_step && saw_upload && saw_artifact_name && saw_select && !saw_build_after_select && !saw_upload_after_select) + } + ' "$CI_FILE"; then + echo "FAIL: swift-package-tests must pin and validate the macOS 15 Ghostty helper before selecting Xcode 26" + exit 1 + fi + + if ! awk ' + /^ release-build:/ { in_job=1; next } + in_job && /^ [^[:space:]#][^:]*:[[:space:]]*(#.*)?$/ { in_job=0 } + + in_job && /- swift-package-tests/ { saw_need=1 } + in_job && /- name: Download universal Ghostty CLI helper/ { saw_download_step=1; next } + in_job && /uses: actions\/download-artifact@/ { saw_download=1 } + in_job && /name:[[:space:]]*cmux-ghostty-cli-helper/ { saw_artifact_name=1 } + in_job && /- name: Install universal Ghostty CLI helper/ { saw_install_step=1; next } + in_job && /\.\/scripts\/install-prebuilt-ghostty-cli-helper\.sh/ { saw_install=1 } END { - exit !(upload_id && upload_continue && upload_action && upload_required && retry_step && retry_if && retry_action && retry_required && retry_overwrite) + exit !(saw_need && saw_download_step && saw_download && saw_artifact_name && saw_install_step && saw_install) } ' "$CI_FILE"; then - echo "FAIL: release-ghostty-cli-helper must retry required Ghostty helper artifact uploads instead of failing on a single transient upload error" + echo "FAIL: release-build must depend on swift-package-tests, download the helper artifact, and install it into the app" + exit 1 + fi + + if grep -Fq "release-ghostty-cli-helper:" "$CI_FILE"; then + echo "FAIL: CI must not queue a separate release-ghostty-cli-helper job" + exit 1 + fi + + echo "PASS: release-build consumes the Ghostty helper artifact built by swift-package-tests" +} + +check_runtime_regressions_collapsed() { + if grep -Fq "ui-regressions:" "$CI_FILE"; then + echo "FAIL: CI must not queue a separate ui-regressions job" + exit 1 + fi + + if ! awk ' + /^ tests-build-and-lag:/ { in_job=1; next } + in_job && /^ [^[:space:]#][^:]*:[[:space:]]*(#.*)?$/ { in_job=0 } + + in_job && /build-for-testing/ { saw_build_for_testing=1 } + in_job && /scripts\/ci\/run-display-ui-regressions\.sh/ { saw_ui_script=1 } + in_job && /kill -9 "\$VDISPLAY_PID"/ { saw_force_kill=1 } + in_job && /scripts\/ci\/virtual-display-lock\.sh reap-strays/ { saw_reap_strays=1 } + in_job && /timeout-minutes:[[:space:]]*75/ { saw_timeout=1 } + + END { exit !(saw_build_for_testing && saw_ui_script && saw_force_kill && saw_reap_strays && saw_timeout) } + ' "$CI_FILE"; then + echo "FAIL: tests-build-and-lag must build once, run display UI regressions from that DerivedData, and clean virtual displays before releasing the lock" + exit 1 + fi + + if ! awk ' + /^run_browser_find_focus\(\) \{/ { in_func=1; next } + in_func && /^}/ { in_func=0 } + in_func && /persistent_display_id="\$\(tr -d/ { saw_display_id_read=1 } + in_func && /CMUX_UI_TEST_TARGET_DISPLAY_ID="\$persistent_display_id"/ { saw_display_env=1 } + END { exit !(saw_display_id_read && saw_display_env) } + ' "$ROOT_DIR/scripts/ci/run-display-ui-regressions.sh"; then + echo "FAIL: browser-find UI regression must target the persistent virtual display" exit 1 fi - echo "PASS: release-ghostty-cli-helper retries required Ghostty helper artifact uploads" + echo "PASS: runtime display regressions are collapsed into tests-build-and-lag" } check_signing_intermediate_imports() { @@ -796,12 +867,9 @@ check_no_bare_github_hosted_runners check_no_self_hosted_fleet_runners check_macos_runner "$CI_FILE" "app-host-unit-tests" check_macos_runner "$CI_FILE" "tests-build-and-lag" -check_macos_runner "$CI_FILE" "release-ghostty-cli-helper" check_macos_runner "$CI_FILE" "release-build" -check_macos_runner "$CI_FILE" "ui-regressions" check_release_build_runner_disk_capacity check_display_runner_identity_guard "$CI_FILE" "tests-build-and-lag" -check_display_runner_identity_guard "$CI_FILE" "ui-regressions" check_build_lag_deriveddata_cache_path # build-ghosttykit.yml @@ -817,7 +885,8 @@ check_e2e_runner_fallbacks check_xcode_selection check_release_build_signal check_release_build_disk_cleanup -check_release_helper_upload_retry +check_release_helper_artifact_from_package_lane +check_runtime_regressions_collapsed check_signing_intermediate_imports check_signing_intermediate_helper_behavior check_sentry_cli_install_portability diff --git a/tests/test_ci_xcode_selection_fast_path.sh b/tests/test_ci_xcode_selection_fast_path.sh new file mode 100755 index 000000000000..c69df5af719f --- /dev/null +++ b/tests/test_ci_xcode_selection_fast_path.sh @@ -0,0 +1,157 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)" +SCRIPT="$ROOT_DIR/scripts/select-ci-xcode.sh" + +tmp_dir="$(mktemp -d)" +trap 'rm -rf "$tmp_dir"' EXIT + +bin_dir="$tmp_dir/bin" +env_file="$tmp_dir/github-env" +xcode_select_log="$tmp_dir/xcode-select.log" +mkdir -p "$bin_dir" +touch "$env_file" "$xcode_select_log" + +cat > "$bin_dir/xcrun" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +case "$*" in + "--sdk macosx --show-sdk-version") + cat "$DEVELOPER_DIR/sdk-version" + ;; + "--sdk macosx --show-sdk-path") + printf '%s\n' "$DEVELOPER_DIR/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk" + ;; + *) + echo "unexpected xcrun args: $*" >&2 + exit 64 + ;; +esac +EOF +chmod +x "$bin_dir/xcrun" + +cat > "$bin_dir/xcode-select" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' "$*" >> "$CMUX_TEST_XCODE_SELECT_LOG" +EOF +chmod +x "$bin_dir/xcode-select" + +cat > "$bin_dir/xcodebuild" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' "Xcode 26.2" "Build version 17C52" +EOF +chmod +x "$bin_dir/xcodebuild" + +pinned_app="$tmp_dir/Xcode_26.2.app" +pinned_developer="$pinned_app/Contents/Developer" +mkdir -p "$pinned_developer" +printf '%s\n' "26.2" > "$pinned_developer/sdk-version" + +output="$( + PATH="$bin_dir:/usr/bin:/bin" \ + GITHUB_ENV="$env_file" \ + CMUX_TEST_XCODE_SELECT_LOG="$xcode_select_log" \ + CMUX_CI_DEVELOPER_DIR="$pinned_developer" \ + CMUX_CI_REQUIRED_MACOS_SDK_MAJOR=26 \ + CMUX_XCODE_APPLICATIONS_DIR="$tmp_dir/no-apps" \ + "$SCRIPT" +)" + +if ! grep -Fq "Selected pinned Xcode (DEVELOPER_DIR): $pinned_developer (macOS SDK 26.2)" <<< "$output"; then + echo "FAIL: pinned developer dir was not selected" + printf '%s\n' "$output" >&2 + exit 1 +fi + +if grep -Fq "Found " <<< "$output"; then + echo "FAIL: pinned developer dir path should skip scanning Xcode apps" + printf '%s\n' "$output" >&2 + exit 1 +fi + +if [[ "$(cat "$env_file")" != "DEVELOPER_DIR=$pinned_developer" ]]; then + echo "FAIL: select-ci-xcode.sh did not export the pinned developer dir" + cat "$env_file" >&2 + exit 1 +fi + +if [[ "$(cat "$xcode_select_log")" != "-s $pinned_developer" ]]; then + echo "FAIL: select-ci-xcode.sh did not point xcode-select at the pinned developer dir" + cat "$xcode_select_log" >&2 + exit 1 +fi + +old_app="$tmp_dir/Xcode_16.4.app" +old_developer="$old_app/Contents/Developer" +mkdir -p "$old_developer" +printf '%s\n' "15.5" > "$old_developer/sdk-version" + +wrong_sdk_output="$( + PATH="$bin_dir:/usr/bin:/bin" \ + GITHUB_ENV="$env_file" \ + CMUX_TEST_XCODE_SELECT_LOG="$xcode_select_log" \ + CMUX_CI_DEVELOPER_DIR="$old_developer" \ + CMUX_CI_REQUIRED_MACOS_SDK_MAJOR=26 \ + "$SCRIPT" 2>&1 >/dev/null +)" && { + echo "FAIL: pinned Xcode with the wrong SDK major should fail" + exit 1 +} + +if ! grep -Fq "required major is 26" <<< "$wrong_sdk_output"; then + echo "FAIL: wrong pinned SDK major failure was not explained" + printf '%s\n' "$wrong_sdk_output" >&2 + exit 1 +fi + +: > "$env_file" +: > "$xcode_select_log" + +scan_output="$( + PATH="$bin_dir:/usr/bin:/bin" \ + GITHUB_ENV="$env_file" \ + CMUX_TEST_XCODE_SELECT_LOG="$xcode_select_log" \ + CMUX_XCODE_APPLICATIONS_DIR="$tmp_dir" \ + CMUX_CI_REQUIRED_MACOS_SDK_MAJOR=15 \ + "$SCRIPT" +)" + +if ! grep -Fq "Selected Xcode (DEVELOPER_DIR): $old_developer (macOS SDK 15.5)" <<< "$scan_output"; then + echo "FAIL: unpinned required-SDK scan did not select the matching SDK 15 Xcode" + printf '%s\n' "$scan_output" >&2 + exit 1 +fi + +if ! grep -Fq "Skipping $pinned_app -> macOS SDK 26.2; required major is 15" <<< "$scan_output"; then + echo "FAIL: unpinned required-SDK scan did not report skipping the non-matching SDK 26 Xcode" + printf '%s\n' "$scan_output" >&2 + exit 1 +fi + +if [[ "$(cat "$env_file")" != "DEVELOPER_DIR=$old_developer" ]]; then + echo "FAIL: unpinned required-SDK scan did not export the matching developer dir" + cat "$env_file" >&2 + exit 1 +fi + +missing_output="$( + PATH="$bin_dir:/usr/bin:/bin" \ + GITHUB_ENV="$env_file" \ + CMUX_TEST_XCODE_SELECT_LOG="$xcode_select_log" \ + CMUX_CI_DEVELOPER_DIR="$tmp_dir/missing/Contents/Developer" \ + "$SCRIPT" 2>&1 >/dev/null +)" && { + echo "FAIL: missing pinned developer dir should fail" + exit 1 +} + +if ! grep -Fq "Pinned Xcode developer dir does not exist" <<< "$missing_output"; then + echo "FAIL: missing pinned developer dir failure was not explained" + printf '%s\n' "$missing_output" >&2 + exit 1 +fi + +echo "PASS: CI Xcode selection fast path"