Retry Warp display regression once before failing CI

.github/actions/run-display-resolution-regression/action.yml

@@ -0,0 +1,121 @@
+name: Run display resolution regression attempt
+description: Run the display resolution churn XCUITest on WarpBuild.
+outputs:
+  test_started:
+    description: Whether the display regression step started.
+    value: ${{ steps.run-display-regression.outputs.test_started }}
+  completed:
+    description: Whether the display regression reached a deterministic pass/fail outcome.
+    value: ${{ steps.run-display-regression.outputs.completed }}
+  passed:
+    description: Whether the display regression passed.
+    value: ${{ steps.mark-passed.outputs.passed }}
+runs:
+  using: composite
+  steps:
+    - name: Select Xcode
+      shell: bash
+      run: |
+        set -euo pipefail
+        if [ -d "/Applications/Xcode.app/Contents/Developer" ]; then
+          XCODE_DIR="/Applications/Xcode.app/Contents/Developer"
+        else
+          XCODE_APP="$(ls -d /Applications/Xcode*.app 2>/dev/null | sort | tail -n 1 || true)"
+          if [ -n "$XCODE_APP" ]; then
+            XCODE_DIR="$XCODE_APP/Contents/Developer"
+          else
+            echo "No Xcode.app found under /Applications" >&2
+            exit 1
+          fi
+        fi
+        echo "DEVELOPER_DIR=$XCODE_DIR" >> "$GITHUB_ENV"
+        export DEVELOPER_DIR="$XCODE_DIR"
+        xcodebuild -version
+        xcrun --sdk macosx --show-sdk-path
+
+    - name: Capture GhosttyKit cache key
+      id: ghosttykit-cache-key
+      shell: bash
+      run: echo "ghostty_sha=$(git -C ghostty rev-parse HEAD)" >> "$GITHUB_OUTPUT"
+
+    - name: Cache GhosttyKit.xcframework
+      id: cache-ghosttykit-display
+      uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4
+      with:
+        path: GhosttyKit.xcframework
+        key: ghosttykit-${{ hashFiles('.gitmodules') }}-${{ steps.ghosttykit-cache-key.outputs.ghostty_sha }}
+
+    - name: Download pre-built GhosttyKit.xcframework
+      if: steps.cache-ghosttykit-display.outputs.cache-hit != 'true'
+      shell: bash
+      run: ./scripts/download-prebuilt-ghosttykit.sh
+
+    - name: Install zig
+      shell: bash
+      run: ./scripts/install-zig.sh
+
+    - name: Cache Swift packages
+      uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4
+      with:
+        path: .ci-source-packages
+        key: spm-ui-display-resolution-${{ hashFiles('GhosttyTabs.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved') }}
+        restore-keys: spm-ui-display-resolution-
+
+    - name: Resolve Swift packages
+      shell: bash
+      run: |
+        set -euo pipefail
+        SOURCE_PACKAGES_DIR="$PWD/.ci-source-packages"
+        mkdir -p "$SOURCE_PACKAGES_DIR"
+
+        for attempt in 1 2 3; do
+          if xcodebuild -project GhosttyTabs.xcodeproj -scheme cmux -configuration Debug \
+            -clonedSourcePackagesDirPath "$SOURCE_PACKAGES_DIR" \
+            -resolvePackageDependencies; then
+            exit 0
+          fi
+          if [ "$attempt" -eq 3 ]; then
+            echo "Failed to resolve Swift packages after 3 attempts" >&2
+            exit 1
+          fi
+          echo "Package resolution failed on attempt $attempt, retrying..."
+          sleep $((attempt * 5))
+        done
+
+    - name: Run display resolution churn UI regression
+      id: run-display-regression
+      shell: bash
+      run: |
+        set -euo pipefail
+        echo "test_started=true" >> "$GITHUB_OUTPUT"
+        SOURCE_PACKAGES_DIR="$PWD/.ci-source-packages"
+        HELPER_PATH="/tmp/create-virtual-display"
+        MANIFEST_PATH="/tmp/cmux-ui-test-display-harness.json"
+
+        rm -f "$MANIFEST_PATH"
+        trap 'rm -f "$MANIFEST_PATH"' EXIT
+
+        clang -framework Foundation -framework CoreGraphics \
+          -o "$HELPER_PATH" scripts/create-virtual-display.m
+
+        printf '{"helperBinaryPath":"%s"}\n' "$HELPER_PATH" >"$MANIFEST_PATH"
+
+        set +e
+        xcodebuild -project GhosttyTabs.xcodeproj -scheme cmux -configuration Debug \
+          -clonedSourcePackagesDirPath "$SOURCE_PACKAGES_DIR" \
+          -disableAutomaticPackageResolution \
+          -destination "platform=macOS" \
+          -only-testing:cmuxUITests/DisplayResolutionRegressionUITests \
+          test
+        EXIT_CODE=$?
+        set -e
+        echo "completed=true" >> "$GITHUB_OUTPUT"
+        if [ "$EXIT_CODE" -ne 0 ]; then
+          exit "$EXIT_CODE"
+        fi
+
+    - name: Mark display regression passed
+      if: ${{ success() }}
+      id: mark-passed
+      shell: bash
+      run: echo "passed=true" >> "$GITHUB_OUTPUT"

.github/actions/run-lag-regression/action.yml

@@ -0,0 +1,178 @@
+name: Run lag regression attempt
+description: Build cmux on WarpBuild and run the workspace churn typing-lag regression.
+outputs:
+  test_started:
+    description: Whether the lag regression step started.
+    value: ${{ steps.run-lag-regression.outputs.test_started }}
+  completed:
+    description: Whether the lag regression reached a deterministic pass/fail outcome.
+    value: ${{ steps.run-lag-regression.outputs.completed }}
+  passed:
+    description: Whether the lag regression passed.
+    value: ${{ steps.mark-passed.outputs.passed }}
+runs:
+  using: composite
+  steps:
+    - name: Select Xcode
+      shell: bash
+      run: |
+        set -euo pipefail
+        if [ -d "/Applications/Xcode.app/Contents/Developer" ]; then
+          XCODE_DIR="/Applications/Xcode.app/Contents/Developer"
+        else
+          XCODE_APP="$(ls -d /Applications/Xcode*.app 2>/dev/null | sort | tail -n 1 || true)"
+          if [ -n "$XCODE_APP" ]; then
+            XCODE_DIR="$XCODE_APP/Contents/Developer"
+          else
+            echo "No Xcode.app found under /Applications" >&2
+            exit 1
+          fi
+        fi
+        echo "DEVELOPER_DIR=$XCODE_DIR" >> "$GITHUB_ENV"
+        export DEVELOPER_DIR="$XCODE_DIR"
+        xcodebuild -version
+
+    - name: Capture GhosttyKit cache key
+      id: ghosttykit-cache-key
+      shell: bash
+      run: echo "ghostty_sha=$(git -C ghostty rev-parse HEAD)" >> "$GITHUB_OUTPUT"
+
+    - name: Cache GhosttyKit.xcframework
+      id: cache-ghosttykit-lag
+      uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4
+      with:
+        path: GhosttyKit.xcframework
+        key: ghosttykit-${{ hashFiles('.gitmodules') }}-${{ steps.ghosttykit-cache-key.outputs.ghostty_sha }}
+
+    - name: Download pre-built GhosttyKit.xcframework
+      if: steps.cache-ghosttykit-lag.outputs.cache-hit != 'true'
+      shell: bash
+      run: ./scripts/download-prebuilt-ghosttykit.sh
+
+    - name: Install zig
+      shell: bash
+      run: ./scripts/install-zig.sh
+
+    - name: Cache DerivedData
+      uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4
+      with:
+        path: ~/Library/Developer/Xcode/DerivedData/GhosttyTabs-*
+        key: deriveddata-build-${{ hashFiles('GhosttyTabs.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved') }}-${{ hashFiles('GhosttyTabs.xcodeproj/project.pbxproj') }}
+        restore-keys: |
+          deriveddata-build-${{ hashFiles('GhosttyTabs.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved') }}-
+          deriveddata-build-
+
+    - name: Cache Swift packages
+      uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4
+      with:
+        path: .ci-source-packages
+        key: spm-build-${{ hashFiles('GhosttyTabs.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved') }}
+        restore-keys: spm-build-
+
+    - name: Resolve Swift packages
+      shell: bash
+      run: |
+        set -euo pipefail
+        SOURCE_PACKAGES_DIR="$PWD/.ci-source-packages"
+        mkdir -p "$SOURCE_PACKAGES_DIR"
+
+        for attempt in 1 2 3; do
+          if xcodebuild -project GhosttyTabs.xcodeproj -scheme cmux -configuration Debug \
+            -clonedSourcePackagesDirPath "$SOURCE_PACKAGES_DIR" \
+            -resolvePackageDependencies; then
+            exit 0
+          fi
+          if [ "$attempt" -eq 3 ]; then
+            echo "Failed to resolve Swift packages after 3 attempts" >&2
+            exit 1
+          fi
+          echo "Package resolution failed on attempt $attempt, retrying..."
+          sleep $((attempt * 5))
+        done
+
+    - name: Build app
+      shell: bash
+      run: |
+        set -euo pipefail
+        SOURCE_PACKAGES_DIR="$PWD/.ci-source-packages"
+        xcodebuild -project GhosttyTabs.xcodeproj -scheme cmux -configuration Debug \
+          -clonedSourcePackagesDirPath "$SOURCE_PACKAGES_DIR" \
+          -disableAutomaticPackageResolution \
+          -destination "platform=macOS" build
+
+    - name: Create virtual display
+      shell: bash
+      run: |
+        set -euo pipefail
+        clang -framework Foundation -framework CoreGraphics \
+          -o /tmp/create-virtual-display scripts/create-virtual-display.m
+        /tmp/create-virtual-display &
+        VDISPLAY_PID=$!
+        echo "VDISPLAY_PID=$VDISPLAY_PID" >> "$GITHUB_ENV"
+        sleep 3
+
+    - name: Run workspace churn typing-lag regression
+      id: run-lag-regression
+      shell: bash
+      run: |
+        set -euo pipefail
+        echo "test_started=true" >> "$GITHUB_OUTPUT"
+
+        APP="$(find "$HOME/Library/Developer/Xcode/DerivedData" -path "*/Build/Products/Debug/cmux DEV.app" -print -quit)"
+        if [ -z "${APP:-}" ] || [ ! -d "$APP" ]; then
+          echo "cmux DEV.app not found in DerivedData" >&2
+          exit 1
+        fi
+
+        TAG="ci-lag"
+        SOCK="/tmp/cmux-debug-${TAG}.sock"
+        BUNDLE_ID="$(
+          /usr/libexec/PlistBuddy -c 'Print :CFBundleIdentifier' "$APP/Contents/Info.plist" 2>/dev/null \
+            || echo 'com.cmuxterm.app.debug'
+        )"
+
+        pkill -x "cmux DEV" || true
+        rm -f "$SOCK" "/tmp/cmux-${TAG}.sock" || true
+        defaults write "$BUNDLE_ID" socketControlMode -string full >/dev/null 2>&1 || true
+
+        CMUX_TAG="$TAG" CMUX_SOCKET_PATH="$SOCK" CMUX_UI_TEST_MODE=1 "$APP/Contents/MacOS/cmux DEV" >/tmp/cmux-ci-lag.log 2>&1 &
+        APP_PID=$!
+        trap 'kill "$APP_PID" >/dev/null 2>&1 || true' EXIT
+
+        for _ in {1..240}; do
+          [ -S "$SOCK" ] && break
+          sleep 0.25
+        done
+        [ -S "$SOCK" ] || { echo "Socket not ready at $SOCK" >&2; exit 1; }
+
+        set +e
+        CMUX_SOCKET_PATH="$SOCK" \
+        CMUX_LAG_MAX_P95_RATIO=1.70 \
+        CMUX_LAG_MAX_AVG_RATIO=1.70 \
+        CMUX_LAG_MIN_BASELINE_P95_MS_FOR_RATIO=6.0 \
+        CMUX_LAG_MIN_BASELINE_AVG_MS_FOR_RATIO=4.0 \
+        CMUX_LAG_MAX_P95_DELTA_MS=20.0 \
+        CMUX_LAG_MAX_AVG_DELTA_MS=12.0 \
+        CMUX_LAG_MAX_CHURN_P95_MS=35 \
+        CMUX_LAG_KEY_EVENTS=180 \
+        python3 tests/test_workspace_churn_up_arrow_lag.py
+        EXIT_CODE=$?
+        set -e
+        echo "completed=true" >> "$GITHUB_OUTPUT"
+        if [ "$EXIT_CODE" -ne 0 ]; then
+          exit "$EXIT_CODE"
+        fi
+
+    - name: Mark lag regression passed
+      if: ${{ success() }}
+      id: mark-passed
+      shell: bash
+      run: echo "passed=true" >> "$GITHUB_OUTPUT"
+
+    - name: Stop virtual display
+      if: ${{ always() }}
+      shell: bash
+      run: |
+        if [ -n "${VDISPLAY_PID:-}" ]; then
+          kill "$VDISPLAY_PID" >/dev/null 2>&1 || true
+        fi