Harden E2E recording and dual-Xcode CI routing

.github/workflows/ci.yml

@@ -709,6 +709,12 @@ jobs:
           CMUX_SOURCE_PACKAGES_DIR="$PWD/.ci-source-packages" \
             ./tests/test_bundled_ghostty_theme_picker_helper.sh
 
+      - name: Set up Node for CLI no-socket regressions
+        if: ${{ matrix.shard == fromJSON(env.CMUX_APP_HOST_FOCUSED_REGRESSION_SHARD) }}
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          node-version: "22"
+
       - name: Run CLI no-socket regressions
         if: ${{ matrix.shard == fromJSON(env.CMUX_APP_HOST_FOCUSED_REGRESSION_SHARD) }}
         run: |
@@ -827,7 +833,10 @@ jobs:
     # transitive skip otherwise marks every macOS job skipped even when
     # linux-preflight itself succeeds. Require the direct needs explicitly.
     if: ${{ !cancelled() && needs.changes.result == 'success' && needs.linux-preflight.result == 'success' && needs.changes.outputs.macos == 'true' }}
-    runs-on: ${{ vars.MACOS_RUNNER_15 || 'warp-macos-15-arm64-6x' }}
+    # This job builds the universal Ghostty helper with the macOS 15 SDK, then
+    # switches to the macOS 26 SDK for package tests. The single-Xcode Tart
+    # image cannot satisfy both toolchains.
+    runs-on: ${{ vars.MACOS_RUNNER_DUAL_XCODE || 'blacksmith-6vcpu-macos-15' }}
     timeout-minutes: 40
     env:
       CMUX_CI_XCODE_APP: ${{ vars.CMUX_CI_XCODE_APP_MACOS_15 }}

.github/workflows/test-e2e.yml

@@ -390,39 +390,79 @@ jobs:
           if [ "$RECORD_VIDEO" = "true" ]; then
             # A persistent AVFoundation capture session prevents
             # XCUIApplication from foregrounding a macOS app on both Warp and
-            # Tart. Capture independent JPEG frames instead, then assemble
-            # them after XCTest exits. No recorder owns the display while the
-            # app is activating.
+            # Tart. Capture independent JPEG frames instead, but arm the
+            # recorder until cmux is actually frontmost. Capturing before the
+            # first activation can leave the app Running Background.
             GUI_USER="$(stat -f %Su /dev/console 2>/dev/null || true)"
             if [ -z "$GUI_USER" ] || [ "$GUI_USER" = "root" ]; then
               echo "::error::No logged-in GUI user is available for screen recording"
               exit 1
             fi
             GUI_UID="$(id -u "$GUI_USER")"
             FRAME_DIR=/tmp/test-recording-frames
+            RECORD_READY=/tmp/test-recording-ready
             rm -rf "$FRAME_DIR"
             mkdir -p "$FRAME_DIR"
+            rm -f "$RECORD_READY"
+
+            printf '%s\n' \
+              '#import <AppKit/AppKit.h>' \
+              '' \
+              'int main(int argc, const char *argv[]) {' \
+              '    @autoreleasepool {' \
+              '        if (argc != 2) {' \
+              '            return 2;' \
+              '        }' \
+              '        NSString *expected = [NSString stringWithUTF8String:argv[1]];' \
+              '        NSString *actual = NSWorkspace.sharedWorkspace.frontmostApplication.bundleIdentifier;' \
+              '        return [actual isEqualToString:expected] ? 0 : 1;' \
+              '    }' \
+              '}' \
+              > /tmp/cmux-frontmost-application.m
+            clang -framework AppKit -o /tmp/cmux-frontmost-application \
+              /tmp/cmux-frontmost-application.m
+
             (
               frame=0
+              frontmost_poll_delay=1
+              : > "$RECORD_READY"
               while true; do
+                if ! sudo -n launchctl asuser "$GUI_UID" sudo -n -H -u "$GUI_USER" \
+                  /tmp/cmux-frontmost-application com.cmuxterm.app.debug; then
+                  sleep "$frontmost_poll_delay"
+                  if [ "$frontmost_poll_delay" -lt 5 ]; then
+                    frontmost_poll_delay=$((frontmost_poll_delay + 1))
+                  fi
+                  continue
+                fi
                 output="$(printf '%s/frame-%06d.jpg' "$FRAME_DIR" "$frame")"
-                sudo -n launchctl asuser "$GUI_UID" sudo -n -H -u "$GUI_USER" \
-                  /usr/sbin/screencapture -x -t jpg -D 1 "$output" || exit 1
+                if ! sudo -n launchctl asuser "$GUI_UID" sudo -n -H -u "$GUI_USER" \
+                  /usr/sbin/screencapture -x -t jpg -D 1 "$output"; then
+                  echo "screencapture failed for frame $frame" >&2
+                  sleep 0.5
+                  continue
+                fi
                 frame=$((frame + 1))
                 sleep 0.5
               done
             ) </dev/null >/tmp/screencapture.log 2>&1 &
             RECORD_PID=$!
             echo "RECORD_PID=$RECORD_PID" >> "$GITHUB_ENV"
-            sleep 2
-
-            if kill -0 "$RECORD_PID" 2>/dev/null && [ -s "$FRAME_DIR/frame-000000.jpg" ]; then
-              echo "Frame capture started (PID $RECORD_PID)"
-            else
-              echo "::error::screen frame capture failed to start"
-              cat /tmp/screencapture.log
+            if ! kill -0 "$RECORD_PID"; then
+              echo "::error::Frame recorder process died before initializing"
+              cat /tmp/screencapture.log || true
+              exit 1
+            fi
+            for _ in {1..50}; do
+              [ -f "$RECORD_READY" ] && break
+              sleep 0.1
+            done
+            if [ ! -f "$RECORD_READY" ]; then
+              echo "::error::Frame recorder failed to initialize"
+              cat /tmp/screencapture.log || true
               exit 1
             fi
+            echo "Frame capture armed for frontmost cmux (PID $RECORD_PID)"
           fi
 
           XCODEBUILD_ENV=(

Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalReplayLifecycle.swift

@@ -302,6 +302,35 @@ extension MobileShellComposite {
         return true
     }
 
+    /// Request an authoritative replacement without racing another replay that
+    /// already owns this surface. An active barrier remains the sole lifecycle
+    /// owner; marking output as dropped makes its acknowledgement launch the
+    /// existing bounded follow-up replay. A surface without an owner gets a new
+    /// barrier before the request starts so live output cannot interleave with
+    /// the replacement.
+    func requestTerminalResync(surfaceID: String, deferBehindActiveReplay: Bool) {
+        guard deferBehindActiveReplay else {
+            requestTerminalReplay(surfaceID: surfaceID)
+            return
+        }
+        guard supportedHostCapabilities.contains(Self.terminalReplayCapability) else {
+            // Legacy hosts answer replay requests but do not support the
+            // replacement acknowledgement needed to release a replay barrier.
+            // Preserve their existing unbarriered recovery path.
+            requestTerminalReplay(surfaceID: surfaceID)
+            return
+        }
+        guard hasTerminalOutputSink(surfaceID: surfaceID) else { return }
+        if terminalReplayBarrierTokensBySurfaceID[surfaceID] != nil {
+            terminalReplayBarrierDroppedOutputSurfaceIDs.insert(surfaceID)
+            terminalReplayBarrierDroppedOutputCountsBySurfaceID[surfaceID, default: 0] &+= 1
+            MobileDebugLog.anchormux("terminal.output.resync_deferred surface=\(surfaceID)")
+            return
+        }
+        let replayBarrierToken = beginTerminalReplayBarrier(surfaceID: surfaceID)
+        requestTerminalReplay(surfaceID: surfaceID, replayBarrierToken: replayBarrierToken)
+    }
+
     func markTerminalReplayInFlight(
         surfaceID: String,
         requestID: UUID,

Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift

@@ -6664,7 +6664,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking {
     func resyncTerminalOutput(
         reason: String,
         restartEventStream: Bool,
-        surfaceIDs requestedSurfaceIDs: [String]? = nil
+        surfaceIDs requestedSurfaceIDs: [String]? = nil, deferBehindActiveReplay: Bool = false
     ) {
         guard remoteClient != nil, connectionState == .connected else { return }
         if restartEventStream {
@@ -6681,7 +6681,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking {
             "sync.resync reason=\(reason) restart=\(restartEventStream) surfaces=\(surfaceIDs.count)"
         )
         for surfaceID in surfaceIDs {
-            requestTerminalReplay(surfaceID: surfaceID)
+            requestTerminalResync(surfaceID: surfaceID, deferBehindActiveReplay: deferBehindActiveReplay)
         }
     }
 
@@ -6732,7 +6732,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking {
         resyncTerminalOutput(
             reason: "input_seq_behind",
             restartEventStream: false,
-            surfaceIDs: [surfaceID]
+            surfaceIDs: [surfaceID], deferBehindActiveReplay: terminalOutputTransport == .rawBytes
         )
     }
 

Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellLegacyReplayCompatibilityTests.swift

@@ -0,0 +1,50 @@
+import Foundation
+import Testing
+@testable import CmuxMobileShell
+
+@MainActor
+@Test func legacyHostResyncDoesNotBlockLiveOutputBehindReplayBarrier() async throws {
+    let clock = TestClock()
+    let router = LivenessHostRouter()
+    await router.setCapabilities(["events.v1", "terminal.render_grid.v1"])
+    let box = TransportBox()
+    let store = try await makeConnectedStore(router: router, box: box, clock: clock)
+
+    let collector = OutputCollector()
+    collector.mount(store: store, surfaceID: "live-terminal")
+    let coldReplayRequested = try await pollUntil {
+        await router.count(of: "mobile.terminal.replay") >= 1
+    }
+    #expect(coldReplayRequested)
+    try await waitForReplayResponsesServed(
+        1,
+        router: router,
+        "the legacy cold replay must settle before testing resync"
+    )
+
+    let transport = try #require(box.get())
+    let liveFrame = try renderGridEventFrame(
+        surfaceID: "live-terminal",
+        seq: 100,
+        text: "legacy-live"
+    )
+    let replayCount = await router.count(of: "mobile.terminal.replay")
+    await router.holdNextReplayResponses()
+    store.requestTerminalResync(surfaceID: "live-terminal", deferBehindActiveReplay: true)
+    let resyncRequested = try await pollUntil {
+        await router.count(of: "mobile.terminal.replay") > replayCount
+    }
+    #expect(resyncRequested)
+
+    await transport.deliver(liveFrame)
+    let liveOutputDelivered = try await pollUntil {
+        collector.lines.contains { $0.contains("legacy-live") }
+    }
+    #expect(
+        liveOutputDelivered,
+        "a host without terminal.replay.v1 must keep delivering live output while its unbarriered resync is pending"
+    )
+
+    await router.releaseAllHeld()
+    collector.unmount()
+}

docs/ci-runners.md

@@ -9,6 +9,7 @@ that takes effect on the next workflow run.
 | ------------------- | ---------------------------------------------------------- | --------------------------- | -------------------------------- |
 | `LINUX_RUNNER`      | every Linux job (`ci.yml` web/typecheck/db, presence, cloud-vm, nightly/ios decide jobs, claude, homebrew, tmux fuzz) | `blacksmith-4vcpu-ubuntu-2404` | `warp-ubuntu-latest-x64-4x`   |
 | `MACOS_RUNNER_15`   | universal Release app builds: nightly, stable release, `release-ghostty-cli-helper`, most macOS defaults | `tart-macos-15` | `warp-macos-15-arm64-6x`         |
+| `MACOS_RUNNER_DUAL_XCODE` | Ghostty helper build with SDK 15 followed by Swift package tests with SDK 26 | `blacksmith-6vcpu-macos-15` | `blacksmith-6vcpu-macos-15` |
 | `MACOS_RUNNER_26`   | macOS 26 compatibility jobs                                | `blacksmith-6vcpu-macos-26` | `blacksmith-6vcpu-macos-26`      |
 | `MACOS_RUNNER_26_RELEASE` | disk-heavy `release-build` universal app             | `blacksmith-6vcpu-macos-26` | `blacksmith-6vcpu-macos-26`      |
 | `MACOS_RUNNER_DISPLAY` | macOS GUI, XCUITest, and virtual-display tests           | `tart-gui`                  | `warp-macos-15-arm64-6x`         |
@@ -46,6 +47,7 @@ fleet recovers.
 ```bash
 gh variable set LINUX_RUNNER          --repo manaflow-ai/cmux -b blacksmith-4vcpu-ubuntu-2404
 gh variable set MACOS_RUNNER_15         --repo manaflow-ai/cmux -b blacksmith-6vcpu-macos-15
+gh variable set MACOS_RUNNER_DUAL_XCODE --repo manaflow-ai/cmux -b blacksmith-6vcpu-macos-15
 gh variable set MACOS_RUNNER_26         --repo manaflow-ai/cmux -b blacksmith-6vcpu-macos-26
 gh variable set MACOS_RUNNER_26_RELEASE --repo manaflow-ai/cmux -b blacksmith-6vcpu-macos-26
 gh variable set MACOS_RUNNER_DISPLAY    --repo manaflow-ai/cmux -b depot-macos-latest
@@ -56,6 +58,7 @@ Restore the self-hosted pool with explicit labels:
 
 ```bash
 gh variable set MACOS_RUNNER_15         --repo manaflow-ai/cmux -b tart-macos-15
+gh variable set MACOS_RUNNER_DUAL_XCODE --repo manaflow-ai/cmux -b blacksmith-6vcpu-macos-15
 gh variable set MACOS_RUNNER_26         --repo manaflow-ai/cmux -b blacksmith-6vcpu-macos-26
 gh variable set MACOS_RUNNER_26_RELEASE --repo manaflow-ai/cmux -b blacksmith-6vcpu-macos-26
 gh variable set MACOS_RUNNER_DISPLAY    --repo manaflow-ai/cmux -b tart-gui