perf(tests): stop the slowest app-host unit tests waiting out real timeouts

.github/workflows/ci-macos.yml

@@ -2052,8 +2052,10 @@ jobs:
           set -euo pipefail
           ./tests/test_bundled_ghostty_theme_picker_helper.sh
 
+      # The notification semantics step below resolves `bun` too, and it runs
+      # on the focused regression shard, so Bun must be set up there as well.
       - name: Set up Bun for Pi extension dispatch regression
-        if: ${{ matrix.shard == fromJSON(env.CMUX_APP_HOST_CLI_REGRESSION_SHARD) }}
+        if: ${{ matrix.shard == fromJSON(env.CMUX_APP_HOST_CLI_REGRESSION_SHARD) || matrix.shard == fromJSON(env.CMUX_APP_HOST_FOCUSED_REGRESSION_SHARD) || contains(inputs.unit_strict_steps, '|Run agent notification semantics|') }}
         uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
         with:
           bun-version: "1.3.6"

.github/workflows/command-palette-search-benchmarks.yml

@@ -0,0 +1,152 @@
+name: command palette search benchmarks
+
+# The wall-clock command-palette search benchmarks are gated out of the sharded
+# app-host unit suite by CMUX_COMMAND_PALETTE_SEARCH_BENCHMARKS (see
+# skipUnlessCommandPaletteSearchBenchmarksAreEnabled in
+# cmuxTests/CommandPaletteNucleoFixtures.swift). A gate with no caller at all is
+# deletion with extra steps: the benchmarks keep compiling and stop running.
+# This workflow is that caller -- dispatch-only while automatic CI is paused, so
+# running the benchmarks is a deliberate act rather than a daily cost. It runs
+# the one script that owns the gate,
+# scripts/test-command-palette-nucleo-ffi.sh, which enables the env var and
+# names both benchmark classes; the script's own BENCH assertions fail the job
+# if any gated benchmark silently skipped.
+on:
+  # Temporarily manual-only beginning 2026-07-13 to pause automatic CI.
+  # tmux-corpus.yml and perf-activation.yml -- the two closest macOS
+  # benchmark nightlies -- both carry this and have their crons removed.
+  # A new cron here would quietly reverse that reduction, so the gate's
+  # caller is dispatch-only until the pause lifts.
+  workflow_dispatch:
+    inputs:
+      runner:
+        description: macOS runner (auto follows MACOS_RUNNER_15)
+        required: false
+        default: auto
+        type: choice
+        options:
+          - auto
+          - blacksmith-6vcpu-macos-15
+          - blacksmith-6vcpu-macos-26
+          - blacksmith-6vcpu-macos-latest
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  command-palette-search-benchmarks:
+    runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || ((!inputs.runner || inputs.runner == 'auto') && (vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15') || inputs.runner) }}
+    timeout-minutes: 60
+    env:
+      # XCTest app-host crashes can leave xcodebuild waiting in Swift's crash
+      # backtracer until the job timeout. Keep crash handling non-interactive
+      # and cheap so xcodebuild can restart/finish the suite.
+      SWIFT_BACKTRACE: "interactive=no,timeout=0s,symbolicate=off,color=no"
+      # Declared here, not in a step, so the always() summary and upload steps
+      # still have a path when an earlier step fails.
+      CMUX_NUCLEO_FFI_LOG: ${{ github.workspace }}/command-palette-search-benchmarks.log
+    steps:
+      - name: Clear stale git locks (self-hosted reused workspace)
+        shell: bash
+        run: |
+          # Self-hosted macOS runners reuse the workspace. A job cancelled or
+          # killed mid-checkout can leave a stale .git/modules/*/index.lock that
+          # fails every later submodule checkout (e.g. ghostty). Clear them first.
+          ws="${GITHUB_WORKSPACE:-$PWD}"
+          rm -f "$ws/.git/index.lock" 2>/dev/null || true
+          if [ -d "$ws/.git/modules" ]; then
+            find "$ws/.git/modules" -type f -name "*.lock" -delete 2>/dev/null || true
+          fi
+
+      - name: Checkout
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+          submodules: recursive
+
+      - name: Select Xcode
+        run: |
+          set -euo pipefail
+          ./scripts/select-ci-xcode.sh
+
+      - name: Setup Bun
+        uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
+        with:
+          bun-version: "1.3.14"
+
+      - name: Cache GhosttyKit.xcframework
+        id: cache-ghosttykit
+        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        with:
+          path: GhosttyKit.xcframework
+          key: ghosttykit-sentry-off-v1-${{ hashFiles('.gitmodules', 'ghostty/**') }}
+
+      - name: Download pre-built GhosttyKit.xcframework
+        if: steps.cache-ghosttykit.outputs.cache-hit != 'true'
+        run: ./scripts/download-prebuilt-ghosttykit.sh
+
+      - name: Install zig
+        run: ./scripts/install-zig-ci.sh
+
+      - name: Install Rust
+        run: ./scripts/install-rust-ci.sh
+
+      - name: Cache Swift packages
+        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        with:
+          path: .ci-source-packages
+          key: command-palette-bench-spm-${{ hashFiles('cmux.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved') }}
+          restore-keys: command-palette-bench-spm-
+
+      - name: Sanitize Swift package cache
+        run: python3 scripts/ci/sanitize-xcode-source-packages-cache.py .ci-source-packages
+
+      - name: Prepare isolated DerivedData
+        run: |
+          set -euo pipefail
+          DERIVED_DATA_PATH="${RUNNER_TEMP}/cmux-derived-data-command-palette-bench-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          rm -rf "$DERIVED_DATA_PATH"
+          mkdir -p "$DERIVED_DATA_PATH"
+          echo "CMUX_NUCLEO_FFI_DERIVED_DATA=$DERIVED_DATA_PATH" >> "$GITHUB_ENV"
+
+      - name: Resolve Swift packages
+        run: |
+          set -euo pipefail
+          SOURCE_PACKAGES_DIR="$PWD/.ci-source-packages"
+          mkdir -p "$SOURCE_PACKAGES_DIR"
+          xcodebuild -project cmux.xcodeproj -scheme cmux-unit -configuration Debug \
+            -derivedDataPath "$CMUX_NUCLEO_FFI_DERIVED_DATA" \
+            -clonedSourcePackagesDirPath "$SOURCE_PACKAGES_DIR" \
+            -resolvePackageDependencies
+
+      - name: Run command palette search benchmarks
+        run: |
+          set -euo pipefail
+          CMUX_NUCLEO_FFI_SOURCE_PACKAGES="$PWD/.ci-source-packages" \
+            ./scripts/test-command-palette-nucleo-ffi.sh
+
+      - name: Write benchmark summary
+        if: always()
+        run: |
+          set -euo pipefail
+          {
+            echo "## Command palette search benchmarks"
+            echo ""
+            if [ -f "$CMUX_NUCLEO_FFI_LOG" ]; then
+              grep 'BENCH cmd+' "$CMUX_NUCLEO_FFI_LOG" | sed 's/^/- `/; s/$/`/' || echo "No BENCH lines were emitted."
+            else
+              echo "The benchmark invocation produced no log."
+            fi
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload benchmark log
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: command-palette-search-benchmarks
+          path: ${{ env.CMUX_NUCLEO_FFI_LOG }}
+          if-no-files-found: ignore

CLI/CMUXCLI+BrowserDownload.swift

@@ -1,3 +1,4 @@
+import CmuxControlSocket
 import Foundation
 
 extension CMUXCLI {
@@ -51,9 +52,11 @@ extension CMUXCLI {
             if let timeoutMs = wait.timeoutMs {
                 params["timeout_ms"] = timeoutMs
             }
-            let requestedTimeoutMs = wait.timeoutMs ?? 10_000
-            let effectiveTimeoutMs = min(requestedTimeoutMs, 120_000)
-            let responseTimeout = Double(max(1, effectiveTimeoutMs)) / 1000.0 + 5.0
+            // The handler's own window plus reply slack, both owned by
+            // BrowserDownloadWaitTimeout so the client cannot give up while the
+            // app is still inside the window it is allowed to wait.
+            let responseTimeout = BrowserDownloadWaitTimeout.standard
+                .clientResponseTimeoutSeconds(requestedMilliseconds: wait.timeoutMs)
             let payload = try client.sendV2(
                 method: "browser.download.wait",
                 params: params,

CLI/CMUXCLI+RestorePreflight.swift

@@ -161,7 +161,9 @@ extension CMUXCLI {
 
         guard try waitForRestorePreflightExit(
             exitQueue,
-            timeout: 10
+            timeout: AgentRestorePreflightInvocation.timeoutSeconds(
+                environment: ProcessInfo.processInfo.environment
+            )
         ) else {
             terminateRestorePreflight(processID, exitQueue: exitQueue)
             throw loggedRestoreError(

CLI/SocketClient+StartupWait.swift

@@ -2,6 +2,14 @@ import CmuxControlSocket
 import Foundation
 
 extension SocketClient {
+    /// Budget the launch-capable commands give a starting app to bind its
+    /// control socket. ``SocketStartupWaiter`` owns both the default window and
+    /// the environment override that narrows it.
+    static let appStartupWaitTimeoutSeconds: TimeInterval =
+        SocketStartupWaiter.appStartupTimeoutSeconds(
+            environment: ProcessInfo.processInfo.environment
+        )
+
     static func waitForConnectableSocket(path: String, timeout: TimeInterval) throws -> SocketClient {
         try waitForConnectableSocket(resolvePath: { path }, timeout: timeout)
     }

CLI/cmux.swift

@@ -4197,19 +4197,11 @@ struct CMUXCLI {
     /// Restored terminals start the app and then race its listener bind. Keep
     /// the implicit restore connection alive long enough for that lifecycle,
     /// while explicit socket paths retain their immediate failure semantics.
-    private static let defaultRestoreSocketStartupTimeoutSeconds: TimeInterval = 45
-    /// Tests that exercise the "still opening" failure set
-    /// `CMUX_RESTORE_SOCKET_STARTUP_TIMEOUT_SECONDS` so they do not wait out
-    /// the full startup budget. It can only shorten the default.
-    private static var restoreSocketStartupTimeoutSeconds: TimeInterval {
-        let key = "CMUX_RESTORE_SOCKET_STARTUP_TIMEOUT_SECONDS"
-        guard let raw = ProcessInfo.processInfo.environment[key]?.trimmingCharacters(in: .whitespacesAndNewlines),
-              let value = TimeInterval(raw),
-              value.isFinite else {
-            return defaultRestoreSocketStartupTimeoutSeconds
-        }
-        return min(max(value, 0.05), defaultRestoreSocketStartupTimeoutSeconds)
-    }
+    ///
+    /// ``SocketStartupWaiter`` owns the default window and its environment
+    /// override, so the CLI and the socket package cannot drift apart.
+    private static let restoreSocketStartupTimeoutSeconds: TimeInterval =
+        SocketClient.appStartupWaitTimeoutSeconds
     // Stable per-user slot for the pinned Cloud VM. This value is intentionally reused as
     // both the backend create idempotency key and the local daemon slot so every open,
     // reconnect, session restore, and mobile attach targets the same provider VM once

Packages/macOS/CMUXAgentLaunch/Sources/CMUXAgentLaunch/AgentRestorePreflightInvocation.swift

@@ -1,3 +1,5 @@
+import Foundation
+
 /// A shell-free subprocess invocation run before the restored process.
 public struct AgentRestorePreflightInvocation: Equatable, Sendable {
     /// The executable token from ``arguments``.
@@ -19,4 +21,35 @@ public struct AgentRestorePreflightInvocation: Equatable, Sendable {
         self.arguments = arguments
         self.environment = environment
     }
+
+    /// Ceiling applied to one preflight invocation.
+    ///
+    /// The restore command runs the preflight before `execve`, so this is the
+    /// product-visible ceiling on "provider setup" before restore reports that
+    /// setup took too long.
+    public static let defaultTimeoutSeconds: Double = 10
+
+    /// Environment key that replaces ``defaultTimeoutSeconds``.
+    ///
+    /// A caller that drives restore non-interactively — a harness, or a
+    /// supervisor that retries on its own schedule — bounds the wait here
+    /// instead of holding the terminal for the full default window.
+    public static let timeoutEnvironmentKey = "CMUX_RESTORE_PREFLIGHT_TIMEOUT_SECONDS"
+
+    /// Resolves the preflight budget for `environment`.
+    ///
+    /// - Parameter environment: Process environment to read the override from.
+    /// - Returns: The override when it parses to a finite positive number of
+    ///   seconds no greater than the default, otherwise
+    ///   ``defaultTimeoutSeconds``.
+    public static func timeoutSeconds(environment: [String: String]) -> Double {
+        guard let raw = environment[timeoutEnvironmentKey]?
+            .trimmingCharacters(in: .whitespacesAndNewlines),
+            let value = Double(raw),
+            value.isFinite,
+            value > 0 else {
+            return defaultTimeoutSeconds
+        }
+        return min(value, defaultTimeoutSeconds)
+    }
 }

Packages/macOS/CmuxControlSocket/Sources/CmuxControlSocket/Policy/BrowserDownloadWaitTimeout.swift

@@ -0,0 +1,71 @@
+public import Foundation
+
+/// Shared timeout window for the `browser.download.wait` control call.
+///
+/// The app-side handler and the command-line client have to agree: the client
+/// has to outwait the window the handler is allowed to spend, or a download
+/// that the app reports on time still fails in the terminal. Both sides read
+/// ``standard`` so the two windows cannot drift apart.
+///
+/// The windows are values rather than fixed constants so a test can hold the
+/// same agreement at a fraction of the wall-clock cost instead of waiting the
+/// shipped window out.
+public struct BrowserDownloadWaitTimeout: Equatable, Sendable {
+    /// The windows the app and the CLI ship with.
+    public static let standard = BrowserDownloadWaitTimeout()
+
+    /// Window the handler waits when the caller sends no `timeout_ms`.
+    public let defaultTimeoutMilliseconds: Int
+
+    /// Ceiling the handler applies to a caller-supplied `timeout_ms`.
+    public let maximumTimeoutMilliseconds: Int
+
+    /// Slack the client adds on top of the handler's window to cover request
+    /// dispatch, the handler's own bookkeeping, and the reply hop.
+    public let clientResponseSlackSeconds: TimeInterval
+
+    /// Creates a window pair the handler and the client both read.
+    ///
+    /// - Parameters:
+    ///   - defaultTimeoutMilliseconds: Window for a caller that sends no
+    ///     `timeout_ms`.
+    ///   - maximumTimeoutMilliseconds: Ceiling for a caller-supplied
+    ///     `timeout_ms`.
+    ///   - clientResponseSlackSeconds: Slack the client adds to the handler's
+    ///     window.
+    public init(
+        defaultTimeoutMilliseconds: Int = 10_000,
+        maximumTimeoutMilliseconds: Int = 120_000,
+        clientResponseSlackSeconds: TimeInterval = 5
+    ) {
+        self.defaultTimeoutMilliseconds = defaultTimeoutMilliseconds
+        self.maximumTimeoutMilliseconds = maximumTimeoutMilliseconds
+        self.clientResponseSlackSeconds = clientResponseSlackSeconds
+    }
+
+    /// Window the handler spends for `requestedMilliseconds`.
+    ///
+    /// - Parameter requestedMilliseconds: The caller's `timeout_ms`, or `nil`
+    ///   when the caller sent none.
+    /// - Returns: The clamped handler window in milliseconds.
+    public func handlerTimeoutMilliseconds(
+        requestedMilliseconds: Int?
+    ) -> Int {
+        let requested = max(1, requestedMilliseconds ?? defaultTimeoutMilliseconds)
+        return min(requested, maximumTimeoutMilliseconds)
+    }
+
+    /// Socket response timeout the client uses for `requestedMilliseconds`.
+    ///
+    /// - Parameter requestedMilliseconds: The caller's `--timeout-ms`, or `nil`
+    ///   when the caller passed none.
+    /// - Returns: The handler window plus ``clientResponseSlackSeconds``.
+    public func clientResponseTimeoutSeconds(
+        requestedMilliseconds: Int?
+    ) -> TimeInterval {
+        let handlerWindow = handlerTimeoutMilliseconds(
+            requestedMilliseconds: requestedMilliseconds
+        )
+        return TimeInterval(handlerWindow) / 1000.0 + clientResponseSlackSeconds
+    }
+}

Packages/macOS/CmuxControlSocket/Sources/CmuxControlSocket/Transport/SocketStartupWaiter.swift

@@ -265,3 +265,35 @@ public struct SocketStartupWaiter {
         }
     }
 }
+
+extension SocketStartupWaiter {
+    /// Window a command-line client gives a launching cmux app to bind its
+    /// control socket before reporting that the app is still opening.
+    public static let appStartupTimeoutDefaultSeconds: TimeInterval = 45
+
+    /// Environment key that replaces ``appStartupTimeoutDefaultSeconds``.
+    ///
+    /// A caller that already knows how long the app may take — a supervised
+    /// relaunch, a harness that never launches the app at all — bounds the
+    /// wait here instead of holding the terminal for the full default window.
+    public static let appStartupTimeoutEnvironmentKey = "CMUX_APP_STARTUP_WAIT_TIMEOUT_SECONDS"
+
+    /// Resolves the app-startup wait budget for `environment`.
+    ///
+    /// - Parameter environment: Process environment to read the override from.
+    /// - Returns: The override when it parses to a finite positive number of
+    ///   seconds no greater than the default window, otherwise
+    ///   ``appStartupTimeoutDefaultSeconds``.
+    public static func appStartupTimeoutSeconds(
+        environment: [String: String]
+    ) -> TimeInterval {
+        guard let raw = environment[appStartupTimeoutEnvironmentKey]?
+            .trimmingCharacters(in: .whitespacesAndNewlines),
+            let value = TimeInterval(raw),
+            value.isFinite,
+            value > 0 else {
+            return appStartupTimeoutDefaultSeconds
+        }
+        return min(value, appStartupTimeoutDefaultSeconds)
+    }
+}