Avoid restore snapshot mutation churn and synchronous index loading

Packages/macOS/CmuxWorkspaces/Sources/CmuxWorkspaces/Session/RestoredProcessDetectionObservation.swift

@@ -0,0 +1,57 @@
+/// Owns the explicit lifecycle observation state for a process-backed restore binding.
+///
+/// A restored binding is protected from empty scans while its terminal has not
+/// yet spawned, because its restore command cannot have run. Once the runtime
+/// spawns, protection lasts ``observationWindow``: a binding whose command never
+/// launches, or whose shell never reports a prompt transition, then retires like
+/// any unobserved binding instead of surviving every relaunch.
+public struct RestoredProcessDetectionObservation: Equatable, Sendable {
+    /// Upper bound for a spawned restored shell to run its queued command.
+    ///
+    /// Measured on the suspending clock so machine sleep does not consume it.
+    public static let observationWindow: Duration = .seconds(300)
+
+    private enum Phase: Equatable, Sendable {
+        case awaitingRuntimeSpawn
+        case runtimeSpawned(at: SuspendingClock.Instant)
+    }
+
+    private var phase: Phase?
+
+    /// Creates a cleared observation policy.
+    public init() {
+        phase = nil
+    }
+
+    /// Arms observation until the runtime spawns and evidence arrives or the window elapses.
+    public mutating func arm() {
+        phase = .awaitingRuntimeSpawn
+    }
+
+    /// Starts the observation window once the restored terminal runtime exists.
+    ///
+    /// - Returns: Whether the observation state changed.
+    @discardableResult
+    public mutating func recordRuntimeSpawn(at now: SuspendingClock.Instant = .now) -> Bool {
+        guard phase == .awaitingRuntimeSpawn else { return false }
+        phase = .runtimeSpawned(at: now)
+        return true
+    }
+
+    /// Returns whether the restore binding remains protected from an empty scan.
+    public func preserves(at now: SuspendingClock.Instant = .now) -> Bool {
+        switch phase {
+        case nil:
+            return false
+        case .awaitingRuntimeSpawn:
+            return true
+        case .runtimeSpawned(let spawnedAt):
+            return spawnedAt.duration(to: now) < Self.observationWindow
+        }
+    }
+
+    /// Clears observation after authoritative process or shell evidence.
+    public mutating func clear() {
+        phase = nil
+    }
+}

Packages/macOS/CmuxWorkspaces/Tests/CmuxWorkspacesTests/RestoredProcessDetectionObservationTests.swift

@@ -0,0 +1,39 @@
+import Testing
+@testable import CmuxWorkspaces
+
+@Suite struct RestoredProcessDetectionObservationTests {
+    @Test func armedLifecycleStatePreservesUntilCleared() {
+        var observation = RestoredProcessDetectionObservation()
+        #expect(!observation.preserves())
+        observation.arm()
+        #expect(observation.preserves())
+        observation.clear()
+        #expect(!observation.preserves())
+    }
+
+    @Test func observationWaitsForTheRuntimeSpawnBeforeItsWindowStarts() {
+        // A paced or unviewed restored terminal may spawn long after restore;
+        // its command cannot have run, so the window must not start early.
+        var observation = RestoredProcessDetectionObservation()
+        observation.arm()
+        let spawnedAt = SuspendingClock.now.advanced(by: .seconds(3_600))
+        #expect(observation.preserves(at: spawnedAt))
+        // `recordRuntimeSpawn` is mutating, so each result is read outside the test macros.
+        let firstSpawn = observation.recordRuntimeSpawn(at: spawnedAt)
+        let repeatedSpawn = observation.recordRuntimeSpawn(at: spawnedAt.advanced(by: .seconds(10)))
+        #expect(firstSpawn)
+        #expect(!repeatedSpawn)
+        let window = RestoredProcessDetectionObservation.observationWindow
+        #expect(observation.preserves(at: spawnedAt.advanced(by: window - .seconds(1))))
+        #expect(!observation.preserves(at: spawnedAt.advanced(by: window)))
+    }
+
+    @Test func aClearedObservationIgnoresALateSpawn() {
+        var observation = RestoredProcessDetectionObservation()
+        observation.arm()
+        observation.clear()
+        let lateSpawn = observation.recordRuntimeSpawn()
+        #expect(!lateSpawn)
+        #expect(!observation.preserves())
+    }
+}

Sources/AppDelegate+RecoverableMainWindowRoutes.swift

@@ -1,12 +1,10 @@
 import AppKit
 import CmuxTerminalCore
 import CmuxTerminal
-
 // The retire sweep is the MainWindowRouteRetiring witness: terminal topology
 // changes prompt a coalesced lifecycle audit through the seam instead of the
 // registry reaching up to AppDelegate.shared.
 extension AppDelegate: MainWindowRouteRetiring {}
-
 extension AppDelegate {
     typealias MainWindowSessionPersistenceRoute = (
         windowId: UUID,
@@ -394,17 +392,63 @@ extension AppDelegate {
     /// routing call stack. Identity checks on both sides of the suspension make
     /// a reattached or explicitly closed route win over the deferred freeze.
     private func scheduleWindowlessRecoverableMainWindowRouteFreeze(
-        _ route: RecoverableMainWindowRoute
+        _ route: RecoverableMainWindowRoute,
+        retryAttempt: Int = 0
     ) {
         let routeTTYDeviceBindings = currentSurfaceTTYDeviceBindings(for: route)
         let windowId = route.windowId
         let routeIdentity = ObjectIdentifier(route)
         let taskToken = UUID()
+        let onWorkerCreated: @MainActor @Sendable (Task<ProcessDetectedResumeIndexes, Never>) -> Void = {
+            [weak self, weak lifecycleCoordinator = mainWindowLifecycleCoordinator] worker in
+            lifecycleCoordinator?.retainWindowlessRecoveryResumeIndexesWorker(
+                worker,
+                onCompleted: { [weak self, weak lifecycleCoordinator] _ in
+                    guard let self, let lifecycleCoordinator else { return }
+                    for retry in lifecycleCoordinator.consumeWindowlessRouteFreezeRetries() {
+                        guard let route = lifecycleCoordinator.orphanedRoute(windowId: retry.windowId),
+                              route.window == nil,
+                              route.frozenWindowSnapshot == nil else {
+                            continue
+                        }
+                        self.scheduleWindowlessRecoverableMainWindowRouteFreeze(
+                            route,
+                            retryAttempt: retry.attempt
+                        )
+                    }
+                }
+            )
+        }
         let task = Task { @MainActor [weak self] in
+            var nextRetryAttempt: Int?
             defer {
+                let workerIsRunning = self?.mainWindowLifecycleCoordinator
+                    .isWindowlessRecoveryResumeIndexesWorkerRunning() == true
                 self?.mainWindowLifecycleCoordinator.releaseWindowlessRouteFreezeTask(
                     windowId: windowId,
-                    token: taskToken
+                    token: taskToken,
+                    retryAttemptWhenWorkerCompletes: workerIsRunning ? nextRetryAttempt : nil
+                )
+                if let nextRetryAttempt, !workerIsRunning,
+                   let self,
+                   let currentRoute = self.mainWindowLifecycleCoordinator.orphanedRoute(windowId: windowId),
+                   ObjectIdentifier(currentRoute) == routeIdentity,
+                   currentRoute.window == nil,
+                   currentRoute.frozenWindowSnapshot == nil {
+                    self.scheduleWindowlessRecoverableMainWindowRouteFreeze(
+                        currentRoute,
+                        retryAttempt: nextRetryAttempt
+                    )
+                }
+            }
+            if retryAttempt > 0 {
+                // Back off before rescanning so an index that stays incomplete
+                // (an undecodable hook store, a scan slower than its deadline)
+                // cannot drive back-to-back process/filesystem scans.
+                try? await Task.sleep(
+                    for: MainWindowLifecycleCoordinator.windowlessRouteFreezeRetryDelay(
+                        attempt: retryAttempt
+                    )
                 )
             }
             guard !Task.isCancelled else { return }
@@ -442,18 +486,14 @@ extension AppDelegate {
                 return
             }
             let lifecycleCoordinator = self?.mainWindowLifecycleCoordinator
-            let resumeIndexes = await self?.mainWindowLifecycleCoordinator
-                .loadWindowlessRecoveryResumeIndexes(
-                    ttyDeviceBindings: ttyDeviceBindings
-                ) { bindings in
-                    await ProcessDetectedResumeIndexes.loadFreshWithDeadline(
-                        ttyDeviceBindings: bindings,
-                        onWorkerCreated: { [weak lifecycleCoordinator] worker in
-                            lifecycleCoordinator?
-                                .retainWindowlessRecoveryResumeIndexesWorker(worker)
-                        }
-                    )
-                }
+            let resumeIndexes = await lifecycleCoordinator?.loadWindowlessRecoveryResumeIndexes(
+                ttyDeviceBindings: ttyDeviceBindings
+            ) { bindings in
+                await ProcessDetectedResumeIndexes.loadFreshWithDeadline(
+                    ttyDeviceBindings: bindings,
+                    onWorkerCreated: onWorkerCreated
+                )
+            }
             guard !Task.isCancelled,
                   self?.mainWindowLifecycleCoordinator.orphanedRoute(
                       windowId: windowId
@@ -463,38 +503,25 @@ extension AppDelegate {
                   self?.windowForMainWindowId(windowId) == nil else {
                 return
             }
-            // A timed-out fresh scan must retain the last cached agent projection;
-            // only its process-detected surface bindings are unavailable.
-            let restorableAgentIndex: RestorableAgentSessionIndex
-            if let detectedAgentIndex = resumeIndexes?.restorableAgentIndex {
-                restorableAgentIndex = detectedAgentIndex
-            } else if let cachedAgentIndex = SharedLiveAgentIndex.shared.index {
-                restorableAgentIndex = cachedAgentIndex
-            } else {
-                // A cold cache still has a persisted projection. Refresh it
-                // off the interactive path, then fall back to that persisted
-                // projection if the ownership-sensitive refresh deadline is
-                // unavailable. Never substitute an empty index merely because
-                // a bounded refresh timed out before this irreversible freeze.
-                if let refreshedAgentIndex = await SharedLiveAgentIndex.shared
-                    .indexRefreshingNow() {
-                    restorableAgentIndex = refreshedAgentIndex
-                } else {
-                    let fallbackTask = Task.detached(priority: .utility) {
-                        RestorableAgentSessionIndex.load()
-                    }
-                    restorableAgentIndex = await fallbackTask.value
-                }
+            // Windowless teardown is irreversible. A timeout or incomplete
+            // process scan leaves the live route intact for a bounded retry;
+            // once the budget is spent the quit/power-off save owns the route.
+            guard let resumeIndexes,
+                  resumeIndexes.isFresh,
+                  resumeIndexes.restorableAgentIndex.isComplete else {
+                nextRetryAttempt = MainWindowLifecycleCoordinator
+                    .nextWindowlessRouteFreezeRetryAttempt(after: retryAttempt)
+                return
             }
             guard !Task.isCancelled else { return }
-            let detectedSurfaceResumeBindingIndex = resumeIndexes?.surfaceResumeBindingIndex
+            let detectedSurfaceResumeBindingIndex = resumeIndexes.surfaceResumeBindingIndex
             if let currentRoute = self?.mainWindowLifecycleCoordinator.orphanedRoute(
                 windowId: windowId
             ), ObjectIdentifier(currentRoute) == routeIdentity {
                 self?.freezeWindowlessRecoverableMainWindowRoute(
                     currentRoute,
-                    restorableAgentIndex: restorableAgentIndex,
-                    surfaceResumeBindingIndex: detectedSurfaceResumeBindingIndex?.isEmpty == false
+                    restorableAgentIndex: resumeIndexes.restorableAgentIndex,
+                    surfaceResumeBindingIndex: detectedSurfaceResumeBindingIndex.isEmpty == false
                         ? detectedSurfaceResumeBindingIndex
                         : nil
                 )
@@ -615,7 +642,9 @@ extension AppDelegate {
     ) -> [MainWindowPersistenceRouteSnapshot] {
         let windowsByWindowId = currentMainWindowsByWindowId()
         let maximumRecoverableRoutes = availableWindowlessPersistenceSlots()
-        if freezeWindowlessRoutes, let suppliedRestorableAgentIndex {
+        if freezeWindowlessRoutes,
+           let suppliedRestorableAgentIndex,
+           suppliedRestorableAgentIndex.isComplete {
             var candidateOrphanedRoutes: [RecoverableMainWindowRoute] = []
             for route in mainWindowLifecycleCoordinator.orphanedRoutes() {
                 guard let snapshot = recoverableMainWindowPersistenceRouteSnapshot(