iOS: stop the foreground reconnect storm (#10482)

Packages/iOS/CmuxMobileShell/Package.swift

@@ -28,6 +28,7 @@ let package = Package(
         .package(path: "../CmuxMobileRPC"),
         .package(path: "../CmuxMobileShellModel"),
         .package(path: "../CmuxMobileSupport"),
+        .package(path: "../CmuxMobileTerminalKit"),
         .package(path: "../CmuxMobileTransport"),
     ],
     targets: [
@@ -43,6 +44,7 @@ let package = Package(
                 "CmuxMobileRPC",
                 "CmuxMobileShellModel",
                 "CmuxMobileSupport",
+                "CmuxMobileTerminalKit",
                 "CmuxMobileTransport",
             ],
             swiftSettings: [
@@ -78,6 +80,8 @@ let package = Package(
                 "CmuxMobilePairedMac",
                 "CmuxMobileRPC",
                 "CmuxMobileShellModel",
+                "CmuxMobileSupport",
+                "CmuxMobileTerminalKit",
                 "CmuxMobileTransport",
             ],
             swiftSettings: [

Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileConnectionRecoveryOwner.swift

@@ -1,4 +1,5 @@
 import CMUXMobileCore
+internal import CmuxMobileSupport
 import Foundation
 
 /// Main-actor authority for one foreground Mac recovery attempt.
@@ -31,6 +32,20 @@ final class MobileConnectionRecoveryOwner {
     private(set) var phase: Phase = .idle
     private(set) var task: Task<Void, Never>?
 
+    /// Backoff and one-shot task for a terminal event stream that ended before
+    /// delivering an event. Keeping this beside the connection-recovery task
+    /// makes owner cancellation invalidate every recovery continuation.
+    private(set) var deadTerminalEventStreamRedialBackoff =
+        MobileDeadStreamRedialBackoff()
+    private var deadTerminalEventStreamRedialTask: Task<Void, Never>?
+    private var deadTerminalEventStreamRedialGeneration = UUID()
+
+    /// Number of barren streams in the current session, used by recovery
+    /// diagnostics without exposing the mutable backoff itself.
+    var deadTerminalEventStreamBarrenCount: Int {
+        deadTerminalEventStreamRedialBackoff.consecutiveBarrenRedials
+    }
+
     var activeAttempt: Attempt? {
         switch phase {
         case .probing(let attempt), .redialing(let attempt),
@@ -64,6 +79,14 @@ final class MobileConnectionRecoveryOwner {
         }
     }
 
+    /// Whether a delayed barren-stream retry is waiting for its deadline.
+    /// Background suspension uses this to park the corresponding recovery
+    /// trigger before cancellation can otherwise lose the wake-up.
+    var hasPendingDeadTerminalEventStreamRedial: Bool {
+        deadTerminalEventStreamRedialTask != nil
+    }
+
+    /// Claims a new probe or redial attempt when no recovery is active.
     func begin(
         trigger: String,
         sourceConnectionGeneration: UUID,
@@ -72,6 +95,7 @@ final class MobileConnectionRecoveryOwner {
         guard !isActive else { return nil }
         task?.cancel()
         task = nil
+        cancelDeadTerminalEventStreamRedial()
         let attempt = Attempt(
             id: UUID(),
             trigger: trigger,
@@ -90,6 +114,7 @@ final class MobileConnectionRecoveryOwner {
         guard case .probing = phase else { return nil }
         task?.cancel()
         task = nil
+        cancelDeadTerminalEventStreamRedial()
         let attempt = Attempt(
             id: UUID(),
             trigger: trigger,
@@ -190,9 +215,67 @@ final class MobileConnectionRecoveryOwner {
         }
     }
 
+    /// Cancels the active connection attempt and any owned dead-stream retry.
     func cancel() {
         task?.cancel()
         task = nil
+        cancelDeadTerminalEventStreamRedial()
         phase = .idle
     }
+
+    /// Claims the next barren-stream redial delay, coalescing while a delayed
+    /// redial is already pending.
+    func nextDeadTerminalEventStreamRedialDelay() -> Duration? {
+        deadTerminalEventStreamRedialBackoff.nextRedialDelay()
+    }
+
+    /// Schedules one cancellable barren-stream retry under this recovery owner.
+    /// The callback runs only if the owner generation is still current.
+    /// - Parameters:
+    ///   - delay: The injected-clock delay before retrying.
+    ///   - clock: Clock used for the genuine retry deadline.
+    ///   - operation: Main-actor recovery callback to invoke after the delay.
+    func scheduleDeadTerminalEventStreamRedial(
+        after delay: Duration,
+        clock: any Clock<Duration>,
+        operation: @escaping @MainActor () -> Void
+    ) {
+        let generation = UUID()
+        deadTerminalEventStreamRedialGeneration = generation
+        deadTerminalEventStreamRedialTask?.cancel()
+        deadTerminalEventStreamRedialTask = Task { @MainActor [weak self] in
+            do {
+                try await clock.sleep(for: delay)
+            } catch {
+                return
+            }
+            guard let self,
+                  !Task.isCancelled,
+                  self.deadTerminalEventStreamRedialGeneration == generation else {
+                return
+            }
+            self.deadTerminalEventStreamRedialTask = nil
+            self.deadTerminalEventStreamRedialBackoff.redialFired()
+            operation()
+        }
+    }
+
+    /// Cancels a pending barren-stream retry while preserving the accumulated
+    /// session streak. Background suspension uses this form so a resumed
+    /// session cannot immediately return to a tight redial loop.
+    @discardableResult
+    func cancelDeadTerminalEventStreamRedial() -> Bool {
+        let wasPending = deadTerminalEventStreamRedialTask != nil
+        deadTerminalEventStreamRedialGeneration = UUID()
+        deadTerminalEventStreamRedialTask?.cancel()
+        deadTerminalEventStreamRedialTask = nil
+        deadTerminalEventStreamRedialBackoff.redialFired()
+        return wasPending
+    }
+
+    /// Resets the barren-stream retry state at a fresh account/session boundary.
+    func resetDeadTerminalEventStreamBackoff() {
+        deadTerminalEventStreamRedialBackoff.reset()
+        cancelDeadTerminalEventStreamRedial()
+    }
 }

Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+ConnectionRecovery.swift

@@ -123,6 +123,11 @@ extension MobileShellComposite {
             // in-flight recovery. The replacement below owns a new generation
             // and is the only attempt allowed to publish a foreground client.
             connectionRecoveryOwner.cancel()
+            // A deliberate connection-method change restarts connectivity from
+            // scratch (it already clears the automatic reconnect backoff), so
+            // clear the barren-stream streak too instead of inheriting a stale
+            // backoff on the fresh method.
+            resetDeadTerminalEventStreamBackoff()
             applyConnectionRecoveryOwnerState()
             invalidateStoredMacReconnectAttempt()
         } else {
@@ -170,7 +175,10 @@ extension MobileShellComposite {
             guard failConnectionRecoveryReplacement(failure: .connectionClosed) else { return }
             connectionState = .disconnected
             macConnectionStatus = .unavailable
-            clearRemoteConnectionContext()
+            clearRemoteConnectionContext(
+                preservingTerminalMirror: true,
+                preservingWorkspaceChanges: true
+            )
             applyConnectionRecoveryOwnerState()
             armAutomaticReconnectRetryAfterFailedAttempt(
                 failure: .connectionClosed,
@@ -192,10 +200,101 @@ extension MobileShellComposite {
         )
     }
 
+    /// Routes a dead terminal-event-stream recovery through a backoff gate so a
+    /// subscription that keeps ending — or being rejected — before delivering
+    /// any event cannot spin the reconnect loop (issue #10482).
+    ///
+    /// A stream that proved itself alive (delivered at least one event) is a
+    /// genuine mid-session drop and recovers immediately. A stream that ended
+    /// barren recovers immediately the first time — a transient blip should
+    /// heal fast — but each subsequent barren stream is redialed on an
+    /// exponential backoff instead of restarting the same failing stream at
+    /// scheduler speed.
+    func recoverDeadTerminalEventStream(
+        trigger: RecoveryTrigger,
+        expectedClient: MobileCoreRPCClient,
+        streamDeliveredEvent: Bool
+    ) {
+        // Listener tasks from an older client can finish after a replacement
+        // has already become current. Reject that callback before it can
+        // consume or cancel the current session's retry budget.
+        guard remoteClient === expectedClient, connectionState == .connected else {
+            return
+        }
+        if streamDeliveredEvent {
+            connectionRecoveryOwner.resetDeadTerminalEventStreamBackoff()
+            recoverDeadConnection(trigger: trigger, expectedClient: expectedClient)
+            return
+        }
+        guard let delay = connectionRecoveryOwner
+            .nextDeadTerminalEventStreamRedialDelay() else {
+            // A delayed redial is already pending; coalesce into it instead of
+            // stacking another dial.
+            return
+        }
+        guard delay > .zero else {
+            recoverDeadConnection(trigger: trigger, expectedClient: expectedClient)
+            return
+        }
+        scheduleDeadTerminalEventStreamRedial(
+            after: delay,
+            trigger: trigger,
+            expectedClient: expectedClient
+        )
+    }
+
+    /// Schedules one owner-managed retry for the exact failing client.
+    private func scheduleDeadTerminalEventStreamRedial(
+        after delay: Duration,
+        trigger: RecoveryTrigger,
+        expectedClient: MobileCoreRPCClient
+    ) {
+        // Hold the session visibly reconnecting (once) during the wait so the
+        // status pill does not flip on every barren stream end.
+        if connectionState == .connected { markMacConnectionReconnecting() }
+        MobileDebugLog.anchormux(
+            "connection.recovery dead-stream backoff trigger=\(trigger.description) "
+                + "delay=\(delay) barren=\(connectionRecoveryOwner.deadTerminalEventStreamBarrenCount)"
+        )
+        connectionRecoveryOwner.scheduleDeadTerminalEventStreamRedial(
+            after: delay,
+            clock: controlPlaneSchedulingClock
+        ) { [weak self] in
+            guard let self,
+                  self.remoteClient === expectedClient,
+                  self.connectionState == .connected else {
+                return
+            }
+            self.recoverDeadConnection(trigger: trigger, expectedClient: expectedClient)
+        }
+    }
+
+    /// Cancel a pending backoff redial. Every `connectionRecoveryOwner.cancel()`
+    /// pairs with this — directly (background suspend), or through
+    /// ``resetDeadTerminalEventStreamBackoff()`` at new-session boundaries
+    /// (sign-out, new pairing, method change) — so the single recovery owner's
+    /// lifecycle also invalidates the dead-stream redial. Clearing the backoff's
+    /// scheduled flag is part of the cancel: a cancelled redial is no longer
+    /// scheduled, so the next barren stream may schedule again instead of
+    /// coalescing into a dead timer. It keeps the barren-stream streak, so a
+    /// suspend/resume of the same session preserves the accrued backoff.
+    @discardableResult
+    func cancelDeadTerminalEventStreamRedial() -> Bool {
+        connectionRecoveryOwner.cancelDeadTerminalEventStreamRedial()
+    }
+
+    /// Clear the dead-stream backoff streak and cancel any pending backoff
+    /// redial. A delivered event or a fresh foreground return proves the path
+    /// can carry traffic, so the next failure should recover fast.
+    func resetDeadTerminalEventStreamBackoff() {
+        connectionRecoveryOwner.resetDeadTerminalEventStreamBackoff()
+    }
+
     /// Replays the most recent recovery trigger that was parked while the
-    /// scene was inactive. Called from `resumeForegroundRefresh()` after the
-    /// foreground recovery passes, so a replay coalesces into any attempt
-    /// they already started instead of stacking a second dial.
+    /// scene was inactive. Called from `resumeForegroundRefresh()` before the
+    /// generic foreground recovery pass so a parked stream-end replay can force
+    /// its subscription resync instead of being coalesced into a probe that
+    /// reports the still-healthy RPC connection and skips that resync.
     func recoverPendingInactiveRecoveryIfNeeded() {
         guard foregroundRefreshIsActive,
               let trigger = pendingInactiveRecoveryTrigger else { return }
@@ -320,7 +419,10 @@ extension MobileShellComposite {
                     // while the fresh stored-Mac dial starts.
                     self.connectionState = .disconnected
                     self.macConnectionStatus = .unavailable
-                    self.clearRemoteConnectionContext()
+                    self.clearRemoteConnectionContext(
+                        preservingTerminalMirror: true,
+                        preservingWorkspaceChanges: true
+                    )
                     self.applyConnectionRecoveryOwnerState()
                     MobileDebugLog.anchormux(
                         "connection.recovery waiting for physical transport drain "
@@ -351,7 +453,10 @@ extension MobileShellComposite {
                 if self.connectionState == .connected {
                     self.connectionState = .disconnected
                     self.macConnectionStatus = .unavailable
-                    self.clearRemoteConnectionContext()
+                    self.clearRemoteConnectionContext(
+                        preservingTerminalMirror: true,
+                        preservingWorkspaceChanges: true
+                    )
                 }
                 self.applyConnectionRecoveryOwnerState()
 

Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+ReconnectRoutes.swift

@@ -417,6 +417,10 @@ extension MobileShellComposite {
         foregroundRefreshLifecycleState = .active
         foregroundRefreshIsActive = true
         foregroundResumeEpoch &+= 1
+        // A fresh foreground return earns a clean fast recovery: clear any
+        // dead-stream backoff accrued before backgrounding so the first probe
+        // or resync is not needlessly delayed (issue #10482).
+        resetDeadTerminalEventStreamBackoff()
         startObservingNetworkPathChanges()
         // Covers stores constructed already-signed-in (no isSignedIn edge) and
         // restarts a subscription torn down while backgrounded.
@@ -433,9 +437,13 @@ extension MobileShellComposite {
         }
         restartActiveMobileBrowserStreams()
         restartActiveMobileSimulatorStreams()
+        // Replay a parked stream-end recovery before the generic foreground
+        // probe. Its healthy result must force a subscription resync; if the
+        // generic probe claimed the owner first, it could complete as healthy
+        // without restarting a listener whose backoff was canceled above.
+        recoverPendingInactiveRecoveryIfNeeded()
         recoverForegroundConnectionIfNeeded(resyncAfterHealthy: shouldResync)
         recoverDisconnectedOnForegroundIfNeeded()
-        recoverPendingInactiveRecoveryIfNeeded()
         resumeSecondaryControlMaintenanceAfterForeground()
         // The foreground Mac's workspace list updates live over the sync stream,
         // but the other Macs are a read-only snapshot. Re-aggregate them on
@@ -454,6 +462,15 @@ extension MobileShellComposite {
         guard foregroundRefreshLifecycleState != .background else { return }
         foregroundRefreshLifecycleState = .background
         foregroundRefreshIsActive = false
+        // A pending dead-stream backoff redial would otherwise fire on resume
+        // with the process's frozen wall clock; foreground recovery re-drives it.
+        if cancelDeadTerminalEventStreamRedial() {
+            // Keep the definitive stream-end trigger alive across suspension.
+            // The connection can remain RPC-healthy while its event listener is
+            // gone, so a generic foreground liveness probe is not sufficient to
+            // guarantee that the listener is restarted.
+            pendingInactiveRecoveryTrigger = .eventStreamEnded
+        }
         if connectionRecoveryOwner.cancelProbing() {
             applyConnectionRecoveryOwnerState()
         }

Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalMirrorLifecycle.swift

@@ -0,0 +1,46 @@
+import CMUXMobileCore
+internal import CmuxMobileTerminalKit
+
+extension MobileShellComposite {
+    /// Marks every mounted terminal mirror as blank at an intentional teardown
+    /// boundary. Surface identifiers can be reused by another Mac or account,
+    /// so retaining the old producer metadata would risk skipping hydration.
+    func invalidateMountedTerminalMirrors() {
+        for surfaceID in terminalByteContinuationsBySurfaceID.keys {
+            var mirrorState = terminalMirrorStatesBySurfaceID[surfaceID]
+                ?? MobileTerminalMirrorState()
+            mirrorState.invalidate()
+            terminalMirrorStatesBySurfaceID[surfaceID] = mirrorState
+        }
+    }
+
+    /// Mark a mounted mirror blank so its next screen-anchored replay hydrates
+    /// the local scrollback from the current producer.
+    func markTerminalMirrorHydrationNeeded(surfaceID: String) {
+        var state = terminalMirrorStatesBySurfaceID[surfaceID]
+            ?? MobileTerminalMirrorState()
+        state.invalidate()
+        terminalMirrorStatesBySurfaceID[surfaceID] = state
+    }
+
+    /// Record the producer identity and history baseline of a delivered frame.
+    @discardableResult
+    func recordTerminalMirrorFrame(_ frame: MobileTerminalRenderGridFrame) -> Bool {
+        var state = terminalMirrorStatesBySurfaceID[frame.surfaceID]
+            ?? MobileTerminalMirrorState()
+        let retainedMirrorWasActive = state.retainedAcrossReconnect
+        state.record(frame)
+        terminalMirrorStatesBySurfaceID[frame.surfaceID] = state
+        return retainedMirrorWasActive && state.hydrationNeeded
+    }
+
+    /// Returns whether a retained mirror's provisional zero-row replay failed
+    /// its producer-identity and history-freshness checks.
+    func terminalMirrorRequiresHydration(
+        surfaceID: String,
+        frame: MobileTerminalRenderGridFrame
+    ) -> Bool {
+        terminalMirrorStatesBySurfaceID[surfaceID]?.requiresHydration(for: frame)
+            ?? true
+    }
+}