From fd4a7e7bcac514dffad7cc291f29349c727e60c4 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 19 Aug 2026 22:42:18 -0700 Subject: [PATCH 01/21] test(ios): reproduce the foreground reconnect storm (#10482) Four store-level regression tests that FAIL on current main, one per acceptance criterion of the dogfood report: - foregroundDeadEventStreamRedialLoopIsRateLimited: a subscription that keeps ending/being-rejected before delivering any event drives an unbounded, back-off-free recoverDeadConnection redial loop. - workspaceChangesChipsSurviveTransientReconnect: a transient disconnect wipes the files-changed chips (51 -> 0), which re-presents the changes hint on every reconnect cycle. - reconnectWithLiveMirrorResumesWithoutFullScrollbackReplay: a reconnect that keeps a live on-screen mirror re-hydrates the full scrollback (max_scrollback_rows 4000) instead of a cheap repaint. - deadStreamStormDoesNotRepeatedlyReplayAndKeepsViewport: the storm never settles onto a backoff, so the main thread stays pinned. Adds LivenessHostRouter.failNextSubscribeRequests and captures max_scrollback_rows on recorded replay requests. Regression policy: this commit is intentionally red; the fix follows. --- .../MobileForegroundReconnectStormTests.swift | 302 ++++++++++++++++++ ...leShellRenderGridLivenessTestSupport.swift | 22 +- 2 files changed, 321 insertions(+), 3 deletions(-) create mode 100644 Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileForegroundReconnectStormTests.swift diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileForegroundReconnectStormTests.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileForegroundReconnectStormTests.swift new file mode 100644 index 000000000000..10cd74010eb3 --- /dev/null +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileForegroundReconnectStormTests.swift @@ -0,0 +1,302 @@ +import CMUXMobileCore +import CmuxMobileChanges +import CmuxMobilePairedMac +import CmuxMobileRPC +import CmuxMobileShellModel +import Foundation +import Testing +@testable import CmuxMobileShell + +// Regression coverage for https://github.com/manaflow-ai/cmux/issues/10482: +// foregrounding the iOS app after a background triggered a reconnect storm. +// A freshly (re)established terminal event subscription that ends — or whose +// enable handshake is rejected — before delivering any event fired +// `recoverDeadConnection(.eventStreamEnded/.subscriptionStartFailed)` with NO +// backoff. The redial succeeds, restarts the same failing stream, and ends +// again at scheduler speed, pinning the main thread (94% CPU), full-replaying +// terminal scrollback (~20MB/burst on cellular), and churning the +// files-changed chip on every cycle. +// +// These tests assert the four acceptance criteria at the store layer: +// B. a repeatedly-barren event stream backs off instead of tight-looping. +// A. the files-changed chips survive a transient reconnect (no 51->0->51). +// C. a reconnect with a live on-screen mirror resumes (no full scrollback +// re-hydration) rather than re-downloading history. +// D. the dead-stream storm does not fire an unbounded number of terminal +// replays (each replay repaints/anchors the grid and resets scroll), and +// the phone's reported viewport geometry survives the reconnect. + +// MARK: - B. Dead event-stream redials are rate-limited (single-flight + backoff) + +@MainActor +@Test func foregroundDeadEventStreamRedialLoopIsRateLimited() async throws { + let router = LivenessHostRouter() + let box = TransportBox() + let clock = TestClock() + let controlClock = ControlPoolManualClock() + let (store, directory) = try await makeStormRecoveryStore( + router: router, + box: box, + clock: clock, + controlClock: controlClock + ) + defer { + Task { await router.releaseAllHeld() } + try? FileManager.default.removeItem(at: directory) + } + + #expect(store.connectionState == .connected) + let subscribeCountBefore = await router.count(of: "mobile.events.subscribe") + + // Model a host that keeps accepting the transport dial but rejects every + // subscription enable. On current main each rejected subscribe fires + // recoverDeadConnection(.subscriptionStartFailed), which redials, restarts + // the stream, and rejects again — a back-off-free loop at scheduler speed. + await router.failNextSubscribeRequests(count: 25) + store.resyncTerminalOutput(reason: "test.deadStreamStorm", restartEventStream: true) + + // A rate-limited recovery parks the redial on the control-plane backoff + // clock instead of spinning. On main no backoff exists, so this never + // becomes true (and the loop burns through every scripted failure). + let backoffEngaged = try await pollUntil { controlClock.sleeperCount >= 1 } + #expect( + backoffEngaged, + "a repeatedly-barren event stream must back off, not redial in a tight loop" + ) + + // Only a couple of immediate redials before the backoff gate holds. On main + // this instead climbs through the whole scripted-failure budget. + let subscribeDelta = await router.count(of: "mobile.events.subscribe") - subscribeCountBefore + #expect( + subscribeDelta <= 5, + "dead-stream redials must be rate-limited; saw \(subscribeDelta) subscribe attempts" + ) + + // Advancing the backoff clock releases exactly one further redial, which — + // still barren — re-parks on a longer backoff instead of resuming the storm. + let subscribeBeforeAdvance = await router.count(of: "mobile.events.subscribe") + controlClock.advance(by: .seconds(60)) + _ = try await pollUntil { + await router.count(of: "mobile.events.subscribe") > subscribeBeforeAdvance + } + let subscribeAfterAdvance = await router.count(of: "mobile.events.subscribe") + #expect( + subscribeAfterAdvance - subscribeBeforeAdvance <= 3, + "each backoff tick must release a bounded redial, not reopen the storm" + ) +} + +// MARK: - A. Files-changed chips survive a transient reconnect + +@MainActor +@Test func workspaceChangesChipsSurviveTransientReconnect() async throws { + let router = LivenessHostRouter() + let box = TransportBox() + let clock = TestClock() + let store = try await makeConnectedStore(router: router, box: box, clock: clock) + + // The "51 files" chip content the review sheet / hint / toolbar all read. + store.setWorkspaceChangeChipsByWorkspaceID([ + "live-workspace": MobileWorkspaceChangesChip( + filesChanged: 51, + additions: 120, + deletions: 8 + ), + ]) + #expect(store.workspaceChangeChipsByWorkspaceID["live-workspace"]?.filesChanged == 51) + + // A transient reconnect flips connectionState .connected -> .disconnected + // -> .connected. On main the disconnect edge wiped every chip to empty + // (filesChanged 51 -> 0) and the reconnect refetch restored it (0 -> 51), + // and that 51->0->51 churn re-presented the files-changed hint every cycle. + store.connectionState = .disconnected + #expect( + store.workspaceChangeChipsByWorkspaceID["live-workspace"]?.filesChanged == 51, + "a transient disconnect must not drop the files-changed chip to zero" + ) + + store.connectionState = .connected + #expect( + store.workspaceChangeChipsByWorkspaceID["live-workspace"]?.filesChanged == 51, + "reconnecting must not churn the files-changed chip content" + ) +} + +// MARK: - C. A reconnect with a live mirror resumes (no full scrollback replay) + +@MainActor +@Test func reconnectWithLiveMirrorResumesWithoutFullScrollbackReplay() async throws { + let router = LivenessHostRouter() + // Screen-anchored render grid is what carries a deep local scrollback, so + // its replay is where the phone chooses full hydration vs a cheap repaint. + await router.setCapabilities([ + "events.v1", + "terminal.render_grid.v1", + "terminal.render_grid.screen_anchor.v1", + "terminal.replay.v1", + ]) + let box = TransportBox() + let clock = TestClock() + let store = try await makeConnectedStore(router: router, box: box, clock: clock) + let surfaceID = "live-terminal" + #expect(try await pollUntil { store.usesScreenAnchoredRenderGrid }) + + await router.enqueueReplayTexts(["cold-replay"]) + let iterator = store.terminalOutputStream(surfaceID: surfaceID).makeAsyncIterator() + _ = iterator + await router.waitForCount(of: "mobile.terminal.replay", atLeast: 1) + + // Cold attach hydrates this device's deep scrollback (the mirror was blank). + let coldReplay = try #require(await router.requests(for: "mobile.terminal.replay").first) + #expect( + (coldReplay.maxScrollbackRows ?? 0) > 0, + "a cold attach must hydrate scrollback" + ) + + // The surface now has a populated on-screen mirror with a delivery cursor, + // exactly as a steady-state terminal does. + store.deliveredTerminalByteEndSeqBySurfaceID[surfaceID] = 100 + #expect(store.deliveredTerminalByteEndSeqBySurfaceID[surfaceID] != nil) + + // Simulate a reconnect: the recovery path clears the live client, which + // resets terminal output tracking (dropping the delivery cursor), then + // installs a fresh one. The on-screen mirror survives — the surface is + // still mounted — so the reconnect should repaint the visible screen, not + // re-download the entire scrollback again. + let replayCountBeforeReconnect = await router.count(of: "mobile.terminal.replay") + store.remoteClient = nil + try installFreshLivenessRemoteClient(on: store, router: router, box: box, clock: clock) + // A real reconnect re-resolves the host capabilities and transport during + // its handshake; the manual client swap above skips that, so restore the + // screen-anchored render-grid state the reconnect would negotiate. + store.terminalOutputTransport = .renderGrid + store.supportedHostCapabilities = [ + "events.v1", + "terminal.render_grid.v1", + "terminal.render_grid.screen_anchor.v1", + "terminal.replay.v1", + ] + #expect(store.usesScreenAnchoredRenderGrid) + store.requestTerminalReplay(surfaceID: surfaceID) + + #expect(await router.waitForCount( + of: "mobile.terminal.replay", + atLeast: replayCountBeforeReconnect + 1 + )) + let reconnectReplay = try #require(await router.requests(for: "mobile.terminal.replay").last) + #expect( + reconnectReplay.maxScrollbackRows == 0, + "a reconnect that keeps a live mirror must resume (max_scrollback_rows 0), not re-hydrate the full scrollback" + ) +} + +// MARK: - D. The storm does not repeatedly replay; viewport geometry survives + +@MainActor +@Test func deadStreamStormDoesNotRepeatedlyReplayAndKeepsViewport() async throws { + let router = LivenessHostRouter() + let box = TransportBox() + let clock = TestClock() + let controlClock = ControlPoolManualClock() + let (store, directory) = try await makeStormRecoveryStore( + router: router, + box: box, + clock: clock, + controlClock: controlClock + ) + defer { + Task { await router.releaseAllHeld() } + try? FileManager.default.removeItem(at: directory) + } + let surfaceID = "live-terminal" + + await router.enqueueReplayTexts(["cold-replay"]) + var iterator = store.terminalOutputStream(surfaceID: surfaceID).makeAsyncIterator() + await router.waitForCount(of: "mobile.terminal.replay", atLeast: 1) + let coldReplayChunk = try #require(await iterator.next()) + store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: coldReplayChunk.streamToken) + + // Pin a viewport geometry; it drives scroll/grid sizing and must survive a + // reconnect so scrolling keeps working afterward. + _ = await store.updateTerminalViewport(surfaceID: surfaceID, columns: 80, rows: 40) + let viewportKey = MobileTerminalViewportKey( + workspaceID: "live-workspace", + terminalID: MobileTerminalPreview.ID(rawValue: surfaceID) + ) + #expect(store.reportedViewportSizesByTerminalKey[viewportKey]?.columns == 80) + + // Drive the dead-stream storm. On main every redial resyncs and re-anchors + // the terminal grid (which resets the user's scroll) with no backoff; the + // main thread never settles, which is what freezes touch scrolling. + await router.failNextSubscribeRequests(count: 25) + store.resyncTerminalOutput(reason: "test.stormReplay", restartEventStream: true) + + // Rate-limited recovery settles onto the backoff clock instead of spinning. + // On main this never happens, so the reconnect loop keeps re-anchoring the + // grid and pinning the main thread. + let backoffEngaged = try await pollUntil { controlClock.sleeperCount >= 1 } + #expect( + backoffEngaged, + "the reconnect loop must settle so the main thread is free for scrolling" + ) + + // Once parked on the backoff, the terminal is not replayed again until the + // backoff clock advances: no ongoing re-anchoring that resets scroll and no + // main-thread churn. (On main the loop never parks, so it keeps replaying.) + let replaysWhenParked = await router.count(of: "mobile.terminal.replay") + try await Task.sleep(nanoseconds: 150_000_000) + #expect( + await router.count(of: "mobile.terminal.replay") == replaysWhenParked, + "a parked reconnect must stop replaying the terminal" + ) + + // The reported viewport geometry survives the reconnect so scrolling works. + #expect( + store.reportedViewportSizesByTerminalKey[viewportKey]?.columns == 80, + "viewport geometry must survive a reconnect" + ) +} + +// MARK: - Support + +@MainActor +private func makeStormRecoveryStore( + router: LivenessHostRouter, + box: TransportBox, + clock: TestClock, + controlClock: ControlPoolManualClock, + probeTimeoutNanoseconds: UInt64 = 200_000_000 +) async throws -> (store: MobileShellComposite, directory: URL) { + let (pairedStore, directory) = try ReconnectRouteSelectionTests() + .makePairedMacStore() + let route = try #require(makeTicket(clock: clock).routes.first) + try await pairedStore.upsert( + macDeviceID: "test-mac", + displayName: "Test Mac", + routes: [route], + instanceTag: "default", + markActive: true, + stackUserID: "user-1", + teamID: nil, + now: clock.now + ) + let store = MobileShellComposite( + runtime: LivenessTestRuntime( + transportFactory: LivenessTransportFactory(router: router, box: box), + now: { clock.now }, + livenessProbeTimeoutNanoseconds: probeTimeoutNanoseconds + ), + isSignedIn: true, + pairedMacStore: pairedStore, + identityProvider: StaticIdentityProvider(userID: "user-1"), + reachability: AlwaysOnlineReachability(), + pairingHintDefaults: UserDefaults( + suiteName: "storm-recovery-\(UUID().uuidString)" + )!, + controlPlaneSchedulingClock: controlClock + ) + #expect(await store.reconnectActiveMacIfAvailable(stackUserID: "user-1")) + #expect(await router.waitForCount(of: "mobile.events.subscribe", atLeast: 1)) + #expect(try await pollUntil { store.connectionState == .connected }) + return (store, directory) +} diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift index 551c934348df..17181ff901a0 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift @@ -32,6 +32,7 @@ actor LivenessHostRouter { var title: String? var attachToken: String? var stackAccessToken: String? + var maxScrollbackRows: Int? } private var recorded: [RecordedRequest] = [] @@ -131,7 +132,8 @@ actor LivenessHostRouter { action: String? = nil, title: String? = nil, attachToken: String? = nil, - stackAccessToken: String? = nil + stackAccessToken: String? = nil, + maxScrollbackRows: Int? = nil ) { recorded.append(RecordedRequest( method: method, @@ -146,7 +148,8 @@ actor LivenessHostRouter { action: action, title: title, attachToken: attachToken, - stackAccessToken: stackAccessToken + stackAccessToken: stackAccessToken, + maxScrollbackRows: maxScrollbackRows )) resumeSatisfiedCountWaiters() } @@ -432,6 +435,18 @@ actor LivenessHostRouter { subscribeErrorCodesByRequestNumber[number] = code } + /// Reject the next `count` `mobile.events.subscribe` acks (relative to the + /// requests already seen), modeling a host that accepts the transport dial + /// but never enables the subscription. This is the exact edge that drives + /// the `subscriptionStartFailed`/`eventStreamEnded` redial loop in + /// https://github.com/manaflow-ai/cmux/issues/10482. + func failNextSubscribeRequests(count: Int, code: String = "subscribe_failed") { + guard count > 0 else { return } + for offset in 1 ... count { + subscribeErrorCodesByRequestNumber[subscribeRequestCount + offset] = code + } + } + /// Return a malformed acknowledgement for the Nth unsubscribe request. func invalidateUnsubscribeRequest(number: Int) { invalidUnsubscribeRequestNumbers.insert(number) @@ -873,7 +888,8 @@ actor LivenessTransport: CmxByteTransport { action: params?["action"] as? String, title: params?["title"] as? String, attachToken: auth?["attach_token"] as? String, - stackAccessToken: auth?["stack_access_token"] as? String + stackAccessToken: auth?["stack_access_token"] as? String, + maxScrollbackRows: (params?["max_scrollback_rows"] as? NSNumber)?.intValue ) // Answer each request concurrently so one held response cannot // head-of-line block later RPCs, matching the Mac host's From 91c2a87b786c0251f9d8e8ea0673d2ea1a16d03c Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 19 Aug 2026 23:06:04 -0700 Subject: [PATCH 02/21] fix(ios): stop the foreground reconnect storm (#10482) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three coordinated fixes for the dogfood-reported storm when foregrounding the iOS app after a background. 1. Rate-limit the dead terminal-event-stream redial edge (root cause). A subscription that ends — or is rejected — before delivering any event drove recoverDeadConnection(.eventStreamEnded/.subscriptionStartFailed) with no backoff: the redial succeeded, restarted the same failing stream, and re-ended at scheduler speed, pinning the main thread at ~94% CPU (which froze scrolling) and full-replaying scrollback every cycle. Route those edges (and the rejected-subscribe-ack edge) through a new MobileDeadStreamRedialBackoff: the first barren stream still recovers immediately, each subsequent barren stream backs off exponentially (1s..30s) on the control-plane clock, and a delivered event or a fresh foreground return clears the streak. The status pill shows Reconnecting once during the wait instead of flipping every cycle. 2. Preserve the files-changed chips across a transient reconnect. The disconnect edge wiped every chip (filesChanged N -> 0) and the reconnect refetch restored it; that N -> 0 -> N churn re-presented the changes hint and re-showed the toolbar chip on every reconnect cycle. Cancel in-flight fetches on disconnect but keep the last-known chips and reuse-window cache; prune/evict still drop chips for workspaces that actually leave the list. 3. Resume the terminal instead of re-hydrating the full scrollback on reconnect. A connection swap clears each surface's delivery cursor, which forced the next screen-anchored replay to re-download the entire local scrollback (~20MB per reconnect on cellular). Remember surfaces whose on-screen mirror survived the swap and request a history-preserving repaint (max_scrollback_rows 0) for them; a genuinely rebuilt-blank surface still hydrates. --- .../MobileDeadStreamRedialBackoff.swift | 54 +++++++++++ ...ileShellComposite+ConnectionRecovery.swift | 90 +++++++++++++++++++ ...MobileShellComposite+ReconnectRoutes.swift | 7 ++ ...ellComposite+WorkspaceChangesPruning.swift | 26 ++++++ .../MobileShellComposite.swift | 79 ++++++++++++++-- 5 files changed, 247 insertions(+), 9 deletions(-) create mode 100644 Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileDeadStreamRedialBackoff.swift diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileDeadStreamRedialBackoff.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileDeadStreamRedialBackoff.swift new file mode 100644 index 000000000000..0581524695af --- /dev/null +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileDeadStreamRedialBackoff.swift @@ -0,0 +1,54 @@ +import Foundation + +/// Backoff for redialing after a terminal event subscription ends — or is +/// rejected — before it ever delivered an event. +/// +/// A healthy reconnect delivers events, which proves the transport carries +/// traffic and clears the streak. A subscription that keeps ending "barren" +/// (no event delivered) is a broken push path; redialing it again immediately +/// only restarts the same failing stream, so the redial → restart → re-end +/// cycle spins at scheduler speed. On iOS that pinned the main thread at ~94% +/// CPU after foregrounding, froze scrolling, and burned cellular data on +/// repeated full replays (https://github.com/manaflow-ai/cmux/issues/10482). +/// +/// The first barren stream still recovers immediately — a genuine transient +/// blip should heal fast — while each subsequent barren stream backs off +/// exponentially so the loop cannot spin. +struct MobileDeadStreamRedialBackoff { + static let initialBackoff: Duration = .seconds(1) + static let maximumBackoff: Duration = .seconds(30) + + private(set) var consecutiveBarrenRedials = 0 + private var nextBackoff = MobileDeadStreamRedialBackoff.initialBackoff + private(set) var isRedialScheduled = false + + /// The delay to wait before redialing after a stream ended barren. Returns + /// `.zero` for the first barren stream (recover immediately), an increasing + /// delay for each subsequent one, or `nil` when a delayed redial is already + /// scheduled — so simultaneous barren signals coalesce onto one timer + /// instead of stacking redials. + mutating func nextRedialDelay() -> Duration? { + guard !isRedialScheduled else { return nil } + consecutiveBarrenRedials += 1 + guard consecutiveBarrenRedials > 1 else { return .zero } + let delay = nextBackoff + nextBackoff = min(nextBackoff * 2, Self.maximumBackoff) + isRedialScheduled = true + return delay + } + + /// A scheduled delayed redial fired; allow the next barren stream to + /// schedule again. + mutating func redialFired() { + isRedialScheduled = false + } + + /// A delivered event (or an intentional teardown / foreground reset) proves + /// the connection is healthy again; clear the barren streak so the next + /// failure recovers fast. + mutating func reset() { + consecutiveBarrenRedials = 0 + nextBackoff = Self.initialBackoff + isRedialScheduled = false + } +} diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+ConnectionRecovery.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+ConnectionRecovery.swift index 93754efc5078..42a3992b0ca2 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+ConnectionRecovery.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+ConnectionRecovery.swift @@ -192,6 +192,96 @@ extension MobileShellComposite { ) } + /// Routes a dead terminal-event-stream recovery through a backoff gate so a + /// subscription that keeps ending — or being rejected — before delivering + /// any event cannot spin the reconnect loop (issue #10482). + /// + /// A stream that proved itself alive (delivered at least one event) is a + /// genuine mid-session drop and recovers immediately. A stream that ended + /// barren recovers immediately the first time — a transient blip should + /// heal fast — but each subsequent barren stream is redialed on an + /// exponential backoff instead of restarting the same failing stream at + /// scheduler speed. + func recoverDeadTerminalEventStream( + trigger: RecoveryTrigger, + expectedClient: MobileCoreRPCClient, + streamDeliveredEvent: Bool + ) { + if streamDeliveredEvent { + deadTerminalEventStreamRedialBackoff.reset() + cancelDeadTerminalEventStreamRedial() + recoverDeadConnection(trigger: trigger, expectedClient: expectedClient) + return + } + guard let delay = deadTerminalEventStreamRedialBackoff.nextRedialDelay() else { + // A delayed redial is already pending; coalesce into it instead of + // stacking another dial. + return + } + guard delay > .zero else { + recoverDeadConnection(trigger: trigger, expectedClient: expectedClient) + return + } + scheduleDeadTerminalEventStreamRedial( + after: delay, + trigger: trigger, + expectedClient: expectedClient + ) + } + + private func scheduleDeadTerminalEventStreamRedial( + after delay: Duration, + trigger: RecoveryTrigger, + expectedClient: MobileCoreRPCClient + ) { + // Hold the session visibly reconnecting (once) during the wait so the + // status pill does not flip on every barren stream end. + if connectionState == .connected { markMacConnectionReconnecting() } + MobileDebugLog.anchormux( + "connection.recovery dead-stream backoff trigger=\(trigger.description) " + + "delay=\(delay) barren=\(deadTerminalEventStreamRedialBackoff.consecutiveBarrenRedials)" + ) + let clock = controlPlaneSchedulingClock + let generation = UUID() + deadTerminalEventStreamRedialTaskGeneration = generation + deadTerminalEventStreamRedialTask?.cancel() + deadTerminalEventStreamRedialTask = Task { @MainActor [weak self] in + do { + try await clock.sleep(for: delay) + } catch { + return + } + guard let self, + !Task.isCancelled, + self.deadTerminalEventStreamRedialTaskGeneration == generation else { + return + } + self.deadTerminalEventStreamRedialTask = nil + self.deadTerminalEventStreamRedialBackoff.redialFired() + guard self.remoteClient === expectedClient, + self.connectionState == .connected else { + return + } + self.recoverDeadConnection(trigger: trigger, expectedClient: expectedClient) + } + } + + /// Cancel a pending backoff redial (foreground resume, teardown, or a + /// stream that proved itself alive supersede it). + func cancelDeadTerminalEventStreamRedial() { + deadTerminalEventStreamRedialTaskGeneration = UUID() + deadTerminalEventStreamRedialTask?.cancel() + deadTerminalEventStreamRedialTask = nil + } + + /// Clear the dead-stream backoff streak and cancel any pending backoff + /// redial. A delivered event or a fresh foreground return proves the path + /// can carry traffic, so the next failure should recover fast. + func resetDeadTerminalEventStreamBackoff() { + deadTerminalEventStreamRedialBackoff.reset() + cancelDeadTerminalEventStreamRedial() + } + /// Replays the most recent recovery trigger that was parked while the /// scene was inactive. Called from `resumeForegroundRefresh()` after the /// foreground recovery passes, so a replay coalesces into any attempt diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+ReconnectRoutes.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+ReconnectRoutes.swift index f784dca5a0a2..b3f9151a89a9 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+ReconnectRoutes.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+ReconnectRoutes.swift @@ -431,6 +431,10 @@ extension MobileShellComposite { foregroundRefreshLifecycleState = .active foregroundRefreshIsActive = true foregroundResumeEpoch &+= 1 + // A fresh foreground return earns a clean fast recovery: clear any + // dead-stream backoff accrued before backgrounding so the first probe + // or resync is not needlessly delayed (issue #10482). + resetDeadTerminalEventStreamBackoff() startObservingNetworkPathChanges() // Covers stores constructed already-signed-in (no isSignedIn edge) and // restarts a subscription torn down while backgrounded. @@ -468,6 +472,9 @@ extension MobileShellComposite { guard foregroundRefreshLifecycleState != .background else { return } foregroundRefreshLifecycleState = .background foregroundRefreshIsActive = false + // A pending dead-stream backoff redial would otherwise fire on resume + // with the process's frozen wall clock; foreground recovery re-drives it. + cancelDeadTerminalEventStreamRedial() if connectionRecoveryOwner.cancelProbing() { applyConnectionRecoveryOwnerState() } diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+WorkspaceChangesPruning.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+WorkspaceChangesPruning.swift index 65c9c35cfc69..acc6a8da0f93 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+WorkspaceChangesPruning.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+WorkspaceChangesPruning.swift @@ -1,4 +1,30 @@ extension MobileShellComposite { + /// Cancel in-flight workspace-changes summary fetches on a transient + /// disconnect WITHOUT discarding the last-known chips or the reuse-window + /// cache. + /// + /// Wiping the chips on every disconnect dropped each workspace's + /// `filesChanged` to zero, and the reconnect refetch restored it — that + /// `N -> 0 -> N` churn re-presented the files-changed hint (and re-showed + /// the toolbar chip) on every reconnect cycle (issue #10482). Chips for + /// workspaces that actually left the list are still pruned by + /// ``pruneWorkspaceChangesSummaryStateToForeground()`` / + /// ``evictWorkspaceChangesSummaryState(workspaceIDs:)`` when the workspace + /// list changes, and a host that stops advertising the capability clears + /// them through the full ``resetWorkspaceChangesState()``. + func suspendWorkspaceChangesSummaryFetchesPreservingChips() { + workspaceChangesSummaryDebounceTask?.cancel() + workspaceChangesSummaryDebounceTask = nil + workspaceChangesSummaryDebounceTaskID = nil + workspaceChangesSummaryFetchTask?.cancel() + workspaceChangesSummaryFetchTask = nil + workspaceChangesSummaryFetchTaskID = nil + workspaceChangesSummaryTrailingTask?.cancel() + workspaceChangesSummaryTrailingTask = nil + workspaceChangesSummaryTrailingTaskID = nil + workspaceChangesSummaryTrailingDeadline = nil + } + func resetWorkspaceChangesState() { workspaceChangesSummaryDebounceTask?.cancel() workspaceChangesSummaryDebounceTask = nil diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift index fdf198f3bc7f..7b71ef77f789 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift @@ -229,7 +229,10 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { simulatorStreamStore?.setSimulatorStreamConnectionStatus( macConnectionStatus == .reconnecting ? .reconnecting : .disconnected ) - resetWorkspaceChangesState() + // Keep the last-known files-changed chips across a transient + // disconnect so a reconnect does not churn them N -> 0 -> N and + // re-present the changes hint on every cycle (issue #10482). + suspendWorkspaceChangesSummaryFetchesPreservingChips() #if DEBUG cancelLatencyProbe() #endif @@ -1169,6 +1172,13 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { /// across re-subscribes) so events arriving during the round-trip are /// consumed, not buffered invisibly behind the await. private var terminalSubscriptionStartTask: Task? + /// Backoff gate for the dead-terminal-event-stream redial edge. A + /// subscription that keeps ending — or being rejected — before delivering + /// any event must not redial in a tight loop (issue #10482). See + /// ``recoverDeadTerminalEventStream(trigger:expectedClient:streamDeliveredEvent:)``. + var deadTerminalEventStreamRedialBackoff = MobileDeadStreamRedialBackoff() + var deadTerminalEventStreamRedialTask: Task? + @ObservationIgnored var deadTerminalEventStreamRedialTaskGeneration = UUID() /// Subscription success is the final validation edge for a replacement /// connection or listener. This snapshot closes the race where an old /// acknowledgement arrives after a newer listener has taken ownership. @@ -1517,6 +1527,15 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { /// full hydration window; steady-state replays (barrier follow-ups, theme /// resets) request none and replay as history-preserving repaints. var terminalMirrorHydrationNeededSurfaceIDs: Set + /// Mounted surfaces that still hold a populated on-screen mirror across a + /// connection swap (reconnect/handoff). Their delivery cursor is cleared + /// with the old client, but the rendered scrollback survives on screen, so + /// their first replay after the swap should repaint the visible screen + /// (history-preserving) rather than re-download the entire scrollback again + /// — the ~20MB cellular burst per reconnect in issue #10482. A genuinely + /// rebuilt-blank surface still forces hydration through + /// ``terminalMirrorHydrationNeededSurfaceIDs``. + var terminalSurfacesRetainingMirrorAcrossReconnect: Set = [] var terminalReplaySurfaceIDsInFlight: Set var terminalReplayRequestIDsInFlightBySurfaceID: [String: UUID] var terminalReplayTasksBySurfaceID: [String: Task] @@ -11040,6 +11059,18 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { private func resetTerminalOutputTracking() { cancelAllTerminalReplayTasks() + // A connection swap clears each surface's delivery cursor below, but a + // mounted surface's rendered scrollback survives on screen. Remember + // which ones so their first post-swap replay resumes (repaints the + // visible screen) instead of re-downloading the whole scrollback + // (issue #10482). Genuinely rebuilt-blank surfaces are excluded because + // they carry no delivery cursor here (their mirror was wiped). + terminalSurfacesRetainingMirrorAcrossReconnect = Set( + terminalByteContinuationsBySurfaceID.keys.filter { + deliveredTerminalByteEndSeqBySurfaceID[$0] != nil + && !terminalMirrorHydrationNeededSurfaceIDs.contains($0) + } + ) effectiveViewportSizesBySurfaceID = [:]; reportedTerminalViewportSizesBySurfaceID = [:] // Keep viewport sequences for the account lifetime. A warm peer keeps // its Mac-side tombstone, while a reconnected peer safely accepts a @@ -13155,6 +13186,11 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { recoversConnectionOnFailure: recoversConnectionOnSubscriptionFailure ) + // Whether this listener generation ever delivered an event. A + // stream that ends without having delivered anything is "barren" + // and must back off before redialing (issue #10482); one that + // delivered proves the path is alive and recovers immediately. + var didDeliverEvent = false // Keep the listener alive without keeping the shell store alive. for await event in stream { guard !Task.isCancelled else { return } @@ -13165,6 +13201,12 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { ) else { return } + if !didDeliverEvent { + didDeliverEvent = true + // The push path carries traffic; clear any dead-stream + // backoff so a later genuine drop recovers fast. + self.resetDeadTerminalEventStreamBackoff() + } // Any yielded envelope proves the transport is still pushing, so // it resets the liveness window (not just render_grid events). self.cancelTerminalInputAckResubscribeRetry() @@ -13238,7 +13280,8 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { self.handleTerminalEventStreamEnded( listenerID: listenerID, client: client, - recoversConnectionOnFailure: recoversEndedStream + recoversConnectionOnFailure: recoversEndedStream, + didDeliverEvent: didDeliverEvent ) } } @@ -13448,9 +13491,13 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { MobileDebugLog.anchormux("sync.subscribe_failed reason=start") self.diagnosticLog?.record(DiagnosticEvent(.error)) if recoversConnectionOnFailure { - self.recoverDeadConnection( + // A rejected enable handshake never delivered an event, so + // it is a barren redial: gate it so a host that keeps + // rejecting the subscription cannot spin the reconnect loop. + self.recoverDeadTerminalEventStream( trigger: .subscriptionStartFailed, - expectedClient: client + expectedClient: client, + streamDeliveredEvent: false ) } return @@ -13487,7 +13534,8 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { private func handleTerminalEventStreamEnded( listenerID: UUID, client: MobileCoreRPCClient, - recoversConnectionOnFailure: Bool + recoversConnectionOnFailure: Bool, + didDeliverEvent: Bool ) { guard !Task.isCancelled, terminalEventListenerID == listenerID, @@ -13512,16 +13560,21 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { mobileShellLog.info("terminal event stream ended before subscribe ack, marking unavailable") MobileDebugLog.anchormux("sync.stream_ended before subscribe ack; failed start") diagnosticLog?.record(DiagnosticEvent(.error)) - recoverDeadConnection( + recoverDeadTerminalEventStream( trigger: .subscriptionStartFailed, - expectedClient: client + expectedClient: client, + streamDeliveredEvent: didDeliverEvent ) return } mobileShellLog.info("terminal event stream ended, redialing stored Mac") MobileDebugLog.anchormux("sync.stream_ended redialing stored Mac") diagnosticLog?.record(DiagnosticEvent(.streamEnded)) - recoverDeadConnection(trigger: .eventStreamEnded, expectedClient: client) + recoverDeadTerminalEventStream( + trigger: .eventStreamEnded, + expectedClient: client, + streamDeliveredEvent: didDeliverEvent + ) } // MARK: - Render-grid liveness watchdog @@ -14358,8 +14411,16 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { // destroys locally accumulated history. if let self, self.usesScreenAnchoredRenderGrid { params["anchor"] = MobileTerminalRenderGridFrame.Anchor.screen.rawValue + // A surface whose mirror survived a connection swap keeps + // its rendered scrollback on screen, so its cleared cursor + // alone must not force a full re-hydration (~20MB per + // reconnect on cellular, issue #10482); repaint the visible + // screen instead. A rebuilt-blank surface still hydrates. + let mirrorSurvivedReconnect = + self.terminalSurfacesRetainingMirrorAcrossReconnect.contains(surfaceID) let needsHydration = - self.deliveredTerminalByteEndSeqBySurfaceID[surfaceID] == nil + (self.deliveredTerminalByteEndSeqBySurfaceID[surfaceID] == nil + && !mirrorSurvivedReconnect) || self.terminalMirrorHydrationNeededSurfaceIDs.contains(surfaceID) params["max_scrollback_rows"] = needsHydration ? MobileTerminalScrollbackPreference.resolve() From a202839481211c5596a8e6424a9d463612e3d4f6 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 19 Aug 2026 23:55:25 -0700 Subject: [PATCH 03/21] fix(ios): tie dead-stream backoff cancellation to the recovery owner (#10482) Address review feedback on PR #10491: - cancelDeadTerminalEventStreamRedial() now clears the backoff's scheduled flag, so a cancelled redial is not left marked scheduled (which would coalesce the next barren stream into a dead timer), and pair it with every connectionRecoveryOwner.cancel() (method change, account boundary, explicit connect, deinit). The single recovery owner's lifecycle now invalidates the pending dead-stream redial instead of leaving an independent task alive. - Test: replace the fixed Task.sleep in the parked-reconnect assertion with a bounded router.waitForCount(recordIssueOnTimeout: false) that asserts no further replay lands. --- .../MobileShellComposite+ConnectionRecovery.swift | 10 ++++++++-- .../CmuxMobileShell/MobileShellComposite.swift | 3 +++ .../MobileForegroundReconnectStormTests.swift | 12 ++++++++---- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+ConnectionRecovery.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+ConnectionRecovery.swift index 42a3992b0ca2..1066ed18f093 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+ConnectionRecovery.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+ConnectionRecovery.swift @@ -123,6 +123,7 @@ extension MobileShellComposite { // in-flight recovery. The replacement below owns a new generation // and is the only attempt allowed to publish a foreground client. connectionRecoveryOwner.cancel() + cancelDeadTerminalEventStreamRedial() applyConnectionRecoveryOwnerState() invalidateStoredMacReconnectAttempt() } else { @@ -266,12 +267,17 @@ extension MobileShellComposite { } } - /// Cancel a pending backoff redial (foreground resume, teardown, or a - /// stream that proved itself alive supersede it). + /// Cancel a pending backoff redial. Every `connectionRecoveryOwner.cancel()` + /// pairs with this so the single recovery owner's lifecycle also invalidates + /// the dead-stream redial (foreground resume and a stream that proved itself + /// alive supersede it too). Clearing the backoff's scheduled flag is part of + /// the cancel: a cancelled redial is no longer scheduled, so the next barren + /// stream may schedule again instead of coalescing into a dead timer. func cancelDeadTerminalEventStreamRedial() { deadTerminalEventStreamRedialTaskGeneration = UUID() deadTerminalEventStreamRedialTask?.cancel() deadTerminalEventStreamRedialTask = nil + deadTerminalEventStreamRedialBackoff.redialFired() } /// Clear the dead-stream backoff streak and cancel any pending backoff diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift index 7b71ef77f789..200f75c1624a 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift @@ -2001,6 +2001,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { connectionMethodObservationTask?.cancel() terminalEventListenerTask?.cancel() terminalSubscriptionStartTask?.cancel() + deadTerminalEventStreamRedialTask?.cancel() renderGridLivenessTimer?.cancel() renderGridLivenessProbeTask?.cancel() terminalInputAckResubscribeRetryTask?.cancel() @@ -2098,6 +2099,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { presencePushRecoveryThrottle.reset() pendingInactiveRecoveryTrigger = nil connectionRecoveryOwner.cancel() + cancelDeadTerminalEventStreamRedial() applyConnectionRecoveryOwnerState() invalidatePairingAttempt() clearMacSwitchAttemptState() @@ -11147,6 +11149,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { // recovery parked while the scene was inactive. pendingInactiveRecoveryTrigger = nil connectionRecoveryOwner.cancel() + cancelDeadTerminalEventStreamRedial() applyConnectionRecoveryOwnerState() invalidateStoredMacReconnectAttempt() connectionGeneration = UUID() diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileForegroundReconnectStormTests.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileForegroundReconnectStormTests.swift index 10cd74010eb3..8f788d55b752 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileForegroundReconnectStormTests.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileForegroundReconnectStormTests.swift @@ -243,12 +243,16 @@ import Testing // Once parked on the backoff, the terminal is not replayed again until the // backoff clock advances: no ongoing re-anchoring that resets scroll and no // main-thread churn. (On main the loop never parks, so it keeps replaying.) + // Wait on the router's arrival signal (bounded) and assert no further replay + // lands, rather than sleeping a fixed interval. let replaysWhenParked = await router.count(of: "mobile.terminal.replay") - try await Task.sleep(nanoseconds: 150_000_000) - #expect( - await router.count(of: "mobile.terminal.replay") == replaysWhenParked, - "a parked reconnect must stop replaying the terminal" + let replayedAgain = await router.waitForCount( + of: "mobile.terminal.replay", + atLeast: replaysWhenParked + 1, + timeoutNanoseconds: 200_000_000, + recordIssueOnTimeout: false ) + #expect(!replayedAgain, "a parked reconnect must stop replaying the terminal") // The reported viewport geometry survives the reconnect so scrolling works. #expect( From f5a95dd9b699e566c10dfc9362fe72128a2c11e3 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Thu, 20 Aug 2026 23:43:46 -0700 Subject: [PATCH 04/21] fix(ios): reset dead-stream backoff streak at session boundaries (#10482) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address CodeRabbit follow-up: cancelDeadTerminalEventStreamRedial() clears the scheduled flag but keeps consecutiveBarrenRedials, so a new session could inherit the previous session's backoff (up to the 30s cap). Use resetDeadTerminalEventStreamBackoff() at the fresh-start boundaries — sign-out, new pairing attempt, and connection-method change — while a same-session background suspend still keeps the accrued streak. Adds a unit test that reset returns the streak to an immediate first redial. --- ...ileShellComposite+ConnectionRecovery.swift | 19 ++++++++++----- .../MobileShellComposite.swift | 10 ++++++-- .../MobileForegroundReconnectStormTests.swift | 23 +++++++++++++++++++ 3 files changed, 44 insertions(+), 8 deletions(-) diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+ConnectionRecovery.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+ConnectionRecovery.swift index 1066ed18f093..8a51da9eb144 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+ConnectionRecovery.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+ConnectionRecovery.swift @@ -123,7 +123,11 @@ extension MobileShellComposite { // in-flight recovery. The replacement below owns a new generation // and is the only attempt allowed to publish a foreground client. connectionRecoveryOwner.cancel() - cancelDeadTerminalEventStreamRedial() + // A deliberate connection-method change restarts connectivity from + // scratch (it already clears the automatic reconnect backoff), so + // clear the barren-stream streak too instead of inheriting a stale + // backoff on the fresh method. + resetDeadTerminalEventStreamBackoff() applyConnectionRecoveryOwnerState() invalidateStoredMacReconnectAttempt() } else { @@ -268,11 +272,14 @@ extension MobileShellComposite { } /// Cancel a pending backoff redial. Every `connectionRecoveryOwner.cancel()` - /// pairs with this so the single recovery owner's lifecycle also invalidates - /// the dead-stream redial (foreground resume and a stream that proved itself - /// alive supersede it too). Clearing the backoff's scheduled flag is part of - /// the cancel: a cancelled redial is no longer scheduled, so the next barren - /// stream may schedule again instead of coalescing into a dead timer. + /// pairs with this — directly (background suspend), or through + /// ``resetDeadTerminalEventStreamBackoff()`` at new-session boundaries + /// (sign-out, new pairing, method change) — so the single recovery owner's + /// lifecycle also invalidates the dead-stream redial. Clearing the backoff's + /// scheduled flag is part of the cancel: a cancelled redial is no longer + /// scheduled, so the next barren stream may schedule again instead of + /// coalescing into a dead timer. It keeps the barren-stream streak, so a + /// suspend/resume of the same session preserves the accrued backoff. func cancelDeadTerminalEventStreamRedial() { deadTerminalEventStreamRedialTaskGeneration = UUID() deadTerminalEventStreamRedialTask?.cancel() diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift index 200f75c1624a..397dfcf06093 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift @@ -2099,7 +2099,10 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { presencePushRecoveryThrottle.reset() pendingInactiveRecoveryTrigger = nil connectionRecoveryOwner.cancel() - cancelDeadTerminalEventStreamRedial() + // A new session boundary (sign-out, new pairing attempt): reset the + // barren-stream streak, not just the pending redial, so the next + // session does not inherit the previous one's backoff (issue #10482). + resetDeadTerminalEventStreamBackoff() applyConnectionRecoveryOwnerState() invalidatePairingAttempt() clearMacSwitchAttemptState() @@ -11149,7 +11152,10 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { // recovery parked while the scene was inactive. pendingInactiveRecoveryTrigger = nil connectionRecoveryOwner.cancel() - cancelDeadTerminalEventStreamRedial() + // A new session boundary (sign-out, new pairing attempt): reset the + // barren-stream streak, not just the pending redial, so the next + // session does not inherit the previous one's backoff (issue #10482). + resetDeadTerminalEventStreamBackoff() applyConnectionRecoveryOwnerState() invalidateStoredMacReconnectAttempt() connectionGeneration = UUID() diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileForegroundReconnectStormTests.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileForegroundReconnectStormTests.swift index 8f788d55b752..afc7fbf6aba4 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileForegroundReconnectStormTests.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileForegroundReconnectStormTests.swift @@ -86,6 +86,29 @@ import Testing ) } +// MARK: - B. Backoff streak resets at a session boundary (does not carry over) + +@Test func deadStreamRedialBackoffResetClearsStreak() { + var backoff = MobileDeadStreamRedialBackoff() + // The first barren stream recovers immediately; each subsequent one backs + // off exponentially, coalescing while a delayed redial is already scheduled. + #expect(backoff.nextRedialDelay() == .zero) + #expect(backoff.nextRedialDelay() == .seconds(1)) + #expect(backoff.nextRedialDelay() == nil) + backoff.redialFired() + #expect(backoff.nextRedialDelay() == .seconds(2)) + backoff.redialFired() + #expect(backoff.nextRedialDelay() == .seconds(4)) + backoff.redialFired() + + // A new-session boundary (sign-out, new pairing, method change) resets the + // streak, so the next session's first barren stream recovers immediately + // instead of inheriting the previous session's accrued backoff (#10482). + backoff.reset() + #expect(backoff.nextRedialDelay() == .zero) + #expect(backoff.nextRedialDelay() == .seconds(1)) +} + // MARK: - A. Files-changed chips survive a transient reconnect @MainActor From 4d09f201c8a5f5b3ab7ab9cbe749712bd1ecee79 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 1 Sep 2026 15:08:28 -0700 Subject: [PATCH 05/21] fix(ios): harden reconnect storm lifecycle state --- Packages/iOS/CmuxMobileShell/Package.swift | 1 + .../MobileConnectionRecoveryOwner.swift | 72 ++++++++ ...ileShellComposite+ConnectionRecovery.swift | 59 +++--- ...ellComposite+TerminalMirrorLifecycle.swift | 42 +++++ ...hellComposite+TerminalOutputDelivery.swift | 12 +- ...obileShellComposite+WorkspaceChanges.swift | 43 ++++- .../MobileShellComposite.swift | 172 ++++++++++++------ .../MobileTerminalMirrorState.swift | 70 +++++++ .../MobileForegroundReconnectStormTests.swift | 70 ++++++- .../MobileReconnectStateLifecycleTests.swift | 158 ++++++++++++++++ ...leShellRenderGridLivenessTestSupport.swift | 47 ++++- .../MobileDeadStreamRedialBackoff.swift | 26 ++- 12 files changed, 658 insertions(+), 114 deletions(-) create mode 100644 Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalMirrorLifecycle.swift create mode 100644 Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileTerminalMirrorState.swift create mode 100644 Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileReconnectStateLifecycleTests.swift rename Packages/iOS/{CmuxMobileShell/Sources/CmuxMobileShell => CmuxMobileSupport/Sources/CmuxMobileSupport}/MobileDeadStreamRedialBackoff.swift (72%) diff --git a/Packages/iOS/CmuxMobileShell/Package.swift b/Packages/iOS/CmuxMobileShell/Package.swift index c6b11e364ea8..bec5dc8a51ef 100644 --- a/Packages/iOS/CmuxMobileShell/Package.swift +++ b/Packages/iOS/CmuxMobileShell/Package.swift @@ -78,6 +78,7 @@ let package = Package( "CmuxMobilePairedMac", "CmuxMobileRPC", "CmuxMobileShellModel", + "CmuxMobileSupport", "CmuxMobileTransport", ], swiftSettings: [ diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileConnectionRecoveryOwner.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileConnectionRecoveryOwner.swift index b242eab5f9e2..c8936b8fb420 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileConnectionRecoveryOwner.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileConnectionRecoveryOwner.swift @@ -1,4 +1,5 @@ import CMUXMobileCore +internal import CmuxMobileSupport import Foundation /// Main-actor authority for one foreground Mac recovery attempt. @@ -31,6 +32,20 @@ final class MobileConnectionRecoveryOwner { private(set) var phase: Phase = .idle private(set) var task: Task? + /// Backoff and one-shot task for a terminal event stream that ended before + /// delivering an event. Keeping this beside the connection-recovery task + /// makes owner cancellation invalidate every recovery continuation. + private(set) var deadTerminalEventStreamRedialBackoff = + MobileDeadStreamRedialBackoff() + private var deadTerminalEventStreamRedialTask: Task? + private var deadTerminalEventStreamRedialGeneration = UUID() + + /// Number of barren streams in the current session, used by recovery + /// diagnostics without exposing the mutable backoff itself. + var deadTerminalEventStreamBarrenCount: Int { + deadTerminalEventStreamRedialBackoff.consecutiveBarrenRedials + } + var activeAttempt: Attempt? { switch phase { case .probing(let attempt), .redialing(let attempt), @@ -72,6 +87,7 @@ final class MobileConnectionRecoveryOwner { guard !isActive else { return nil } task?.cancel() task = nil + cancelDeadTerminalEventStreamRedial() let attempt = Attempt( id: UUID(), trigger: trigger, @@ -90,6 +106,7 @@ final class MobileConnectionRecoveryOwner { guard case .probing = phase else { return nil } task?.cancel() task = nil + cancelDeadTerminalEventStreamRedial() let attempt = Attempt( id: UUID(), trigger: trigger, @@ -190,9 +207,64 @@ final class MobileConnectionRecoveryOwner { } } + /// Cancels the active connection attempt and any owned dead-stream retry. func cancel() { task?.cancel() task = nil + cancelDeadTerminalEventStreamRedial() phase = .idle } + + /// Claims the next barren-stream redial delay, coalescing while a delayed + /// redial is already pending. + func nextDeadTerminalEventStreamRedialDelay() -> Duration? { + deadTerminalEventStreamRedialBackoff.nextRedialDelay() + } + + /// Schedules one cancellable barren-stream retry under this recovery owner. + /// The callback runs only if the owner generation is still current. + /// - Parameters: + /// - delay: The injected-clock delay before retrying. + /// - clock: Clock used for the genuine retry deadline. + /// - operation: Main-actor recovery callback to invoke after the delay. + func scheduleDeadTerminalEventStreamRedial( + after delay: Duration, + clock: any Clock, + operation: @escaping @MainActor () -> Void + ) { + let generation = UUID() + deadTerminalEventStreamRedialGeneration = generation + deadTerminalEventStreamRedialTask?.cancel() + deadTerminalEventStreamRedialTask = Task { @MainActor [weak self] in + do { + try await clock.sleep(for: delay) + } catch { + return + } + guard let self, + !Task.isCancelled, + self.deadTerminalEventStreamRedialGeneration == generation else { + return + } + self.deadTerminalEventStreamRedialTask = nil + self.deadTerminalEventStreamRedialBackoff.redialFired() + operation() + } + } + + /// Cancels a pending barren-stream retry while preserving the accumulated + /// session streak. Background suspension uses this form so a resumed + /// session cannot immediately return to a tight redial loop. + func cancelDeadTerminalEventStreamRedial() { + deadTerminalEventStreamRedialGeneration = UUID() + deadTerminalEventStreamRedialTask?.cancel() + deadTerminalEventStreamRedialTask = nil + deadTerminalEventStreamRedialBackoff.redialFired() + } + + /// Resets the barren-stream retry state at a fresh account/session boundary. + func resetDeadTerminalEventStreamBackoff() { + deadTerminalEventStreamRedialBackoff.reset() + cancelDeadTerminalEventStreamRedial() + } } diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+ConnectionRecovery.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+ConnectionRecovery.swift index 8a51da9eb144..fea449e731c8 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+ConnectionRecovery.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+ConnectionRecovery.swift @@ -175,7 +175,10 @@ extension MobileShellComposite { guard failConnectionRecoveryReplacement(failure: .connectionClosed) else { return } connectionState = .disconnected macConnectionStatus = .unavailable - clearRemoteConnectionContext() + clearRemoteConnectionContext( + preservingTerminalMirror: true, + preservingWorkspaceChanges: true + ) applyConnectionRecoveryOwnerState() armAutomaticReconnectRetryAfterFailedAttempt( failure: .connectionClosed, @@ -212,13 +215,19 @@ extension MobileShellComposite { expectedClient: MobileCoreRPCClient, streamDeliveredEvent: Bool ) { + // Listener tasks from an older client can finish after a replacement + // has already become current. Reject that callback before it can + // consume or cancel the current session's retry budget. + guard remoteClient === expectedClient, connectionState == .connected else { + return + } if streamDeliveredEvent { - deadTerminalEventStreamRedialBackoff.reset() - cancelDeadTerminalEventStreamRedial() + connectionRecoveryOwner.resetDeadTerminalEventStreamBackoff() recoverDeadConnection(trigger: trigger, expectedClient: expectedClient) return } - guard let delay = deadTerminalEventStreamRedialBackoff.nextRedialDelay() else { + guard let delay = connectionRecoveryOwner + .nextDeadTerminalEventStreamRedialDelay() else { // A delayed redial is already pending; coalesce into it instead of // stacking another dial. return @@ -244,26 +253,14 @@ extension MobileShellComposite { if connectionState == .connected { markMacConnectionReconnecting() } MobileDebugLog.anchormux( "connection.recovery dead-stream backoff trigger=\(trigger.description) " - + "delay=\(delay) barren=\(deadTerminalEventStreamRedialBackoff.consecutiveBarrenRedials)" + + "delay=\(delay) barren=\(connectionRecoveryOwner.deadTerminalEventStreamBarrenCount)" ) - let clock = controlPlaneSchedulingClock - let generation = UUID() - deadTerminalEventStreamRedialTaskGeneration = generation - deadTerminalEventStreamRedialTask?.cancel() - deadTerminalEventStreamRedialTask = Task { @MainActor [weak self] in - do { - try await clock.sleep(for: delay) - } catch { - return - } + connectionRecoveryOwner.scheduleDeadTerminalEventStreamRedial( + after: delay, + clock: controlPlaneSchedulingClock + ) { [weak self] in guard let self, - !Task.isCancelled, - self.deadTerminalEventStreamRedialTaskGeneration == generation else { - return - } - self.deadTerminalEventStreamRedialTask = nil - self.deadTerminalEventStreamRedialBackoff.redialFired() - guard self.remoteClient === expectedClient, + self.remoteClient === expectedClient, self.connectionState == .connected else { return } @@ -281,18 +278,14 @@ extension MobileShellComposite { /// coalescing into a dead timer. It keeps the barren-stream streak, so a /// suspend/resume of the same session preserves the accrued backoff. func cancelDeadTerminalEventStreamRedial() { - deadTerminalEventStreamRedialTaskGeneration = UUID() - deadTerminalEventStreamRedialTask?.cancel() - deadTerminalEventStreamRedialTask = nil - deadTerminalEventStreamRedialBackoff.redialFired() + connectionRecoveryOwner.cancelDeadTerminalEventStreamRedial() } /// Clear the dead-stream backoff streak and cancel any pending backoff /// redial. A delivered event or a fresh foreground return proves the path /// can carry traffic, so the next failure should recover fast. func resetDeadTerminalEventStreamBackoff() { - deadTerminalEventStreamRedialBackoff.reset() - cancelDeadTerminalEventStreamRedial() + connectionRecoveryOwner.resetDeadTerminalEventStreamBackoff() } /// Replays the most recent recovery trigger that was parked while the @@ -423,7 +416,10 @@ extension MobileShellComposite { // while the fresh stored-Mac dial starts. self.connectionState = .disconnected self.macConnectionStatus = .unavailable - self.clearRemoteConnectionContext() + self.clearRemoteConnectionContext( + preservingTerminalMirror: true, + preservingWorkspaceChanges: true + ) self.applyConnectionRecoveryOwnerState() MobileDebugLog.anchormux( "connection.recovery waiting for physical transport drain " @@ -454,7 +450,10 @@ extension MobileShellComposite { if self.connectionState == .connected { self.connectionState = .disconnected self.macConnectionStatus = .unavailable - self.clearRemoteConnectionContext() + self.clearRemoteConnectionContext( + preservingTerminalMirror: true, + preservingWorkspaceChanges: true + ) } self.applyConnectionRecoveryOwnerState() diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalMirrorLifecycle.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalMirrorLifecycle.swift new file mode 100644 index 000000000000..e499347c7676 --- /dev/null +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalMirrorLifecycle.swift @@ -0,0 +1,42 @@ +import CMUXMobileCore + +extension MobileShellComposite { + /// Marks every mounted terminal mirror as blank at an intentional teardown + /// boundary. Surface identifiers can be reused by another Mac or account, + /// so retaining the old producer metadata would risk skipping hydration. + func invalidateMountedTerminalMirrors() { + for surfaceID in terminalByteContinuationsBySurfaceID.keys { + var mirrorState = terminalMirrorStatesBySurfaceID[surfaceID] + ?? MobileTerminalMirrorState() + mirrorState.invalidate() + terminalMirrorStatesBySurfaceID[surfaceID] = mirrorState + } + } + + /// Mark a mounted mirror blank so its next screen-anchored replay hydrates + /// the local scrollback from the current producer. + func markTerminalMirrorHydrationNeeded(surfaceID: String) { + var state = terminalMirrorStatesBySurfaceID[surfaceID] + ?? MobileTerminalMirrorState() + state.invalidate() + terminalMirrorStatesBySurfaceID[surfaceID] = state + } + + /// Record the producer identity and history baseline of a delivered frame. + func recordTerminalMirrorFrame(_ frame: MobileTerminalRenderGridFrame) { + var state = terminalMirrorStatesBySurfaceID[frame.surfaceID] + ?? MobileTerminalMirrorState() + state.record(frame) + terminalMirrorStatesBySurfaceID[frame.surfaceID] = state + } + + /// Returns whether a retained mirror's provisional zero-row replay failed + /// its producer-identity and history-freshness checks. + func terminalMirrorRequiresHydration( + surfaceID: String, + frame: MobileTerminalRenderGridFrame + ) -> Bool { + terminalMirrorStatesBySurfaceID[surfaceID]?.requiresHydration(for: frame) + ?? true + } +} diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift index 1c24da81db43..949f120106ca 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift @@ -17,6 +17,8 @@ extension MobileShellComposite { return true } + /// Updates per-surface screen and mirror metadata after an authoritative + /// render-grid frame has been accepted for delivery. func recordTerminalRenderGridDelivery(_ renderGrid: MobileTerminalRenderGridFrame) { // The toolbar observes this dictionary via `isAlternateScreen`; same-value // writes would re-fire observers for every delivered render-grid frame. @@ -33,6 +35,7 @@ extension MobileShellComposite { } else if renderGrid.activeScreen == .primary { terminalAlternateRenderGridBaselineSurfaceIDs.remove(renderGrid.surfaceID) } + recordTerminalMirrorFrame(renderGrid) } /// Record the screen-anchor history that the next live delta must link to. @@ -346,9 +349,6 @@ extension MobileShellComposite { fullReplacement: renderGrid.full ) recordTerminalRenderGridHistoryContinuity(renderGrid) - if renderGrid.full, renderGrid.scrollbackRows > 0 { - terminalMirrorHydrationNeededSurfaceIDs.remove(renderGrid.surfaceID) - } #if DEBUG MobileLatencyTrace.stamp( "gate", @@ -647,7 +647,7 @@ extension MobileShellComposite { // Rebuilt surface: nothing pre-barrier is visible anymore. rebaseTerminalReplayStaleFloor(surfaceID: surfaceID) terminalAlternateRenderGridBaselineSurfaceIDs.remove(surfaceID) - terminalMirrorHydrationNeededSurfaceIDs.insert(surfaceID) + markTerminalMirrorHydrationNeeded(surfaceID: surfaceID) MobileDebugLog.anchormux("terminal.output.reset surface=\(surfaceID)") requestTerminalReplay(surfaceID: surfaceID, replayBarrierToken: replayBarrierToken) } @@ -666,7 +666,7 @@ extension MobileShellComposite { deliveredTerminalByteEndSeqBySurfaceID.removeValue(forKey: surfaceID) terminalRenderGridHistoryContinuityBySurfaceID.removeValue(forKey: surfaceID) terminalRenderGridRevisionContinuityBySurfaceID.removeValue(forKey: surfaceID) - terminalMirrorHydrationNeededSurfaceIDs.insert(surfaceID) + markTerminalMirrorHydrationNeeded(surfaceID: surfaceID) terminalAlternateRenderGridBaselineSurfaceIDs.remove(surfaceID) terminalFullReplacementSeqBySurfaceID.removeValue(forKey: surfaceID) terminalFullReplacementGenerationBySurfaceID.removeValue(forKey: surfaceID) @@ -715,7 +715,7 @@ extension MobileShellComposite { let replayBarrierToken = beginTerminalReplayBarrier(surfaceID: surfaceID) rebaseTerminalReplayStaleFloor(surfaceID: surfaceID) terminalAlternateRenderGridBaselineSurfaceIDs.remove(surfaceID) - terminalMirrorHydrationNeededSurfaceIDs.insert(surfaceID) + markTerminalMirrorHydrationNeeded(surfaceID: surfaceID) MobileDebugLog.anchormux("terminal.output.replay_requested surface=\(surfaceID)") requestTerminalReplay(surfaceID: surfaceID, replayBarrierToken: replayBarrierToken) } diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+WorkspaceChanges.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+WorkspaceChanges.swift index ebecc34459f8..e8da136668f0 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+WorkspaceChanges.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+WorkspaceChanges.swift @@ -61,6 +61,23 @@ extension MobileShellComposite { workspaceIDs: [String], force: Bool = false ) async { + await fetchWorkspaceChangesSummaries( + workspaceIDs: workspaceIDs, + force: force, + taskID: nil + ) + } + + /// Fetches one orchestrated summary pass while holding its task-generation + /// ownership across every suspension point. + func fetchWorkspaceChangesSummaries( + workspaceIDs: [String], + force: Bool, + taskID: UUID? + ) async { + guard taskID == nil || workspaceChangesSummaryFetchTaskID == taskID else { + return + } let startedAt = appDiagnosticNow() recordAppEvent( .changesSummaryLoadStarted, @@ -101,7 +118,10 @@ extension MobileShellComposite { for batch in plan.batches { guard !Task.isCancelled, remoteClient === client, - connectionState == .connected else { return } + connectionState == .connected, + taskID == nil || workspaceChangesSummaryFetchTaskID == taskID else { + return + } do { guard let summaryRequest = MobileWorkspaceChangesSummaryRequest( workspaceIDs: batch, @@ -119,7 +139,11 @@ extension MobileShellComposite { ) let data = try await client.sendRequest(request) let response = try MobileWorkspaceChangesSummariesResponse.decode(data) - guard remoteClient === client, connectionState == .connected else { return } + guard remoteClient === client, + connectionState == .connected, + taskID == nil || workspaceChangesSummaryFetchTaskID == taskID else { + return + } let batchFetchedAt = runtime?.now() ?? Date() let currentWorkspaceSet = pruneWorkspaceChangesSummaryStateToForeground() let retainedBatch = currentWorkspaceSet.workspaceIDs(retaining: batch) @@ -137,6 +161,9 @@ extension MobileShellComposite { chips.removeValue(forKey: summary.workspaceID) } } + guard taskID == nil || workspaceChangesSummaryFetchTaskID == taskID else { + return + } setWorkspaceChangeChipsByWorkspaceID(chips) loadedSummaryCount += response.summaries.count MobileDebugLog.anchormux( @@ -157,11 +184,18 @@ extension MobileShellComposite { ) } catch { MobileDebugLog.anchormux("changes.summary error \(error)") - guard !Task.isCancelled, remoteClient === client else { return } + guard !Task.isCancelled, + remoteClient === client, + taskID == nil || workspaceChangesSummaryFetchTaskID == taskID else { + return + } lastFailure = DiagnosticFailureKind.classify(error) _ = disconnectForAuthorizationFailureIfNeeded(error) } } + guard taskID == nil || workspaceChangesSummaryFetchTaskID == taskID else { + return + } rescheduleWorkspaceChangesSummaryTrailingTask() if let lastFailure { recordAppEvent( @@ -398,7 +432,8 @@ extension MobileShellComposite { if !workspaceIDs.isEmpty { await self.fetchWorkspaceChangesSummaries( workspaceIDs: workspaceIDs, - force: force + force: force, + taskID: taskID ) } guard self.workspaceChangesSummaryFetchTaskID == taskID else { return } diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift index 397dfcf06093..131e2c04dea4 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift @@ -549,6 +549,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { correlationID: foregroundMacDeviceID, count: supportedHostCapabilities.count ) + guard !isResettingTerminalOutputTracking else { return } if workspaceChangesCapable { scheduleWorkspaceChangesSummaryRefresh() } else { @@ -556,6 +557,9 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { } } } + /// Suppresses capability-driven workspace-chip eviction while a transient + /// client swap clears terminal transport state. + @ObservationIgnored private var isResettingTerminalOutputTracking = false /// Authenticated phone-forwarding readiness from the focused Mac. `nil` /// means no attached Mac has proved same-account ownership and exposed the /// independent Mac privacy gate. @@ -1172,13 +1176,6 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { /// across re-subscribes) so events arriving during the round-trip are /// consumed, not buffered invisibly behind the await. private var terminalSubscriptionStartTask: Task? - /// Backoff gate for the dead-terminal-event-stream redial edge. A - /// subscription that keeps ending — or being rejected — before delivering - /// any event must not redial in a tight loop (issue #10482). See - /// ``recoverDeadTerminalEventStream(trigger:expectedClient:streamDeliveredEvent:)``. - var deadTerminalEventStreamRedialBackoff = MobileDeadStreamRedialBackoff() - var deadTerminalEventStreamRedialTask: Task? - @ObservationIgnored var deadTerminalEventStreamRedialTaskGeneration = UUID() /// Subscription success is the final validation edge for a replacement /// connection or listener. This snapshot closes the race where an old /// acknowledgement arrives after a newer listener has taken ownership. @@ -1522,20 +1519,10 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { /// repaints invisible to the history chain) and delivery requests a full /// replay instead of patching. var terminalRenderGridRevisionContinuityBySurfaceID: [String: MobileTerminalRenderGridRevisionContinuity] - /// Surfaces whose local mirror lost (or never had) its deep scrollback: - /// cold attach and post-rebuild resets. Only these replays request the - /// full hydration window; steady-state replays (barrier follow-ups, theme - /// resets) request none and replay as history-preserving repaints. - var terminalMirrorHydrationNeededSurfaceIDs: Set - /// Mounted surfaces that still hold a populated on-screen mirror across a - /// connection swap (reconnect/handoff). Their delivery cursor is cleared - /// with the old client, but the rendered scrollback survives on screen, so - /// their first replay after the swap should repaint the visible screen - /// (history-preserving) rather than re-download the entire scrollback again - /// — the ~20MB cellular burst per reconnect in issue #10482. A genuinely - /// rebuilt-blank surface still forces hydration through - /// ``terminalMirrorHydrationNeededSurfaceIDs``. - var terminalSurfacesRetainingMirrorAcrossReconnect: Set = [] + /// Per-mounted-surface mirror lifecycle. Keeping hydration, reconnect + /// retention, and producer freshness metadata together prevents a stale + /// surface ID from borrowing another terminal's scrollback. + @ObservationIgnored var terminalMirrorStatesBySurfaceID: [String: MobileTerminalMirrorState] var terminalReplaySurfaceIDsInFlight: Set var terminalReplayRequestIDsInFlightBySurfaceID: [String: UUID] var terminalReplayTasksBySurfaceID: [String: Task] @@ -1932,7 +1919,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { self.terminalActiveScreenBySurfaceID = [:] self.terminalRenderGridHistoryContinuityBySurfaceID = [:] self.terminalRenderGridRevisionContinuityBySurfaceID = [:] - self.terminalMirrorHydrationNeededSurfaceIDs = [] + self.terminalMirrorStatesBySurfaceID = [:] self.terminalReplaySurfaceIDsInFlight = [] self.terminalReplayRequestIDsInFlightBySurfaceID = [:] self.terminalReplayTasksBySurfaceID = [:] @@ -2001,7 +1988,6 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { connectionMethodObservationTask?.cancel() terminalEventListenerTask?.cancel() terminalSubscriptionStartTask?.cancel() - deadTerminalEventStreamRedialTask?.cancel() renderGridLivenessTimer?.cancel() renderGridLivenessProbeTask?.cancel() terminalInputAckResubscribeRetryTask?.cancel() @@ -2031,6 +2017,13 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { } } + /// Creates a lightweight shell composition for previews and unit tests. + /// - Parameters: + /// - runtime: Optional injected transport/runtime implementation. + /// - terminalInputAckResubscribeClock: Clock for terminal ACK retries. + /// - controlPlaneSchedulingClock: Clock for recovery and control retries. + /// - workspaceChangesSchedulingClock: Clock for workspace-summary debouncing. + /// - Returns: A shell store backed by the preview workspace fixture. public static func preview( runtime: (any MobileSyncRuntime)? = nil, // In-memory so previews and package tests never share persisted @@ -2040,13 +2033,15 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { browserStreamEvents: (any BrowserStreamEventReceiving)? = nil, simulatorStreamStore: MobileSimulatorStreamStore? = nil, terminalInputAckResubscribeClock: any Clock = ContinuousClock(), - controlPlaneSchedulingClock: any Clock = ContinuousClock() + controlPlaneSchedulingClock: any Clock = ContinuousClock(), + workspaceChangesSchedulingClock: any Clock = ContinuousClock() ) -> CMUXMobileShellStore { CMUXMobileShellStore( runtime: runtime, workspaces: PreviewMobileHost.workspaces, deliveredNotificationClearer: NoopDeliveredNotificationClearer(), lastTabStore: lastTabStore, + workspaceChangesSchedulingClock: workspaceChangesSchedulingClock, controlPlaneSchedulingClock: controlPlaneSchedulingClock, terminalInputAckResubscribeClock: terminalInputAckResubscribeClock, browserStreamEvents: browserStreamEvents, @@ -2103,6 +2098,8 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { // barren-stream streak, not just the pending redial, so the next // session does not inherit the previous one's backoff (issue #10482). resetDeadTerminalEventStreamBackoff() + invalidateMountedTerminalMirrors() + resetWorkspaceChangesState() applyConnectionRecoveryOwnerState() invalidatePairingAttempt() clearMacSwitchAttemptState() @@ -4784,6 +4781,8 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { if connectionState != .connected { clearActiveConnectionContext() macConnectionStatus = .unavailable + invalidateMountedTerminalMirrors() + resetWorkspaceChangesState() replaceRemoteClient(with: nil) } clearPairingError() @@ -10631,7 +10630,17 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { connectedHostName = "" } - func clearRemoteConnectionContext(preservingOtherMacWorkspaceState: Bool = false) { + /// Clears the active remote connection while optionally retaining state for + /// a same-session transient reconnect. + /// - Parameters: + /// - preservingOtherMacWorkspaceState: Keep secondary-Mac workspace rows. + /// - preservingTerminalMirror: Keep mounted mirror metadata for validation. + /// - preservingWorkspaceChanges: Keep last-known files-changed chips. + func clearRemoteConnectionContext( + preservingOtherMacWorkspaceState: Bool = false, + preservingTerminalMirror: Bool = false, + preservingWorkspaceChanges: Bool = false + ) { connectionGeneration = UUID() connectionAttemptGeneration = UUID() // Capture the tagged foreground key BEFORE the identity clears below: @@ -10656,7 +10665,13 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { removeControlCapability(ifMatching: focused) macConnectionRegistry.setFocusedConnection(nil, for: focused.ownerKey) } + if !preservingTerminalMirror { + invalidateMountedTerminalMirrors() + } replaceRemoteClient(with: nil) + if !preservingWorkspaceChanges { + resetWorkspaceChangesState() + } foregroundMacDeviceID = nil if !preservingOtherMacWorkspaceState { // Cancel the live secondary subscriptions (slice 3) and keep only the @@ -11020,6 +11035,8 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { macConnectionStatus = .unavailable foregroundMacDeviceID = nil clearActiveConnectionContext() + invalidateMountedTerminalMirrors() + resetWorkspaceChangesState() replaceRemoteClient(with: nil) } @@ -11065,17 +11082,25 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { private func resetTerminalOutputTracking() { cancelAllTerminalReplayTasks() // A connection swap clears each surface's delivery cursor below, but a - // mounted surface's rendered scrollback survives on screen. Remember - // which ones so their first post-swap replay resumes (repaints the - // visible screen) instead of re-downloading the whole scrollback - // (issue #10482). Genuinely rebuilt-blank surfaces are excluded because - // they carry no delivery cursor here (their mirror was wiped). - terminalSurfacesRetainingMirrorAcrossReconnect = Set( - terminalByteContinuationsBySurfaceID.keys.filter { - deliveredTerminalByteEndSeqBySurfaceID[$0] != nil - && !terminalMirrorHydrationNeededSurfaceIDs.contains($0) - } - ) + // mounted surface's rendered scrollback survives on screen. Carry that + // fact in the same per-surface lifecycle record used by replay + // hydration, and retain the producer metadata needed to validate the + // first post-swap frame before trusting a zero-row replay. + let mountedSurfaceIDs = Set(terminalByteContinuationsBySurfaceID.keys) + terminalMirrorStatesBySurfaceID = terminalMirrorStatesBySurfaceID.filter { + mountedSurfaceIDs.contains($0.key) + } + for surfaceID in mountedSurfaceIDs { + var mirrorState = terminalMirrorStatesBySurfaceID[surfaceID] + ?? MobileTerminalMirrorState() + mirrorState.prepareForReconnect( + hasDeliveredFrame: + deliveredTerminalByteEndSeqBySurfaceID[surfaceID] != nil + || terminalPreBarrierDeliveredEndSeqBySurfaceID[surfaceID] != nil + || !mirrorState.hydrationNeeded + ) + terminalMirrorStatesBySurfaceID[surfaceID] = mirrorState + } effectiveViewportSizesBySurfaceID = [:]; reportedTerminalViewportSizesBySurfaceID = [:] // Keep viewport sequences for the account lifetime. A warm peer keeps // its Mac-side tombstone, while a reconnected peer safely accepts a @@ -11098,7 +11123,9 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { diagnosedTerminalOutputSurfaceIDs = [] terminalRenderGridHistoryContinuityBySurfaceID = [:] terminalRenderGridRevisionContinuityBySurfaceID = [:] - terminalMirrorHydrationNeededSurfaceIDs = [] + terminalMirrorStatesBySurfaceID = terminalMirrorStatesBySurfaceID.filter { + terminalByteContinuationsBySurfaceID.keys.contains($0.key) + } terminalReplaySurfaceIDsInFlight = [] terminalReplayRequestIDsInFlightBySurfaceID = [:] cancelAllTerminalReplayBarrierWatchdogs() @@ -11123,7 +11150,9 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { terminalScrollbackPrefetchStatesBySurfaceID = [:] terminalOutputTransport = .rawBytes deactivateAllTerminalLanes() + isResettingTerminalOutputTracking = true supportedHostCapabilities = [] + isResettingTerminalOutputTracking = false phonePushMacStatus = nil caffeineStatus = nil isCaffeineMutationInFlight = false @@ -11156,6 +11185,8 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { // barren-stream streak, not just the pending redial, so the next // session does not inherit the previous one's backoff (issue #10482). resetDeadTerminalEventStreamBackoff() + invalidateMountedTerminalMirrors() + resetWorkspaceChangesState() applyConnectionRecoveryOwnerState() invalidateStoredMacReconnectAttempt() connectionGeneration = UUID() @@ -14100,6 +14131,9 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { terminalByteContinuationsBySurfaceID[surfaceID] = continuation terminalOutputStreamTokensBySurfaceID[surfaceID] = streamToken terminalOutputConsumerOwnerIDsBySurfaceID[surfaceID] = ownerID + // A new consumer owns a new mirror lifecycle, even when the public + // surface identifier is reused after an older stream terminated. + terminalMirrorStatesBySurfaceID[surfaceID] = MobileTerminalMirrorState() terminalOutputQueuesBySurfaceID[surfaceID] = TerminalOutputDeliveryQueue() deliveredTerminalByteEndSeqBySurfaceID.removeValue(forKey: surfaceID) terminalPreBarrierDeliveredEndSeqBySurfaceID.removeValue(forKey: surfaceID) @@ -14165,7 +14199,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { terminalActiveScreenBySurfaceID.removeValue(forKey: surfaceID) terminalRenderGridHistoryContinuityBySurfaceID.removeValue(forKey: surfaceID) terminalRenderGridRevisionContinuityBySurfaceID.removeValue(forKey: surfaceID) - terminalMirrorHydrationNeededSurfaceIDs.remove(surfaceID) + terminalMirrorStatesBySurfaceID.removeValue(forKey: surfaceID) diagnosedTerminalOutputSurfaceIDs.remove(surfaceID) recordAppEvent( .terminalUnmounted, @@ -14399,6 +14433,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { generation: terminalViewportGeneration(for: surfaceID)) } let replayTask = Task { @MainActor [weak self] in let replayResult: Result + var requestedMirrorReuse = false do { var params: [String: Any] = [ "workspace_id": remoteWorkspaceID.rawValue, @@ -14413,24 +14448,16 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { } } // Screen-anchored replays hydrate this device's deep local - // scrollback only when the mirror has none (cold attach, a - // rebuilt-blank surface). Steady-state replays request no - // scrollback and replay as history-preserving repaints, so - // replay-barrier churn during streaming stays cheap and never - // destroys locally accumulated history. + // scrollback only when the per-surface mirror lifecycle says + // it is missing or stale. A retained mirror starts with a + // provisional zero-row replay; the response is validated + // against its producer epoch/history before that optimization + // is accepted. if let self, self.usesScreenAnchoredRenderGrid { params["anchor"] = MobileTerminalRenderGridFrame.Anchor.screen.rawValue - // A surface whose mirror survived a connection swap keeps - // its rendered scrollback on screen, so its cleared cursor - // alone must not force a full re-hydration (~20MB per - // reconnect on cellular, issue #10482); repaint the visible - // screen instead. A rebuilt-blank surface still hydrates. - let mirrorSurvivedReconnect = - self.terminalSurfacesRetainingMirrorAcrossReconnect.contains(surfaceID) - let needsHydration = - (self.deliveredTerminalByteEndSeqBySurfaceID[surfaceID] == nil - && !mirrorSurvivedReconnect) - || self.terminalMirrorHydrationNeededSurfaceIDs.contains(surfaceID) + let needsHydration = self.terminalMirrorStatesBySurfaceID[surfaceID]?.hydrationNeeded + ?? true + requestedMirrorReuse = !needsHydration params["max_scrollback_rows"] = needsHydration ? MobileTerminalScrollbackPreference.resolve() : 0 @@ -14499,6 +14526,43 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { return } } + let retainedMirrorIsStale = requestedMirrorReuse && ( + renderGrid == nil + || (renderGrid.map { + self.terminalMirrorRequiresHydration( + surfaceID: surfaceID, + frame: $0 + ) + } ?? false) + ) + if retainedMirrorIsStale { + // The producer changed epoch/history while the phone was + // disconnected. Do not paint a zero-row frame onto a + // mirror whose deep scrollback is no longer aligned; turn + // the same replay generation into one full hydration. + self.markTerminalMirrorHydrationNeeded(surfaceID: surfaceID) + self.clearTerminalReplayInFlightIfCurrent( + surfaceID: surfaceID, + requestID: replayRequestID + ) + guard let retryToken = self.prepareTerminalReplayFailureRetry( + surfaceID: surfaceID, + replayBarrierToken: replayBarrierTokenForRequest + ) else { + self.clearTerminalReplayBarrierIfCurrent( + surfaceID: surfaceID, + token: replayBarrierTokenForRequest, + reason: "mirror_freshness" + ) + return + } + transferredInFlightToRetry = true + self.requestTerminalReplay( + surfaceID: surfaceID, + replayBarrierToken: retryToken + ) + return + } #if DEBUG let seq = replaySeq ?? 0 let cols = payload?.columns ?? -1 diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileTerminalMirrorState.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileTerminalMirrorState.swift new file mode 100644 index 000000000000..5e118a9bd204 --- /dev/null +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileTerminalMirrorState.swift @@ -0,0 +1,70 @@ +import CMUXMobileCore +import Foundation + +/// Lifecycle state for one mounted terminal mirror. +/// +/// The state is the single source of truth for whether a surface needs +/// scrollback hydration and whether its rendered mirror may be reused after a +/// connection swap. Producer identity and history metadata make the reuse +/// decision fail closed when the Mac recreated the surface or history moved. +struct MobileTerminalMirrorState: Sendable { + var hydrationNeeded = true + var retainedAcrossReconnect = false + private(set) var renderEpoch: String? + private(set) var historyRows: UInt64? + private(set) var rowSpaceRevision: UInt64? + + /// Marks the mirror as blank and requiring a full screen-anchored replay. + mutating func invalidate() { + hydrationNeeded = true + retainedAcrossReconnect = false + renderEpoch = nil + historyRows = nil + rowSpaceRevision = nil + } + + /// Carries a populated mounted mirror across a connection swap only when + /// its last delivered frame proved that hydration had completed. + mutating func prepareForReconnect(hasDeliveredFrame: Bool) { + retainedAcrossReconnect = hasDeliveredFrame && !hydrationNeeded + hydrationNeeded = !retainedAcrossReconnect + } + + /// Records producer metadata from a delivered frame. A full frame with no + /// retained history still completes hydration; deltas never do. + mutating func record(_ frame: MobileTerminalRenderGridFrame) { + if retainedAcrossReconnect && !frame.full { + return + } + renderEpoch = frame.renderEpoch.isEmpty ? nil : frame.renderEpoch + historyRows = frame.historyRows + rowSpaceRevision = frame.rowSpaceRevision + let hydrationSatisfied = retainedAcrossReconnect + || frame.anchor != .screen + || frame.scrollbackRows > 0 + || frame.historyRows == 0 + || frame.activeScreen == .alternate + if frame.full, hydrationSatisfied { + hydrationNeeded = false + retainedAcrossReconnect = false + } + } + + /// Returns whether a provisional zero-row replay is unsafe for this mirror. + /// A changed producer epoch, history count, or row-space revision means the + /// local scrollback can no longer be trusted and must be rehydrated. + func requiresHydration(for frame: MobileTerminalRenderGridFrame) -> Bool { + guard retainedAcrossReconnect else { return hydrationNeeded } + guard let renderEpoch, + let historyRows, + let rowSpaceRevision, + !frame.renderEpoch.isEmpty, + let frameHistoryRows = frame.historyRows, + let frameRowSpaceRevision = frame.rowSpaceRevision else { + return true + } + return renderEpoch != frame.renderEpoch + || historyRows != frameHistoryRows + || rowSpaceRevision != frameRowSpaceRevision + } +} diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileForegroundReconnectStormTests.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileForegroundReconnectStormTests.swift index afc7fbf6aba4..5bd37518a380 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileForegroundReconnectStormTests.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileForegroundReconnectStormTests.swift @@ -3,6 +3,7 @@ import CmuxMobileChanges import CmuxMobilePairedMac import CmuxMobileRPC import CmuxMobileShellModel +import CmuxMobileSupport import Foundation import Testing @testable import CmuxMobileShell @@ -99,7 +100,6 @@ import Testing #expect(backoff.nextRedialDelay() == .seconds(2)) backoff.redialFired() #expect(backoff.nextRedialDelay() == .seconds(4)) - backoff.redialFired() // A new-session boundary (sign-out, new pairing, method change) resets the // streak, so the next session's first barren stream recovers immediately @@ -143,6 +143,14 @@ import Testing store.workspaceChangeChipsByWorkspaceID["live-workspace"]?.filesChanged == 51, "reconnecting must not churn the files-changed chip content" ) + + // Capability reset is part of replacing the client. It must not evict the + // last-known chip snapshot while a transient reconnect is in progress. + store.remoteClient = nil + #expect( + store.workspaceChangeChipsByWorkspaceID["live-workspace"]?.filesChanged == 51, + "client replacement must preserve files-changed chips" + ) } // MARK: - C. A reconnect with a live mirror resumes (no full scrollback replay) @@ -164,9 +172,29 @@ import Testing let surfaceID = "live-terminal" #expect(try await pollUntil { store.usesScreenAnchoredRenderGrid }) - await router.enqueueReplayTexts(["cold-replay"]) - let iterator = store.terminalOutputStream(surfaceID: surfaceID).makeAsyncIterator() - _ = iterator + let coldFrame = try MobileTerminalRenderGridFrame( + surfaceID: surfaceID, + stateSeq: 100, + renderEpoch: "epoch-1", + renderRevision: 1, + columns: 80, + rows: 4, + full: true, + rowSpans: [ + MobileTerminalRenderGridFrame.RowSpan( + row: 0, + column: 0, + styleID: 0, + text: "cold-replay" + ), + ], + anchor: .screen, + scrollbackRows: 20, + historyRows: 20, + rowSpaceRevision: 1 + ) + await router.enqueueReplayRenderGrid(coldFrame) + var iterator = store.terminalOutputStream(surfaceID: surfaceID).makeAsyncIterator() await router.waitForCount(of: "mobile.terminal.replay", atLeast: 1) // Cold attach hydrates this device's deep scrollback (the mirror was blank). @@ -176,10 +204,14 @@ import Testing "a cold attach must hydrate scrollback" ) - // The surface now has a populated on-screen mirror with a delivery cursor, - // exactly as a steady-state terminal does. - store.deliveredTerminalByteEndSeqBySurfaceID[surfaceID] = 100 - #expect(store.deliveredTerminalByteEndSeqBySurfaceID[surfaceID] != nil) + // The surface now has a populated on-screen mirror with a delivery cursor + // and producer history identity, exactly as a steady-state terminal does. + let coldChunk = try #require(await iterator.next()) + store.terminalOutputDidProcess( + surfaceID: surfaceID, + streamToken: coldChunk.streamToken + ) + #expect(store.deliveredTerminalByteEndSeqBySurfaceID[surfaceID] == 100) // Simulate a reconnect: the recovery path clears the live client, which // resets terminal output tracking (dropping the delivery cursor), then @@ -187,6 +219,28 @@ import Testing // still mounted — so the reconnect should repaint the visible screen, not // re-download the entire scrollback again. let replayCountBeforeReconnect = await router.count(of: "mobile.terminal.replay") + let warmFrame = try MobileTerminalRenderGridFrame( + surfaceID: surfaceID, + stateSeq: 101, + renderEpoch: "epoch-1", + renderRevision: 2, + columns: 80, + rows: 4, + full: true, + rowSpans: [ + MobileTerminalRenderGridFrame.RowSpan( + row: 0, + column: 0, + styleID: 0, + text: "warm-replay" + ), + ], + anchor: .screen, + scrollbackRows: 0, + historyRows: 20, + rowSpaceRevision: 1 + ) + await router.enqueueReplayRenderGrid(warmFrame) store.remoteClient = nil try installFreshLivenessRemoteClient(on: store, router: router, box: box, clock: clock) // A real reconnect re-resolves the host capabilities and transport during diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileReconnectStateLifecycleTests.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileReconnectStateLifecycleTests.swift new file mode 100644 index 000000000000..2c2aaf37ccb7 --- /dev/null +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileReconnectStateLifecycleTests.swift @@ -0,0 +1,158 @@ +import CMUXMobileCore +import Foundation +import Testing +@testable import CmuxMobileShell + +// Review-found lifecycle regressions around issue #10482: stale asynchronous +// workspace summaries and public terminal surface-ID reuse. + +@Test func retainedMirrorFreshnessFailsClosed() throws { + var state = MobileTerminalMirrorState() + let delivered = try MobileTerminalRenderGridFrame( + surfaceID: "surface", + stateSeq: 10, + renderEpoch: "epoch-1", + renderRevision: 1, + columns: 80, + rows: 4, + full: true, + rowSpans: [], + anchor: .screen, + scrollbackRows: 20, + historyRows: 20, + rowSpaceRevision: 1 + ) + state.record(delivered) + state.prepareForReconnect(hasDeliveredFrame: true) + + var sameHistory = delivered + sameHistory.stateSeq = 11 + sameHistory.renderRevision = 2 + #expect(!state.requiresHydration(for: sameHistory)) + + var advancedHistory = sameHistory + advancedHistory.historyRows = 21 + #expect(state.requiresHydration(for: advancedHistory)) + + var replacedProducer = sameHistory + replacedProducer.renderEpoch = "epoch-2" + #expect(state.requiresHydration(for: replacedProducer)) +} + +@MainActor +@Test func canceledWorkspaceSummaryCannotPublishStaleChips() async throws { + let router = LivenessHostRouter() + await router.setCapabilities([ + "events.v1", + "terminal.render_grid.v1", + "terminal.replay.v1", + "workspace.changes.v1", + ]) + let box = TransportBox() + let clock = TestClock() + let summaryClock = ControlPoolManualClock() + let store = try await makeConnectedStore( + router: router, + box: box, + clock: clock, + workspaceChangesSchedulingClock: summaryClock + ) + // Keep the automatic capability-triggered debounce from becoming the + // request under test; this pass owns an explicit generation below. + store.suspendWorkspaceChangesSummaryFetchesPreservingChips() + store.setWorkspaceChangeChipsByWorkspaceID([ + "live-workspace": MobileWorkspaceChangesChip( + filesChanged: 51, + additions: 120, + deletions: 8 + ), + ]) + let responseData = try JSONSerialization.data(withJSONObject: [ + "summaries": [[ + "workspace_id": "live-workspace", + "is_repo": true, + "files_changed": 99, + "additions": 200, + "deletions": 1, + ]], + ]) + await router.enqueueWorkspaceChangesSummaryResponse(jsonData: responseData) + await router.holdNextWorkspaceChangesSummaryRequests() + + let taskID = UUID() + store.workspaceChangesSummaryFetchTaskID = taskID + let fetchTask = Task { @MainActor in + await store.fetchWorkspaceChangesSummaries( + workspaceIDs: ["live-workspace"], + force: true, + taskID: taskID + ) + } + #expect(await router.waitForCount( + of: "mobile.workspace.changes.summary", + atLeast: 1 + )) + + // Supersede the in-flight request while its client/state identity still + // matches. The post-await task-ID guard must prevent its 99-file response + // from overwriting the authoritative 51-file chip. + store.workspaceChangesSummaryFetchTaskID = UUID() + await router.releaseAllHeld() + await fetchTask.value + #expect( + store.workspaceChangeChipsByWorkspaceID["live-workspace"]?.filesChanged == 51, + "a canceled summary generation must not publish stale chips" + ) +} + +@MainActor +@Test func terminalSurfaceIDReuseStartsFreshHydration() async throws { + let router = LivenessHostRouter() + await router.setCapabilities([ + "events.v1", + "terminal.render_grid.v1", + "terminal.render_grid.screen_anchor.v1", + "terminal.replay.v1", + ]) + let box = TransportBox() + let clock = TestClock() + let store = try await makeConnectedStore(router: router, box: box, clock: clock) + let surfaceID = "live-terminal" + let frame = try MobileTerminalRenderGridFrame( + surfaceID: surfaceID, + stateSeq: 10, + renderEpoch: "epoch-1", + renderRevision: 1, + columns: 80, + rows: 4, + full: true, + rowSpans: [], + anchor: .screen, + scrollbackRows: 20, + historyRows: 20, + rowSpaceRevision: 1 + ) + await router.enqueueReplayRenderGrid(frame) + let collector = OutputCollector() + collector.mount(store: store, surfaceID: surfaceID) + #expect(await router.waitForCount(of: "mobile.terminal.replay", atLeast: 1)) + #expect(try await pollUntil { !collector.lines.isEmpty }) + + // Ending the first consumer removes its per-surface lifecycle record. + collector.unmount() + #expect(try await pollUntil { !store.hasTerminalOutputSink(surfaceID: surfaceID) }) + + let replayCountBeforeRemount = await router.count(of: "mobile.terminal.replay") + await router.enqueueReplayRenderGrid(frame) + collector.mount(store: store, surfaceID: surfaceID) + #expect(await router.waitForCount( + of: "mobile.terminal.replay", + atLeast: replayCountBeforeRemount + 1 + )) + let remountReplay = try #require(await router.requests(for: "mobile.terminal.replay").last) + #expect( + (remountReplay.maxScrollbackRows ?? 0) > 0, + "reusing a surface ID after unregistration must hydrate a fresh mirror" + ) + collector.unmount() +} diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift index 17181ff901a0..9782a1601c83 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift @@ -50,6 +50,9 @@ actor LivenessHostRouter { private var workspaceListRequestCount = 0 private var heldWorkspaceListRequestNumbers: Set = [] private var workspaceListErrorCodesByRequestNumber: [Int: String] = [:] + private var workspaceChangesSummaryRequestCount = 0 + private var heldWorkspaceChangesSummaryRequestNumbers: Set = [] + private var workspaceChangesSummaryResponses: [[String: Any]] = [] private var subscribeRequestCount = 0 private var probeRequestCount = 0 private var heldSubscribeRequestNumbers: Set = [] @@ -375,6 +378,32 @@ actor LivenessHostRouter { heldWorkspaceListRequestNumbers.insert(number) } + /// Hold one workspace-changes summary response so a test can invalidate + /// the owning fetch generation before the response publishes. + func holdWorkspaceChangesSummaryRequest(number: Int) { + heldWorkspaceChangesSummaryRequestNumbers.insert(number) + } + + /// Hold the next workspace-changes summary response relative to requests + /// already observed by the scripted host. + func holdNextWorkspaceChangesSummaryRequests(count: Int = 1) { + guard count > 0 else { return } + for offset in 1 ... count { + heldWorkspaceChangesSummaryRequestNumbers.insert( + workspaceChangesSummaryRequestCount + offset + ) + } + } + + /// Queue a JSON result for the next workspace-changes summary request. + func enqueueWorkspaceChangesSummaryResponse(jsonData: Data) { + guard let object = (try? JSONSerialization.jsonObject(with: jsonData)) + as? [String: Any] else { + return + } + workspaceChangesSummaryResponses.append(object) + } + func failWorkspaceListRequest( number: Int, code: String = "workspace_list_failed" @@ -494,6 +523,7 @@ actor LivenessHostRouter { heldHostStatusRequestNumbers = [] delayedHostStatusRequestNumbers = [] heldWorkspaceListRequestNumbers = [] + heldWorkspaceChangesSummaryRequestNumbers = [] heldSubscribeRequestNumbers = [] heldProbeRequestNumbers = [] delayedSubscribeRequestNumbers = [] @@ -571,6 +601,17 @@ actor LivenessHostRouter { return try? Self.resultFrame(id: id, result: [ "workspaces": workspaces, ]) + case "mobile.workspace.changes.summary": + workspaceChangesSummaryRequestCount += 1 + if heldWorkspaceChangesSummaryRequestNumbers.contains( + workspaceChangesSummaryRequestCount + ) { + await park() + } + let result: [String: Any] = workspaceChangesSummaryResponses.isEmpty + ? ["summaries": []] + : workspaceChangesSummaryResponses.removeFirst() + return try? Self.resultFrame(id: id, result: result) case "mobile.host.status": hostStatusRequestCount += 1 if heldHostStatusRequestNumbers.contains(hostStatusRequestCount) { @@ -1024,7 +1065,8 @@ func makeConnectedStore( clock: TestClock, probeTimeoutNanoseconds: UInt64 = 200_000_000, inputAckRetryClock: any Clock = ContinuousClock(), - controlPlaneSchedulingClock: any Clock = ContinuousClock() + controlPlaneSchedulingClock: any Clock = ContinuousClock(), + workspaceChangesSchedulingClock: any Clock = ContinuousClock() ) async throws -> MobileShellComposite { let runtime = LivenessTestRuntime( transportFactory: LivenessTransportFactory(router: router, box: box), @@ -1034,7 +1076,8 @@ func makeConnectedStore( let store = MobileShellComposite.preview( runtime: runtime, terminalInputAckResubscribeClock: inputAckRetryClock, - controlPlaneSchedulingClock: controlPlaneSchedulingClock + controlPlaneSchedulingClock: controlPlaneSchedulingClock, + workspaceChangesSchedulingClock: workspaceChangesSchedulingClock ) store.signIn() let ticket = try makeTicket(clock: clock) diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileDeadStreamRedialBackoff.swift b/Packages/iOS/CmuxMobileSupport/Sources/CmuxMobileSupport/MobileDeadStreamRedialBackoff.swift similarity index 72% rename from Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileDeadStreamRedialBackoff.swift rename to Packages/iOS/CmuxMobileSupport/Sources/CmuxMobileSupport/MobileDeadStreamRedialBackoff.swift index 0581524695af..84b0b91bcd7a 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileDeadStreamRedialBackoff.swift +++ b/Packages/iOS/CmuxMobileSupport/Sources/CmuxMobileSupport/MobileDeadStreamRedialBackoff.swift @@ -1,7 +1,6 @@ import Foundation -/// Backoff for redialing after a terminal event subscription ends — or is -/// rejected — before it ever delivered an event. +/// Backoff for redialing after a terminal event subscription ends before delivery. /// /// A healthy reconnect delivers events, which proves the transport carries /// traffic and clears the streak. A subscription that keeps ending "barren" @@ -14,20 +13,27 @@ import Foundation /// The first barren stream still recovers immediately — a genuine transient /// blip should heal fast — while each subsequent barren stream backs off /// exponentially so the loop cannot spin. -struct MobileDeadStreamRedialBackoff { - static let initialBackoff: Duration = .seconds(1) - static let maximumBackoff: Duration = .seconds(30) +public struct MobileDeadStreamRedialBackoff: Sendable { + /// Delay used for the second consecutive barren stream. + public static let initialBackoff: Duration = .seconds(1) + /// Upper bound for a repeated barren-stream delay. + public static let maximumBackoff: Duration = .seconds(30) - private(set) var consecutiveBarrenRedials = 0 + /// Number of consecutive barren streams observed since the last reset. + public private(set) var consecutiveBarrenRedials = 0 private var nextBackoff = MobileDeadStreamRedialBackoff.initialBackoff - private(set) var isRedialScheduled = false + /// Whether a delayed redial is currently outstanding. + public private(set) var isRedialScheduled = false + + /// Creates an empty barren-stream backoff. + public init() {} /// The delay to wait before redialing after a stream ended barren. Returns /// `.zero` for the first barren stream (recover immediately), an increasing /// delay for each subsequent one, or `nil` when a delayed redial is already /// scheduled — so simultaneous barren signals coalesce onto one timer /// instead of stacking redials. - mutating func nextRedialDelay() -> Duration? { + public mutating func nextRedialDelay() -> Duration? { guard !isRedialScheduled else { return nil } consecutiveBarrenRedials += 1 guard consecutiveBarrenRedials > 1 else { return .zero } @@ -39,14 +45,14 @@ struct MobileDeadStreamRedialBackoff { /// A scheduled delayed redial fired; allow the next barren stream to /// schedule again. - mutating func redialFired() { + public mutating func redialFired() { isRedialScheduled = false } /// A delivered event (or an intentional teardown / foreground reset) proves /// the connection is healthy again; clear the barren streak so the next /// failure recovers fast. - mutating func reset() { + public mutating func reset() { consecutiveBarrenRedials = 0 nextBackoff = Self.initialBackoff isRedialScheduled = false From 25b406380d1e8c0e2c83590fd16c731652c4b096 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 1 Sep 2026 15:16:06 -0700 Subject: [PATCH 06/21] fix(ios): order render-grid fixture arguments --- .../MobileForegroundReconnectStormTests.swift | 4 ++-- .../MobileReconnectStateLifecycleTests.swift | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileForegroundReconnectStormTests.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileForegroundReconnectStormTests.swift index 5bd37518a380..f2be23dec632 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileForegroundReconnectStormTests.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileForegroundReconnectStormTests.swift @@ -188,8 +188,8 @@ import Testing text: "cold-replay" ), ], - anchor: .screen, scrollbackRows: 20, + anchor: .screen, historyRows: 20, rowSpaceRevision: 1 ) @@ -235,8 +235,8 @@ import Testing text: "warm-replay" ), ], - anchor: .screen, scrollbackRows: 0, + anchor: .screen, historyRows: 20, rowSpaceRevision: 1 ) diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileReconnectStateLifecycleTests.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileReconnectStateLifecycleTests.swift index 2c2aaf37ccb7..32cd2d5e234e 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileReconnectStateLifecycleTests.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileReconnectStateLifecycleTests.swift @@ -17,8 +17,8 @@ import Testing rows: 4, full: true, rowSpans: [], - anchor: .screen, scrollbackRows: 20, + anchor: .screen, historyRows: 20, rowSpaceRevision: 1 ) @@ -127,8 +127,8 @@ import Testing rows: 4, full: true, rowSpans: [], - anchor: .screen, scrollbackRows: 20, + anchor: .screen, historyRows: 20, rowSpaceRevision: 1 ) From c15c3b299eb2125b992f85acc62001d40a65f914 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 1 Sep 2026 15:22:41 -0700 Subject: [PATCH 07/21] test(ios): make mirror remount coverage deterministic --- .../MobileReconnectStateLifecycleTests.swift | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileReconnectStateLifecycleTests.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileReconnectStateLifecycleTests.swift index 32cd2d5e234e..bc71394bded6 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileReconnectStateLifecycleTests.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileReconnectStateLifecycleTests.swift @@ -138,13 +138,14 @@ import Testing #expect(await router.waitForCount(of: "mobile.terminal.replay", atLeast: 1)) #expect(try await pollUntil { !collector.lines.isEmpty }) - // Ending the first consumer removes its per-surface lifecycle record. - collector.unmount() - #expect(try await pollUntil { !store.hasTerminalOutputSink(surfaceID: surfaceID) }) - + // A second mount reuses the public surface ID while the old stream's + // asynchronous termination callback is still in flight. Registration is + // the lifecycle boundary: it must replace the old per-surface state rather + // than inherit any retained mirror marker from that ID. let replayCountBeforeRemount = await router.count(of: "mobile.terminal.replay") await router.enqueueReplayRenderGrid(frame) - collector.mount(store: store, surfaceID: surfaceID) + let replacementCollector = OutputCollector() + replacementCollector.mount(store: store, surfaceID: surfaceID) #expect(await router.waitForCount( of: "mobile.terminal.replay", atLeast: replayCountBeforeRemount + 1 @@ -152,7 +153,8 @@ import Testing let remountReplay = try #require(await router.requests(for: "mobile.terminal.replay").last) #expect( (remountReplay.maxScrollbackRows ?? 0) > 0, - "reusing a surface ID after unregistration must hydrate a fresh mirror" + "reusing a surface ID for a new mount must hydrate a fresh mirror" ) collector.unmount() + replacementCollector.unmount() } From 8da6f9b386ccd05acbb9c443431ed6991d8d4d00 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 1 Sep 2026 16:08:47 -0700 Subject: [PATCH 08/21] refactor(ios): place mirror state in terminal kit --- Packages/iOS/CmuxMobileShell/Package.swift | 3 ++ ...ellComposite+TerminalMirrorLifecycle.swift | 1 + .../MobileShellComposite.swift | 1 + .../MobileReconnectStateLifecycleTests.swift | 33 ----------------- .../MobileTerminalMirrorState.swift | 33 ++++++++++------- .../MobileTerminalMirrorStateTests.swift | 36 +++++++++++++++++++ 6 files changed, 62 insertions(+), 45 deletions(-) rename Packages/iOS/{CmuxMobileShell/Sources/CmuxMobileShell => CmuxMobileTerminalKit/Sources/CmuxMobileTerminalKit}/MobileTerminalMirrorState.swift (63%) create mode 100644 Packages/iOS/CmuxMobileTerminalKit/Tests/CmuxMobileTerminalKitTests/MobileTerminalMirrorStateTests.swift diff --git a/Packages/iOS/CmuxMobileShell/Package.swift b/Packages/iOS/CmuxMobileShell/Package.swift index bec5dc8a51ef..d52c3bac7730 100644 --- a/Packages/iOS/CmuxMobileShell/Package.swift +++ b/Packages/iOS/CmuxMobileShell/Package.swift @@ -28,6 +28,7 @@ let package = Package( .package(path: "../CmuxMobileRPC"), .package(path: "../CmuxMobileShellModel"), .package(path: "../CmuxMobileSupport"), + .package(path: "../CmuxMobileTerminalKit"), .package(path: "../CmuxMobileTransport"), ], targets: [ @@ -43,6 +44,7 @@ let package = Package( "CmuxMobileRPC", "CmuxMobileShellModel", "CmuxMobileSupport", + "CmuxMobileTerminalKit", "CmuxMobileTransport", ], swiftSettings: [ @@ -79,6 +81,7 @@ let package = Package( "CmuxMobileRPC", "CmuxMobileShellModel", "CmuxMobileSupport", + "CmuxMobileTerminalKit", "CmuxMobileTransport", ], swiftSettings: [ diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalMirrorLifecycle.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalMirrorLifecycle.swift index e499347c7676..aaf63f3655b4 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalMirrorLifecycle.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalMirrorLifecycle.swift @@ -1,4 +1,5 @@ import CMUXMobileCore +internal import CmuxMobileTerminalKit extension MobileShellComposite { /// Marks every mounted terminal mirror as blank at an intentional teardown diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift index 131e2c04dea4..cbd59a1e75aa 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift @@ -6,6 +6,7 @@ public import CmuxMobilePairedMac public import CmuxMobileRPC public import CmuxMobileShellModel internal import CmuxMobileSupport +internal import CmuxMobileTerminalKit public import CmuxMobileTransport public import Foundation import Observation diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileReconnectStateLifecycleTests.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileReconnectStateLifecycleTests.swift index bc71394bded6..13ed15ae7a9e 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileReconnectStateLifecycleTests.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileReconnectStateLifecycleTests.swift @@ -6,39 +6,6 @@ import Testing // Review-found lifecycle regressions around issue #10482: stale asynchronous // workspace summaries and public terminal surface-ID reuse. -@Test func retainedMirrorFreshnessFailsClosed() throws { - var state = MobileTerminalMirrorState() - let delivered = try MobileTerminalRenderGridFrame( - surfaceID: "surface", - stateSeq: 10, - renderEpoch: "epoch-1", - renderRevision: 1, - columns: 80, - rows: 4, - full: true, - rowSpans: [], - scrollbackRows: 20, - anchor: .screen, - historyRows: 20, - rowSpaceRevision: 1 - ) - state.record(delivered) - state.prepareForReconnect(hasDeliveredFrame: true) - - var sameHistory = delivered - sameHistory.stateSeq = 11 - sameHistory.renderRevision = 2 - #expect(!state.requiresHydration(for: sameHistory)) - - var advancedHistory = sameHistory - advancedHistory.historyRows = 21 - #expect(state.requiresHydration(for: advancedHistory)) - - var replacedProducer = sameHistory - replacedProducer.renderEpoch = "epoch-2" - #expect(state.requiresHydration(for: replacedProducer)) -} - @MainActor @Test func canceledWorkspaceSummaryCannotPublishStaleChips() async throws { let router = LivenessHostRouter() diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileTerminalMirrorState.swift b/Packages/iOS/CmuxMobileTerminalKit/Sources/CmuxMobileTerminalKit/MobileTerminalMirrorState.swift similarity index 63% rename from Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileTerminalMirrorState.swift rename to Packages/iOS/CmuxMobileTerminalKit/Sources/CmuxMobileTerminalKit/MobileTerminalMirrorState.swift index 5e118a9bd204..80a737e6de04 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileTerminalMirrorState.swift +++ b/Packages/iOS/CmuxMobileTerminalKit/Sources/CmuxMobileTerminalKit/MobileTerminalMirrorState.swift @@ -1,5 +1,4 @@ import CMUXMobileCore -import Foundation /// Lifecycle state for one mounted terminal mirror. /// @@ -7,15 +6,19 @@ import Foundation /// scrollback hydration and whether its rendered mirror may be reused after a /// connection swap. Producer identity and history metadata make the reuse /// decision fail closed when the Mac recreated the surface or history moved. -struct MobileTerminalMirrorState: Sendable { - var hydrationNeeded = true +public struct MobileTerminalMirrorState: Sendable { + /// Whether the next authoritative replay must include scrollback rows. + public private(set) var hydrationNeeded = true var retainedAcrossReconnect = false private(set) var renderEpoch: String? private(set) var historyRows: UInt64? private(set) var rowSpaceRevision: UInt64? + /// Creates a new mirror state that requires a cold hydration replay. + public init() {} + /// Marks the mirror as blank and requiring a full screen-anchored replay. - mutating func invalidate() { + public mutating func invalidate() { hydrationNeeded = true retainedAcrossReconnect = false renderEpoch = nil @@ -23,16 +26,19 @@ struct MobileTerminalMirrorState: Sendable { rowSpaceRevision = nil } - /// Carries a populated mounted mirror across a connection swap only when - /// its last delivered frame proved that hydration had completed. - mutating func prepareForReconnect(hasDeliveredFrame: Bool) { + /// Carries a populated mounted mirror across a connection swap when the + /// last delivered frame proved that hydration had completed. + /// - Parameter hasDeliveredFrame: Whether the mounted surface has delivered + /// an authoritative frame that can remain visible during reconnect. + public mutating func prepareForReconnect(hasDeliveredFrame: Bool) { retainedAcrossReconnect = hasDeliveredFrame && !hydrationNeeded hydrationNeeded = !retainedAcrossReconnect } - /// Records producer metadata from a delivered frame. A full frame with no - /// retained history still completes hydration; deltas never do. - mutating func record(_ frame: MobileTerminalRenderGridFrame) { + /// Records producer metadata from a delivered render-grid frame. A full + /// frame with no retained history still completes hydration; deltas never do. + /// - Parameter frame: The accepted authoritative frame. + public mutating func record(_ frame: MobileTerminalRenderGridFrame) { if retainedAcrossReconnect && !frame.full { return } @@ -50,10 +56,13 @@ struct MobileTerminalMirrorState: Sendable { } } - /// Returns whether a provisional zero-row replay is unsafe for this mirror. + /// Determines whether a retained mirror must be rehydrated for a response. /// A changed producer epoch, history count, or row-space revision means the /// local scrollback can no longer be trusted and must be rehydrated. - func requiresHydration(for frame: MobileTerminalRenderGridFrame) -> Bool { + /// - Parameter frame: The candidate replay frame returned by the producer. + /// - Returns: `true` when producer identity or history freshness is unknown + /// or changed; otherwise `false` for a safe zero-row repaint. + public func requiresHydration(for frame: MobileTerminalRenderGridFrame) -> Bool { guard retainedAcrossReconnect else { return hydrationNeeded } guard let renderEpoch, let historyRows, diff --git a/Packages/iOS/CmuxMobileTerminalKit/Tests/CmuxMobileTerminalKitTests/MobileTerminalMirrorStateTests.swift b/Packages/iOS/CmuxMobileTerminalKit/Tests/CmuxMobileTerminalKitTests/MobileTerminalMirrorStateTests.swift new file mode 100644 index 000000000000..c4850494833c --- /dev/null +++ b/Packages/iOS/CmuxMobileTerminalKit/Tests/CmuxMobileTerminalKitTests/MobileTerminalMirrorStateTests.swift @@ -0,0 +1,36 @@ +import CMUXMobileCore +import Testing +@testable import CmuxMobileTerminalKit + +@Test func retainedMirrorFreshnessFailsClosed() throws { + var state = MobileTerminalMirrorState() + let delivered = try MobileTerminalRenderGridFrame( + surfaceID: "surface", + stateSeq: 10, + renderEpoch: "epoch-1", + renderRevision: 1, + columns: 80, + rows: 4, + full: true, + rowSpans: [], + scrollbackRows: 20, + anchor: .screen, + historyRows: 20, + rowSpaceRevision: 1 + ) + state.record(delivered) + state.prepareForReconnect(hasDeliveredFrame: true) + + var sameHistory = delivered + sameHistory.stateSeq = 11 + sameHistory.renderRevision = 2 + #expect(!state.requiresHydration(for: sameHistory)) + + var advancedHistory = sameHistory + advancedHistory.historyRows = 21 + #expect(state.requiresHydration(for: advancedHistory)) + + var replacedProducer = sameHistory + replacedProducer.renderEpoch = "epoch-2" + #expect(state.requiresHydration(for: replacedProducer)) +} From 077dcec7eb60ee8adb61bdbf2f5664bf59efee98 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 1 Sep 2026 16:22:05 -0700 Subject: [PATCH 09/21] docs(ios): document reconnect lifecycle seams --- .../CmuxMobileShell/MobileConnectionRecoveryOwner.swift | 1 + .../MobileShellComposite+ConnectionRecovery.swift | 1 + .../MobileShellComposite+WorkspaceChanges.swift | 1 + .../Sources/CmuxMobileShell/MobileShellComposite.swift | 7 ++++--- .../MobileForegroundReconnectStormTests.swift | 6 ++++++ .../MobileReconnectStateLifecycleTests.swift | 2 ++ .../MobileShellRenderGridLivenessTestSupport.swift | 2 ++ .../MobileTerminalMirrorStateTests.swift | 1 + 8 files changed, 18 insertions(+), 3 deletions(-) diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileConnectionRecoveryOwner.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileConnectionRecoveryOwner.swift index c8936b8fb420..1703e5ce9809 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileConnectionRecoveryOwner.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileConnectionRecoveryOwner.swift @@ -79,6 +79,7 @@ final class MobileConnectionRecoveryOwner { } } + /// Claims a new probe or redial attempt when no recovery is active. func begin( trigger: String, sourceConnectionGeneration: UUID, diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+ConnectionRecovery.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+ConnectionRecovery.swift index fea449e731c8..16ec05750484 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+ConnectionRecovery.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+ConnectionRecovery.swift @@ -243,6 +243,7 @@ extension MobileShellComposite { ) } + /// Schedules one owner-managed retry for the exact failing client. private func scheduleDeadTerminalEventStreamRedial( after delay: Duration, trigger: RecoveryTrigger, diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+WorkspaceChanges.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+WorkspaceChanges.swift index e8da136668f0..e2df192ebeb9 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+WorkspaceChanges.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+WorkspaceChanges.swift @@ -414,6 +414,7 @@ extension MobileShellComposite { return remoteClient } + /// Starts one generation-owned summary pass and drains any trailing request. private func startWorkspaceChangesSummaryFetch( scope initialScope: WorkspaceChangesSummaryRefreshScope, force initialForce: Bool diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift index cbd59a1e75aa..02c44322a798 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift @@ -11080,6 +11080,8 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { cancelAllTerminalReplayTasks() } + /// Resets client-scoped terminal delivery state while retaining mounted + /// mirror metadata long enough to validate a same-session reconnect. private func resetTerminalOutputTracking() { cancelAllTerminalReplayTasks() // A connection swap clears each surface's delivery cursor below, but a @@ -11124,9 +11126,6 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { diagnosedTerminalOutputSurfaceIDs = [] terminalRenderGridHistoryContinuityBySurfaceID = [:] terminalRenderGridRevisionContinuityBySurfaceID = [:] - terminalMirrorStatesBySurfaceID = terminalMirrorStatesBySurfaceID.filter { - terminalByteContinuationsBySurfaceID.keys.contains($0.key) - } terminalReplaySurfaceIDsInFlight = [] terminalReplayRequestIDsInFlightBySurfaceID = [:] cancelAllTerminalReplayBarrierWatchdogs() @@ -14106,6 +14105,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { return bytes } + /// Installs a fresh output consumer and resets its per-mount delivery state. @discardableResult private func registerTerminalOutput( surfaceID: String, @@ -14162,6 +14162,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { return streamToken } + /// Removes the exact output consumer generation and all per-mount state. private func unregisterTerminalOutput(surfaceID: String, streamToken: UUID) { guard terminalOutputStreamTokensBySurfaceID[surfaceID] == streamToken else { return } terminalLaneOutputReadySurfaceIDs.remove(surfaceID) diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileForegroundReconnectStormTests.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileForegroundReconnectStormTests.swift index f2be23dec632..67f7f8fb0b78 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileForegroundReconnectStormTests.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileForegroundReconnectStormTests.swift @@ -29,6 +29,7 @@ import Testing // MARK: - B. Dead event-stream redials are rate-limited (single-flight + backoff) +/// Verifies repeated barren subscriptions park on bounded backoff. @MainActor @Test func foregroundDeadEventStreamRedialLoopIsRateLimited() async throws { let router = LivenessHostRouter() @@ -89,6 +90,7 @@ import Testing // MARK: - B. Backoff streak resets at a session boundary (does not carry over) +/// Verifies a session reset clears both the scheduled flag and retry streak. @Test func deadStreamRedialBackoffResetClearsStreak() { var backoff = MobileDeadStreamRedialBackoff() // The first barren stream recovers immediately; each subsequent one backs @@ -111,6 +113,7 @@ import Testing // MARK: - A. Files-changed chips survive a transient reconnect +/// Verifies transient disconnect and client replacement preserve change chips. @MainActor @Test func workspaceChangesChipsSurviveTransientReconnect() async throws { let router = LivenessHostRouter() @@ -155,6 +158,7 @@ import Testing // MARK: - C. A reconnect with a live mirror resumes (no full scrollback replay) +/// Verifies a fresh retained mirror resumes without re-downloading scrollback. @MainActor @Test func reconnectWithLiveMirrorResumesWithoutFullScrollbackReplay() async throws { let router = LivenessHostRouter() @@ -269,6 +273,7 @@ import Testing // MARK: - D. The storm does not repeatedly replay; viewport geometry survives +/// Verifies storm suppression preserves replay bounds and viewport geometry. @MainActor @Test func deadStreamStormDoesNotRepeatedlyReplayAndKeepsViewport() async throws { let router = LivenessHostRouter() @@ -340,6 +345,7 @@ import Testing // MARK: - Support +/// Builds a paired, connected shell using the scripted storm-recovery host. @MainActor private func makeStormRecoveryStore( router: LivenessHostRouter, diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileReconnectStateLifecycleTests.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileReconnectStateLifecycleTests.swift index 13ed15ae7a9e..456a1caa05cd 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileReconnectStateLifecycleTests.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileReconnectStateLifecycleTests.swift @@ -6,6 +6,7 @@ import Testing // Review-found lifecycle regressions around issue #10482: stale asynchronous // workspace summaries and public terminal surface-ID reuse. +/// Verifies a superseded workspace summary cannot publish stale chips. @MainActor @Test func canceledWorkspaceSummaryCannotPublishStaleChips() async throws { let router = LivenessHostRouter() @@ -72,6 +73,7 @@ import Testing ) } +/// Verifies a same-ID replacement mount starts with fresh hydration state. @MainActor @Test func terminalSurfaceIDReuseStartsFreshHydration() async throws { let router = LivenessHostRouter() diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift index 9782a1601c83..419d11dfabda 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift @@ -122,6 +122,7 @@ actor LivenessHostRouter { heldSyncFetchRequestNumbers.insert(number) } + /// Records one RPC request and wakes count waiters satisfied by its arrival. func record( method: String?, topics: [String]?, @@ -1058,6 +1059,7 @@ func waitForReplayResponsesServed( #expect(settled, "\(message)") } +/// Builds a connected preview shell against the scripted liveness transport. @MainActor func makeConnectedStore( router: LivenessHostRouter, diff --git a/Packages/iOS/CmuxMobileTerminalKit/Tests/CmuxMobileTerminalKitTests/MobileTerminalMirrorStateTests.swift b/Packages/iOS/CmuxMobileTerminalKit/Tests/CmuxMobileTerminalKitTests/MobileTerminalMirrorStateTests.swift index c4850494833c..c47a2f3c11e7 100644 --- a/Packages/iOS/CmuxMobileTerminalKit/Tests/CmuxMobileTerminalKitTests/MobileTerminalMirrorStateTests.swift +++ b/Packages/iOS/CmuxMobileTerminalKit/Tests/CmuxMobileTerminalKitTests/MobileTerminalMirrorStateTests.swift @@ -2,6 +2,7 @@ import CMUXMobileCore import Testing @testable import CmuxMobileTerminalKit +/// Verifies producer or history changes force retained mirrors to hydrate. @Test func retainedMirrorFreshnessFailsClosed() throws { var state = MobileTerminalMirrorState() let delivered = try MobileTerminalRenderGridFrame( From e299c994179adeed37409c960faf6ba507c37de0 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 1 Sep 2026 16:25:47 -0700 Subject: [PATCH 10/21] fix(ios): export mirror state core dependency --- .../CmuxMobileTerminalKit/MobileTerminalMirrorState.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Packages/iOS/CmuxMobileTerminalKit/Sources/CmuxMobileTerminalKit/MobileTerminalMirrorState.swift b/Packages/iOS/CmuxMobileTerminalKit/Sources/CmuxMobileTerminalKit/MobileTerminalMirrorState.swift index 80a737e6de04..d187c1ab8a8a 100644 --- a/Packages/iOS/CmuxMobileTerminalKit/Sources/CmuxMobileTerminalKit/MobileTerminalMirrorState.swift +++ b/Packages/iOS/CmuxMobileTerminalKit/Sources/CmuxMobileTerminalKit/MobileTerminalMirrorState.swift @@ -1,4 +1,4 @@ -import CMUXMobileCore +public import CMUXMobileCore /// Lifecycle state for one mounted terminal mirror. /// From 2414d12e1ce354b8285e5c8f6bf1002128bc3644 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 8 Sep 2026 07:51:23 -0700 Subject: [PATCH 11/21] test(ios): cover reconnect state invalidation --- .../MobileReconnectStateLifecycleTests.swift | 36 +++++++++++++++++++ .../MobileTerminalMirrorStateTests.swift | 31 ++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileReconnectStateLifecycleTests.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileReconnectStateLifecycleTests.swift index 456a1caa05cd..0928ae2a28db 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileReconnectStateLifecycleTests.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileReconnectStateLifecycleTests.swift @@ -73,6 +73,42 @@ import Testing ) } +/// Verifies canceling a summary task releases the policy's single-flight gate. +@MainActor +@Test func transientDisconnectAllowsWorkspaceSummaryRefreshToRestart() async throws { + let router = LivenessHostRouter() + let box = TransportBox() + let clock = TestClock() + let summaryClock = ControlPoolManualClock() + let store = try await makeConnectedStore( + router: router, + box: box, + clock: clock, + workspaceChangesSchedulingClock: summaryClock + ) + + store.suspendWorkspaceChangesSummaryFetchesPreservingChips() + _ = store.workspaceChangesSummaryRefreshSchedulePolicy.schedule( + scope: .fullSnapshot, + force: false + ) + _ = try #require( + store.workspaceChangesSummaryRefreshSchedulePolicy.beginFetchAfterDebounce() + ) + #expect(store.workspaceChangesSummaryRefreshSchedulePolicy.isFetchInFlight) + + store.suspendWorkspaceChangesSummaryFetchesPreservingChips() + + #expect(!store.workspaceChangesSummaryRefreshSchedulePolicy.isFetchInFlight) + #expect( + store.workspaceChangesSummaryRefreshSchedulePolicy.schedule( + scope: .fullSnapshot, + force: false + ), + "a reconnect must be able to schedule a fresh summary pass" + ) +} + /// Verifies a same-ID replacement mount starts with fresh hydration state. @MainActor @Test func terminalSurfaceIDReuseStartsFreshHydration() async throws { diff --git a/Packages/iOS/CmuxMobileTerminalKit/Tests/CmuxMobileTerminalKitTests/MobileTerminalMirrorStateTests.swift b/Packages/iOS/CmuxMobileTerminalKit/Tests/CmuxMobileTerminalKitTests/MobileTerminalMirrorStateTests.swift index c47a2f3c11e7..13c6d9f57c40 100644 --- a/Packages/iOS/CmuxMobileTerminalKit/Tests/CmuxMobileTerminalKitTests/MobileTerminalMirrorStateTests.swift +++ b/Packages/iOS/CmuxMobileTerminalKit/Tests/CmuxMobileTerminalKitTests/MobileTerminalMirrorStateTests.swift @@ -35,3 +35,34 @@ import Testing replacedProducer.renderEpoch = "epoch-2" #expect(state.requiresHydration(for: replacedProducer)) } + +/// Verifies a live full frame cannot replace retained metadata with a stale +/// producer and make the old local scrollback look reusable. +@Test func retainedMirrorRejectsLiveFullFrameWithChangedProducer() throws { + var state = MobileTerminalMirrorState() + let delivered = try MobileTerminalRenderGridFrame( + surfaceID: "surface", + stateSeq: 10, + renderEpoch: "epoch-1", + renderRevision: 1, + columns: 80, + rows: 4, + full: true, + rowSpans: [], + scrollbackRows: 20, + anchor: .screen, + historyRows: 20, + rowSpaceRevision: 1 + ) + state.record(delivered) + state.prepareForReconnect(hasDeliveredFrame: true) + + var replacement = delivered + replacement.renderEpoch = "epoch-2" + replacement.historyRows = 21 + state.record(replacement) + + #expect(state.hydrationNeeded) + #expect(!state.retainedAcrossReconnect) + #expect(state.requiresHydration(for: delivered)) +} From 4da5488cad1ef0beadd3691fc432e5e6ad9d26be Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 8 Sep 2026 07:51:57 -0700 Subject: [PATCH 12/21] fix(ios): validate reconnect mirror freshness --- ...ellComposite+TerminalMirrorLifecycle.swift | 5 +++- ...hellComposite+TerminalOutputDelivery.swift | 15 ++++++++--- ...ellComposite+WorkspaceChangesPruning.swift | 3 +++ .../MobileTerminalMirrorState.swift | 26 ++++++++++++++----- 4 files changed, 39 insertions(+), 10 deletions(-) diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalMirrorLifecycle.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalMirrorLifecycle.swift index aaf63f3655b4..3ca660639f96 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalMirrorLifecycle.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalMirrorLifecycle.swift @@ -24,11 +24,14 @@ extension MobileShellComposite { } /// Record the producer identity and history baseline of a delivered frame. - func recordTerminalMirrorFrame(_ frame: MobileTerminalRenderGridFrame) { + @discardableResult + func recordTerminalMirrorFrame(_ frame: MobileTerminalRenderGridFrame) -> Bool { var state = terminalMirrorStatesBySurfaceID[frame.surfaceID] ?? MobileTerminalMirrorState() + let retainedMirrorWasActive = state.retainedAcrossReconnect state.record(frame) terminalMirrorStatesBySurfaceID[frame.surfaceID] = state + return retainedMirrorWasActive && state.hydrationNeeded } /// Returns whether a retained mirror's provisional zero-row replay failed diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift index 949f120106ca..e3a38e30c060 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift @@ -19,7 +19,8 @@ extension MobileShellComposite { /// Updates per-surface screen and mirror metadata after an authoritative /// render-grid frame has been accepted for delivery. - func recordTerminalRenderGridDelivery(_ renderGrid: MobileTerminalRenderGridFrame) { + @discardableResult + func recordTerminalRenderGridDelivery(_ renderGrid: MobileTerminalRenderGridFrame) -> Bool { // The toolbar observes this dictionary via `isAlternateScreen`; same-value // writes would re-fire observers for every delivered render-grid frame. if terminalActiveScreenBySurfaceID[renderGrid.surfaceID] != renderGrid.activeScreen { @@ -35,7 +36,7 @@ extension MobileShellComposite { } else if renderGrid.activeScreen == .primary { terminalAlternateRenderGridBaselineSurfaceIDs.remove(renderGrid.surfaceID) } - recordTerminalMirrorFrame(renderGrid) + return recordTerminalMirrorFrame(renderGrid) } /// Record the screen-anchor history that the next live delta must link to. @@ -342,7 +343,15 @@ extension MobileShellComposite { terminalReplayBarrierAckCoveredDroppedOutputCountsBySurfaceID[renderGrid.surfaceID] = terminalReplayBarrierDroppedOutputCountsBySurfaceID[renderGrid.surfaceID] ?? 0 } - recordTerminalRenderGridDelivery(renderGrid) + let retainedMirrorNeedsHydration = recordTerminalRenderGridDelivery(renderGrid) + if source == "event", + retainedMirrorNeedsHydration, + terminalReplayBarrierTokensBySurfaceID[renderGrid.surfaceID] == nil { + // A producer change can arrive as a live full frame before the + // reconnect replay response. Keep that screen visible, but open a + // full replay barrier so its old local scrollback is not trusted. + terminalOutputNeedsReplay(surfaceID: renderGrid.surfaceID) + } markTerminalBytesDelivered( surfaceID: renderGrid.surfaceID, endSeq: renderGrid.stateSeq, diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+WorkspaceChangesPruning.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+WorkspaceChangesPruning.swift index acc6a8da0f93..cd0f64ed3523 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+WorkspaceChangesPruning.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+WorkspaceChangesPruning.swift @@ -23,6 +23,9 @@ extension MobileShellComposite { workspaceChangesSummaryTrailingTask = nil workspaceChangesSummaryTrailingTaskID = nil workspaceChangesSummaryTrailingDeadline = nil + // The canceled task cannot reach `fetchCompleted()`, so clear the + // single-flight marker while preserving fetched timestamps and chips. + workspaceChangesSummaryRefreshSchedulePolicy.reset() } func resetWorkspaceChangesState() { diff --git a/Packages/iOS/CmuxMobileTerminalKit/Sources/CmuxMobileTerminalKit/MobileTerminalMirrorState.swift b/Packages/iOS/CmuxMobileTerminalKit/Sources/CmuxMobileTerminalKit/MobileTerminalMirrorState.swift index d187c1ab8a8a..6fa44b46a041 100644 --- a/Packages/iOS/CmuxMobileTerminalKit/Sources/CmuxMobileTerminalKit/MobileTerminalMirrorState.swift +++ b/Packages/iOS/CmuxMobileTerminalKit/Sources/CmuxMobileTerminalKit/MobileTerminalMirrorState.swift @@ -37,10 +37,18 @@ public struct MobileTerminalMirrorState: Sendable { /// Records producer metadata from a delivered render-grid frame. A full /// frame with no retained history still completes hydration; deltas never do. + /// A retained mirror accepts a live full frame only when its producer + /// identity and history metadata still match the visible mirror. If those + /// values changed, the visible screen may still be useful, but its + /// scrollback must be rehydrated before a zero-row repaint is trusted. /// - Parameter frame: The accepted authoritative frame. public mutating func record(_ frame: MobileTerminalRenderGridFrame) { - if retainedAcrossReconnect && !frame.full { - return + if retainedAcrossReconnect { + guard frame.full else { return } + guard matchesRetainedBaseline(frame) else { + invalidate() + return + } } renderEpoch = frame.renderEpoch.isEmpty ? nil : frame.renderEpoch historyRows = frame.historyRows @@ -64,16 +72,22 @@ public struct MobileTerminalMirrorState: Sendable { /// or changed; otherwise `false` for a safe zero-row repaint. public func requiresHydration(for frame: MobileTerminalRenderGridFrame) -> Bool { guard retainedAcrossReconnect else { return hydrationNeeded } + return !matchesRetainedBaseline(frame) + } + + private func matchesRetainedBaseline( + _ frame: MobileTerminalRenderGridFrame + ) -> Bool { guard let renderEpoch, let historyRows, let rowSpaceRevision, !frame.renderEpoch.isEmpty, let frameHistoryRows = frame.historyRows, let frameRowSpaceRevision = frame.rowSpaceRevision else { - return true + return false } - return renderEpoch != frame.renderEpoch - || historyRows != frameHistoryRows - || rowSpaceRevision != frameRowSpaceRevision + return renderEpoch == frame.renderEpoch + && historyRows == frameHistoryRows + && rowSpaceRevision == frameRowSpaceRevision } } From 7957ec67e099fc586a023b74f67b59a4480f0287 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 8 Sep 2026 08:08:14 -0700 Subject: [PATCH 13/21] fix(ios): keep hydration replay authoritative --- ...leShellComposite+TerminalOutputDelivery.swift | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift index e3a38e30c060..6325a9502125 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift @@ -344,6 +344,16 @@ extension MobileShellComposite { terminalReplayBarrierDroppedOutputCountsBySurfaceID[renderGrid.surfaceID] ?? 0 } let retainedMirrorNeedsHydration = recordTerminalRenderGridDelivery(renderGrid) + markTerminalBytesDelivered( + surfaceID: renderGrid.surfaceID, + endSeq: renderGrid.stateSeq, + // A retained mirror's changed-producer frame paints the current + // screen but is not a trustworthy scrollback replacement. Keeping + // it out of the full-replacement generation prevents the + // same-sequence hydration replay from being rejected as stale. + fullReplacement: renderGrid.full && !retainedMirrorNeedsHydration + ) + recordTerminalRenderGridHistoryContinuity(renderGrid) if source == "event", retainedMirrorNeedsHydration, terminalReplayBarrierTokensBySurfaceID[renderGrid.surfaceID] == nil { @@ -352,12 +362,6 @@ extension MobileShellComposite { // full replay barrier so its old local scrollback is not trusted. terminalOutputNeedsReplay(surfaceID: renderGrid.surfaceID) } - markTerminalBytesDelivered( - surfaceID: renderGrid.surfaceID, - endSeq: renderGrid.stateSeq, - fullReplacement: renderGrid.full - ) - recordTerminalRenderGridHistoryContinuity(renderGrid) #if DEBUG MobileLatencyTrace.stamp( "gate", From e5f4da2e86eed6e5f7727280d0ae9564459573f1 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 8 Sep 2026 09:17:49 -0700 Subject: [PATCH 14/21] fix(ios): preserve reconnect hint and producer freshness --- .../WorkspaceDetailView+Surfaces.swift | 2 +- ...WorkspaceDetailView+WorkspaceChanges.swift | 10 ++++- .../MobileTerminalMirrorState.swift | 22 ++++++++-- .../MobileTerminalMirrorStateTests.swift | 41 +++++++++++++++++++ 4 files changed, 69 insertions(+), 6 deletions(-) diff --git a/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/WorkspaceDetailView+Surfaces.swift b/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/WorkspaceDetailView+Surfaces.swift index 8badb767658d..87ffed9ee910 100644 --- a/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/WorkspaceDetailView+Surfaces.swift +++ b/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/WorkspaceDetailView+Surfaces.swift @@ -81,7 +81,7 @@ extension WorkspaceDetailView { } } .safeAreaInset(edge: .top, spacing: 0) { - if workspaceChangesHint != nil { + if workspaceChangesAreAvailable, workspaceChangesHint != nil { WorkspaceChangesHintBanner( openChanges: openWorkspaceChanges, dismiss: dismissWorkspaceChangesHint diff --git a/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/WorkspaceDetailView+WorkspaceChanges.swift b/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/WorkspaceDetailView+WorkspaceChanges.swift index 331b48149a37..613cce96d39d 100644 --- a/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/WorkspaceDetailView+WorkspaceChanges.swift +++ b/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/WorkspaceDetailView+WorkspaceChanges.swift @@ -55,11 +55,17 @@ extension WorkspaceDetailView { } func refreshWorkspaceChangesHint() { - guard !UITestConfig.hideWorkspaceChangesHintForScreenshots, - workspaceChangesAreAvailable else { + guard !UITestConfig.hideWorkspaceChangesHintForScreenshots else { workspaceChangesHint = nil return } + // A transient reconnect toggles both capability and connection gates. + // Keep the already-presented hint in local view state across that + // transport churn; the banner itself is hidden while unavailable and + // reuses this same state after the handshake. A dismissed hint is nil + // and remains suppressed by the store-backed eligibility check. + guard workspaceChangesAreAvailable, + workspaceChangesHint == nil else { return } workspaceChangesHint = store.workspaceChangesHint( workspaceID: workspace.rpcWorkspaceID.rawValue ) diff --git a/Packages/iOS/CmuxMobileTerminalKit/Sources/CmuxMobileTerminalKit/MobileTerminalMirrorState.swift b/Packages/iOS/CmuxMobileTerminalKit/Sources/CmuxMobileTerminalKit/MobileTerminalMirrorState.swift index 6fa44b46a041..5b5ebffdf1b4 100644 --- a/Packages/iOS/CmuxMobileTerminalKit/Sources/CmuxMobileTerminalKit/MobileTerminalMirrorState.swift +++ b/Packages/iOS/CmuxMobileTerminalKit/Sources/CmuxMobileTerminalKit/MobileTerminalMirrorState.swift @@ -50,14 +50,26 @@ public struct MobileTerminalMirrorState: Sendable { return } } - renderEpoch = frame.renderEpoch.isEmpty ? nil : frame.renderEpoch - historyRows = frame.historyRows - rowSpaceRevision = frame.rowSpaceRevision let hydrationSatisfied = retainedAcrossReconnect || frame.anchor != .screen || frame.scrollbackRows > 0 || frame.historyRows == 0 || frame.activeScreen == .alternate + if frame.full, + hasKnownProducerMetadata, + !matchesRetainedBaseline(frame), + !hydrationSatisfied { + // A producer change after a retained replay is still unsafe even + // after the replay cleared `retainedAcrossReconnect`. A live full + // frame without scrollback can paint the screen while leaving the + // local primary history owned by the retired producer. Invalidate + // that baseline so the next authoritative replay hydrates it. + invalidate() + return + } + renderEpoch = frame.renderEpoch.isEmpty ? nil : frame.renderEpoch + historyRows = frame.historyRows + rowSpaceRevision = frame.rowSpaceRevision if frame.full, hydrationSatisfied { hydrationNeeded = false retainedAcrossReconnect = false @@ -90,4 +102,8 @@ public struct MobileTerminalMirrorState: Sendable { && historyRows == frameHistoryRows && rowSpaceRevision == frameRowSpaceRevision } + + private var hasKnownProducerMetadata: Bool { + renderEpoch != nil || historyRows != nil || rowSpaceRevision != nil + } } diff --git a/Packages/iOS/CmuxMobileTerminalKit/Tests/CmuxMobileTerminalKitTests/MobileTerminalMirrorStateTests.swift b/Packages/iOS/CmuxMobileTerminalKit/Tests/CmuxMobileTerminalKitTests/MobileTerminalMirrorStateTests.swift index 13c6d9f57c40..21b9afdca7ea 100644 --- a/Packages/iOS/CmuxMobileTerminalKit/Tests/CmuxMobileTerminalKitTests/MobileTerminalMirrorStateTests.swift +++ b/Packages/iOS/CmuxMobileTerminalKit/Tests/CmuxMobileTerminalKitTests/MobileTerminalMirrorStateTests.swift @@ -66,3 +66,44 @@ import Testing #expect(!state.retainedAcrossReconnect) #expect(state.requiresHydration(for: delivered)) } + +/// Verifies a producer change after a matching zero-row replay cannot keep the +/// old producer's scrollback baseline marked as hydrated. +@Test func liveProducerChangeAfterRetainedReplayRequiresHydration() throws { + var state = MobileTerminalMirrorState() + let delivered = try MobileTerminalRenderGridFrame( + surfaceID: "surface", + stateSeq: 10, + renderEpoch: "epoch-1", + renderRevision: 1, + columns: 80, + rows: 4, + full: true, + rowSpans: [], + scrollbackRows: 20, + anchor: .screen, + historyRows: 20, + rowSpaceRevision: 1 + ) + state.record(delivered) + state.prepareForReconnect(hasDeliveredFrame: true) + + var replay = delivered + replay.stateSeq = 11 + replay.renderRevision = 2 + replay.scrollbackRows = 0 + replay.scrollbackSpans = [] + state.record(replay) + #expect(!state.hydrationNeeded) + + var replacement = replay + replacement.stateSeq = 12 + replacement.renderRevision = 3 + replacement.renderEpoch = "epoch-2" + replacement.historyRows = 21 + replacement.rowSpaceRevision = 2 + state.record(replacement) + + #expect(state.hydrationNeeded) + #expect(state.requiresHydration(for: replacement)) +} From 1bc98f7a7c4334d95f86b1c146c2268a96196639 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 8 Sep 2026 09:21:09 -0700 Subject: [PATCH 15/21] fix(ios): keep workspace hint stable across reconnects --- .../WorkspaceChangesHintRefreshPolicy.swift | 21 ++++++++ ...WorkspaceDetailView+WorkspaceChanges.swift | 15 +++--- ...rkspaceChangesHintRefreshPolicyTests.swift | 49 +++++++++++++++++++ 3 files changed, 76 insertions(+), 9 deletions(-) create mode 100644 Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/WorkspaceChangesHintRefreshPolicy.swift create mode 100644 Packages/iOS/CmuxMobileShellUI/Tests/CmuxMobileShellUITests/WorkspaceChangesHintRefreshPolicyTests.swift diff --git a/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/WorkspaceChangesHintRefreshPolicy.swift b/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/WorkspaceChangesHintRefreshPolicy.swift new file mode 100644 index 000000000000..17bc5ec31911 --- /dev/null +++ b/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/WorkspaceChangesHintRefreshPolicy.swift @@ -0,0 +1,21 @@ +#if os(iOS) +import CmuxMobileShell + +/// Keeps a one-time workspace-changes hint stable across transient transport +/// state changes while still allowing an eligible detail view to arm it. +enum WorkspaceChangesHintRefreshPolicy { + /// Returns the hint that should remain mounted after an eligibility update. + /// + /// A disconnect temporarily makes the capability gate unavailable. That + /// must not erase an already-presented hint, because the reconnect would + /// otherwise present the same hint again when the gates recover. + static func next( + current: MobileWorkspaceChangesHint?, + isAvailable: Bool, + candidate: MobileWorkspaceChangesHint? + ) -> MobileWorkspaceChangesHint? { + guard isAvailable, current == nil else { return current } + return candidate + } +} +#endif diff --git a/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/WorkspaceDetailView+WorkspaceChanges.swift b/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/WorkspaceDetailView+WorkspaceChanges.swift index 613cce96d39d..2d6c43b2a9fa 100644 --- a/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/WorkspaceDetailView+WorkspaceChanges.swift +++ b/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/WorkspaceDetailView+WorkspaceChanges.swift @@ -59,15 +59,12 @@ extension WorkspaceDetailView { workspaceChangesHint = nil return } - // A transient reconnect toggles both capability and connection gates. - // Keep the already-presented hint in local view state across that - // transport churn; the banner itself is hidden while unavailable and - // reuses this same state after the handshake. A dismissed hint is nil - // and remains suppressed by the store-backed eligibility check. - guard workspaceChangesAreAvailable, - workspaceChangesHint == nil else { return } - workspaceChangesHint = store.workspaceChangesHint( - workspaceID: workspace.rpcWorkspaceID.rawValue + workspaceChangesHint = WorkspaceChangesHintRefreshPolicy.next( + current: workspaceChangesHint, + isAvailable: workspaceChangesAreAvailable, + candidate: store.workspaceChangesHint( + workspaceID: workspace.rpcWorkspaceID.rawValue + ) ) } } diff --git a/Packages/iOS/CmuxMobileShellUI/Tests/CmuxMobileShellUITests/WorkspaceChangesHintRefreshPolicyTests.swift b/Packages/iOS/CmuxMobileShellUI/Tests/CmuxMobileShellUITests/WorkspaceChangesHintRefreshPolicyTests.swift new file mode 100644 index 000000000000..8a90935ff0a6 --- /dev/null +++ b/Packages/iOS/CmuxMobileShellUI/Tests/CmuxMobileShellUITests/WorkspaceChangesHintRefreshPolicyTests.swift @@ -0,0 +1,49 @@ +import CmuxMobileShell +import Testing +@testable import CmuxMobileShellUI + +@Test func hintSurvivesTransientReconnectWithoutRearming() { + let hint = MobileWorkspaceChangesHint( + workspaceID: "workspace-a", + workspaceChangesCapable: true, + chip: MobileWorkspaceChangesChip(filesChanged: 2, additions: 3, deletions: 1), + isDismissed: false + ) + var current = WorkspaceChangesHintRefreshPolicy.next( + current: nil, + isAvailable: true, + candidate: hint + ) + #expect(current == hint) + + current = WorkspaceChangesHintRefreshPolicy.next( + current: current, + isAvailable: false, + candidate: nil + ) + #expect(current == hint) + + current = WorkspaceChangesHintRefreshPolicy.next( + current: current, + isAvailable: true, + candidate: hint + ) + #expect(current == hint) +} + +@Test func unavailableDetailDoesNotArmAHint() { + let hint = MobileWorkspaceChangesHint( + workspaceID: "workspace-a", + workspaceChangesCapable: true, + chip: MobileWorkspaceChangesChip(filesChanged: 2, additions: 3, deletions: 1), + isDismissed: false + ) + + #expect( + WorkspaceChangesHintRefreshPolicy.next( + current: nil, + isAvailable: false, + candidate: hint + ) == nil + ) +} From 969130dc9de9420fe0853aae366adf08bddd5345 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 8 Sep 2026 09:37:44 -0700 Subject: [PATCH 16/21] fix(ios): expose mirror retention state to shell --- .../CmuxMobileTerminalKit/MobileTerminalMirrorState.swift | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Packages/iOS/CmuxMobileTerminalKit/Sources/CmuxMobileTerminalKit/MobileTerminalMirrorState.swift b/Packages/iOS/CmuxMobileTerminalKit/Sources/CmuxMobileTerminalKit/MobileTerminalMirrorState.swift index 5b5ebffdf1b4..db3ee6c93a3b 100644 --- a/Packages/iOS/CmuxMobileTerminalKit/Sources/CmuxMobileTerminalKit/MobileTerminalMirrorState.swift +++ b/Packages/iOS/CmuxMobileTerminalKit/Sources/CmuxMobileTerminalKit/MobileTerminalMirrorState.swift @@ -9,7 +9,11 @@ public import CMUXMobileCore public struct MobileTerminalMirrorState: Sendable { /// Whether the next authoritative replay must include scrollback rows. public private(set) var hydrationNeeded = true - var retainedAcrossReconnect = false + /// Whether the mounted mirror is currently eligible for a zero-row replay. + /// + /// The shell owns the replay barrier and needs to observe this lifecycle + /// state, while all mutations remain inside this value type. + public private(set) var retainedAcrossReconnect = false private(set) var renderEpoch: String? private(set) var historyRows: UInt64? private(set) var rowSpaceRevision: UInt64? From f4db660c5724e093b140712b9b18a484e9777577 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 8 Sep 2026 09:47:15 -0700 Subject: [PATCH 17/21] test(ios): avoid mutating policy in assertion macros --- .../MobileReconnectStateLifecycleTests.swift | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileReconnectStateLifecycleTests.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileReconnectStateLifecycleTests.swift index 0928ae2a28db..5ccc5f0b20f8 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileReconnectStateLifecycleTests.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileReconnectStateLifecycleTests.swift @@ -88,23 +88,24 @@ import Testing ) store.suspendWorkspaceChangesSummaryFetchesPreservingChips() - _ = store.workspaceChangesSummaryRefreshSchedulePolicy.schedule( + let scheduledBeforeFetch = store.workspaceChangesSummaryRefreshSchedulePolicy.schedule( scope: .fullSnapshot, force: false ) - _ = try #require( - store.workspaceChangesSummaryRefreshSchedulePolicy.beginFetchAfterDebounce() - ) + #expect(scheduledBeforeFetch) + let fetchRequest = store.workspaceChangesSummaryRefreshSchedulePolicy.beginFetchAfterDebounce() + try #require(fetchRequest) #expect(store.workspaceChangesSummaryRefreshSchedulePolicy.isFetchInFlight) store.suspendWorkspaceChangesSummaryFetchesPreservingChips() #expect(!store.workspaceChangesSummaryRefreshSchedulePolicy.isFetchInFlight) + let scheduledAfterDisconnect = store.workspaceChangesSummaryRefreshSchedulePolicy.schedule( + scope: .fullSnapshot, + force: false + ) #expect( - store.workspaceChangesSummaryRefreshSchedulePolicy.schedule( - scope: .fullSnapshot, - force: false - ), + scheduledAfterDisconnect, "a reconnect must be able to schedule a fresh summary pass" ) } From 62c3fbedc2d4910ca2dd3ce8fdbac8ba29f72bc4 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 8 Sep 2026 10:03:17 -0700 Subject: [PATCH 18/21] fix(ios): accept same-producer history growth after replay --- .../MobileTerminalMirrorState.swift | 15 ++++++- .../MobileTerminalMirrorStateTests.swift | 42 +++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/Packages/iOS/CmuxMobileTerminalKit/Sources/CmuxMobileTerminalKit/MobileTerminalMirrorState.swift b/Packages/iOS/CmuxMobileTerminalKit/Sources/CmuxMobileTerminalKit/MobileTerminalMirrorState.swift index db3ee6c93a3b..3aa5c2703822 100644 --- a/Packages/iOS/CmuxMobileTerminalKit/Sources/CmuxMobileTerminalKit/MobileTerminalMirrorState.swift +++ b/Packages/iOS/CmuxMobileTerminalKit/Sources/CmuxMobileTerminalKit/MobileTerminalMirrorState.swift @@ -45,6 +45,9 @@ public struct MobileTerminalMirrorState: Sendable { /// identity and history metadata still match the visible mirror. If those /// values changed, the visible screen may still be useful, but its /// scrollback must be rehydrated before a zero-row repaint is trusted. + /// Once a retained replay has completed, normal history growth and + /// row-space changes are accepted; a producer identity change still + /// invalidates the local scrollback baseline. /// - Parameter frame: The accepted authoritative frame. public mutating func record(_ frame: MobileTerminalRenderGridFrame) { if retainedAcrossReconnect { @@ -61,7 +64,7 @@ public struct MobileTerminalMirrorState: Sendable { || frame.activeScreen == .alternate if frame.full, hasKnownProducerMetadata, - !matchesRetainedBaseline(frame), + !matchesRetainedProducer(frame), !hydrationSatisfied { // A producer change after a retained replay is still unsafe even // after the replay cleared `retainedAcrossReconnect`. A live full @@ -107,6 +110,16 @@ public struct MobileTerminalMirrorState: Sendable { && rowSpaceRevision == frameRowSpaceRevision } + private func matchesRetainedProducer( + _ frame: MobileTerminalRenderGridFrame + ) -> Bool { + guard let renderEpoch, + !frame.renderEpoch.isEmpty else { + return false + } + return renderEpoch == frame.renderEpoch + } + private var hasKnownProducerMetadata: Bool { renderEpoch != nil || historyRows != nil || rowSpaceRevision != nil } diff --git a/Packages/iOS/CmuxMobileTerminalKit/Tests/CmuxMobileTerminalKitTests/MobileTerminalMirrorStateTests.swift b/Packages/iOS/CmuxMobileTerminalKit/Tests/CmuxMobileTerminalKitTests/MobileTerminalMirrorStateTests.swift index 21b9afdca7ea..b4a7f93c3614 100644 --- a/Packages/iOS/CmuxMobileTerminalKit/Tests/CmuxMobileTerminalKitTests/MobileTerminalMirrorStateTests.swift +++ b/Packages/iOS/CmuxMobileTerminalKit/Tests/CmuxMobileTerminalKitTests/MobileTerminalMirrorStateTests.swift @@ -107,3 +107,45 @@ import Testing #expect(state.hydrationNeeded) #expect(state.requiresHydration(for: replacement)) } + +/// Verifies normal history growth after a retained replay does not invalidate +/// an otherwise healthy same-producer mirror. +@Test func sameProducerHistoryGrowthAfterRetainedReplayStaysHydrated() throws { + var state = MobileTerminalMirrorState() + let delivered = try MobileTerminalRenderGridFrame( + surfaceID: "surface", + stateSeq: 10, + renderEpoch: "epoch-1", + renderRevision: 1, + columns: 80, + rows: 4, + full: true, + rowSpans: [], + scrollbackRows: 20, + anchor: .screen, + historyRows: 20, + rowSpaceRevision: 1 + ) + state.record(delivered) + state.prepareForReconnect(hasDeliveredFrame: true) + + var replay = delivered + replay.stateSeq = 11 + replay.renderRevision = 2 + replay.scrollbackRows = 0 + replay.scrollbackSpans = [] + state.record(replay) + #expect(!state.hydrationNeeded) + + var historyGrowth = replay + historyGrowth.stateSeq = 12 + historyGrowth.renderRevision = 3 + historyGrowth.historyRows = 21 + historyGrowth.rowSpaceRevision = 2 + historyGrowth.scrollbackRows = 0 + historyGrowth.scrollbackSpans = [] + state.record(historyGrowth) + + #expect(!state.hydrationNeeded) + #expect(!state.requiresHydration(for: historyGrowth)) +} From 6d7e54bcc796853c895a0c5580b61b3704f2dbd6 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 8 Sep 2026 10:24:06 -0700 Subject: [PATCH 19/21] fix(ios): require primary scrollback for hydration --- .../MobileTerminalMirrorState.swift | 14 +++--- .../MobileTerminalMirrorStateTests.swift | 47 +++++++++++++++++++ 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/Packages/iOS/CmuxMobileTerminalKit/Sources/CmuxMobileTerminalKit/MobileTerminalMirrorState.swift b/Packages/iOS/CmuxMobileTerminalKit/Sources/CmuxMobileTerminalKit/MobileTerminalMirrorState.swift index 3aa5c2703822..633d47c747c7 100644 --- a/Packages/iOS/CmuxMobileTerminalKit/Sources/CmuxMobileTerminalKit/MobileTerminalMirrorState.swift +++ b/Packages/iOS/CmuxMobileTerminalKit/Sources/CmuxMobileTerminalKit/MobileTerminalMirrorState.swift @@ -40,7 +40,9 @@ public struct MobileTerminalMirrorState: Sendable { } /// Records producer metadata from a delivered render-grid frame. A full - /// frame with no retained history still completes hydration; deltas never do. + /// screen-anchored primary frame carrying scrollback (or explicitly + /// reporting no history) completes hydration; deltas and frames that do + /// not describe primary scrollback never do. /// A retained mirror accepts a live full frame only when its producer /// identity and history metadata still match the visible mirror. If those /// values changed, the visible screen may still be useful, but its @@ -57,11 +59,11 @@ public struct MobileTerminalMirrorState: Sendable { return } } - let hydrationSatisfied = retainedAcrossReconnect - || frame.anchor != .screen - || frame.scrollbackRows > 0 - || frame.historyRows == 0 - || frame.activeScreen == .alternate + let hydrationSatisfied = retainedAcrossReconnect || ( + frame.anchor == .screen + && frame.activeScreen == .primary + && (frame.scrollbackRows > 0 || frame.historyRows == 0) + ) if frame.full, hasKnownProducerMetadata, !matchesRetainedProducer(frame), diff --git a/Packages/iOS/CmuxMobileTerminalKit/Tests/CmuxMobileTerminalKitTests/MobileTerminalMirrorStateTests.swift b/Packages/iOS/CmuxMobileTerminalKit/Tests/CmuxMobileTerminalKitTests/MobileTerminalMirrorStateTests.swift index b4a7f93c3614..bb0266b6c899 100644 --- a/Packages/iOS/CmuxMobileTerminalKit/Tests/CmuxMobileTerminalKitTests/MobileTerminalMirrorStateTests.swift +++ b/Packages/iOS/CmuxMobileTerminalKit/Tests/CmuxMobileTerminalKitTests/MobileTerminalMirrorStateTests.swift @@ -149,3 +149,50 @@ import Testing #expect(!state.hydrationNeeded) #expect(!state.requiresHydration(for: historyGrowth)) } + +/// Verifies an alternate-screen frame cannot satisfy primary scrollback +/// hydration when the mirror has no retained baseline. +@Test func alternateScreenFrameKeepsPrimaryHydrationPending() throws { + var state = MobileTerminalMirrorState() + let alternate = try MobileTerminalRenderGridFrame( + surfaceID: "surface", + stateSeq: 10, + renderEpoch: "epoch-1", + renderRevision: 1, + columns: 80, + rows: 4, + full: true, + rowSpans: [], + activeScreen: .alternate, + scrollbackRows: 0, + anchor: .screen, + historyRows: 20, + rowSpaceRevision: 1 + ) + state.record(alternate) + + #expect(state.hydrationNeeded) +} + +/// Verifies a viewport-anchored full frame cannot stand in for primary +/// scrollback hydration when it carries no history rows. +@Test func viewportFrameKeepsPrimaryHydrationPending() throws { + var state = MobileTerminalMirrorState() + let viewport = try MobileTerminalRenderGridFrame( + surfaceID: "surface", + stateSeq: 10, + renderEpoch: "epoch-1", + renderRevision: 1, + columns: 80, + rows: 4, + full: true, + rowSpans: [], + scrollbackRows: 0, + anchor: .viewport, + historyRows: 20, + rowSpaceRevision: 1 + ) + state.record(viewport) + + #expect(state.hydrationNeeded) +} From 593af477490d92bb0ea89e4ba2368dc8324821df Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 8 Sep 2026 10:26:46 -0700 Subject: [PATCH 20/21] test(ios): avoid unused tuple assertion warning --- .../MobileReconnectStateLifecycleTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileReconnectStateLifecycleTests.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileReconnectStateLifecycleTests.swift index 5ccc5f0b20f8..d128b874aa15 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileReconnectStateLifecycleTests.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileReconnectStateLifecycleTests.swift @@ -94,7 +94,7 @@ import Testing ) #expect(scheduledBeforeFetch) let fetchRequest = store.workspaceChangesSummaryRefreshSchedulePolicy.beginFetchAfterDebounce() - try #require(fetchRequest) + #expect(fetchRequest != nil) #expect(store.workspaceChangesSummaryRefreshSchedulePolicy.isFetchInFlight) store.suspendWorkspaceChangesSummaryFetchesPreservingChips() From 2cecebc1584438e2f31ccc022887225c811972b1 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 8 Sep 2026 12:18:52 -0700 Subject: [PATCH 21/21] fix(ios): preserve recovery wakeups across backgrounding --- .../MobileConnectionRecoveryOwner.swift | 12 ++++- ...ileShellComposite+ConnectionRecovery.swift | 10 ++-- ...MobileShellComposite+ReconnectRoutes.swift | 14 +++++- .../MobileForegroundReconnectStormTests.swift | 50 +++++++++++++++++++ .../WorkspaceChangesHintRefreshPolicy.swift | 6 ++- ...rkspaceChangesHintRefreshPolicyTests.swift | 17 +++++++ 6 files changed, 100 insertions(+), 9 deletions(-) diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileConnectionRecoveryOwner.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileConnectionRecoveryOwner.swift index 1703e5ce9809..e68d07c6d972 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileConnectionRecoveryOwner.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileConnectionRecoveryOwner.swift @@ -79,6 +79,13 @@ final class MobileConnectionRecoveryOwner { } } + /// Whether a delayed barren-stream retry is waiting for its deadline. + /// Background suspension uses this to park the corresponding recovery + /// trigger before cancellation can otherwise lose the wake-up. + var hasPendingDeadTerminalEventStreamRedial: Bool { + deadTerminalEventStreamRedialTask != nil + } + /// Claims a new probe or redial attempt when no recovery is active. func begin( trigger: String, @@ -256,11 +263,14 @@ final class MobileConnectionRecoveryOwner { /// Cancels a pending barren-stream retry while preserving the accumulated /// session streak. Background suspension uses this form so a resumed /// session cannot immediately return to a tight redial loop. - func cancelDeadTerminalEventStreamRedial() { + @discardableResult + func cancelDeadTerminalEventStreamRedial() -> Bool { + let wasPending = deadTerminalEventStreamRedialTask != nil deadTerminalEventStreamRedialGeneration = UUID() deadTerminalEventStreamRedialTask?.cancel() deadTerminalEventStreamRedialTask = nil deadTerminalEventStreamRedialBackoff.redialFired() + return wasPending } /// Resets the barren-stream retry state at a fresh account/session boundary. diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+ConnectionRecovery.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+ConnectionRecovery.swift index a9f8e5d18932..59440e242f45 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+ConnectionRecovery.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+ConnectionRecovery.swift @@ -278,7 +278,8 @@ extension MobileShellComposite { /// scheduled, so the next barren stream may schedule again instead of /// coalescing into a dead timer. It keeps the barren-stream streak, so a /// suspend/resume of the same session preserves the accrued backoff. - func cancelDeadTerminalEventStreamRedial() { + @discardableResult + func cancelDeadTerminalEventStreamRedial() -> Bool { connectionRecoveryOwner.cancelDeadTerminalEventStreamRedial() } @@ -290,9 +291,10 @@ extension MobileShellComposite { } /// Replays the most recent recovery trigger that was parked while the - /// scene was inactive. Called from `resumeForegroundRefresh()` after the - /// foreground recovery passes, so a replay coalesces into any attempt - /// they already started instead of stacking a second dial. + /// scene was inactive. Called from `resumeForegroundRefresh()` before the + /// generic foreground recovery pass so a parked stream-end replay can force + /// its subscription resync instead of being coalesced into a probe that + /// reports the still-healthy RPC connection and skips that resync. func recoverPendingInactiveRecoveryIfNeeded() { guard foregroundRefreshIsActive, let trigger = pendingInactiveRecoveryTrigger else { return } diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+ReconnectRoutes.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+ReconnectRoutes.swift index b018b2d18682..703b29a33cbc 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+ReconnectRoutes.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+ReconnectRoutes.swift @@ -437,9 +437,13 @@ extension MobileShellComposite { } restartActiveMobileBrowserStreams() restartActiveMobileSimulatorStreams() + // Replay a parked stream-end recovery before the generic foreground + // probe. Its healthy result must force a subscription resync; if the + // generic probe claimed the owner first, it could complete as healthy + // without restarting a listener whose backoff was canceled above. + recoverPendingInactiveRecoveryIfNeeded() recoverForegroundConnectionIfNeeded(resyncAfterHealthy: shouldResync) recoverDisconnectedOnForegroundIfNeeded() - recoverPendingInactiveRecoveryIfNeeded() resumeSecondaryControlMaintenanceAfterForeground() // The foreground Mac's workspace list updates live over the sync stream, // but the other Macs are a read-only snapshot. Re-aggregate them on @@ -460,7 +464,13 @@ extension MobileShellComposite { foregroundRefreshIsActive = false // A pending dead-stream backoff redial would otherwise fire on resume // with the process's frozen wall clock; foreground recovery re-drives it. - cancelDeadTerminalEventStreamRedial() + if cancelDeadTerminalEventStreamRedial() { + // Keep the definitive stream-end trigger alive across suspension. + // The connection can remain RPC-healthy while its event listener is + // gone, so a generic foreground liveness probe is not sufficient to + // guarantee that the listener is restarted. + pendingInactiveRecoveryTrigger = .eventStreamEnded + } if connectionRecoveryOwner.cancelProbing() { applyConnectionRecoveryOwnerState() } diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileForegroundReconnectStormTests.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileForegroundReconnectStormTests.swift index 67f7f8fb0b78..133869eee922 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileForegroundReconnectStormTests.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileForegroundReconnectStormTests.swift @@ -111,6 +111,56 @@ import Testing #expect(backoff.nextRedialDelay() == .seconds(1)) } +/// A background transition must not lose a delayed dead-stream wake-up while +/// the underlying RPC connection still reports healthy. The parked trigger is +/// replayed first on foreground so the healthy probe also restarts the ended +/// event listener instead of silently completing with the stale stream. +@MainActor +@Test func backgroundCancelsDeadStreamBackoffAndForegroundRestartsListener() async throws { + let router = LivenessHostRouter() + let box = TransportBox() + let clock = TestClock() + let controlClock = ControlPoolManualClock() + let (store, directory) = try await makeStormRecoveryStore( + router: router, + box: box, + clock: clock, + controlClock: controlClock + ) + defer { + Task { await router.releaseAllHeld() } + try? FileManager.default.removeItem(at: directory) + } + + // Consume the immediate first retry, then arm the delayed second retry. + #expect(store.connectionRecoveryOwner.nextDeadTerminalEventStreamRedialDelay() == .zero) + let delay = try #require( + store.connectionRecoveryOwner.nextDeadTerminalEventStreamRedialDelay() + ) + #expect(delay > .zero) + store.connectionRecoveryOwner.scheduleDeadTerminalEventStreamRedial( + after: delay, + clock: controlClock + ) {} + #expect(store.connectionRecoveryOwner.hasPendingDeadTerminalEventStreamRedial) + + let subscribeCount = await router.count(of: "mobile.events.subscribe") + store.suspendForegroundRefresh() + + #expect(!store.connectionRecoveryOwner.hasPendingDeadTerminalEventStreamRedial) + #expect( + store.pendingInactiveRecoveryTrigger?.description == "eventStreamEnded" + ) + + store.resumeForegroundRefresh() + + #expect(await router.waitForCount( + of: "mobile.events.subscribe", + atLeast: subscribeCount + 1 + )) + #expect(store.pendingInactiveRecoveryTrigger == nil) +} + // MARK: - A. Files-changed chips survive a transient reconnect /// Verifies transient disconnect and client replacement preserve change chips. diff --git a/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/WorkspaceChangesHintRefreshPolicy.swift b/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/WorkspaceChangesHintRefreshPolicy.swift index 17bc5ec31911..143110165955 100644 --- a/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/WorkspaceChangesHintRefreshPolicy.swift +++ b/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/WorkspaceChangesHintRefreshPolicy.swift @@ -8,13 +8,15 @@ enum WorkspaceChangesHintRefreshPolicy { /// /// A disconnect temporarily makes the capability gate unavailable. That /// must not erase an already-presented hint, because the reconnect would - /// otherwise present the same hint again when the gates recover. + /// otherwise present the same hint again when the gates recover. Once the + /// detail is available again, however, the candidate is authoritative: a + /// missing candidate means the workspace no longer has eligible changes. static func next( current: MobileWorkspaceChangesHint?, isAvailable: Bool, candidate: MobileWorkspaceChangesHint? ) -> MobileWorkspaceChangesHint? { - guard isAvailable, current == nil else { return current } + guard isAvailable else { return current } return candidate } } diff --git a/Packages/iOS/CmuxMobileShellUI/Tests/CmuxMobileShellUITests/WorkspaceChangesHintRefreshPolicyTests.swift b/Packages/iOS/CmuxMobileShellUI/Tests/CmuxMobileShellUITests/WorkspaceChangesHintRefreshPolicyTests.swift index 8a90935ff0a6..2a4780ebe6e0 100644 --- a/Packages/iOS/CmuxMobileShellUI/Tests/CmuxMobileShellUITests/WorkspaceChangesHintRefreshPolicyTests.swift +++ b/Packages/iOS/CmuxMobileShellUI/Tests/CmuxMobileShellUITests/WorkspaceChangesHintRefreshPolicyTests.swift @@ -44,6 +44,23 @@ import Testing current: nil, isAvailable: false, candidate: hint + ) == nil + ) +} + +@Test func availableDetailClearsHintWhenChangesDisappear() { + let hint = MobileWorkspaceChangesHint( + workspaceID: "workspace-a", + workspaceChangesCapable: true, + chip: MobileWorkspaceChangesChip(filesChanged: 2, additions: 3, deletions: 1), + isDismissed: false + ) + + #expect( + WorkspaceChangesHintRefreshPolicy.next( + current: hint, + isAvailable: true, + candidate: nil ) == nil ) }