From 07cee3d069b3dd54f4d95aee821ba253f2f8c955 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 01:50:44 -0700 Subject: [PATCH 01/56] Add failing iOS terminal output reset test --- .../TerminalOutputDeliveryQueueTests.swift | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift index 4b8db691f153..43fb96de7fc7 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift @@ -41,6 +41,35 @@ import Testing #expect(String(decoding: secondChunk.data, as: UTF8.self) == "new-second") } +@MainActor +@Test func terminalOutputResetDropsStalledBacklogAndInvalidatesOldAcks() async throws { + let store = MobileShellComposite.preview() + let surfaceID = "terminal" + + var iterator = store.terminalOutputStream(surfaceID: surfaceID).makeAsyncIterator() + store.deliverTerminalBytes(Data("stalled-first".utf8), surfaceID: surfaceID) + let stalledChunk = try #require(await iterator.next()) + store.deliverTerminalBytes(Data("stale-second".utf8), surfaceID: surfaceID) + + #expect(store.terminalOutputQueuesBySurfaceID[surfaceID]?.pendingCount == 1) + + store.terminalOutputDidReset(surfaceID: surfaceID, streamToken: stalledChunk.streamToken) + + #expect(store.terminalOutputQueuesBySurfaceID[surfaceID]?.isIdle == true) + + store.deliverTerminalBytes(Data("fresh-after-reset".utf8), surfaceID: surfaceID) + let freshChunk = try #require(await iterator.next()) + #expect(String(decoding: freshChunk.data, as: UTF8.self) == "fresh-after-reset") + #expect(freshChunk.streamToken != stalledChunk.streamToken) + + store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: stalledChunk.streamToken) + store.deliverTerminalBytes(Data("after-stale-ack".utf8), surfaceID: surfaceID) + store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: freshChunk.streamToken) + + let afterStaleAck = try #require(await iterator.next()) + #expect(String(decoding: afterStaleAck.data, as: UTF8.self) == "after-stale-ack") +} + @Test func terminalOutputQueueCoalescesReplaceableViewportFramesBehindBackpressure() { var queue = TerminalOutputDeliveryQueue() let inFlight = TerminalOutputDelivery(bytes: Data("in-flight".utf8), replaceable: false) From c38219973c5e05a4f76ffe7539f74b761fa00474 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 02:02:17 -0700 Subject: [PATCH 02/56] Recover iOS terminal render pipeline stalls --- ...hellComposite+TerminalOutputDelivery.swift | 29 ++ .../MobileShellComposite.swift | 2 +- .../MobileTerminalOutputSinking.swift | 12 + .../GhosttySurfaceRepresentable.swift | 33 ++- .../GhosttySurfaceRegistry.swift | 5 +- .../GhosttySurfaceView.swift | 257 ++++++++++++++---- .../GhosttySurfaceWorkQueue.swift | 23 ++ .../SurfaceOperationWaiter.swift | 16 ++ 8 files changed, 319 insertions(+), 58 deletions(-) create mode 100644 Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceWorkQueue.swift create mode 100644 Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/SurfaceOperationWaiter.swift diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift index 95004923b6e7..62837fed1481 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift @@ -1,4 +1,5 @@ import CMUXMobileCore +internal import CmuxMobileDiagnostics import CmuxMobileShellModel public import Foundation @@ -43,7 +44,13 @@ extension MobileShellComposite { let streamToken = terminalOutputStreamTokensBySurfaceID[surfaceID] else { return } var queue = terminalOutputQueuesBySurfaceID[surfaceID] ?? TerminalOutputDeliveryQueue() let immediate = queue.enqueue(delivery) + let pendingCount = queue.pendingCount terminalOutputQueuesBySurfaceID[surfaceID] = queue + if pendingCount >= 32, pendingCount.isMultiple(of: 32) { + MobileDebugLog.anchormux( + "terminal.output.pending surface=\(surfaceID) depth=\(pendingCount)" + ) + } if let immediate { continuation.yield( MobileTerminalOutputChunk( @@ -72,4 +79,26 @@ extension MobileShellComposite { viewportPolicy: next.viewportPolicy )) } + + /// Abandon the current yielded terminal-output chunk after the local render + /// surface reset. The abandoned bytes may have been applied to the old + /// Ghostty surface or may still be behind a wedged worker queue, so continuing + /// to drain the old pending queue would replay stale deltas into the rebuilt + /// surface. Reset the queue, invalidate stale acks, then request a fresh + /// authoritative replay from the Mac. + public func terminalOutputDidReset(surfaceID: String, streamToken: UUID) { + guard terminalOutputStreamTokensBySurfaceID[surfaceID] == streamToken, + terminalOutputQueuesBySurfaceID[surfaceID] != nil else { return } + terminalOutputQueuesBySurfaceID[surfaceID] = TerminalOutputDeliveryQueue() + terminalOutputStreamTokensBySurfaceID[surfaceID] = UUID() + MobileDebugLog.anchormux("terminal.output.reset surface=\(surfaceID)") + requestTerminalReplay(surfaceID: surfaceID) + } + + /// Ask the Mac to replay the authoritative terminal state for a surface. + public func terminalOutputNeedsReplay(surfaceID: String) { + guard terminalByteContinuationsBySurfaceID[surfaceID] != nil else { return } + MobileDebugLog.anchormux("terminal.output.replay_requested surface=\(surfaceID)") + requestTerminalReplay(surfaceID: surfaceID) + } } diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift index 3e476c13a0ed..aaaee7a87214 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift @@ -6510,7 +6510,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { /// resume. The VT snapshot and raw byte ring remain fallbacks, but neither /// is the target architecture: a byte tail is not a complete screen state /// for TUIs, and a VT export is still a replay stream rather than state. - private func requestTerminalReplay(surfaceID: String) { + func requestTerminalReplay(surfaceID: String) { guard let client = remoteClient else { #if DEBUG mobileShellLog.error("CMUX_REPLAY skip surface=\(surfaceID, privacy: .public) reason=no_remote_client") diff --git a/Packages/iOS/CmuxMobileShellModel/Sources/CmuxMobileShellModel/MobileTerminalOutputSinking.swift b/Packages/iOS/CmuxMobileShellModel/Sources/CmuxMobileShellModel/MobileTerminalOutputSinking.swift index baeec82e0e1c..7b2bf990f9f4 100644 --- a/Packages/iOS/CmuxMobileShellModel/Sources/CmuxMobileShellModel/MobileTerminalOutputSinking.swift +++ b/Packages/iOS/CmuxMobileShellModel/Sources/CmuxMobileShellModel/MobileTerminalOutputSinking.swift @@ -49,4 +49,16 @@ public protocol MobileTerminalOutputSinking: Sendable { /// - Parameter surfaceID: The terminal surface identifier. /// - Parameter streamToken: The token carried by the yielded chunk. @MainActor func terminalOutputDidProcess(surfaceID: String, streamToken: UUID) + + /// Abandon the current yielded chunk after the local renderer was reset. + /// + /// The sink must drop stale pending output, invalidate the old stream token, + /// and request an authoritative replay for the same surface. + /// - Parameter surfaceID: The terminal surface identifier. + /// - Parameter streamToken: The token carried by the abandoned chunk. + @MainActor func terminalOutputDidReset(surfaceID: String, streamToken: UUID) + + /// Request an authoritative replay without an abandoned in-flight chunk. + /// - Parameter surfaceID: The terminal surface identifier. + @MainActor func terminalOutputNeedsReplay(surfaceID: String) } diff --git a/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/GhosttySurfaceRepresentable.swift b/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/GhosttySurfaceRepresentable.swift index 8163f4540a0d..fd7586fd8b77 100644 --- a/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/GhosttySurfaceRepresentable.swift +++ b/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/GhosttySurfaceRepresentable.swift @@ -145,20 +145,41 @@ struct GhosttySurfaceRepresentable: UIViewRepresentable { if chunk.data.isEmpty { surfaceView.useNaturalViewSize() } else { - await surfaceView.useNaturalViewSizeAndWait() + let applied = await surfaceView.useNaturalViewSizeAndWait() + guard applied else { + store.terminalOutputDidReset( + surfaceID: surfaceID, + streamToken: chunk.streamToken + ) + continue + } } case .remoteGrid(let columns, let rows): self.activeViewportPolicy = .remoteGrid(columns: columns, rows: rows) if chunk.data.isEmpty { surfaceView.applyViewSize(cols: columns, rows: rows) } else { - await surfaceView.applyViewSizeAndWait(cols: columns, rows: rows) + let applied = await surfaceView.applyViewSizeAndWait(cols: columns, rows: rows) + guard applied else { + store.terminalOutputDidReset( + surfaceID: surfaceID, + streamToken: chunk.streamToken + ) + continue + } } case nil: break } if !chunk.data.isEmpty { - await surfaceView.processOutputAndWait(chunk.data) + let applied = await surfaceView.processOutputAndWait(chunk.data) + guard applied else { + store.terminalOutputDidReset( + surfaceID: surfaceID, + streamToken: chunk.streamToken + ) + continue + } } store.terminalOutputDidProcess( surfaceID: surfaceID, @@ -402,6 +423,12 @@ struct GhosttySurfaceRepresentable: UIViewRepresentable { } } + func ghosttySurfaceViewDidResetRenderPipeline(_ surfaceView: GhosttySurfaceView) { + Task { @MainActor [weak store, surfaceID] in + store?.terminalOutputNeedsReplay(surfaceID: surfaceID) + } + } + /// Walk up from `view` to the nearest owning `UIViewController`, then to /// its top-most presented controller, so a sheet presents above whatever /// is already on screen. diff --git a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceRegistry.swift b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceRegistry.swift index 2adb524e703a..36e51bc9c0b5 100644 --- a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceRegistry.swift +++ b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceRegistry.swift @@ -94,10 +94,11 @@ extension GhosttySurfaceView { && candidate.window != nil && !candidate.isHidden && candidate.alpha > 0.01 } - guard let surface = matchingView?.surface else { return nil } + guard let matchingView, + let surface = matchingView.surface else { return nil } let handle = CopyableTextSurfaceHandle(surface: surface) return await withCheckedContinuation { continuation in - outputQueue.async { + matchingView.outputQueue.async { // SCREEN = scrollback + all written rows. Fall back to the // viewport-only read if the screen read fails outright. let text = surfaceText(handle.surface, pointTag: GHOSTTY_POINT_SCREEN) diff --git a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift index 2d285bf5b12f..9251c1028f13 100644 --- a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift +++ b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift @@ -75,6 +75,9 @@ public protocol GhosttySurfaceViewDelegate: AnyObject { /// reveal-after-hide and the present-while-suppressed paths so the draft and its /// focus return together. Optional. func ghosttySurfaceViewDidRequestComposerFocus(_ surfaceView: GhosttySurfaceView) + /// The local Ghostty render pipeline was rebuilt after a stuck render/output + /// operation. The host should replay authoritative terminal state. + func ghosttySurfaceViewDidResetRenderPipeline(_ surfaceView: GhosttySurfaceView) } public extension GhosttySurfaceViewDelegate { @@ -86,6 +89,8 @@ public extension GhosttySurfaceViewDelegate { func ghosttySurfaceViewDidRequestComposerToggle(_ surfaceView: GhosttySurfaceView) {} /// Default no-op so hosts without a composer can ignore the focus request. func ghosttySurfaceViewDidRequestComposerFocus(_ surfaceView: GhosttySurfaceView) {} + /// Default no-op so hosts without terminal-output replay can ignore renderer resets. + func ghosttySurfaceViewDidResetRenderPipeline(_ surfaceView: GhosttySurfaceView) {} } @MainActor @@ -579,7 +584,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { /// reset/save/restore actions. Owned by the surface (constructed at init) /// rather than reached through a singleton, so it is injectable in tests. private let zoomPreference = MobileTerminalZoomPreference() - private let bridge = GhosttySurfaceBridge() + private var bridge = GhosttySurfaceBridge() private let prefersSnapshotFallbackRendering = false var onFocusInputRequestedForTesting: (() -> Void)? private var surfaceTitle: String? @@ -607,6 +612,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { /// symptom. Coalescing caps the backlog: while a render is in flight, mark /// `needsAnotherRender` and re-enqueue exactly one when it completes. private var renderInFlight: Bool = false + private var renderInFlightSince: CFTimeInterval? private var needsAnotherRender: Bool = false /// True while the app is inactive/backgrounded. On iOS `render_now` /// produces a frame synchronously on `outputQueue` and acquires a @@ -647,10 +653,11 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { /// Internal (not private) so the copyable-text extension in /// `GhosttySurfaceCopyableText.swift` can enqueue its surface read with /// the same FIFO-before-dispose ordering discipline. - static let outputQueue = DispatchQueue( - label: "dev.cmux.GhosttySurfaceView.output", - qos: .userInitiated - ) + var outputQueue = GhosttySurfaceWorkQueue(generation: 0) + private var outputQueueGeneration: UInt64 = 0 + private var isRecoveringRenderPipeline = false + private static let renderPipelineStallDeadline: CFTimeInterval = 2.0 + private static let outputApplyTimeoutNanoseconds: UInt64 = 2_000_000_000 private static let scrollMechanicsContentHeight: CGFloat = 1_000_000 private var scrollMechanicsIsRecentering = false private var lastScrollMechanicsOffsetY: CGFloat? @@ -788,6 +795,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { }() private(set) var surface: ghostty_surface_t? + private var surfaceGeneration: UInt64 = 0 private var lastReportedSize: TerminalGridSize? /// Latest natural grid awaiting a debounced report to the Mac. The display /// link sends it only after the grid has held steady for @@ -1112,6 +1120,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { private func resumeRendering() { renderingSuspended = false renderInFlight = false + renderInFlightSince = nil needsAnotherRender = false guard let surface, window != nil else { return } ghostty_surface_set_occlusion(surface, true) // true = visible @@ -2058,7 +2067,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { // An absolute `set_font_size:` keeps libghostty in lockstep // with `liveFontSize`, which we keep inside [minimumSize, maximumSize]. let action = "set_font_size:\(target)" - Self.outputQueue.async { + outputQueue.async { action.withCString { pointer in _ = ghostty_surface_binding_action(surface, pointer, UInt(action.utf8.count)) } @@ -2258,20 +2267,47 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { /// per-surface backpressure without blocking the main actor while Ghostty /// consumes the chunk. /// - Parameter data: VT or PTY bytes to feed into the surface. - public func processOutputAndWait(_ data: Data) async { + /// - Returns: `true` when the bytes reached the current surface generation, + /// or `false` when the caller should reset its delivery queue and replay. + @discardableResult + public func processOutputAndWait(_ data: Data) async -> Bool { await withCheckedContinuation { continuation in - processOutput(data) { - continuation.resume() + let waiter = SurfaceOperationWaiter() + let startedAt = CACurrentMediaTime() + let timeoutTask = Task { @MainActor [weak self] in + do { + // Genuine apply deadline: if libghostty or the worker queue is + // wedged, unblock the shell stream and rebuild below. + try await Task.sleep(nanoseconds: Self.outputApplyTimeoutNanoseconds) + } catch { + return + } + guard let self else { + _ = waiter.resume(returning: false, continuation: continuation) + return + } + let elapsedMs = Int((CACurrentMediaTime() - startedAt) * 1000) + if waiter.resume(returning: false, continuation: continuation) { + MobileDebugLog.anchormux( + "output.apply.TIMEOUT bytes=\(data.count) elapsedMs=\(elapsedMs)" + ) + self.recoverRenderPipeline(reason: "output_timeout", stalledMs: elapsedMs) + } + } + processOutput(data) { applied in + if waiter.resume(returning: applied, continuation: continuation) { + timeoutTask.cancel() + } } } } private func processOutput( _ data: Data, - completion: (@MainActor @Sendable () -> Void)? + completion: (@MainActor @Sendable (Bool) -> Void)? ) { guard let surface, !isDismantled else { - completion?() + completion?(true) return } #if DEBUG @@ -2289,6 +2325,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { } #endif let forwarded = Self.forwardDaemonOutputBytes(data) + let generation = surfaceGeneration // Track the host's cursor-visible mode (DECTCEM) straight from the VT // bytes the surface is about to apply, so the cursor overlay can match a // TUI that hides the cursor. nil = this delta carried no DECTCEM, so the @@ -2301,7 +2338,8 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { // scene-update watchdog (0x8BADF00D) kills the app. It must run off // the main thread. Feed it on a serial background queue (order // preserved) and hop back to main only for the Swift-side UI state. - Self.outputQueue.async { [weak self] in + let workQueue = outputQueue + workQueue.async { [weak self] in forwarded.withUnsafeBytes { buffer in guard let baseAddress = buffer.baseAddress else { return } let pointer = baseAddress.assumingMemoryBound(to: CChar.self) @@ -2319,14 +2357,18 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { // main. Off-main reads can never trip the main-thread watchdog. var accessibilityText: String? let a11yNow = CACurrentMediaTime() - if a11yNow - Self.lastAccessibilityTextTime > 0.5 { - Self.lastAccessibilityTextTime = a11yNow + if a11yNow - workQueue.lastAccessibilityTextTime > 0.5 { + workQueue.lastAccessibilityTextTime = a11yNow accessibilityText = Self.accessibilitySurfaceText(surface) } #endif DispatchQueue.main.async { guard let self, !self.isDismantled else { - completion?() + completion?(true) + return + } + guard self.surfaceGeneration == generation else { + completion?(false) return } self.needsDraw = true @@ -2355,7 +2397,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { } self.onOutputProcessedForTesting?() #endif - completion?() + completion?(true) } } } @@ -2373,7 +2415,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { // `process_output` also preserves ordering. The return was already // discarded. let action = "scroll_to_bottom" - Self.outputQueue.async { + outputQueue.async { action.withCString { pointer in _ = ghostty_surface_binding_action(surface, pointer, UInt(action.utf8.count)) } @@ -2517,11 +2559,6 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { } } - /// Throttle stamp for the off-main accessibility-label read in - /// `processOutput`. Accessed only on the serial `outputQueue`, so the - /// unchecked mutation is safe. - nonisolated(unsafe) fileprivate static var lastAccessibilityTextTime: CFTimeInterval = 0 - /// Off-main equivalent of ``accessibilityRenderedTextForTesting()`` that /// reads via the raw surface handle so it can run on the serial output queue /// (alongside `process_output`) instead of the main thread. See the call @@ -2571,7 +2608,9 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { guard let surface else { return } GhosttySurfaceView.unregister(surface: surface) self.surface = nil - bridge.detach() + let currentBridge = bridge + let currentQueue = outputQueue + currentBridge.detach() // Free on the SAME serial `outputQueue` that runs `process_output`, // `render_now`, and `binding_action` (all of which capture this C // surface pointer), not a separate queue. FIFO ordering guarantees the @@ -2582,13 +2621,65 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { // work from being enqueued once `surface` is nil, so only the bounded // backlog drains before the free. (Retain the bridge across the hop; it // owns the userdata libghostty still references until the free.) + enqueueSurfaceFree(surface, bridge: currentBridge, on: currentQueue) + } + + private func enqueueSurfaceFree( + _ surface: ghostty_surface_t, + bridge: GhosttySurfaceBridge, + on queue: GhosttySurfaceWorkQueue + ) { let retainedBridge = Unmanaged.passRetained(bridge) - Self.outputQueue.async { + queue.async { ghostty_surface_free(surface) retainedBridge.release() } } + @discardableResult + private func recoverRenderPipeline(reason: String, stalledMs: Int) -> Bool { + guard !isRecoveringRenderPipeline, + !isDismantled, + surface != nil else { + return false + } + isRecoveringRenderPipeline = true + defer { isRecoveringRenderPipeline = false } + + MobileDebugLog.anchormux( + "render.recover reason=\(reason) stalledMs=\(stalledMs) generation=\(surfaceGeneration)" + ) + + stopDisplayLink() + let oldSurface = surface + let oldBridge = bridge + let oldQueue = outputQueue + if let oldSurface { + GhosttySurfaceView.unregister(surface: oldSurface) + enqueueSurfaceFree(oldSurface, bridge: oldBridge, on: oldQueue) + } + oldBridge.detach() + + surface = nil + renderInFlight = false + renderInFlightSince = nil + needsAnotherRender = false + needsDraw = true + cellPixelSize = .zero + lastRenderRect = .zero + lastAppliedContentScale = 0 + + surfaceGeneration &+= 1 + outputQueueGeneration &+= 1 + outputQueue = GhosttySurfaceWorkQueue(generation: outputQueueGeneration) + bridge = GhosttySurfaceBridge() + bridge.attach(to: self) + + initializeSurface() + delegate?.ghosttySurfaceViewDidResetRenderPipeline(self) + return true + } + private var preferredScreenScale: CGFloat { if let screen = window?.windowScene?.screen { return screen.scale @@ -2687,6 +2778,13 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { ) } #endif + if let renderInFlightSince { + let stalledMs = Int((CACurrentMediaTime() - renderInFlightSince) * 1000) + if stalledMs >= Int(Self.renderPipelineStallDeadline * 1000), + recoverRenderPipeline(reason: "render_in_flight", stalledMs: stalledMs) { + return + } + } // Apply at most one coalesced zoom per frame. This only changes the // font; the geometry resync is deferred until zoom settles. let appliedZoom = applyPendingFontSizeIfNeeded() @@ -2803,8 +2901,10 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { return } renderInFlight = true + renderInFlightSince = CACurrentMediaTime() + let generation = surfaceGeneration let enqueuedAt = CACurrentMediaTime() - Self.outputQueue.async { [weak self] in + outputQueue.async { [weak self] in // Queue LAG = how long this render waited behind other ops. If this // climbs into hundreds of ms the queue is backlogged (the freeze). let lagMs = (CACurrentMediaTime() - enqueuedAt) * 1000 @@ -2812,7 +2912,9 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { ghostty_surface_render_now(surface) DispatchQueue.main.async { guard let self else { return } + guard self.surfaceGeneration == generation else { return } self.renderInFlight = false + self.renderInFlightSince = nil guard !self.isDismantled else { self.needsAnotherRender = false return @@ -2978,11 +3080,18 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { applyViewSize(cols: cols, rows: rows, confirmedViewportEcho: false) } - public func applyViewSizeAndWait(cols: Int, rows: Int) async { + /// Apply the daemon's authoritative rendering grid and wait until libghostty + /// accepts the geometry for the current surface generation. + /// - Parameter cols: The authoritative terminal column count. + /// - Parameter rows: The authoritative terminal row count. + /// - Returns: `false` when the surface reset before the geometry applied. + @discardableResult + public func applyViewSizeAndWait(cols: Int, rows: Int) async -> Bool { let changed = updateEffectiveGrid(cols: cols, rows: rows, confirmedViewportEcho: false) if changed || needsGeometrySync { - await syncSurfaceGeometryAndWait(shouldReassertNaturalSize: false) + return await syncSurfaceGeometryAndWait(shouldReassertNaturalSize: false) } + return true } public func applyConfirmedViewSize(cols: Int, rows: Int) { @@ -3020,11 +3129,16 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { setNeedsGeometrySync(reassertNaturalSize: false) } - public func useNaturalViewSizeAndWait() async { + /// Return to the phone's natural viewport capacity and wait until libghostty + /// accepts the geometry for the current surface generation. + /// - Returns: `false` when the surface reset before the geometry applied. + @discardableResult + public func useNaturalViewSizeAndWait() async -> Bool { let changed = clearEffectiveGrid() if changed || needsGeometrySync { - await syncSurfaceGeometryAndWait(shouldReassertNaturalSize: false) + return await syncSurfaceGeometryAndWait(shouldReassertNaturalSize: false) } + return true } private func clearEffectiveGrid() -> Bool { @@ -3081,22 +3195,44 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { let pinnedSize: CGSize? } - private func syncSurfaceGeometryAndWait(shouldReassertNaturalSize: Bool = true) async { + private func syncSurfaceGeometryAndWait(shouldReassertNaturalSize: Bool = true) async -> Bool { needsGeometrySync = false pendingGeometryReassert = false await withCheckedContinuation { continuation in - syncSurfaceGeometry(shouldReassertNaturalSize: shouldReassertNaturalSize) { - continuation.resume() + let waiter = SurfaceOperationWaiter() + let startedAt = CACurrentMediaTime() + let timeoutTask = Task { @MainActor [weak self] in + do { + // Genuine geometry deadline: the shell stream must not wait + // forever on a libghostty worker queue that stopped draining. + try await Task.sleep(nanoseconds: Self.outputApplyTimeoutNanoseconds) + } catch { + return + } + guard let self else { + _ = waiter.resume(returning: false, continuation: continuation) + return + } + let elapsedMs = Int((CACurrentMediaTime() - startedAt) * 1000) + if waiter.resume(returning: false, continuation: continuation) { + MobileDebugLog.anchormux("geometry.apply.TIMEOUT elapsedMs=\(elapsedMs)") + self.recoverRenderPipeline(reason: "geometry_timeout", stalledMs: elapsedMs) + } + } + syncSurfaceGeometry(shouldReassertNaturalSize: shouldReassertNaturalSize) { applied in + if waiter.resume(returning: applied, continuation: continuation) { + timeoutTask.cancel() + } } } } private func syncSurfaceGeometry( shouldReassertNaturalSize: Bool = true, - completion: (@MainActor @Sendable () -> Void)? = nil + completion: (@MainActor @Sendable (Bool) -> Void)? = nil ) { guard let surface else { - completion?() + completion?(true) return } @@ -3144,8 +3280,9 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { let eff = effectiveGrid let pushContentScale = abs(lastAppliedContentScale - scale) > 0.001 if pushContentScale { lastAppliedContentScale = scale } + let generation = surfaceGeneration - Self.outputQueue.async { [weak self] in + outputQueue.async { [weak self] in if pushContentScale { ghostty_surface_set_content_scale(surface, scale, scale) } @@ -3185,14 +3322,22 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { ) let result = GeometryResult(cellPixelSize: cell, naturalSize: natural, pinnedSize: pinnedSize) DispatchQueue.main.async { - self?.applyGeometryResult( + guard let self else { + completion?(true) + return + } + guard self.surfaceGeneration == generation else { + completion?(false) + return + } + self.applyGeometryResult( result, scale: scale, containerW: containerW, containerH: containerH, shouldReassertNaturalSize: shouldReassertNaturalSize ) - completion?() + completion?(true) } } } @@ -3378,7 +3523,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { ios: ghostty_platform_ios_s(uiview: Unmanaged.passUnretained(self).toOpaque()) ) surfaceConfig.scale_factor = preferredScreenScale - surfaceConfig.font_size = fontSize + surfaceConfig.font_size = liveFontSize surfaceConfig.context = GHOSTTY_SURFACE_CONTEXT_WINDOW surfaceConfig.io_mode = GHOSTTY_SURFACE_IO_MANUAL surfaceConfig.io_write_cb = { userdata, buf, len in @@ -3615,7 +3760,12 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { guard view.window != nil, !view.isHidden, view.alpha > 0.01, let surface = view.surface else { continue } let grid = view.effectiveGrid.map { "\($0.cols)x\($0.rows)" } ?? "?" - pending.append(VisibleSnapshotRequest(grid: grid, font: Int(view.liveFontSize), surface: surface)) + pending.append(VisibleSnapshotRequest( + grid: grid, + font: Int(view.liveFontSize), + surface: surface, + queue: view.outputQueue + )) } if pending.isEmpty { return "===== visible terminal: (no on-screen surface) =====" @@ -3624,30 +3774,32 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { // queue stuck mid-`process_output`, a plain `.sync` here would freeze the // whole app exactly when the user taps Copy Debug Logs to capture that // bug. Time out and ship the logs without the snapshot instead. - let holder = VisibleSnapshotHolder() // This synchronous DEV-only "Copy Debug Logs" path reads the viewport off // the serial output queue and must give up after a deadline if a render // wedge holds it; an actor/await cannot express the bounded synchronous // wait the synchronous caller needs. // carve-out justification: one-shot cross-queue completion signal with a // bounded wait, not a lock guarding shared state. - let done = DispatchSemaphore(value: 0) - outputQueue.async { - var built: [String] = [] - for item in pending { + let deadline = DispatchTime.now() + 0.6 + var sections: [String] = [] + for item in pending { + let holder = VisibleSnapshotHolder() + // carve-out justification: one-shot cross-queue completion signal with a bounded wait. + let done = DispatchSemaphore(value: 0) + item.queue.async { let text = surfaceText(item.surface, pointTag: GHOSTTY_POINT_VIEWPORT) ?? "(unavailable)" - built.append( + holder.sections = [ "===== visible terminal · grid=\(item.grid) · font=\(item.font) =====\n" + text - ) + ] + done.signal() } - holder.sections = built - done.signal() - } - if done.wait(timeout: .now() + 0.6) == .timedOut { - return "===== visible terminal: (snapshot skipped — render busy) =====" + if done.wait(timeout: deadline) == .timedOut { + return "===== visible terminal: (snapshot skipped — render busy) =====" + } + sections.append(contentsOf: holder.sections) } - return holder.sections.joined(separator: "\n\n") + return sections.joined(separator: "\n\n") } private func handleBell() { @@ -3720,6 +3872,7 @@ private struct VisibleSnapshotRequest: @unchecked Sendable { let grid: String let font: Int let surface: ghostty_surface_t + let queue: GhosttySurfaceWorkQueue } /// Carrier for the snapshot text produced off `GhosttySurfaceView.outputQueue`. diff --git a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceWorkQueue.swift b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceWorkQueue.swift new file mode 100644 index 000000000000..999dec0ba9c0 --- /dev/null +++ b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceWorkQueue.swift @@ -0,0 +1,23 @@ +import Foundation + +/// Owns the serial libghostty work queue for one surface generation. +/// All mutable state is accessed only from `queue`; main-actor code replaces whole instances on recovery. +final class GhosttySurfaceWorkQueue: @unchecked Sendable { + let queue: DispatchQueue + #if DEBUG + /// Accessed only from ``queue`` while producing DEBUG accessibility snapshots. + var lastAccessibilityTextTime: CFTimeInterval = 0 + #endif + + init(generation: UInt64) { + // carve-out justification: serial event-delivery queue for low-level libghostty C calls; not used as a lock. + queue = DispatchQueue( + label: "dev.cmux.GhosttySurfaceView.output.\(generation)", + qos: .userInitiated + ) + } + + func async(_ work: @escaping @Sendable () -> Void) { + queue.async(execute: work) + } +} diff --git a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/SurfaceOperationWaiter.swift b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/SurfaceOperationWaiter.swift new file mode 100644 index 000000000000..1a5cc15ccd06 --- /dev/null +++ b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/SurfaceOperationWaiter.swift @@ -0,0 +1,16 @@ +import Foundation + +@MainActor +final class SurfaceOperationWaiter { + private var didResume = false + + func resume( + returning result: Bool, + continuation: CheckedContinuation + ) -> Bool { + guard !didResume else { return false } + didResume = true + continuation.resume(returning: result) + return true + } +} From b53da2b14fe0ddcfe81734da630a2de8eefbfd13 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 02:07:07 -0700 Subject: [PATCH 03/56] Update Swift file length budget --- .github/swift-file-length-budget.tsv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/swift-file-length-budget.tsv b/.github/swift-file-length-budget.tsv index 12299781ba6e..99fafbdc597a 100644 --- a/.github/swift-file-length-budget.tsv +++ b/.github/swift-file-length-budget.tsv @@ -3,7 +3,7 @@ # Reduce counts as files shrink. CI fails if tracked files exceed this budget. 34427 CLI/cmux.swift 17978 Sources/AppDelegate.swift -16521 Sources/ContentView.swift +16500 Sources/ContentView.swift 14224 Sources/TerminalController.swift 12913 Sources/Workspace.swift 12348 cmuxTests/AppDelegateShortcutRoutingTests.swift @@ -27,8 +27,8 @@ 4121 Sources/BrowserWindowPortal.swift 3934 Sources/Feed/FeedPanelView.swift 3926 cmuxTests/TabManagerUnitTests.swift +3900 Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift 3896 cmuxTests/WindowAndDragTests.swift -3747 Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift 3668 cmuxTests/CLIGenericHookPersistenceTests.swift 3397 Sources/CmuxConfig.swift 3364 cmuxTests/TabManagerSessionSnapshotTests.swift From 5cab06632daaef7cafb3179f42e22691b2481337 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 02:15:56 -0700 Subject: [PATCH 04/56] Address terminal replay barrier review --- .github/swift-file-length-budget.tsv | 4 +- ...hellComposite+TerminalOutputDelivery.swift | 47 +++++++++++++---- .../MobileShellComposite.swift | 51 +++++++++++++++++-- .../TerminalOutputDeliveryQueueTests.swift | 31 ++++++++--- .../GhosttySurfaceView.swift | 26 +++++++++- 5 files changed, 136 insertions(+), 23 deletions(-) diff --git a/.github/swift-file-length-budget.tsv b/.github/swift-file-length-budget.tsv index 99fafbdc597a..69476c8249c7 100644 --- a/.github/swift-file-length-budget.tsv +++ b/.github/swift-file-length-budget.tsv @@ -14,7 +14,7 @@ 8016 CLI/cmux_open.swift 7366 cmuxTests/WorkspaceUnitTests.swift 7218 cmuxTests/WorkspaceRemoteConnectionTests.swift -7022 Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift +7065 Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift 6359 cmuxTests/SessionPersistenceTests.swift 6217 cmuxTests/GhosttyConfigTests.swift 6180 Sources/TabManager.swift @@ -27,7 +27,7 @@ 4121 Sources/BrowserWindowPortal.swift 3934 Sources/Feed/FeedPanelView.swift 3926 cmuxTests/TabManagerUnitTests.swift -3900 Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift +3924 Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift 3896 cmuxTests/WindowAndDragTests.swift 3668 cmuxTests/CLIGenericHookPersistenceTests.swift 3397 Sources/CmuxConfig.swift diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift index 62837fed1481..d5ec68b041e9 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift @@ -5,25 +5,35 @@ public import Foundation extension MobileShellComposite { /// Yield a raw PTY byte chunk to the surface stream, if one is attached. - func deliverTerminalBytes(_ bytes: Data, surfaceID: String) { + func deliverTerminalBytes( + _ bytes: Data, + surfaceID: String, + bypassReplayBarrier: Bool = false + ) { deliverTerminalOutput( TerminalOutputDelivery( bytes: bytes, replaceable: false, viewportPolicy: .natural ), - surfaceID: surfaceID + surfaceID: surfaceID, + bypassReplayBarrier: bypassReplayBarrier ) } - func deliverTerminalRenderGrid(_ frame: MobileTerminalRenderGridFrame, surfaceID: String) { + func deliverTerminalRenderGrid( + _ frame: MobileTerminalRenderGridFrame, + surfaceID: String, + bypassReplayBarrier: Bool = false + ) { deliverTerminalOutput( TerminalOutputDelivery( renderGrid: frame, replaceable: frame.isReplaceableViewportPatchForMobileDelivery, viewportPolicy: frame.mobileViewportPolicy ), - surfaceID: surfaceID + surfaceID: surfaceID, + bypassReplayBarrier: bypassReplayBarrier ) } @@ -39,13 +49,26 @@ extension MobileShellComposite { ) } - private func deliverTerminalOutput(_ delivery: TerminalOutputDelivery, surfaceID: String) { + private func deliverTerminalOutput( + _ delivery: TerminalOutputDelivery, + surfaceID: String, + bypassReplayBarrier: Bool = false + ) { guard let continuation = terminalByteContinuationsBySurfaceID[surfaceID], let streamToken = terminalOutputStreamTokensBySurfaceID[surfaceID] else { return } + if terminalReplayBarrierTokensBySurfaceID[surfaceID] != nil, !bypassReplayBarrier { + MobileDebugLog.anchormux("terminal.output.drop_replay_barrier surface=\(surfaceID)") + return + } var queue = terminalOutputQueuesBySurfaceID[surfaceID] ?? TerminalOutputDeliveryQueue() let immediate = queue.enqueue(delivery) let pendingCount = queue.pendingCount terminalOutputQueuesBySurfaceID[surfaceID] = queue + if bypassReplayBarrier, + immediate != nil, + terminalReplayBarrierTokensBySurfaceID[surfaceID] != nil { + terminalReplayBarrierAckStreamTokensBySurfaceID[surfaceID] = streamToken + } if pendingCount >= 32, pendingCount.isMultiple(of: 32) { MobileDebugLog.anchormux( "terminal.output.pending surface=\(surfaceID) depth=\(pendingCount)" @@ -68,6 +91,11 @@ extension MobileShellComposite { var queue = terminalOutputQueuesBySurfaceID[surfaceID] else { return } let next = queue.completeInFlight() terminalOutputQueuesBySurfaceID[surfaceID] = queue + if terminalReplayBarrierAckStreamTokensBySurfaceID[surfaceID] == streamToken { + terminalReplayBarrierAckStreamTokensBySurfaceID.removeValue(forKey: surfaceID) + terminalReplayBarrierTokensBySurfaceID.removeValue(forKey: surfaceID) + MobileDebugLog.anchormux("terminal.output.replay_barrier_cleared surface=\(surfaceID)") + } guard let next, let continuation = terminalByteContinuationsBySurfaceID[surfaceID], terminalOutputStreamTokensBySurfaceID[surfaceID] == streamToken else { @@ -89,16 +117,17 @@ extension MobileShellComposite { public func terminalOutputDidReset(surfaceID: String, streamToken: UUID) { guard terminalOutputStreamTokensBySurfaceID[surfaceID] == streamToken, terminalOutputQueuesBySurfaceID[surfaceID] != nil else { return } - terminalOutputQueuesBySurfaceID[surfaceID] = TerminalOutputDeliveryQueue() - terminalOutputStreamTokensBySurfaceID[surfaceID] = UUID() + let replayBarrierToken = beginTerminalReplayBarrier(surfaceID: surfaceID) MobileDebugLog.anchormux("terminal.output.reset surface=\(surfaceID)") - requestTerminalReplay(surfaceID: surfaceID) + requestTerminalReplay(surfaceID: surfaceID, replayBarrierToken: replayBarrierToken) } /// Ask the Mac to replay the authoritative terminal state for a surface. public func terminalOutputNeedsReplay(surfaceID: String) { guard terminalByteContinuationsBySurfaceID[surfaceID] != nil else { return } + let replayBarrierToken = beginTerminalReplayBarrier(surfaceID: surfaceID) MobileDebugLog.anchormux("terminal.output.replay_requested surface=\(surfaceID)") - requestTerminalReplay(surfaceID: surfaceID) + requestTerminalReplay(surfaceID: surfaceID, replayBarrierToken: replayBarrierToken) } + } diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift index aaaee7a87214..017ef8ffe7c2 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift @@ -715,6 +715,8 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { private var pendingTerminalByteEndSeqBySurfaceID: [String: UInt64] private var terminalActiveScreenBySurfaceID: [String: MobileTerminalRenderGridFrame.Screen] private var terminalReplaySurfaceIDsInFlight: Set + var terminalReplayBarrierTokensBySurfaceID: [String: UUID] + var terminalReplayBarrierAckStreamTokensBySurfaceID: [String: UUID] private var terminalOutputTransport: TerminalOutputTransport var terminalByteContinuationsBySurfaceID: [String: AsyncStream.Continuation] var terminalOutputStreamTokensBySurfaceID: [String: UUID] @@ -906,6 +908,8 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { self.pendingTerminalByteEndSeqBySurfaceID = [:] self.terminalActiveScreenBySurfaceID = [:] self.terminalReplaySurfaceIDsInFlight = [] + self.terminalReplayBarrierTokensBySurfaceID = [:] + self.terminalReplayBarrierAckStreamTokensBySurfaceID = [:] self.terminalOutputTransport = .rawBytes self.terminalByteContinuationsBySurfaceID = [:] self.terminalOutputStreamTokensBySurfaceID = [:] @@ -4933,6 +4937,8 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { pendingTerminalByteEndSeqBySurfaceID = [:] terminalActiveScreenBySurfaceID = [:] terminalReplaySurfaceIDsInFlight = [] + terminalReplayBarrierTokensBySurfaceID = [:] + terminalReplayBarrierAckStreamTokensBySurfaceID = [:] terminalOutputQueuesBySurfaceID = [:] terminalOutputStreamTokensBySurfaceID = terminalOutputStreamTokensBySurfaceID.mapValues { _ in UUID() } terminalScrollQueueTokensBySurfaceID = [:] @@ -6292,6 +6298,18 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { } } + func beginTerminalReplayBarrier(surfaceID: String) -> UUID { + terminalOutputQueuesBySurfaceID[surfaceID] = TerminalOutputDeliveryQueue() + terminalOutputStreamTokensBySurfaceID[surfaceID] = UUID() + deliveredTerminalByteEndSeqBySurfaceID.removeValue(forKey: surfaceID) + pendingTerminalByteEndSeqBySurfaceID.removeValue(forKey: surfaceID) + terminalActiveScreenBySurfaceID.removeValue(forKey: surfaceID) + let token = UUID() + terminalReplayBarrierTokensBySurfaceID[surfaceID] = token + terminalReplayBarrierAckStreamTokensBySurfaceID.removeValue(forKey: surfaceID) + return token + } + private enum RenderGridEventDeliveryDecision { case deliver case advisory(requestReplay: Bool, updateTrackedScreen: Bool, deliverViewportPolicy: Bool) @@ -6385,6 +6403,8 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { terminalByteContinuationsBySurfaceID.removeValue(forKey: surfaceID) terminalOutputStreamTokensBySurfaceID.removeValue(forKey: surfaceID) terminalOutputQueuesBySurfaceID.removeValue(forKey: surfaceID) + terminalReplayBarrierTokensBySurfaceID.removeValue(forKey: surfaceID) + terminalReplayBarrierAckStreamTokensBySurfaceID.removeValue(forKey: surfaceID) terminalScrollQueueTokensBySurfaceID.removeValue(forKey: surfaceID) terminalScrollQueuesBySurfaceID.removeValue(forKey: surfaceID) terminalScrollbackPrefetchStatesBySurfaceID.removeValue(forKey: surfaceID) @@ -6510,7 +6530,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { /// resume. The VT snapshot and raw byte ring remain fallbacks, but neither /// is the target architecture: a byte tail is not a complete screen state /// for TUIs, and a VT export is still a replay stream rather than state. - func requestTerminalReplay(surfaceID: String) { + func requestTerminalReplay(surfaceID: String, replayBarrierToken: UUID? = nil) { guard let client = remoteClient else { #if DEBUG mobileShellLog.error("CMUX_REPLAY skip surface=\(surfaceID, privacy: .public) reason=no_remote_client") @@ -6524,7 +6544,11 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { return } let remoteWorkspaceID = remoteWorkspaceID(for: workspaceID) - guard !terminalReplaySurfaceIDsInFlight.contains(surfaceID) else { + guard replayBarrierToken != nil || terminalReplayBarrierTokensBySurfaceID[surfaceID] == nil else { + MobileDebugLog.anchormux("CMUX_REPLAY skip surface=\(surfaceID) reason=barrier_active") + return + } + guard replayBarrierToken != nil || !terminalReplaySurfaceIDsInFlight.contains(surfaceID) else { #if DEBUG mobileShellLog.info("CMUX_REPLAY skip surface=\(surfaceID, privacy: .public) reason=in_flight") #endif @@ -6550,6 +6574,12 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { let decodedRenderGrid = payload?.renderGrid let renderGrid = decodedRenderGrid?.surfaceID == surfaceID ? decodedRenderGrid : nil let replaySeq = renderGrid?.stateSeq ?? payload?.sequence + if let replayBarrierToken { + guard self.terminalReplayBarrierTokensBySurfaceID[surfaceID] == replayBarrierToken else { + MobileDebugLog.anchormux("CMUX_REPLAY barrier_stale surface=\(surfaceID)") + return + } + } #if DEBUG let seq = replaySeq ?? 0 let cols = payload?.columns ?? -1 @@ -6578,13 +6608,26 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { } if let renderGrid { self.terminalActiveScreenBySurfaceID[surfaceID] = renderGrid.activeScreen - self.deliverTerminalRenderGrid(renderGrid, surfaceID: surfaceID) + self.deliverTerminalRenderGrid( + renderGrid, + surfaceID: surfaceID, + bypassReplayBarrier: replayBarrierToken != nil + ) return } guard let deliverBytes, !deliverBytes.isEmpty else { + if replayBarrierToken != nil { + self.terminalReplayBarrierAckStreamTokensBySurfaceID.removeValue(forKey: surfaceID) + self.terminalReplayBarrierTokensBySurfaceID.removeValue(forKey: surfaceID) + MobileDebugLog.anchormux("terminal.output.replay_barrier_cleared_empty surface=\(surfaceID)") + } return } - self.deliverTerminalBytes(deliverBytes, surfaceID: surfaceID) + self.deliverTerminalBytes( + deliverBytes, + surfaceID: surfaceID, + bypassReplayBarrier: replayBarrierToken != nil + ) } catch { mobileShellLog.error("CMUX_REPLAY failed surface=\(surfaceID, privacy: .public) error=\(String(describing: error), privacy: .public)") // The replay request is the view-only/foreground-resume path. A diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift index 43fb96de7fc7..9d065dc4c78a 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift @@ -56,18 +56,35 @@ import Testing store.terminalOutputDidReset(surfaceID: surfaceID, streamToken: stalledChunk.streamToken) #expect(store.terminalOutputQueuesBySurfaceID[surfaceID]?.isIdle == true) + #expect(store.terminalReplayBarrierTokensBySurfaceID[surfaceID] != nil) - store.deliverTerminalBytes(Data("fresh-after-reset".utf8), surfaceID: surfaceID) - let freshChunk = try #require(await iterator.next()) - #expect(String(decoding: freshChunk.data, as: UTF8.self) == "fresh-after-reset") - #expect(freshChunk.streamToken != stalledChunk.streamToken) + store.deliverTerminalBytes(Data("live-before-replay".utf8), surfaceID: surfaceID) + #expect(store.terminalOutputQueuesBySurfaceID[surfaceID]?.isIdle == true) store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: stalledChunk.streamToken) + + store.deliverTerminalBytes( + Data("authoritative-replay".utf8), + surfaceID: surfaceID, + bypassReplayBarrier: true + ) + let replayChunk = try #require(await iterator.next()) + #expect(String(decoding: replayChunk.data, as: UTF8.self) == "authoritative-replay") + #expect(replayChunk.streamToken != stalledChunk.streamToken) + + store.deliverTerminalBytes(Data("live-before-replay-ack".utf8), surfaceID: surfaceID) + #expect(store.terminalOutputQueuesBySurfaceID[surfaceID]?.pendingCount == 0) + store.deliverTerminalBytes(Data("after-stale-ack".utf8), surfaceID: surfaceID) - store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: freshChunk.streamToken) + #expect(store.terminalOutputQueuesBySurfaceID[surfaceID]?.pendingCount == 0) + + store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: replayChunk.streamToken) + #expect(store.terminalReplayBarrierTokensBySurfaceID[surfaceID] == nil) + + store.deliverTerminalBytes(Data("after-replay-ack".utf8), surfaceID: surfaceID) - let afterStaleAck = try #require(await iterator.next()) - #expect(String(decoding: afterStaleAck.data, as: UTF8.self) == "after-stale-ack") + let afterReplayAck = try #require(await iterator.next()) + #expect(String(decoding: afterReplayAck.data, as: UTF8.self) == "after-replay-ack") } @Test func terminalOutputQueueCoalescesReplaceableViewportFramesBehindBackpressure() { diff --git a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift index 9251c1028f13..1a72f5dad3f2 100644 --- a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift +++ b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift @@ -656,6 +656,9 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { var outputQueue = GhosttySurfaceWorkQueue(generation: 0) private var outputQueueGeneration: UInt64 = 0 private var isRecoveringRenderPipeline = false + private var renderPipelineRecoveryCount = 0 + private var renderPipelineStoppedAfterRecoveryLimit = false + private static let maxRenderPipelineRecoveries = 1 private static let renderPipelineStallDeadline: CFTimeInterval = 2.0 private static let outputApplyTimeoutNanoseconds: UInt64 = 2_000_000_000 private static let scrollMechanicsContentHeight: CGFloat = 1_000_000 @@ -2271,6 +2274,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { /// or `false` when the caller should reset its delivery queue and replay. @discardableResult public func processOutputAndWait(_ data: Data) async -> Bool { + guard !renderPipelineStoppedAfterRecoveryLimit else { return false } await withCheckedContinuation { continuation in let waiter = SurfaceOperationWaiter() let startedAt = CACurrentMediaTime() @@ -2306,6 +2310,10 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { _ data: Data, completion: (@MainActor @Sendable (Bool) -> Void)? ) { + guard !renderPipelineStoppedAfterRecoveryLimit else { + completion?(false) + return + } guard let surface, !isDismantled else { completion?(true) return @@ -2643,8 +2651,20 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { surface != nil else { return false } + guard renderPipelineRecoveryCount < Self.maxRenderPipelineRecoveries else { + renderPipelineStoppedAfterRecoveryLimit = true + stopDisplayLink() + renderInFlight = false + renderInFlightSince = nil + needsAnotherRender = false + MobileDebugLog.anchormux( + "render.recover.limit reason=\(reason) stalledMs=\(stalledMs) generation=\(surfaceGeneration)" + ) + return false + } isRecoveringRenderPipeline = true defer { isRecoveringRenderPipeline = false } + renderPipelineRecoveryCount += 1 MobileDebugLog.anchormux( "render.recover reason=\(reason) stalledMs=\(stalledMs) generation=\(surfaceGeneration)" @@ -2893,7 +2913,10 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { // now bounded in libghostty (so a foreground stall self-heals as a // skipped frame the display link re-drives), but we still gate on // suspension; `resumeRendering` clears it on the next active transition. - guard !renderingSuspended, let surface, !isDismantled else { return } + guard !renderPipelineStoppedAfterRecoveryLimit, + !renderingSuspended, + let surface, + !isDismantled else { return } // Coalesce: never let more than one render_now sit on the serial queue. // (Called on main from the display link.) if renderInFlight { @@ -3196,6 +3219,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { } private func syncSurfaceGeometryAndWait(shouldReassertNaturalSize: Bool = true) async -> Bool { + guard !renderPipelineStoppedAfterRecoveryLimit else { return false } needsGeometrySync = false pendingGeometryReassert = false await withCheckedContinuation { continuation in From d470c26042f209638ace691cd230b6b52bc6026e Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 02:23:46 -0700 Subject: [PATCH 05/56] Fix replay barrier reset races --- .github/swift-file-length-budget.tsv | 2 +- ...hellComposite+TerminalOutputDelivery.swift | 19 +++++++----- .../MobileShellComposite.swift | 30 ++++++++++++------- .../TerminalOutputDeliveryQueueTests.swift | 18 +++++++++-- .../GhosttySurfaceView.swift | 6 ++-- 5 files changed, 50 insertions(+), 25 deletions(-) diff --git a/.github/swift-file-length-budget.tsv b/.github/swift-file-length-budget.tsv index 69476c8249c7..c1b7ef7e7f94 100644 --- a/.github/swift-file-length-budget.tsv +++ b/.github/swift-file-length-budget.tsv @@ -14,7 +14,7 @@ 8016 CLI/cmux_open.swift 7366 cmuxTests/WorkspaceUnitTests.swift 7218 cmuxTests/WorkspaceRemoteConnectionTests.swift -7065 Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift +7075 Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift 6359 cmuxTests/SessionPersistenceTests.swift 6217 cmuxTests/GhosttyConfigTests.swift 6180 Sources/TabManager.swift diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift index d5ec68b041e9..8a036a7aa646 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift @@ -5,12 +5,13 @@ public import Foundation extension MobileShellComposite { /// Yield a raw PTY byte chunk to the surface stream, if one is attached. + @discardableResult func deliverTerminalBytes( _ bytes: Data, surfaceID: String, bypassReplayBarrier: Bool = false - ) { - deliverTerminalOutput( + ) -> Bool { + return deliverTerminalOutput( TerminalOutputDelivery( bytes: bytes, replaceable: false, @@ -21,12 +22,13 @@ extension MobileShellComposite { ) } + @discardableResult func deliverTerminalRenderGrid( _ frame: MobileTerminalRenderGridFrame, surfaceID: String, bypassReplayBarrier: Bool = false - ) { - deliverTerminalOutput( + ) -> Bool { + return deliverTerminalOutput( TerminalOutputDelivery( renderGrid: frame, replaceable: frame.isReplaceableViewportPatchForMobileDelivery, @@ -38,7 +40,7 @@ extension MobileShellComposite { } func deliverTerminalViewportPolicy(_ policy: MobileTerminalOutputViewportPolicy, surfaceID: String) { - deliverTerminalOutput( + _ = deliverTerminalOutput( TerminalOutputDelivery( bytes: Data(), replaceable: true, @@ -53,12 +55,12 @@ extension MobileShellComposite { _ delivery: TerminalOutputDelivery, surfaceID: String, bypassReplayBarrier: Bool = false - ) { + ) -> Bool { guard let continuation = terminalByteContinuationsBySurfaceID[surfaceID], - let streamToken = terminalOutputStreamTokensBySurfaceID[surfaceID] else { return } + let streamToken = terminalOutputStreamTokensBySurfaceID[surfaceID] else { return false } if terminalReplayBarrierTokensBySurfaceID[surfaceID] != nil, !bypassReplayBarrier { MobileDebugLog.anchormux("terminal.output.drop_replay_barrier surface=\(surfaceID)") - return + return false } var queue = terminalOutputQueuesBySurfaceID[surfaceID] ?? TerminalOutputDeliveryQueue() let immediate = queue.enqueue(delivery) @@ -83,6 +85,7 @@ extension MobileShellComposite { ) ) } + return true } /// Mark the current yielded terminal-output chunk as applied by the iOS surface. diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift index 017ef8ffe7c2..4b7647d90154 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift @@ -6352,7 +6352,6 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { : .deliver switch deliveryDecision { case .deliver: - terminalActiveScreenBySurfaceID[renderGrid.surfaceID] = renderGrid.activeScreen break case .advisory(let requestReplay, let updateTrackedScreen, let deliverViewportPolicy): if updateTrackedScreen { @@ -6369,8 +6368,9 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { } return } + guard deliverTerminalRenderGrid(renderGrid, surfaceID: renderGrid.surfaceID) else { return } + terminalActiveScreenBySurfaceID[renderGrid.surfaceID] = renderGrid.activeScreen markTerminalBytesDelivered(surfaceID: renderGrid.surfaceID, endSeq: renderGrid.stateSeq) - deliverTerminalRenderGrid(renderGrid, surfaceID: renderGrid.surfaceID) } private static func terminalSnapshotReplacementBytes(_ snapshotBytes: Data) -> Data { @@ -6603,16 +6603,17 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { deliverBytes = bytes MobileDebugLog.anchormux("CMUX_REPLAY raw_tail surface=\(surfaceID) bytes=\(bytes?.count ?? -1) seq=\(replaySeq ?? 0)") } - if let replaySeq { - self.markTerminalBytesDelivered(surfaceID: surfaceID, endSeq: replaySeq) - } if let renderGrid { - self.terminalActiveScreenBySurfaceID[surfaceID] = renderGrid.activeScreen - self.deliverTerminalRenderGrid( + let accepted = self.deliverTerminalRenderGrid( renderGrid, surfaceID: surfaceID, bypassReplayBarrier: replayBarrierToken != nil ) + guard accepted else { return } + self.terminalActiveScreenBySurfaceID[surfaceID] = renderGrid.activeScreen + if let replaySeq { + self.markTerminalBytesDelivered(surfaceID: surfaceID, endSeq: replaySeq) + } return } guard let deliverBytes, !deliverBytes.isEmpty else { @@ -6623,13 +6624,22 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { } return } - self.deliverTerminalBytes( + let accepted = self.deliverTerminalBytes( deliverBytes, surfaceID: surfaceID, bypassReplayBarrier: replayBarrierToken != nil ) + if accepted, let replaySeq { + self.markTerminalBytesDelivered(surfaceID: surfaceID, endSeq: replaySeq) + } } catch { mobileShellLog.error("CMUX_REPLAY failed surface=\(surfaceID, privacy: .public) error=\(String(describing: error), privacy: .public)") + if let replayBarrierToken, + self.terminalReplayBarrierTokensBySurfaceID[surfaceID] == replayBarrierToken { + self.terminalReplayBarrierAckStreamTokensBySurfaceID.removeValue(forKey: surfaceID) + self.terminalReplayBarrierTokensBySurfaceID.removeValue(forKey: surfaceID) + MobileDebugLog.anchormux("terminal.output.replay_barrier_cleared_failed surface=\(surfaceID)") + } // The replay request is the view-only/foreground-resume path. A // definitive auth failure here (after the RPC layer's // force-refresh-and-retry already gave up) must drive the re-auth @@ -6754,11 +6764,11 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { } let overlap = deliveredSeq - seq let deliverBytes = Data(bytes.dropFirst(Int(overlap))) - deliverTerminalBytes(deliverBytes, surfaceID: surfaceID) + guard deliverTerminalBytes(deliverBytes, surfaceID: surfaceID) else { return } markTerminalBytesDelivered(surfaceID: surfaceID, endSeq: endSeq) return } - deliverTerminalBytes(bytes, surfaceID: surfaceID) + guard deliverTerminalBytes(bytes, surfaceID: surfaceID) else { return } markTerminalBytesDelivered(surfaceID: surfaceID, endSeq: endSeq) } diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift index 9d065dc4c78a..b39ef542fc89 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift @@ -58,7 +58,11 @@ import Testing #expect(store.terminalOutputQueuesBySurfaceID[surfaceID]?.isIdle == true) #expect(store.terminalReplayBarrierTokensBySurfaceID[surfaceID] != nil) - store.deliverTerminalBytes(Data("live-before-replay".utf8), surfaceID: surfaceID) + let liveBeforeReplayAccepted = store.deliverTerminalBytes( + Data("live-before-replay".utf8), + surfaceID: surfaceID + ) + #expect(liveBeforeReplayAccepted == false) #expect(store.terminalOutputQueuesBySurfaceID[surfaceID]?.isIdle == true) store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: stalledChunk.streamToken) @@ -72,10 +76,18 @@ import Testing #expect(String(decoding: replayChunk.data, as: UTF8.self) == "authoritative-replay") #expect(replayChunk.streamToken != stalledChunk.streamToken) - store.deliverTerminalBytes(Data("live-before-replay-ack".utf8), surfaceID: surfaceID) + let liveBeforeReplayAckAccepted = store.deliverTerminalBytes( + Data("live-before-replay-ack".utf8), + surfaceID: surfaceID + ) + #expect(liveBeforeReplayAckAccepted == false) #expect(store.terminalOutputQueuesBySurfaceID[surfaceID]?.pendingCount == 0) - store.deliverTerminalBytes(Data("after-stale-ack".utf8), surfaceID: surfaceID) + let afterStaleAckAccepted = store.deliverTerminalBytes( + Data("after-stale-ack".utf8), + surfaceID: surfaceID + ) + #expect(afterStaleAckAccepted == false) #expect(store.terminalOutputQueuesBySurfaceID[surfaceID]?.pendingCount == 0) store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: replayChunk.streamToken) diff --git a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift index 1a72f5dad3f2..6a91b5ab8ce7 100644 --- a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift +++ b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift @@ -2274,7 +2274,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { /// or `false` when the caller should reset its delivery queue and replay. @discardableResult public func processOutputAndWait(_ data: Data) async -> Bool { - guard !renderPipelineStoppedAfterRecoveryLimit else { return false } + guard !renderPipelineStoppedAfterRecoveryLimit else { return true } await withCheckedContinuation { continuation in let waiter = SurfaceOperationWaiter() let startedAt = CACurrentMediaTime() @@ -2311,7 +2311,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { completion: (@MainActor @Sendable (Bool) -> Void)? ) { guard !renderPipelineStoppedAfterRecoveryLimit else { - completion?(false) + completion?(true) return } guard let surface, !isDismantled else { @@ -3219,7 +3219,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { } private func syncSurfaceGeometryAndWait(shouldReassertNaturalSize: Bool = true) async -> Bool { - guard !renderPipelineStoppedAfterRecoveryLimit else { return false } + guard !renderPipelineStoppedAfterRecoveryLimit else { return true } needsGeometrySync = false pendingGeometryReassert = false await withCheckedContinuation { continuation in From dc045c259b470066c117b7394ab5c5a459798587 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 02:25:41 -0700 Subject: [PATCH 06/56] Address iOS render recovery review feedback --- .../CmuxMobileShellUI/GhosttySurfaceRepresentable.swift | 3 ++- .../Sources/CmuxMobileTerminal/GhosttySurfaceView.swift | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/GhosttySurfaceRepresentable.swift b/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/GhosttySurfaceRepresentable.swift index fd7586fd8b77..30f6e1389cc1 100644 --- a/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/GhosttySurfaceRepresentable.swift +++ b/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/GhosttySurfaceRepresentable.swift @@ -424,7 +424,8 @@ struct GhosttySurfaceRepresentable: UIViewRepresentable { } func ghosttySurfaceViewDidResetRenderPipeline(_ surfaceView: GhosttySurfaceView) { - Task { @MainActor [weak store, surfaceID] in + Task { @MainActor [weak self, weak store, surfaceID] in + guard let self, self.surfaceView === surfaceView else { return } store?.terminalOutputNeedsReplay(surfaceID: surfaceID) } } diff --git a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift index 6a91b5ab8ce7..7abee6cda77a 100644 --- a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift +++ b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift @@ -3804,9 +3804,9 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { // wait the synchronous caller needs. // carve-out justification: one-shot cross-queue completion signal with a // bounded wait, not a lock guarding shared state. - let deadline = DispatchTime.now() + 0.6 var sections: [String] = [] for item in pending { + let deadline = DispatchTime.now() + 0.6 let holder = VisibleSnapshotHolder() // carve-out justification: one-shot cross-queue completion signal with a bounded wait. let done = DispatchSemaphore(value: 0) From d9f28958cdf617a2b960731025a6fa8c36e8ab22 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 02:37:23 -0700 Subject: [PATCH 07/56] Remove sleep-based render deadlines --- .github/swift-file-length-budget.tsv | 2 +- .../WorkspaceDetailView.swift | 6 +- .../GhosttySurfaceView.swift | 317 ++++++++++++------ .../SurfaceOperationWaiter.swift | 16 - 4 files changed, 222 insertions(+), 119 deletions(-) delete mode 100644 Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/SurfaceOperationWaiter.swift diff --git a/.github/swift-file-length-budget.tsv b/.github/swift-file-length-budget.tsv index c1b7ef7e7f94..b65afee165de 100644 --- a/.github/swift-file-length-budget.tsv +++ b/.github/swift-file-length-budget.tsv @@ -25,9 +25,9 @@ 4483 Sources/cmuxApp.swift 4367 cmuxTests/BrowserPanelTests.swift 4121 Sources/BrowserWindowPortal.swift +4043 Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift 3934 Sources/Feed/FeedPanelView.swift 3926 cmuxTests/TabManagerUnitTests.swift -3924 Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift 3896 cmuxTests/WindowAndDragTests.swift 3668 cmuxTests/CLIGenericHookPersistenceTests.swift 3397 Sources/CmuxConfig.swift diff --git a/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/WorkspaceDetailView.swift b/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/WorkspaceDetailView.swift index 815439484977..a603f0a16be8 100644 --- a/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/WorkspaceDetailView.swift +++ b/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/WorkspaceDetailView.swift @@ -493,8 +493,8 @@ struct WorkspaceDetailView: View { private func copyDebugLogsFromMenu() { // Include "what the user sees" (the visible terminal text) above the // debug log so a pasted bug report shows the on-screen content too. - let terminalText = GhosttySurfaceView.visibleTerminalSnapshot() Task { @MainActor in + let terminalText = await GhosttySurfaceView.visibleTerminalSnapshot() let count = await MobileDebugLog.shared.copyToPasteboard(prepending: terminalText) UINotificationFeedbackGenerator().notificationOccurred(.success) NSLog("cmux.terminal copied %d debug log lines + visible terminal to pasteboard", count) @@ -620,10 +620,10 @@ struct WorkspaceDetailView: View { // Only the agent path reads the terminal/debug snapshots; reading them is // cheap and harmless on the email path, but skip the work when unused. // `visibleTerminalSnapshot()` reads off the output queue with a bounded - // wait (never a main-thread `ghostty_surface_read_text`, which blanks the + // async deadline (never a main-thread `ghostty_surface_read_text`, which blanks the // terminal). The debug-log snapshot is awaited from its actor. - let terminalText = routesToAgent ? GhosttySurfaceView.visibleTerminalSnapshot() : "" Task { @MainActor in + let terminalText = routesToAgent ? await GhosttySurfaceView.visibleTerminalSnapshot() : "" let debugLogText = routesToAgent ? await MobileDebugLog.shared.sink.snapshotWithCount().1 : "" let outcome = await store.submitFeedback( message: note, diff --git a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift index 7abee6cda77a..98ed39bdf1d6 100644 --- a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift +++ b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift @@ -575,8 +575,8 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { private var zoomOverlayShown = false /// Media time of the last zoom interaction (pinch step, zoom button, or HUD /// tap). The display link fades the HUD once this is older than - /// `zoomOverlayVisibleDuration`. Time-based off the per-frame callback, not a - /// timer/`Task.sleep`, so it honors the no-sleep rule and tracks real + /// `zoomOverlayVisibleDuration`. Time-based off the per-frame callback, not + /// a sleeping timer task, so it honors the no-sleep rule and tracks real /// elapsed time regardless of frame rate. private var zoomOverlayLastInteraction: CFTimeInterval = 0 private static let zoomOverlayVisibleDuration: CFTimeInterval = 2.5 @@ -660,7 +660,12 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { private var renderPipelineStoppedAfterRecoveryLimit = false private static let maxRenderPipelineRecoveries = 1 private static let renderPipelineStallDeadline: CFTimeInterval = 2.0 - private static let outputApplyTimeoutNanoseconds: UInt64 = 2_000_000_000 + private static let outputApplyTimeout: CFTimeInterval = 2.0 + private static let visibleSnapshotTimeout: CFTimeInterval = 0.6 + private var nextSurfaceOperationID: UInt64 = 0 + private var pendingOutputApply: PendingSurfaceOperation? + private var pendingGeometryApply: PendingSurfaceOperation? + private var pendingVisibleSnapshot: PendingVisibleSnapshot? private static let scrollMechanicsContentHeight: CGFloat = 1_000_000 private var scrollMechanicsIsRecentering = false private var lastScrollMechanicsOffsetY: CGFloat? @@ -1570,7 +1575,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { /// toolbar stays visible regardless, so closing the composer just collapses its /// band. Gating the re-focus on `keyboardVisible` makes both directions /// correct. - /// No sleep / `asyncAfter`: the `become` is issued synchronously here. + /// No deferred timer task: the `become` is issued synchronously here. public func setComposerActive(_ active: Bool) { guard composerActive != active else { return } composerActive = active @@ -2276,36 +2281,141 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { public func processOutputAndWait(_ data: Data) async -> Bool { guard !renderPipelineStoppedAfterRecoveryLimit else { return true } await withCheckedContinuation { continuation in - let waiter = SurfaceOperationWaiter() - let startedAt = CACurrentMediaTime() - let timeoutTask = Task { @MainActor [weak self] in - do { - // Genuine apply deadline: if libghostty or the worker queue is - // wedged, unblock the shell stream and rebuild below. - try await Task.sleep(nanoseconds: Self.outputApplyTimeoutNanoseconds) - } catch { - return - } - guard let self else { - _ = waiter.resume(returning: false, continuation: continuation) - return - } - let elapsedMs = Int((CACurrentMediaTime() - startedAt) * 1000) - if waiter.resume(returning: false, continuation: continuation) { - MobileDebugLog.anchormux( - "output.apply.TIMEOUT bytes=\(data.count) elapsedMs=\(elapsedMs)" - ) - self.recoverRenderPipeline(reason: "output_timeout", stalledMs: elapsedMs) - } - } + let operationID = registerPendingOutputApply( + byteCount: data.count, + continuation: continuation + ) processOutput(data) { applied in - if waiter.resume(returning: applied, continuation: continuation) { - timeoutTask.cancel() - } + self.completePendingOutputApply(id: operationID, returning: applied) } } } + private func makeSurfaceOperationID() -> UInt64 { + nextSurfaceOperationID &+= 1 + return nextSurfaceOperationID + } + + private func ensureSurfaceOperationDeadlinePump() { + guard window != nil, displayLink == nil, !renderingSuspended else { return } + startDisplayLink() + } + + private func registerPendingOutputApply( + byteCount: Int, + continuation: CheckedContinuation + ) -> UInt64 { + let operationID = makeSurfaceOperationID() + if let existing = pendingOutputApply { + pendingOutputApply = nil + let elapsedMs = Int((CACurrentMediaTime() - existing.startedAt) * 1000) + MobileDebugLog.anchormux("output.apply.OVERLAP elapsedMs=\(elapsedMs)") + existing.continuation.resume(returning: false) + } + pendingOutputApply = PendingSurfaceOperation( + id: operationID, + startedAt: CACurrentMediaTime(), + byteCount: byteCount, + continuation: continuation + ) + ensureSurfaceOperationDeadlinePump() + return operationID + } + + @discardableResult + private func completePendingOutputApply(id: UInt64, returning result: Bool) -> Bool { + guard let pending = pendingOutputApply, pending.id == id else { return false } + pendingOutputApply = nil + pending.continuation.resume(returning: result) + return true + } + + private func registerPendingGeometryApply( + continuation: CheckedContinuation + ) -> UInt64 { + let operationID = makeSurfaceOperationID() + if let existing = pendingGeometryApply { + pendingGeometryApply = nil + let elapsedMs = Int((CACurrentMediaTime() - existing.startedAt) * 1000) + MobileDebugLog.anchormux("geometry.apply.OVERLAP elapsedMs=\(elapsedMs)") + existing.continuation.resume(returning: false) + } + pendingGeometryApply = PendingSurfaceOperation( + id: operationID, + startedAt: CACurrentMediaTime(), + byteCount: nil, + continuation: continuation + ) + ensureSurfaceOperationDeadlinePump() + return operationID + } + + @discardableResult + private func completePendingGeometryApply(id: UInt64, returning result: Bool) -> Bool { + guard let pending = pendingGeometryApply, pending.id == id else { return false } + pendingGeometryApply = nil + pending.continuation.resume(returning: result) + return true + } + + @discardableResult + private func checkSurfaceOperationDeadlines(now: CFTimeInterval) -> Bool { + if let pending = pendingOutputApply, + now - pending.startedAt >= Self.outputApplyTimeout { + pendingOutputApply = nil + let elapsedMs = Int((now - pending.startedAt) * 1000) + MobileDebugLog.anchormux( + "output.apply.TIMEOUT bytes=\(pending.byteCount ?? 0) elapsedMs=\(elapsedMs)" + ) + let recovered = recoverRenderPipeline(reason: "output_timeout", stalledMs: elapsedMs) + pending.continuation.resume(returning: renderPipelineStoppedAfterRecoveryLimit) + return recovered + } + + if let pending = pendingGeometryApply, + now - pending.startedAt >= Self.outputApplyTimeout { + pendingGeometryApply = nil + let elapsedMs = Int((now - pending.startedAt) * 1000) + MobileDebugLog.anchormux("geometry.apply.TIMEOUT elapsedMs=\(elapsedMs)") + let recovered = recoverRenderPipeline(reason: "geometry_timeout", stalledMs: elapsedMs) + pending.continuation.resume(returning: renderPipelineStoppedAfterRecoveryLimit) + return recovered + } + + if let pending = pendingVisibleSnapshot, + now - pending.startedAt >= Self.visibleSnapshotTimeout { + pendingVisibleSnapshot = nil + pending.continuation.resume(returning: nil) + } + return false + } + + private func completePendingSurfaceOperations(returning result: Bool) { + if let pending = pendingOutputApply { + pendingOutputApply = nil + pending.continuation.resume(returning: result) + } + if let pending = pendingGeometryApply { + pendingGeometryApply = nil + pending.continuation.resume(returning: result) + } + skipPendingVisibleSnapshot() + } + + private func skipPendingVisibleSnapshot() { + guard let pending = pendingVisibleSnapshot else { return } + pendingVisibleSnapshot = nil + pending.continuation.resume(returning: nil) + } + + @discardableResult + private func completePendingVisibleSnapshot(id: UInt64, returning section: String?) -> Bool { + guard let pending = pendingVisibleSnapshot, pending.id == id else { return false } + pendingVisibleSnapshot = nil + pending.continuation.resume(returning: section) + return true + } + private func processOutput( _ data: Data, completion: (@MainActor @Sendable (Bool) -> Void)? @@ -2657,6 +2767,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { renderInFlight = false renderInFlightSince = nil needsAnotherRender = false + completePendingSurfaceOperations(returning: true) MobileDebugLog.anchormux( "render.recover.limit reason=\(reason) stalledMs=\(stalledMs) generation=\(surfaceGeneration)" ) @@ -2669,6 +2780,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { MobileDebugLog.anchormux( "render.recover reason=\(reason) stalledMs=\(stalledMs) generation=\(surfaceGeneration)" ) + completePendingSurfaceOperations(returning: false) stopDisplayLink() let oldSurface = surface @@ -2772,6 +2884,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { @objc func handleDisplayLinkFire() { guard let surface else { return } + let now = CACurrentMediaTime() #if DEBUG // Main-thread liveness heartbeat + presented-surface state. Time-gated, // no behavior change. The `contents`/size fields let an IDLE blank be @@ -2781,7 +2894,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { // content itself is empty (sync/producer). `sinceOutput` ties a blank // to a render-grid stream gap or rules it out. CALayer reads only — no // libghostty call, so no futex/main-thread-wedge risk. - let nowHeartbeat = CACurrentMediaTime() + let nowHeartbeat = now if nowHeartbeat - lastHeartbeatTime >= 2.0 { lastHeartbeatTime = nowHeartbeat let renderLayer = (layer.sublayers ?? []).first(where: { isGhosttyRendererLayer($0) }) @@ -2798,8 +2911,11 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { ) } #endif + if checkSurfaceOperationDeadlines(now: now) { + return + } if let renderInFlightSince { - let stalledMs = Int((CACurrentMediaTime() - renderInFlightSince) * 1000) + let stalledMs = Int((now - renderInFlightSince) * 1000) if stalledMs >= Int(Self.renderPipelineStallDeadline * 1000), recoverRenderPipeline(reason: "render_in_flight", stalledMs: stalledMs) { return @@ -2833,7 +2949,6 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { pendingGeometryReassert = false syncSurfaceGeometry(shouldReassertNaturalSize: reassert) } - let now = CACurrentMediaTime() let blinkChanged = cursorBlinkState.advance(now: now) // Draw on content/cursor changes, and for a short bounded burst after // any geometry change. iOS has no renderer-side vsync, so a frame is @@ -2885,7 +3000,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { // Fade the zoom HUD once interaction has been quiet. Uses real elapsed // time off the continuous display link (no timer / sleep). if zoomOverlayShown, - CACurrentMediaTime() - zoomOverlayLastInteraction > Self.zoomOverlayVisibleDuration { + now - zoomOverlayLastInteraction > Self.zoomOverlayVisibleDuration { fadeOutZoomOverlay() } } @@ -3223,30 +3338,9 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { needsGeometrySync = false pendingGeometryReassert = false await withCheckedContinuation { continuation in - let waiter = SurfaceOperationWaiter() - let startedAt = CACurrentMediaTime() - let timeoutTask = Task { @MainActor [weak self] in - do { - // Genuine geometry deadline: the shell stream must not wait - // forever on a libghostty worker queue that stopped draining. - try await Task.sleep(nanoseconds: Self.outputApplyTimeoutNanoseconds) - } catch { - return - } - guard let self else { - _ = waiter.resume(returning: false, continuation: continuation) - return - } - let elapsedMs = Int((CACurrentMediaTime() - startedAt) * 1000) - if waiter.resume(returning: false, continuation: continuation) { - MobileDebugLog.anchormux("geometry.apply.TIMEOUT elapsedMs=\(elapsedMs)") - self.recoverRenderPipeline(reason: "geometry_timeout", stalledMs: elapsedMs) - } - } + let operationID = registerPendingGeometryApply(continuation: continuation) syncSurfaceGeometry(shouldReassertNaturalSize: shouldReassertNaturalSize) { applied in - if waiter.resume(returning: applied, continuation: continuation) { - timeoutTask.cancel() - } + self.completePendingGeometryApply(id: operationID, returning: applied) } } } @@ -3769,7 +3863,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { /// terminal surface, for the DEV "Copy Debug Logs" action so a bug report /// pairs the on-screen content with the debug log. Reads the VIEWPORT /// (visible grid only, not scrollback) via libghostty. - public static func visibleTerminalSnapshot() -> String { + public static func visibleTerminalSnapshot() async -> String { registeredSurfaceViews = registeredSurfaceViews.filter { $0.value.value != nil } // Collect the main-actor state + surface pointers first, then read the // viewport text on the serial output queue. `ghostty_surface_read_text` @@ -3777,55 +3871,68 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { // reading it on the MAIN thread here contends that lock during a render // storm and stalls the present — tapping Copy Debug Logs would itself // blank the terminal. The output queue is never concurrent with - // `process_output`, so the read can't wedge. No `main.sync` runs on that - // queue, so this `.sync` cannot deadlock. + // `process_output`, so the read can't wedge. The await is bounded by + // the surface's display-link deadline so this diagnostic path does not + // add a sleeping timer task or block the main actor. var pending: [VisibleSnapshotRequest] = [] for view in registeredSurfaceViews.values.compactMap(\.value) { guard view.window != nil, !view.isHidden, view.alpha > 0.01, let surface = view.surface else { continue } let grid = view.effectiveGrid.map { "\($0.cols)x\($0.rows)" } ?? "?" pending.append(VisibleSnapshotRequest( + view: view, grid: grid, font: Int(view.liveFontSize), - surface: surface, - queue: view.outputQueue + surface: surface )) } if pending.isEmpty { return "===== visible terminal: (no on-screen surface) =====" } - // Read on the output queue, but bound the wait. If a render wedge has the - // queue stuck mid-`process_output`, a plain `.sync` here would freeze the - // whole app exactly when the user taps Copy Debug Logs to capture that - // bug. Time out and ship the logs without the snapshot instead. - // This synchronous DEV-only "Copy Debug Logs" path reads the viewport off - // the serial output queue and must give up after a deadline if a render - // wedge holds it; an actor/await cannot express the bounded synchronous - // wait the synchronous caller needs. - // carve-out justification: one-shot cross-queue completion signal with a - // bounded wait, not a lock guarding shared state. var sections: [String] = [] for item in pending { - let deadline = DispatchTime.now() + 0.6 - let holder = VisibleSnapshotHolder() - // carve-out justification: one-shot cross-queue completion signal with a bounded wait. - let done = DispatchSemaphore(value: 0) - item.queue.async { - let text = surfaceText(item.surface, pointTag: GHOSTTY_POINT_VIEWPORT) ?? "(unavailable)" - holder.sections = [ - "===== visible terminal · grid=\(item.grid) · font=\(item.font) =====\n" - + text - ] - done.signal() - } - if done.wait(timeout: deadline) == .timedOut { + guard let section = await item.view.visibleSnapshotSection( + surface: item.surface, + grid: item.grid, + font: item.font + ) else { return "===== visible terminal: (snapshot skipped — render busy) =====" } - sections.append(contentsOf: holder.sections) + sections.append(section) } return sections.joined(separator: "\n\n") } + private func visibleSnapshotSection( + surface: ghostty_surface_t, + grid: String, + font: Int + ) async -> String? { + await withCheckedContinuation { continuation in + let operationID = makeSurfaceOperationID() + if let existing = pendingVisibleSnapshot { + pendingVisibleSnapshot = nil + existing.continuation.resume(returning: nil) + } + pendingVisibleSnapshot = PendingVisibleSnapshot( + id: operationID, + startedAt: CACurrentMediaTime(), + continuation: continuation + ) + ensureSurfaceOperationDeadlinePump() + let queue = outputQueue + let read = VisibleSnapshotRead(surface: surface, grid: grid, font: font) + queue.async { + let text = Self.surfaceText(read.surface, pointTag: GHOSTTY_POINT_VIEWPORT) ?? "(unavailable)" + let section = "===== visible terminal · grid=\(read.grid) · font=\(read.font) =====\n" + + text + DispatchQueue.main.async { [weak self] in + self?.completePendingVisibleSnapshot(id: operationID, returning: section) + } + } + } + } + private func handleBell() { UINotificationFeedbackGenerator().notificationOccurred(.warning) NotificationCenter.default.post( @@ -3886,27 +3993,39 @@ extension GhosttySurfaceView: UIScrollViewDelegate { } } +/// One output/geometry operation awaiting either its output-queue completion or +/// the display-link deadline that rebuilds the stalled render pipeline. +private struct PendingSurfaceOperation { + let id: UInt64 + let startedAt: CFTimeInterval + let byteCount: Int? + let continuation: CheckedContinuation +} + +/// One visible-terminal snapshot read awaiting output-queue completion or its +/// display-link deadline. A timeout skips only the diagnostic snapshot. +private struct PendingVisibleSnapshot { + let id: UInt64 + let startedAt: CFTimeInterval + let continuation: CheckedContinuation +} + /// One surface's request for the bounded visible-terminal snapshot. -/// -/// The `ghostty_surface_t` is a C pointer that the snapshot only dereferences on -/// `GhosttySurfaceView.outputQueue` (the queue that owns `process_output`) and -/// never mutates, so carrying it across the queue hop is safe — hence -/// `@unchecked Sendable`. -private struct VisibleSnapshotRequest: @unchecked Sendable { +private struct VisibleSnapshotRequest { + let view: GhosttySurfaceView let grid: String let font: Int let surface: ghostty_surface_t - let queue: GhosttySurfaceWorkQueue } -/// Carrier for the snapshot text produced off `GhosttySurfaceView.outputQueue`. +/// Raw surface read payload captured by the off-main output queue. /// -/// `sections` is written exactly once on that queue before its semaphore is -/// signaled and read by the caller only after the matching wait, so the two -/// accesses never overlap — hence `@unchecked Sendable`. On the timeout path the -/// caller never reads it, leaving the queue task the sole accessor. -private final class VisibleSnapshotHolder: @unchecked Sendable { - var sections: [String] = [] +/// The C surface pointer is dereferenced only on `GhosttySurfaceWorkQueue`, +/// which is the same FIFO queue that owns `process_output` and surface free. +private struct VisibleSnapshotRead: @unchecked Sendable { + let surface: ghostty_surface_t + let grid: String + let font: Int } private class DisplayLinkProxy { diff --git a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/SurfaceOperationWaiter.swift b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/SurfaceOperationWaiter.swift deleted file mode 100644 index 1a5cc15ccd06..000000000000 --- a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/SurfaceOperationWaiter.swift +++ /dev/null @@ -1,16 +0,0 @@ -import Foundation - -@MainActor -final class SurfaceOperationWaiter { - private var didResume = false - - func resume( - returning result: Bool, - continuation: CheckedContinuation - ) -> Bool { - guard !didResume else { return false } - didResume = true - continuation.resume(returning: result) - return true - } -} From a66725117ba3732532a39255cd8efcfd26481a64 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 02:44:55 -0700 Subject: [PATCH 08/56] Clear replay barriers on replay abort --- .github/swift-file-length-budget.tsv | 2 +- .../MobileShellComposite.swift | 76 +++++++++++++++---- .../TerminalOutputDeliveryQueueTests.swift | 32 +++++++- .../GhosttySurfaceView.swift | 4 +- 4 files changed, 95 insertions(+), 19 deletions(-) diff --git a/.github/swift-file-length-budget.tsv b/.github/swift-file-length-budget.tsv index b65afee165de..0fcbdb85e4b3 100644 --- a/.github/swift-file-length-budget.tsv +++ b/.github/swift-file-length-budget.tsv @@ -14,7 +14,7 @@ 8016 CLI/cmux_open.swift 7366 cmuxTests/WorkspaceUnitTests.swift 7218 cmuxTests/WorkspaceRemoteConnectionTests.swift -7075 Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift +7125 Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift 6359 cmuxTests/SessionPersistenceTests.swift 6217 cmuxTests/GhosttyConfigTests.swift 6180 Sources/TabManager.swift diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift index 4b7647d90154..bde527e9eb0f 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift @@ -6310,6 +6310,22 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { return token } + @discardableResult + private func clearTerminalReplayBarrierIfCurrent( + surfaceID: String, + token: UUID?, + reason: String + ) -> Bool { + guard let token, + terminalReplayBarrierTokensBySurfaceID[surfaceID] == token else { + return false + } + terminalReplayBarrierAckStreamTokensBySurfaceID.removeValue(forKey: surfaceID) + terminalReplayBarrierTokensBySurfaceID.removeValue(forKey: surfaceID) + MobileDebugLog.anchormux("terminal.output.replay_barrier_cleared_\(reason) surface=\(surfaceID)") + return true + } + private enum RenderGridEventDeliveryDecision { case deliver case advisory(requestReplay: Bool, updateTrackedScreen: Bool, deliverViewportPolicy: Bool) @@ -6532,12 +6548,22 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { /// for TUIs, and a VT export is still a replay stream rather than state. func requestTerminalReplay(surfaceID: String, replayBarrierToken: UUID? = nil) { guard let client = remoteClient else { + clearTerminalReplayBarrierIfCurrent( + surfaceID: surfaceID, + token: replayBarrierToken, + reason: "no_remote_client" + ) #if DEBUG mobileShellLog.error("CMUX_REPLAY skip surface=\(surfaceID, privacy: .public) reason=no_remote_client") #endif return } guard let workspaceID = workspaceID(forTerminalID: surfaceID) else { + clearTerminalReplayBarrierIfCurrent( + surfaceID: surfaceID, + token: replayBarrierToken, + reason: "workspace_not_found" + ) #if DEBUG mobileShellLog.error("CMUX_REPLAY skip surface=\(surfaceID, privacy: .public) reason=workspace_not_found") #endif @@ -6567,7 +6593,14 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { ] ) let data = try await client.sendRequest(request) - guard self.remoteClient === client else { return } + guard self.remoteClient === client else { + self.clearTerminalReplayBarrierIfCurrent( + surfaceID: surfaceID, + token: replayBarrierToken, + reason: "stale_client" + ) + return + } let payload = try? MobileTerminalReplayResponse.decode(data) let bytes = payload?.dataBase64.flatMap { Data(base64Encoded: $0) } let snapshotBytes = payload?.snapshotBase64.flatMap { Data(base64Encoded: $0) } @@ -6590,6 +6623,11 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { let deliveredSeq = self.deliveredTerminalByteEndSeqBySurfaceID[surfaceID], deliveredSeq > replaySeq { MobileDebugLog.anchormux("CMUX_REPLAY stale surface=\(surfaceID) delivered=\(deliveredSeq) replay=\(replaySeq)") + self.clearTerminalReplayBarrierIfCurrent( + surfaceID: surfaceID, + token: replayBarrierToken, + reason: "stale_sequence" + ) return } let deliverBytes: Data? @@ -6609,7 +6647,14 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { surfaceID: surfaceID, bypassReplayBarrier: replayBarrierToken != nil ) - guard accepted else { return } + guard accepted else { + self.clearTerminalReplayBarrierIfCurrent( + surfaceID: surfaceID, + token: replayBarrierToken, + reason: "not_delivered" + ) + return + } self.terminalActiveScreenBySurfaceID[surfaceID] = renderGrid.activeScreen if let replaySeq { self.markTerminalBytesDelivered(surfaceID: surfaceID, endSeq: replaySeq) @@ -6617,11 +6662,11 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { return } guard let deliverBytes, !deliverBytes.isEmpty else { - if replayBarrierToken != nil { - self.terminalReplayBarrierAckStreamTokensBySurfaceID.removeValue(forKey: surfaceID) - self.terminalReplayBarrierTokensBySurfaceID.removeValue(forKey: surfaceID) - MobileDebugLog.anchormux("terminal.output.replay_barrier_cleared_empty surface=\(surfaceID)") - } + self.clearTerminalReplayBarrierIfCurrent( + surfaceID: surfaceID, + token: replayBarrierToken, + reason: "empty" + ) return } let accepted = self.deliverTerminalBytes( @@ -6631,15 +6676,20 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { ) if accepted, let replaySeq { self.markTerminalBytesDelivered(surfaceID: surfaceID, endSeq: replaySeq) + } else if !accepted { + self.clearTerminalReplayBarrierIfCurrent( + surfaceID: surfaceID, + token: replayBarrierToken, + reason: "not_delivered" + ) } } catch { mobileShellLog.error("CMUX_REPLAY failed surface=\(surfaceID, privacy: .public) error=\(String(describing: error), privacy: .public)") - if let replayBarrierToken, - self.terminalReplayBarrierTokensBySurfaceID[surfaceID] == replayBarrierToken { - self.terminalReplayBarrierAckStreamTokensBySurfaceID.removeValue(forKey: surfaceID) - self.terminalReplayBarrierTokensBySurfaceID.removeValue(forKey: surfaceID) - MobileDebugLog.anchormux("terminal.output.replay_barrier_cleared_failed surface=\(surfaceID)") - } + self.clearTerminalReplayBarrierIfCurrent( + surfaceID: surfaceID, + token: replayBarrierToken, + reason: "failed" + ) // The replay request is the view-only/foreground-resume path. A // definitive auth failure here (after the RPC layer's // force-refresh-and-retry already gave up) must drive the re-auth diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift index b39ef542fc89..7fedd1752ac2 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift @@ -42,7 +42,7 @@ import Testing } @MainActor -@Test func terminalOutputResetDropsStalledBacklogAndInvalidatesOldAcks() async throws { +@Test func terminalReplayBarrierDropsStalledBacklogAndInvalidatesOldAcks() async throws { let store = MobileShellComposite.preview() let surfaceID = "terminal" @@ -53,10 +53,10 @@ import Testing #expect(store.terminalOutputQueuesBySurfaceID[surfaceID]?.pendingCount == 1) - store.terminalOutputDidReset(surfaceID: surfaceID, streamToken: stalledChunk.streamToken) + let replayBarrierToken = store.beginTerminalReplayBarrier(surfaceID: surfaceID) #expect(store.terminalOutputQueuesBySurfaceID[surfaceID]?.isIdle == true) - #expect(store.terminalReplayBarrierTokensBySurfaceID[surfaceID] != nil) + #expect(store.terminalReplayBarrierTokensBySurfaceID[surfaceID] == replayBarrierToken) let liveBeforeReplayAccepted = store.deliverTerminalBytes( Data("live-before-replay".utf8), @@ -99,6 +99,32 @@ import Testing #expect(String(decoding: afterReplayAck.data, as: UTF8.self) == "after-replay-ack") } +@MainActor +@Test func terminalOutputResetClearsBarrierWhenReplayCannotStart() async throws { + let store = MobileShellComposite.preview() + let surfaceID = "terminal" + + var iterator = store.terminalOutputStream(surfaceID: surfaceID).makeAsyncIterator() + store.deliverTerminalBytes(Data("stalled-first".utf8), surfaceID: surfaceID) + let stalledChunk = try #require(await iterator.next()) + store.deliverTerminalBytes(Data("stale-second".utf8), surfaceID: surfaceID) + + #expect(store.terminalOutputQueuesBySurfaceID[surfaceID]?.pendingCount == 1) + + store.terminalOutputDidReset(surfaceID: surfaceID, streamToken: stalledChunk.streamToken) + + #expect(store.terminalOutputQueuesBySurfaceID[surfaceID]?.isIdle == true) + #expect(store.terminalReplayBarrierTokensBySurfaceID[surfaceID] == nil) + + store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: stalledChunk.streamToken) + + let accepted = store.deliverTerminalBytes(Data("after-aborted-replay".utf8), surfaceID: surfaceID) + #expect(accepted == true) + + let afterAbort = try #require(await iterator.next()) + #expect(String(decoding: afterAbort.data, as: UTF8.self) == "after-aborted-replay") +} + @Test func terminalOutputQueueCoalescesReplaceableViewportFramesBehindBackpressure() { var queue = TerminalOutputDeliveryQueue() let inFlight = TerminalOutputDelivery(bytes: Data("in-flight".utf8), replaceable: false) diff --git a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift index 98ed39bdf1d6..1016fb280af8 100644 --- a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift +++ b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift @@ -2280,7 +2280,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { @discardableResult public func processOutputAndWait(_ data: Data) async -> Bool { guard !renderPipelineStoppedAfterRecoveryLimit else { return true } - await withCheckedContinuation { continuation in + return await withCheckedContinuation { continuation in let operationID = registerPendingOutputApply( byteCount: data.count, continuation: continuation @@ -3337,7 +3337,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { guard !renderPipelineStoppedAfterRecoveryLimit else { return true } needsGeometrySync = false pendingGeometryReassert = false - await withCheckedContinuation { continuation in + return await withCheckedContinuation { continuation in let operationID = registerPendingGeometryApply(continuation: continuation) syncSurfaceGeometry(shouldReassertNaturalSize: shouldReassertNaturalSize) { applied in self.completePendingGeometryApply(id: operationID, returning: applied) From de50f842d8a50ff8a4656fc91d3a5636d2bbf4c0 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 02:57:42 -0700 Subject: [PATCH 09/56] Address replay barrier recovery feedback --- .github/swift-file-length-budget.tsv | 4 +- ...hellComposite+TerminalOutputDelivery.swift | 7 ++ .../MobileShellComposite.swift | 6 ++ ...leShellRenderGridLivenessTestSupport.swift | 15 +++- .../TerminalOutputDeliveryQueueTests.swift | 55 +++++++++++++ .../GhosttySurfaceView.swift | 77 ++++++++++--------- 6 files changed, 123 insertions(+), 41 deletions(-) diff --git a/.github/swift-file-length-budget.tsv b/.github/swift-file-length-budget.tsv index 0fcbdb85e4b3..189d07faa20c 100644 --- a/.github/swift-file-length-budget.tsv +++ b/.github/swift-file-length-budget.tsv @@ -14,7 +14,7 @@ 8016 CLI/cmux_open.swift 7366 cmuxTests/WorkspaceUnitTests.swift 7218 cmuxTests/WorkspaceRemoteConnectionTests.swift -7125 Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift +7131 Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift 6359 cmuxTests/SessionPersistenceTests.swift 6217 cmuxTests/GhosttyConfigTests.swift 6180 Sources/TabManager.swift @@ -25,7 +25,7 @@ 4483 Sources/cmuxApp.swift 4367 cmuxTests/BrowserPanelTests.swift 4121 Sources/BrowserWindowPortal.swift -4043 Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift +4044 Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift 3934 Sources/Feed/FeedPanelView.swift 3926 cmuxTests/TabManagerUnitTests.swift 3896 cmuxTests/WindowAndDragTests.swift diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift index 8a036a7aa646..6f78decfd571 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift @@ -59,6 +59,7 @@ extension MobileShellComposite { guard let continuation = terminalByteContinuationsBySurfaceID[surfaceID], let streamToken = terminalOutputStreamTokensBySurfaceID[surfaceID] else { return false } if terminalReplayBarrierTokensBySurfaceID[surfaceID] != nil, !bypassReplayBarrier { + terminalReplayBarrierDroppedOutputSurfaceIDs.insert(surfaceID) MobileDebugLog.anchormux("terminal.output.drop_replay_barrier surface=\(surfaceID)") return false } @@ -98,6 +99,12 @@ extension MobileShellComposite { terminalReplayBarrierAckStreamTokensBySurfaceID.removeValue(forKey: surfaceID) terminalReplayBarrierTokensBySurfaceID.removeValue(forKey: surfaceID) MobileDebugLog.anchormux("terminal.output.replay_barrier_cleared surface=\(surfaceID)") + if terminalReplayBarrierDroppedOutputSurfaceIDs.remove(surfaceID) != nil { + let replayBarrierToken = beginTerminalReplayBarrier(surfaceID: surfaceID) + MobileDebugLog.anchormux("terminal.output.replay_followup surface=\(surfaceID)") + requestTerminalReplay(surfaceID: surfaceID, replayBarrierToken: replayBarrierToken) + return + } } guard let next, let continuation = terminalByteContinuationsBySurfaceID[surfaceID], diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift index bde527e9eb0f..8d13a9ace822 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift @@ -717,6 +717,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { private var terminalReplaySurfaceIDsInFlight: Set var terminalReplayBarrierTokensBySurfaceID: [String: UUID] var terminalReplayBarrierAckStreamTokensBySurfaceID: [String: UUID] + var terminalReplayBarrierDroppedOutputSurfaceIDs: Set private var terminalOutputTransport: TerminalOutputTransport var terminalByteContinuationsBySurfaceID: [String: AsyncStream.Continuation] var terminalOutputStreamTokensBySurfaceID: [String: UUID] @@ -910,6 +911,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { self.terminalReplaySurfaceIDsInFlight = [] self.terminalReplayBarrierTokensBySurfaceID = [:] self.terminalReplayBarrierAckStreamTokensBySurfaceID = [:] + self.terminalReplayBarrierDroppedOutputSurfaceIDs = [] self.terminalOutputTransport = .rawBytes self.terminalByteContinuationsBySurfaceID = [:] self.terminalOutputStreamTokensBySurfaceID = [:] @@ -4939,6 +4941,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { terminalReplaySurfaceIDsInFlight = [] terminalReplayBarrierTokensBySurfaceID = [:] terminalReplayBarrierAckStreamTokensBySurfaceID = [:] + terminalReplayBarrierDroppedOutputSurfaceIDs = [] terminalOutputQueuesBySurfaceID = [:] terminalOutputStreamTokensBySurfaceID = terminalOutputStreamTokensBySurfaceID.mapValues { _ in UUID() } terminalScrollQueueTokensBySurfaceID = [:] @@ -6307,6 +6310,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { let token = UUID() terminalReplayBarrierTokensBySurfaceID[surfaceID] = token terminalReplayBarrierAckStreamTokensBySurfaceID.removeValue(forKey: surfaceID) + terminalReplayBarrierDroppedOutputSurfaceIDs.remove(surfaceID) return token } @@ -6322,6 +6326,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { } terminalReplayBarrierAckStreamTokensBySurfaceID.removeValue(forKey: surfaceID) terminalReplayBarrierTokensBySurfaceID.removeValue(forKey: surfaceID) + terminalReplayBarrierDroppedOutputSurfaceIDs.remove(surfaceID) MobileDebugLog.anchormux("terminal.output.replay_barrier_cleared_\(reason) surface=\(surfaceID)") return true } @@ -6421,6 +6426,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { terminalOutputQueuesBySurfaceID.removeValue(forKey: surfaceID) terminalReplayBarrierTokensBySurfaceID.removeValue(forKey: surfaceID) terminalReplayBarrierAckStreamTokensBySurfaceID.removeValue(forKey: surfaceID) + terminalReplayBarrierDroppedOutputSurfaceIDs.remove(surfaceID) terminalScrollQueueTokensBySurfaceID.removeValue(forKey: surfaceID) terminalScrollQueuesBySurfaceID.removeValue(forKey: surfaceID) terminalScrollbackPrefetchStatesBySurfaceID.removeValue(forKey: surfaceID) diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift index e09f84a16ee8..9377c83f400d 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift @@ -67,6 +67,7 @@ actor LivenessHostRouter { private var hasActiveSubscription = false private var heldContinuations: [CheckedContinuation] = [] private var capabilities = ["events.v1", "terminal.bytes.v1", "terminal.render_grid.v1", "terminal.replay.v1"] + private var replayTexts: [String] = [] func record(method: String?, topics: [String]?) { recorded.append(RecordedRequest(method: method, topics: topics)) @@ -87,6 +88,10 @@ actor LivenessHostRouter { self.capabilities = capabilities } + func enqueueReplayTexts(_ texts: [String]) { + replayTexts.append(contentsOf: texts) + } + /// Hold every `mobile.events.subscribe` response until released. func setHoldSubscribe(_ hold: Bool) { holdSubscribe = hold @@ -169,7 +174,15 @@ actor LivenessHostRouter { "topics": ["workspace.updated", "terminal.render_grid"], "already_subscribed": alreadySubscribed, ]) - case "mobile.events.unsubscribe", "mobile.terminal.replay", "mobile.terminal.viewport": + case "mobile.terminal.replay": + guard !replayTexts.isEmpty else { + return try? Self.resultFrame(id: id, result: [:]) + } + let text = replayTexts.removeFirst() + return try? Self.resultFrame(id: id, result: [ + "data_b64": Data(text.utf8).base64EncodedString(), + ]) + case "mobile.events.unsubscribe", "mobile.terminal.viewport": return try? Self.resultFrame(id: id, result: [:]) case "terminal.input": return try? Self.resultFrame(id: id, result: [ diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift index 7fedd1752ac2..28494fbce263 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift @@ -125,6 +125,61 @@ import Testing #expect(String(decoding: afterAbort.data, as: UTF8.self) == "after-aborted-replay") } +@MainActor +@Test func terminalReplayBarrierRequestsFollowUpWhenLiveOutputDropsBeforeAck() async throws { + let router = LivenessHostRouter() + let box = TransportBox() + let clock = TestClock() + let store = try await makeConnectedStore(router: router, box: box, clock: clock) + let surfaceID = "live-terminal" + + await router.enqueueReplayTexts(["cold-replay", "first-replay", "follow-up-replay"]) + var iterator = store.terminalOutputStream(surfaceID: surfaceID).makeAsyncIterator() + let sawMountReplay = try await pollUntil { await router.count(of: "mobile.terminal.replay") >= 1 } + #expect(sawMountReplay, "mounting a sink must arm the cold-attach replay") + let coldReplayChunk = try #require(await iterator.next()) + #expect(String(decoding: coldReplayChunk.data, as: UTF8.self) == "cold-replay") + store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: coldReplayChunk.streamToken) + let replayCountAfterMount = await router.count(of: "mobile.terminal.replay") + + store.deliverTerminalBytes(Data("stalled-first".utf8), surfaceID: surfaceID) + let stalledChunk = try #require(await iterator.next()) + store.deliverTerminalBytes(Data("stale-second".utf8), surfaceID: surfaceID) + + store.terminalOutputDidReset(surfaceID: surfaceID, streamToken: stalledChunk.streamToken) + let sawResetReplay = try await pollUntil { + await router.count(of: "mobile.terminal.replay") >= replayCountAfterMount + 1 + } + #expect(sawResetReplay, "reset must request an authoritative replay") + + let replayChunk = try #require(await iterator.next()) + #expect(String(decoding: replayChunk.data, as: UTF8.self) == "first-replay") + + let acceptedDuringBarrier = store.deliverTerminalBytes( + Data("live-during-barrier".utf8), + surfaceID: surfaceID + ) + #expect(acceptedDuringBarrier == false) + #expect(store.terminalReplayBarrierDroppedOutputSurfaceIDs.contains(surfaceID)) + + store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: replayChunk.streamToken) + let sawFollowUpReplay = try await pollUntil { + await router.count(of: "mobile.terminal.replay") >= replayCountAfterMount + 2 + } + #expect(sawFollowUpReplay, "live output dropped during a replay barrier must trigger a follow-up replay") + + let followUpChunk = try #require(await iterator.next()) + #expect(String(decoding: followUpChunk.data, as: UTF8.self) == "follow-up-replay") + #expect(!store.terminalReplayBarrierDroppedOutputSurfaceIDs.contains(surfaceID)) + + store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: followUpChunk.streamToken) + #expect(store.terminalReplayBarrierTokensBySurfaceID[surfaceID] == nil) + + store.deliverTerminalBytes(Data("after-follow-up".utf8), surfaceID: surfaceID) + let afterFollowUp = try #require(await iterator.next()) + #expect(String(decoding: afterFollowUp.data, as: UTF8.self) == "after-follow-up") +} + @Test func terminalOutputQueueCoalescesReplaceableViewportFramesBehindBackpressure() { var queue = TerminalOutputDeliveryQueue() let inFlight = TerminalOutputDelivery(bytes: Data("in-flight".utf8), replaceable: false) diff --git a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift index 1016fb280af8..0737796a2ceb 100644 --- a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift +++ b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift @@ -655,10 +655,6 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { /// the same FIFO-before-dispose ordering discipline. var outputQueue = GhosttySurfaceWorkQueue(generation: 0) private var outputQueueGeneration: UInt64 = 0 - private var isRecoveringRenderPipeline = false - private var renderPipelineRecoveryCount = 0 - private var renderPipelineStoppedAfterRecoveryLimit = false - private static let maxRenderPipelineRecoveries = 1 private static let renderPipelineStallDeadline: CFTimeInterval = 2.0 private static let outputApplyTimeout: CFTimeInterval = 2.0 private static let visibleSnapshotTimeout: CFTimeInterval = 0.6 @@ -2279,7 +2275,6 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { /// or `false` when the caller should reset its delivery queue and replay. @discardableResult public func processOutputAndWait(_ data: Data) async -> Bool { - guard !renderPipelineStoppedAfterRecoveryLimit else { return true } return await withCheckedContinuation { continuation in let operationID = registerPendingOutputApply( byteCount: data.count, @@ -2367,8 +2362,12 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { MobileDebugLog.anchormux( "output.apply.TIMEOUT bytes=\(pending.byteCount ?? 0) elapsedMs=\(elapsedMs)" ) - let recovered = recoverRenderPipeline(reason: "output_timeout", stalledMs: elapsedMs) - pending.continuation.resume(returning: renderPipelineStoppedAfterRecoveryLimit) + let recovered = recoverRenderPipeline( + reason: "output_timeout", + stalledMs: elapsedMs, + replay: .callerWillRequestReplay + ) + pending.continuation.resume(returning: false) return recovered } @@ -2377,8 +2376,12 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { pendingGeometryApply = nil let elapsedMs = Int((now - pending.startedAt) * 1000) MobileDebugLog.anchormux("geometry.apply.TIMEOUT elapsedMs=\(elapsedMs)") - let recovered = recoverRenderPipeline(reason: "geometry_timeout", stalledMs: elapsedMs) - pending.continuation.resume(returning: renderPipelineStoppedAfterRecoveryLimit) + let recovered = recoverRenderPipeline( + reason: "geometry_timeout", + stalledMs: elapsedMs, + replay: .callerWillRequestReplay + ) + pending.continuation.resume(returning: false) return recovered } @@ -2390,16 +2393,21 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { return false } - private func completePendingSurfaceOperations(returning result: Bool) { + @discardableResult + private func completePendingSurfaceOperations(returning result: Bool) -> Bool { + var completed = false if let pending = pendingOutputApply { pendingOutputApply = nil pending.continuation.resume(returning: result) + completed = true } if let pending = pendingGeometryApply { pendingGeometryApply = nil pending.continuation.resume(returning: result) + completed = true } skipPendingVisibleSnapshot() + return completed } private func skipPendingVisibleSnapshot() { @@ -2420,10 +2428,6 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { _ data: Data, completion: (@MainActor @Sendable (Bool) -> Void)? ) { - guard !renderPipelineStoppedAfterRecoveryLimit else { - completion?(true) - return - } guard let surface, !isDismantled else { completion?(true) return @@ -2755,32 +2759,20 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { } @discardableResult - private func recoverRenderPipeline(reason: String, stalledMs: Int) -> Bool { - guard !isRecoveringRenderPipeline, - !isDismantled, + private func recoverRenderPipeline( + reason: String, + stalledMs: Int, + replay: RenderPipelineRecoveryReplay + ) -> Bool { + guard !isDismantled, surface != nil else { return false } - guard renderPipelineRecoveryCount < Self.maxRenderPipelineRecoveries else { - renderPipelineStoppedAfterRecoveryLimit = true - stopDisplayLink() - renderInFlight = false - renderInFlightSince = nil - needsAnotherRender = false - completePendingSurfaceOperations(returning: true) - MobileDebugLog.anchormux( - "render.recover.limit reason=\(reason) stalledMs=\(stalledMs) generation=\(surfaceGeneration)" - ) - return false - } - isRecoveringRenderPipeline = true - defer { isRecoveringRenderPipeline = false } - renderPipelineRecoveryCount += 1 MobileDebugLog.anchormux( "render.recover reason=\(reason) stalledMs=\(stalledMs) generation=\(surfaceGeneration)" ) - completePendingSurfaceOperations(returning: false) + let completedFailedOperation = completePendingSurfaceOperations(returning: false) stopDisplayLink() let oldSurface = surface @@ -2808,7 +2800,9 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { bridge.attach(to: self) initializeSurface() - delegate?.ghosttySurfaceViewDidResetRenderPipeline(self) + if replay == .delegateWhenNoCaller && !completedFailedOperation { + delegate?.ghosttySurfaceViewDidResetRenderPipeline(self) + } return true } @@ -2917,7 +2911,11 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { if let renderInFlightSince { let stalledMs = Int((now - renderInFlightSince) * 1000) if stalledMs >= Int(Self.renderPipelineStallDeadline * 1000), - recoverRenderPipeline(reason: "render_in_flight", stalledMs: stalledMs) { + recoverRenderPipeline( + reason: "render_in_flight", + stalledMs: stalledMs, + replay: .delegateWhenNoCaller + ) { return } } @@ -3028,8 +3026,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { // now bounded in libghostty (so a foreground stall self-heals as a // skipped frame the display link re-drives), but we still gate on // suspension; `resumeRendering` clears it on the next active transition. - guard !renderPipelineStoppedAfterRecoveryLimit, - !renderingSuspended, + guard !renderingSuspended, let surface, !isDismantled else { return } // Coalesce: never let more than one render_now sit on the serial queue. @@ -3334,7 +3331,6 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { } private func syncSurfaceGeometryAndWait(shouldReassertNaturalSize: Bool = true) async -> Bool { - guard !renderPipelineStoppedAfterRecoveryLimit else { return true } needsGeometrySync = false pendingGeometryReassert = false return await withCheckedContinuation { continuation in @@ -3993,6 +3989,11 @@ extension GhosttySurfaceView: UIScrollViewDelegate { } } +private enum RenderPipelineRecoveryReplay { + case callerWillRequestReplay + case delegateWhenNoCaller +} + /// One output/geometry operation awaiting either its output-queue completion or /// the display-link deadline that rebuilds the stalled render pipeline. private struct PendingSurfaceOperation { From 9f03583ad77c132b891107785b33d389b477d169 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 03:11:20 -0700 Subject: [PATCH 10/56] Add replay failure regression test --- .../MobileShellComposite.swift | 2 +- ...leShellRenderGridLivenessTestSupport.swift | 9 ++++ .../TerminalOutputDeliveryQueueTests.swift | 54 +++++++++++++++++++ 3 files changed, 64 insertions(+), 1 deletion(-) diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift index 8d13a9ace822..854d9b30c643 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift @@ -714,7 +714,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { private var deliveredTerminalByteEndSeqBySurfaceID: [String: UInt64] private var pendingTerminalByteEndSeqBySurfaceID: [String: UInt64] private var terminalActiveScreenBySurfaceID: [String: MobileTerminalRenderGridFrame.Screen] - private var terminalReplaySurfaceIDsInFlight: Set + var terminalReplaySurfaceIDsInFlight: Set var terminalReplayBarrierTokensBySurfaceID: [String: UUID] var terminalReplayBarrierAckStreamTokensBySurfaceID: [String: UUID] var terminalReplayBarrierDroppedOutputSurfaceIDs: Set diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift index 9377c83f400d..f04d0e674ccb 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift @@ -68,6 +68,7 @@ actor LivenessHostRouter { private var heldContinuations: [CheckedContinuation] = [] private var capabilities = ["events.v1", "terminal.bytes.v1", "terminal.render_grid.v1", "terminal.replay.v1"] private var replayTexts: [String] = [] + private var replayFailuresRemaining = 0 func record(method: String?, topics: [String]?) { recorded.append(RecordedRequest(method: method, topics: topics)) @@ -92,6 +93,10 @@ actor LivenessHostRouter { replayTexts.append(contentsOf: texts) } + func failNextReplay(count: Int = 1) { + replayFailuresRemaining += count + } + /// Hold every `mobile.events.subscribe` response until released. func setHoldSubscribe(_ hold: Bool) { holdSubscribe = hold @@ -175,6 +180,10 @@ actor LivenessHostRouter { "already_subscribed": alreadySubscribed, ]) case "mobile.terminal.replay": + if replayFailuresRemaining > 0 { + replayFailuresRemaining -= 1 + return try? Self.errorFrame(id: id, message: "replay failed") + } guard !replayTexts.isEmpty else { return try? Self.resultFrame(id: id, result: [:]) } diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift index 28494fbce263..9ca817c9dc2b 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift @@ -180,6 +180,60 @@ import Testing #expect(String(decoding: afterFollowUp.data, as: UTF8.self) == "after-follow-up") } +@MainActor +@Test func terminalReplayBarrierRetriesAfterReplayFailureWithDroppedOutput() async throws { + let router = LivenessHostRouter() + let box = TransportBox() + let clock = TestClock() + let store = try await makeConnectedStore(router: router, box: box, clock: clock) + let surfaceID = "live-terminal" + + await router.enqueueReplayTexts(["cold-replay", "retry-replay"]) + var iterator = store.terminalOutputStream(surfaceID: surfaceID).makeAsyncIterator() + let sawMountReplay = try await pollUntil { await router.count(of: "mobile.terminal.replay") >= 1 } + #expect(sawMountReplay, "mounting a sink must arm the cold-attach replay") + let coldReplayChunk = try #require(await iterator.next()) + #expect(String(decoding: coldReplayChunk.data, as: UTF8.self) == "cold-replay") + store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: coldReplayChunk.streamToken) + let replayCountAfterMount = await router.count(of: "mobile.terminal.replay") + + await router.failNextReplay() + let replayBarrierToken = store.beginTerminalReplayBarrier(surfaceID: surfaceID) + let firstDropAccepted = store.deliverTerminalBytes( + Data("live-during-failed-replay".utf8), + surfaceID: surfaceID + ) + #expect(firstDropAccepted == false) + + let sawFailedReplay = try await pollUntil { + await router.count(of: "mobile.terminal.replay") >= replayCountAfterMount + 1 + } + #expect(sawFailedReplay, "first dropped live output should request a replay") + + let failureSettled = try await pollUntil { + !store.terminalReplaySurfaceIDsInFlight.contains(surfaceID) + && store.terminalReplayBarrierTokensBySurfaceID[surfaceID] == replayBarrierToken + } + #expect(failureSettled, "failed replay with dropped output must keep the barrier active") + #expect(store.terminalReplayBarrierDroppedOutputSurfaceIDs.contains(surfaceID)) + + let retryDropAccepted = store.deliverTerminalBytes( + Data("live-after-failed-replay".utf8), + surfaceID: surfaceID + ) + #expect(retryDropAccepted == false) + let sawRetryReplay = try await pollUntil { + await router.count(of: "mobile.terminal.replay") >= replayCountAfterMount + 2 + } + #expect(sawRetryReplay, "next dropped live output should retry the preserved barrier replay") + + let retryReplayChunk = try #require(await iterator.next()) + #expect(String(decoding: retryReplayChunk.data, as: UTF8.self) == "retry-replay") + store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: retryReplayChunk.streamToken) + #expect(store.terminalReplayBarrierTokensBySurfaceID[surfaceID] == nil) + #expect(!store.terminalReplayBarrierDroppedOutputSurfaceIDs.contains(surfaceID)) +} + @Test func terminalOutputQueueCoalescesReplaceableViewportFramesBehindBackpressure() { var queue = TerminalOutputDeliveryQueue() let inFlight = TerminalOutputDelivery(bytes: Data("in-flight".utf8), replaceable: false) From 5b2d48d263e1e25ffbf28c94b8e378966695a3ad Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 03:16:24 -0700 Subject: [PATCH 11/56] Bound iOS terminal render recovery retries --- .github/swift-file-length-budget.tsv | 4 +- ...hellComposite+TerminalOutputDelivery.swift | 21 ++++++++- .../MobileShellComposite.swift | 47 ++++++++++++++++--- .../GhosttySurfaceView.swift | 26 +++++++++- 4 files changed, 86 insertions(+), 12 deletions(-) diff --git a/.github/swift-file-length-budget.tsv b/.github/swift-file-length-budget.tsv index 189d07faa20c..4857274633fc 100644 --- a/.github/swift-file-length-budget.tsv +++ b/.github/swift-file-length-budget.tsv @@ -14,7 +14,7 @@ 8016 CLI/cmux_open.swift 7366 cmuxTests/WorkspaceUnitTests.swift 7218 cmuxTests/WorkspaceRemoteConnectionTests.swift -7131 Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift +7166 Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift 6359 cmuxTests/SessionPersistenceTests.swift 6217 cmuxTests/GhosttyConfigTests.swift 6180 Sources/TabManager.swift @@ -25,7 +25,7 @@ 4483 Sources/cmuxApp.swift 4367 cmuxTests/BrowserPanelTests.swift 4121 Sources/BrowserWindowPortal.swift -4044 Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift +4066 Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift 3934 Sources/Feed/FeedPanelView.swift 3926 cmuxTests/TabManagerUnitTests.swift 3896 cmuxTests/WindowAndDragTests.swift diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift index 6f78decfd571..b450ba472feb 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift @@ -58,9 +58,20 @@ extension MobileShellComposite { ) -> Bool { guard let continuation = terminalByteContinuationsBySurfaceID[surfaceID], let streamToken = terminalOutputStreamTokensBySurfaceID[surfaceID] else { return false } - if terminalReplayBarrierTokensBySurfaceID[surfaceID] != nil, !bypassReplayBarrier { + if let replayBarrierToken = terminalReplayBarrierTokensBySurfaceID[surfaceID], + !bypassReplayBarrier { terminalReplayBarrierDroppedOutputSurfaceIDs.insert(surfaceID) MobileDebugLog.anchormux("terminal.output.drop_replay_barrier surface=\(surfaceID)") + if remoteClient != nil, + terminalReplayBarrierAckStreamTokensBySurfaceID[surfaceID] == nil, + !terminalReplaySurfaceIDsInFlight.contains(surfaceID) { + MobileDebugLog.anchormux("terminal.output.replay_retry_after_drop surface=\(surfaceID)") + requestTerminalReplay( + surfaceID: surfaceID, + replayBarrierToken: replayBarrierToken, + coversReplayBarrierDroppedOutput: true + ) + } return false } var queue = terminalOutputQueuesBySurfaceID[surfaceID] ?? TerminalOutputDeliveryQueue() @@ -96,10 +107,12 @@ extension MobileShellComposite { let next = queue.completeInFlight() terminalOutputQueuesBySurfaceID[surfaceID] = queue if terminalReplayBarrierAckStreamTokensBySurfaceID[surfaceID] == streamToken { + let replayAckCoversDroppedOutput = terminalReplayBarrierAckCoversDroppedOutputSurfaceIDs.remove(surfaceID) != nil terminalReplayBarrierAckStreamTokensBySurfaceID.removeValue(forKey: surfaceID) terminalReplayBarrierTokensBySurfaceID.removeValue(forKey: surfaceID) MobileDebugLog.anchormux("terminal.output.replay_barrier_cleared surface=\(surfaceID)") - if terminalReplayBarrierDroppedOutputSurfaceIDs.remove(surfaceID) != nil { + if terminalReplayBarrierDroppedOutputSurfaceIDs.remove(surfaceID) != nil, + !replayAckCoversDroppedOutput { let replayBarrierToken = beginTerminalReplayBarrier(surfaceID: surfaceID) MobileDebugLog.anchormux("terminal.output.replay_followup surface=\(surfaceID)") requestTerminalReplay(surfaceID: surfaceID, replayBarrierToken: replayBarrierToken) @@ -127,6 +140,10 @@ extension MobileShellComposite { public func terminalOutputDidReset(surfaceID: String, streamToken: UUID) { guard terminalOutputStreamTokensBySurfaceID[surfaceID] == streamToken, terminalOutputQueuesBySurfaceID[surfaceID] != nil else { return } + guard terminalReplayBarrierTokensBySurfaceID[surfaceID] == nil else { + MobileDebugLog.anchormux("terminal.output.reset_barrier_active surface=\(surfaceID)") + return + } let replayBarrierToken = beginTerminalReplayBarrier(surfaceID: surfaceID) MobileDebugLog.anchormux("terminal.output.reset surface=\(surfaceID)") requestTerminalReplay(surfaceID: surfaceID, replayBarrierToken: replayBarrierToken) diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift index 854d9b30c643..6c6eebdbb396 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift @@ -718,6 +718,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { var terminalReplayBarrierTokensBySurfaceID: [String: UUID] var terminalReplayBarrierAckStreamTokensBySurfaceID: [String: UUID] var terminalReplayBarrierDroppedOutputSurfaceIDs: Set + var terminalReplayBarrierAckCoversDroppedOutputSurfaceIDs: Set private var terminalOutputTransport: TerminalOutputTransport var terminalByteContinuationsBySurfaceID: [String: AsyncStream.Continuation] var terminalOutputStreamTokensBySurfaceID: [String: UUID] @@ -912,6 +913,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { self.terminalReplayBarrierTokensBySurfaceID = [:] self.terminalReplayBarrierAckStreamTokensBySurfaceID = [:] self.terminalReplayBarrierDroppedOutputSurfaceIDs = [] + self.terminalReplayBarrierAckCoversDroppedOutputSurfaceIDs = [] self.terminalOutputTransport = .rawBytes self.terminalByteContinuationsBySurfaceID = [:] self.terminalOutputStreamTokensBySurfaceID = [:] @@ -4942,6 +4944,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { terminalReplayBarrierTokensBySurfaceID = [:] terminalReplayBarrierAckStreamTokensBySurfaceID = [:] terminalReplayBarrierDroppedOutputSurfaceIDs = [] + terminalReplayBarrierAckCoversDroppedOutputSurfaceIDs = [] terminalOutputQueuesBySurfaceID = [:] terminalOutputStreamTokensBySurfaceID = terminalOutputStreamTokensBySurfaceID.mapValues { _ in UUID() } terminalScrollQueueTokensBySurfaceID = [:] @@ -6311,6 +6314,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { terminalReplayBarrierTokensBySurfaceID[surfaceID] = token terminalReplayBarrierAckStreamTokensBySurfaceID.removeValue(forKey: surfaceID) terminalReplayBarrierDroppedOutputSurfaceIDs.remove(surfaceID) + terminalReplayBarrierAckCoversDroppedOutputSurfaceIDs.remove(surfaceID) return token } @@ -6318,15 +6322,22 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { private func clearTerminalReplayBarrierIfCurrent( surfaceID: String, token: UUID?, - reason: String + reason: String, + preserveDroppedOutput: Bool = false ) -> Bool { guard let token, terminalReplayBarrierTokensBySurfaceID[surfaceID] == token else { return false } + if preserveDroppedOutput, + terminalReplayBarrierDroppedOutputSurfaceIDs.contains(surfaceID) { + MobileDebugLog.anchormux("terminal.output.replay_barrier_preserved_\(reason) surface=\(surfaceID)") + return false + } terminalReplayBarrierAckStreamTokensBySurfaceID.removeValue(forKey: surfaceID) terminalReplayBarrierTokensBySurfaceID.removeValue(forKey: surfaceID) terminalReplayBarrierDroppedOutputSurfaceIDs.remove(surfaceID) + terminalReplayBarrierAckCoversDroppedOutputSurfaceIDs.remove(surfaceID) MobileDebugLog.anchormux("terminal.output.replay_barrier_cleared_\(reason) surface=\(surfaceID)") return true } @@ -6427,6 +6438,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { terminalReplayBarrierTokensBySurfaceID.removeValue(forKey: surfaceID) terminalReplayBarrierAckStreamTokensBySurfaceID.removeValue(forKey: surfaceID) terminalReplayBarrierDroppedOutputSurfaceIDs.remove(surfaceID) + terminalReplayBarrierAckCoversDroppedOutputSurfaceIDs.remove(surfaceID) terminalScrollQueueTokensBySurfaceID.removeValue(forKey: surfaceID) terminalScrollQueuesBySurfaceID.removeValue(forKey: surfaceID) terminalScrollbackPrefetchStatesBySurfaceID.removeValue(forKey: surfaceID) @@ -6552,7 +6564,11 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { /// resume. The VT snapshot and raw byte ring remain fallbacks, but neither /// is the target architecture: a byte tail is not a complete screen state /// for TUIs, and a VT export is still a replay stream rather than state. - func requestTerminalReplay(surfaceID: String, replayBarrierToken: UUID? = nil) { + func requestTerminalReplay( + surfaceID: String, + replayBarrierToken: UUID? = nil, + coversReplayBarrierDroppedOutput: Bool = false + ) { guard let client = remoteClient else { clearTerminalReplayBarrierIfCurrent( surfaceID: surfaceID, @@ -6657,10 +6673,18 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { self.clearTerminalReplayBarrierIfCurrent( surfaceID: surfaceID, token: replayBarrierToken, - reason: "not_delivered" + reason: "not_delivered", + preserveDroppedOutput: true ) return } + if self.terminalReplayBarrierAckStreamTokensBySurfaceID[surfaceID] != nil { + if coversReplayBarrierDroppedOutput { + self.terminalReplayBarrierAckCoversDroppedOutputSurfaceIDs.insert(surfaceID) + } else { + self.terminalReplayBarrierAckCoversDroppedOutputSurfaceIDs.remove(surfaceID) + } + } self.terminalActiveScreenBySurfaceID[surfaceID] = renderGrid.activeScreen if let replaySeq { self.markTerminalBytesDelivered(surfaceID: surfaceID, endSeq: replaySeq) @@ -6671,7 +6695,8 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { self.clearTerminalReplayBarrierIfCurrent( surfaceID: surfaceID, token: replayBarrierToken, - reason: "empty" + reason: "empty", + preserveDroppedOutput: true ) return } @@ -6680,13 +6705,22 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { surfaceID: surfaceID, bypassReplayBarrier: replayBarrierToken != nil ) + if accepted, + self.terminalReplayBarrierAckStreamTokensBySurfaceID[surfaceID] != nil { + if coversReplayBarrierDroppedOutput { + self.terminalReplayBarrierAckCoversDroppedOutputSurfaceIDs.insert(surfaceID) + } else { + self.terminalReplayBarrierAckCoversDroppedOutputSurfaceIDs.remove(surfaceID) + } + } if accepted, let replaySeq { self.markTerminalBytesDelivered(surfaceID: surfaceID, endSeq: replaySeq) } else if !accepted { self.clearTerminalReplayBarrierIfCurrent( surfaceID: surfaceID, token: replayBarrierToken, - reason: "not_delivered" + reason: "not_delivered", + preserveDroppedOutput: true ) } } catch { @@ -6694,7 +6728,8 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { self.clearTerminalReplayBarrierIfCurrent( surfaceID: surfaceID, token: replayBarrierToken, - reason: "failed" + reason: "failed", + preserveDroppedOutput: true ) // The replay request is the view-only/foreground-resume path. A // definitive auth failure here (after the RPC layer's diff --git a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift index 0737796a2ceb..c0ca741a9c71 100644 --- a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift +++ b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift @@ -655,6 +655,8 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { /// the same FIFO-before-dispose ordering discipline. var outputQueue = GhosttySurfaceWorkQueue(generation: 0) private var outputQueueGeneration: UInt64 = 0 + private var pendingSurfaceFreeCount = 0 + private static let maxPendingSurfaceFrees = 1 private static let renderPipelineStallDeadline: CFTimeInterval = 2.0 private static let outputApplyTimeout: CFTimeInterval = 2.0 private static let visibleSnapshotTimeout: CFTimeInterval = 0.6 @@ -2749,12 +2751,16 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { private func enqueueSurfaceFree( _ surface: ghostty_surface_t, bridge: GhosttySurfaceBridge, - on queue: GhosttySurfaceWorkQueue + on queue: GhosttySurfaceWorkQueue, + completion: (@MainActor @Sendable () -> Void)? = nil ) { let retainedBridge = Unmanaged.passRetained(bridge) queue.async { ghostty_surface_free(surface) retainedBridge.release() + if let completion { + Task { @MainActor in completion() } + } } } @@ -2768,6 +2774,15 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { surface != nil else { return false } + guard pendingSurfaceFreeCount < Self.maxPendingSurfaceFrees else { + renderInFlight = false + renderInFlightSince = nil + needsAnotherRender = false + MobileDebugLog.anchormux( + "render.recover.blocked reason=\(reason) stalledMs=\(stalledMs) pendingFrees=\(pendingSurfaceFreeCount)" + ) + return false + } MobileDebugLog.anchormux( "render.recover reason=\(reason) stalledMs=\(stalledMs) generation=\(surfaceGeneration)" @@ -2780,7 +2795,14 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { let oldQueue = outputQueue if let oldSurface { GhosttySurfaceView.unregister(surface: oldSurface) - enqueueSurfaceFree(oldSurface, bridge: oldBridge, on: oldQueue) + pendingSurfaceFreeCount += 1 + enqueueSurfaceFree(oldSurface, bridge: oldBridge, on: oldQueue) { [weak self] in + guard let self else { return } + self.pendingSurfaceFreeCount = max(0, self.pendingSurfaceFreeCount - 1) + MobileDebugLog.anchormux( + "render.recover.free_drained pendingFrees=\(self.pendingSurfaceFreeCount)" + ) + } } oldBridge.detach() From 1bccdcc3cb56c6c76620ed76f21766fbe12421e4 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 03:21:51 -0700 Subject: [PATCH 12/56] Address iOS render recovery review feedback --- .github/swift-file-length-budget.tsv | 3 +- ...leShellRenderGridLivenessTestSupport.swift | 35 ++++++++++++++++ .../TerminalOutputDeliveryQueueTests.swift | 41 +++++++++---------- .../GhosttySurfaceView.swift | 39 ++++++++++++++---- 4 files changed, 87 insertions(+), 31 deletions(-) diff --git a/.github/swift-file-length-budget.tsv b/.github/swift-file-length-budget.tsv index 4857274633fc..9e7c95027c3d 100644 --- a/.github/swift-file-length-budget.tsv +++ b/.github/swift-file-length-budget.tsv @@ -25,7 +25,7 @@ 4483 Sources/cmuxApp.swift 4367 cmuxTests/BrowserPanelTests.swift 4121 Sources/BrowserWindowPortal.swift -4066 Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift +4087 Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift 3934 Sources/Feed/FeedPanelView.swift 3926 cmuxTests/TabManagerUnitTests.swift 3896 cmuxTests/WindowAndDragTests.swift @@ -234,6 +234,7 @@ 514 Packages/macOS/CmuxSwiftRender/Sources/CmuxSwiftRender/ExpressionEvaluator.swift 514 cmuxUITests/UpdatePillUITests.swift 513 Packages/iOS/CmuxAgentChatUI/Sources/CmuxAgentChatUI/Transcript/ChatTranscriptTableView.swift +513 Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift 509 Packages/macOS/CMUXAgentLaunch/Sources/CMUXAgentLaunch/AgentLaunchSanitizerAdditionalPolicies.swift 507 Sources/TerminalControllerTopSupport.swift 506 Sources/App/MainWindowVisibilityController.swift diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift index f04d0e674ccb..739c5cab75b5 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift @@ -58,7 +58,14 @@ actor LivenessHostRouter { var topics: [String]? } + private struct CountWaiter { + var method: String + var expectedCount: Int + var continuation: CheckedContinuation + } + private var recorded: [RecordedRequest] = [] + private var countWaiters: [CountWaiter] = [] private var hostStatusRequestCount = 0 private var heldHostStatusRequestNumbers: Set = [] private var subscribeRequestCount = 0 @@ -72,12 +79,40 @@ actor LivenessHostRouter { func record(method: String?, topics: [String]?) { recorded.append(RecordedRequest(method: method, topics: topics)) + resumeSatisfiedCountWaiters() } func count(of method: String) -> Int { recorded.filter { $0.method == method }.count } + func waitForCount(of method: String, atLeast expectedCount: Int) async { + guard count(of: method) < expectedCount else { return } + await withCheckedContinuation { continuation in + countWaiters.append(CountWaiter( + method: method, + expectedCount: expectedCount, + continuation: continuation + )) + } + } + + private func resumeSatisfiedCountWaiters() { + var remaining: [CountWaiter] = [] + var ready: [CheckedContinuation] = [] + for waiter in countWaiters { + if count(of: waiter.method) >= waiter.expectedCount { + ready.append(waiter.continuation) + } else { + remaining.append(waiter) + } + } + countWaiters = remaining + for continuation in ready { + continuation.resume(returning: ()) + } + } + func topics(for method: String) -> [[String]] { recorded.compactMap { request in guard request.method == method else { return nil } diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift index 9ca817c9dc2b..68480b5ea7a0 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift @@ -135,8 +135,7 @@ import Testing await router.enqueueReplayTexts(["cold-replay", "first-replay", "follow-up-replay"]) var iterator = store.terminalOutputStream(surfaceID: surfaceID).makeAsyncIterator() - let sawMountReplay = try await pollUntil { await router.count(of: "mobile.terminal.replay") >= 1 } - #expect(sawMountReplay, "mounting a sink must arm the cold-attach replay") + await router.waitForCount(of: "mobile.terminal.replay", atLeast: 1) let coldReplayChunk = try #require(await iterator.next()) #expect(String(decoding: coldReplayChunk.data, as: UTF8.self) == "cold-replay") store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: coldReplayChunk.streamToken) @@ -147,10 +146,7 @@ import Testing store.deliverTerminalBytes(Data("stale-second".utf8), surfaceID: surfaceID) store.terminalOutputDidReset(surfaceID: surfaceID, streamToken: stalledChunk.streamToken) - let sawResetReplay = try await pollUntil { - await router.count(of: "mobile.terminal.replay") >= replayCountAfterMount + 1 - } - #expect(sawResetReplay, "reset must request an authoritative replay") + await router.waitForCount(of: "mobile.terminal.replay", atLeast: replayCountAfterMount + 1) let replayChunk = try #require(await iterator.next()) #expect(String(decoding: replayChunk.data, as: UTF8.self) == "first-replay") @@ -163,10 +159,7 @@ import Testing #expect(store.terminalReplayBarrierDroppedOutputSurfaceIDs.contains(surfaceID)) store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: replayChunk.streamToken) - let sawFollowUpReplay = try await pollUntil { - await router.count(of: "mobile.terminal.replay") >= replayCountAfterMount + 2 - } - #expect(sawFollowUpReplay, "live output dropped during a replay barrier must trigger a follow-up replay") + await router.waitForCount(of: "mobile.terminal.replay", atLeast: replayCountAfterMount + 2) let followUpChunk = try #require(await iterator.next()) #expect(String(decoding: followUpChunk.data, as: UTF8.self) == "follow-up-replay") @@ -190,8 +183,7 @@ import Testing await router.enqueueReplayTexts(["cold-replay", "retry-replay"]) var iterator = store.terminalOutputStream(surfaceID: surfaceID).makeAsyncIterator() - let sawMountReplay = try await pollUntil { await router.count(of: "mobile.terminal.replay") >= 1 } - #expect(sawMountReplay, "mounting a sink must arm the cold-attach replay") + await router.waitForCount(of: "mobile.terminal.replay", atLeast: 1) let coldReplayChunk = try #require(await iterator.next()) #expect(String(decoding: coldReplayChunk.data, as: UTF8.self) == "cold-replay") store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: coldReplayChunk.streamToken) @@ -205,12 +197,9 @@ import Testing ) #expect(firstDropAccepted == false) - let sawFailedReplay = try await pollUntil { - await router.count(of: "mobile.terminal.replay") >= replayCountAfterMount + 1 - } - #expect(sawFailedReplay, "first dropped live output should request a replay") + await router.waitForCount(of: "mobile.terminal.replay", atLeast: replayCountAfterMount + 1) - let failureSettled = try await pollUntil { + let failureSettled = await waitForReplayBarrierFailureToSettle { !store.terminalReplaySurfaceIDsInFlight.contains(surfaceID) && store.terminalReplayBarrierTokensBySurfaceID[surfaceID] == replayBarrierToken } @@ -222,10 +211,7 @@ import Testing surfaceID: surfaceID ) #expect(retryDropAccepted == false) - let sawRetryReplay = try await pollUntil { - await router.count(of: "mobile.terminal.replay") >= replayCountAfterMount + 2 - } - #expect(sawRetryReplay, "next dropped live output should retry the preserved barrier replay") + await router.waitForCount(of: "mobile.terminal.replay", atLeast: replayCountAfterMount + 2) let retryReplayChunk = try #require(await iterator.next()) #expect(String(decoding: retryReplayChunk.data, as: UTF8.self) == "retry-replay") @@ -234,6 +220,19 @@ import Testing #expect(!store.terminalReplayBarrierDroppedOutputSurfaceIDs.contains(surfaceID)) } +@MainActor +private func waitForReplayBarrierFailureToSettle( + _ condition: @MainActor () -> Bool +) async -> Bool { + for _ in 0..<1_000 { + if condition() { + return true + } + await Task.yield() + } + return condition() +} + @Test func terminalOutputQueueCoalescesReplaceableViewportFramesBehindBackpressure() { var queue = TerminalOutputDeliveryQueue() let inFlight = TerminalOutputDelivery(bytes: Data("in-flight".utf8), replaceable: false) diff --git a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift index c0ca741a9c71..9df59decd453 100644 --- a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift +++ b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift @@ -656,6 +656,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { var outputQueue = GhosttySurfaceWorkQueue(generation: 0) private var outputQueueGeneration: UInt64 = 0 private var pendingSurfaceFreeCount = 0 + private var renderPipelineRecoveryBlocked = false private static let maxPendingSurfaceFrees = 1 private static let renderPipelineStallDeadline: CFTimeInterval = 2.0 private static let outputApplyTimeout: CFTimeInterval = 2.0 @@ -2604,6 +2605,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { /// tree. Does not set ``isDismantled`` so a transient detach can re-attach /// and resume; only ``prepareForDismantle()`` marks the surface dead. private func prepareForReuseAfterDetach() { + renderPipelineRecoveryBlocked = false resignInput() stopKeyboardHeightAnimation() stopDisplayLink() @@ -2775,6 +2777,9 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { return false } guard pendingSurfaceFreeCount < Self.maxPendingSurfaceFrees else { + renderPipelineRecoveryBlocked = true + stopDisplayLink() + completePendingSurfaceOperations(returning: false) renderInFlight = false renderInFlightSince = nil needsAnotherRender = false @@ -2860,6 +2865,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { guard let app = runtime?.app else { return } surface = makeSurface(app: app) if let surface { + renderPipelineRecoveryBlocked = false GhosttySurfaceView.register(surface: surface, for: self) if let config = runtime?.config { applyBackgroundColorFromConfig(config) @@ -2874,7 +2880,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { } private func startDisplayLink() { - guard displayLink == nil else { return } + guard displayLink == nil, !renderPipelineRecoveryBlocked else { return } let link = CADisplayLink(target: DisplayLinkProxy(target: self), selector: #selector(DisplayLinkProxy.handleDisplayLink)) link.preferredFrameRateRange = CAFrameRateRange(minimum: 30, maximum: 120, preferred: 120) link.add(to: .main, forMode: .common) @@ -3901,7 +3907,8 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { view: view, grid: grid, font: Int(view.liveFontSize), - surface: surface + surface: surface, + generation: view.surfaceGeneration )) } if pending.isEmpty { @@ -3911,6 +3918,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { for item in pending { guard let section = await item.view.visibleSnapshotSection( surface: item.surface, + generation: item.generation, grid: item.grid, font: item.font ) else { @@ -3923,9 +3931,14 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { private func visibleSnapshotSection( surface: ghostty_surface_t, + generation: UInt64, grid: String, font: Int ) async -> String? { + guard self.surface == surface, + surfaceGeneration == generation else { + return nil + } await withCheckedContinuation { continuation in let operationID = makeSurfaceOperationID() if let existing = pendingVisibleSnapshot { @@ -3939,13 +3952,19 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { ) ensureSurfaceOperationDeadlinePump() let queue = outputQueue - let read = VisibleSnapshotRead(surface: surface, grid: grid, font: font) + let read = VisibleSnapshotRead(surface: surface, generation: generation, grid: grid, font: font) queue.async { let text = Self.surfaceText(read.surface, pointTag: GHOSTTY_POINT_VIEWPORT) ?? "(unavailable)" let section = "===== visible terminal · grid=\(read.grid) · font=\(read.font) =====\n" + text DispatchQueue.main.async { [weak self] in - self?.completePendingVisibleSnapshot(id: operationID, returning: section) + guard let view = self else { return } + guard view.surface == read.surface, + view.surfaceGeneration == read.generation else { + view.completePendingVisibleSnapshot(id: operationID, returning: nil) + return + } + view.completePendingVisibleSnapshot(id: operationID, returning: section) } } } @@ -4011,14 +4030,14 @@ extension GhosttySurfaceView: UIScrollViewDelegate { } } -private enum RenderPipelineRecoveryReplay { +nonisolated private enum RenderPipelineRecoveryReplay { case callerWillRequestReplay case delegateWhenNoCaller } /// One output/geometry operation awaiting either its output-queue completion or /// the display-link deadline that rebuilds the stalled render pipeline. -private struct PendingSurfaceOperation { +nonisolated private struct PendingSurfaceOperation { let id: UInt64 let startedAt: CFTimeInterval let byteCount: Int? @@ -4027,26 +4046,28 @@ private struct PendingSurfaceOperation { /// One visible-terminal snapshot read awaiting output-queue completion or its /// display-link deadline. A timeout skips only the diagnostic snapshot. -private struct PendingVisibleSnapshot { +nonisolated private struct PendingVisibleSnapshot { let id: UInt64 let startedAt: CFTimeInterval let continuation: CheckedContinuation } /// One surface's request for the bounded visible-terminal snapshot. -private struct VisibleSnapshotRequest { +nonisolated private struct VisibleSnapshotRequest { let view: GhosttySurfaceView let grid: String let font: Int let surface: ghostty_surface_t + let generation: UInt64 } /// Raw surface read payload captured by the off-main output queue. /// /// The C surface pointer is dereferenced only on `GhosttySurfaceWorkQueue`, /// which is the same FIFO queue that owns `process_output` and surface free. -private struct VisibleSnapshotRead: @unchecked Sendable { +nonisolated private struct VisibleSnapshotRead: @unchecked Sendable { let surface: ghostty_surface_t + let generation: UInt64 let grid: String let font: Int } From 45c89dab1311cc441a8bb373991029c357ad28ec Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 03:28:34 -0700 Subject: [PATCH 13/56] Fix replay ack reset recovery --- ...hellComposite+TerminalOutputDelivery.swift | 10 ++++- .../TerminalOutputDeliveryQueueTests.swift | 39 +++++++++++++++++++ .../GhosttySurfaceView.swift | 2 +- 3 files changed, 48 insertions(+), 3 deletions(-) diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift index b450ba472feb..43bf4199aa08 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift @@ -140,8 +140,14 @@ extension MobileShellComposite { public func terminalOutputDidReset(surfaceID: String, streamToken: UUID) { guard terminalOutputStreamTokensBySurfaceID[surfaceID] == streamToken, terminalOutputQueuesBySurfaceID[surfaceID] != nil else { return } - guard terminalReplayBarrierTokensBySurfaceID[surfaceID] == nil else { - MobileDebugLog.anchormux("terminal.output.reset_barrier_active surface=\(surfaceID)") + if terminalReplayBarrierTokensBySurfaceID[surfaceID] != nil { + guard terminalReplayBarrierAckStreamTokensBySurfaceID[surfaceID] == streamToken else { + MobileDebugLog.anchormux("terminal.output.reset_barrier_active surface=\(surfaceID)") + return + } + let replayBarrierToken = beginTerminalReplayBarrier(surfaceID: surfaceID) + MobileDebugLog.anchormux("terminal.output.reset_replay_ack surface=\(surfaceID)") + requestTerminalReplay(surfaceID: surfaceID, replayBarrierToken: replayBarrierToken) return } let replayBarrierToken = beginTerminalReplayBarrier(surfaceID: surfaceID) diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift index 68480b5ea7a0..5f5ce497e50d 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift @@ -220,6 +220,45 @@ import Testing #expect(!store.terminalReplayBarrierDroppedOutputSurfaceIDs.contains(surfaceID)) } +@MainActor +@Test func terminalReplayBarrierRetriesWhenReplayAckChunkResets() async throws { + let router = LivenessHostRouter() + let box = TransportBox() + let clock = TestClock() + let store = try await makeConnectedStore(router: router, box: box, clock: clock) + let surfaceID = "live-terminal" + + await router.enqueueReplayTexts(["cold-replay", "first-replay", "retry-replay"]) + var iterator = store.terminalOutputStream(surfaceID: surfaceID).makeAsyncIterator() + await router.waitForCount(of: "mobile.terminal.replay", atLeast: 1) + let coldReplayChunk = try #require(await iterator.next()) + #expect(String(decoding: coldReplayChunk.data, as: UTF8.self) == "cold-replay") + store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: coldReplayChunk.streamToken) + let replayCountAfterMount = await router.count(of: "mobile.terminal.replay") + + store.deliverTerminalBytes(Data("stalled-first".utf8), surfaceID: surfaceID) + let stalledChunk = try #require(await iterator.next()) + store.deliverTerminalBytes(Data("stale-second".utf8), surfaceID: surfaceID) + + store.terminalOutputDidReset(surfaceID: surfaceID, streamToken: stalledChunk.streamToken) + await router.waitForCount(of: "mobile.terminal.replay", atLeast: replayCountAfterMount + 1) + + let replayChunk = try #require(await iterator.next()) + #expect(String(decoding: replayChunk.data, as: UTF8.self) == "first-replay") + let firstBarrierToken = try #require(store.terminalReplayBarrierTokensBySurfaceID[surfaceID]) + + store.terminalOutputDidReset(surfaceID: surfaceID, streamToken: replayChunk.streamToken) + await router.waitForCount(of: "mobile.terminal.replay", atLeast: replayCountAfterMount + 2) + + let retryReplayChunk = try #require(await iterator.next()) + #expect(String(decoding: retryReplayChunk.data, as: UTF8.self) == "retry-replay") + let retryBarrierToken = try #require(store.terminalReplayBarrierTokensBySurfaceID[surfaceID]) + #expect(retryBarrierToken != firstBarrierToken) + + store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: retryReplayChunk.streamToken) + #expect(store.terminalReplayBarrierTokensBySurfaceID[surfaceID] == nil) +} + @MainActor private func waitForReplayBarrierFailureToSettle( _ condition: @MainActor () -> Bool diff --git a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift index 9df59decd453..374ccdaf04c3 100644 --- a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift +++ b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift @@ -3939,7 +3939,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { surfaceGeneration == generation else { return nil } - await withCheckedContinuation { continuation in + return await withCheckedContinuation { continuation in let operationID = makeSurfaceOperationID() if let existing = pendingVisibleSnapshot { pendingVisibleSnapshot = nil From 495693827a1fd922142b4dc4470f8ad7bcbec956 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 03:40:13 -0700 Subject: [PATCH 14/56] Track replay barrier drops by generation --- .github/swift-file-length-budget.tsv | 2 +- ...hellComposite+TerminalOutputDelivery.swift | 16 ++++-- .../MobileShellComposite.swift | 36 +++++++----- .../TerminalOutputDeliveryQueueTests.swift | 56 +++++++++++++++++++ 4 files changed, 91 insertions(+), 19 deletions(-) diff --git a/.github/swift-file-length-budget.tsv b/.github/swift-file-length-budget.tsv index 9e7c95027c3d..d98938b44b3d 100644 --- a/.github/swift-file-length-budget.tsv +++ b/.github/swift-file-length-budget.tsv @@ -14,7 +14,7 @@ 8016 CLI/cmux_open.swift 7366 cmuxTests/WorkspaceUnitTests.swift 7218 cmuxTests/WorkspaceRemoteConnectionTests.swift -7166 Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift +7174 Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift 6359 cmuxTests/SessionPersistenceTests.swift 6217 cmuxTests/GhosttyConfigTests.swift 6180 Sources/TabManager.swift diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift index 43bf4199aa08..46459fa73c2d 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift @@ -61,6 +61,8 @@ extension MobileShellComposite { if let replayBarrierToken = terminalReplayBarrierTokensBySurfaceID[surfaceID], !bypassReplayBarrier { terminalReplayBarrierDroppedOutputSurfaceIDs.insert(surfaceID) + let droppedOutputCount = (terminalReplayBarrierDroppedOutputCountsBySurfaceID[surfaceID] ?? 0) &+ 1 + terminalReplayBarrierDroppedOutputCountsBySurfaceID[surfaceID] = droppedOutputCount MobileDebugLog.anchormux("terminal.output.drop_replay_barrier surface=\(surfaceID)") if remoteClient != nil, terminalReplayBarrierAckStreamTokensBySurfaceID[surfaceID] == nil, @@ -69,7 +71,7 @@ extension MobileShellComposite { requestTerminalReplay( surfaceID: surfaceID, replayBarrierToken: replayBarrierToken, - coversReplayBarrierDroppedOutput: true + coveredReplayBarrierDroppedOutputCount: droppedOutputCount ) } return false @@ -107,12 +109,18 @@ extension MobileShellComposite { let next = queue.completeInFlight() terminalOutputQueuesBySurfaceID[surfaceID] = queue if terminalReplayBarrierAckStreamTokensBySurfaceID[surfaceID] == streamToken { - let replayAckCoversDroppedOutput = terminalReplayBarrierAckCoversDroppedOutputSurfaceIDs.remove(surfaceID) != nil + let coveredDroppedOutputCount = + terminalReplayBarrierAckCoveredDroppedOutputCountsBySurfaceID.removeValue(forKey: surfaceID) + let currentDroppedOutputCount = terminalReplayBarrierDroppedOutputCountsBySurfaceID[surfaceID] ?? 0 + let needsFollowUpReplay = coveredDroppedOutputCount.map { + currentDroppedOutputCount > $0 + } ?? true terminalReplayBarrierAckStreamTokensBySurfaceID.removeValue(forKey: surfaceID) terminalReplayBarrierTokensBySurfaceID.removeValue(forKey: surfaceID) MobileDebugLog.anchormux("terminal.output.replay_barrier_cleared surface=\(surfaceID)") - if terminalReplayBarrierDroppedOutputSurfaceIDs.remove(surfaceID) != nil, - !replayAckCoversDroppedOutput { + let droppedOutputDuringBarrier = terminalReplayBarrierDroppedOutputSurfaceIDs.remove(surfaceID) != nil + terminalReplayBarrierDroppedOutputCountsBySurfaceID.removeValue(forKey: surfaceID) + if droppedOutputDuringBarrier, needsFollowUpReplay { let replayBarrierToken = beginTerminalReplayBarrier(surfaceID: surfaceID) MobileDebugLog.anchormux("terminal.output.replay_followup surface=\(surfaceID)") requestTerminalReplay(surfaceID: surfaceID, replayBarrierToken: replayBarrierToken) diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift index 6c6eebdbb396..244b82e214fc 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift @@ -718,7 +718,8 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { var terminalReplayBarrierTokensBySurfaceID: [String: UUID] var terminalReplayBarrierAckStreamTokensBySurfaceID: [String: UUID] var terminalReplayBarrierDroppedOutputSurfaceIDs: Set - var terminalReplayBarrierAckCoversDroppedOutputSurfaceIDs: Set + var terminalReplayBarrierDroppedOutputCountsBySurfaceID: [String: UInt64] + var terminalReplayBarrierAckCoveredDroppedOutputCountsBySurfaceID: [String: UInt64] private var terminalOutputTransport: TerminalOutputTransport var terminalByteContinuationsBySurfaceID: [String: AsyncStream.Continuation] var terminalOutputStreamTokensBySurfaceID: [String: UUID] @@ -913,7 +914,8 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { self.terminalReplayBarrierTokensBySurfaceID = [:] self.terminalReplayBarrierAckStreamTokensBySurfaceID = [:] self.terminalReplayBarrierDroppedOutputSurfaceIDs = [] - self.terminalReplayBarrierAckCoversDroppedOutputSurfaceIDs = [] + self.terminalReplayBarrierDroppedOutputCountsBySurfaceID = [:] + self.terminalReplayBarrierAckCoveredDroppedOutputCountsBySurfaceID = [:] self.terminalOutputTransport = .rawBytes self.terminalByteContinuationsBySurfaceID = [:] self.terminalOutputStreamTokensBySurfaceID = [:] @@ -4944,7 +4946,8 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { terminalReplayBarrierTokensBySurfaceID = [:] terminalReplayBarrierAckStreamTokensBySurfaceID = [:] terminalReplayBarrierDroppedOutputSurfaceIDs = [] - terminalReplayBarrierAckCoversDroppedOutputSurfaceIDs = [] + terminalReplayBarrierDroppedOutputCountsBySurfaceID = [:] + terminalReplayBarrierAckCoveredDroppedOutputCountsBySurfaceID = [:] terminalOutputQueuesBySurfaceID = [:] terminalOutputStreamTokensBySurfaceID = terminalOutputStreamTokensBySurfaceID.mapValues { _ in UUID() } terminalScrollQueueTokensBySurfaceID = [:] @@ -6314,7 +6317,8 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { terminalReplayBarrierTokensBySurfaceID[surfaceID] = token terminalReplayBarrierAckStreamTokensBySurfaceID.removeValue(forKey: surfaceID) terminalReplayBarrierDroppedOutputSurfaceIDs.remove(surfaceID) - terminalReplayBarrierAckCoversDroppedOutputSurfaceIDs.remove(surfaceID) + terminalReplayBarrierDroppedOutputCountsBySurfaceID.removeValue(forKey: surfaceID) + terminalReplayBarrierAckCoveredDroppedOutputCountsBySurfaceID.removeValue(forKey: surfaceID) return token } @@ -6337,7 +6341,8 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { terminalReplayBarrierAckStreamTokensBySurfaceID.removeValue(forKey: surfaceID) terminalReplayBarrierTokensBySurfaceID.removeValue(forKey: surfaceID) terminalReplayBarrierDroppedOutputSurfaceIDs.remove(surfaceID) - terminalReplayBarrierAckCoversDroppedOutputSurfaceIDs.remove(surfaceID) + terminalReplayBarrierDroppedOutputCountsBySurfaceID.removeValue(forKey: surfaceID) + terminalReplayBarrierAckCoveredDroppedOutputCountsBySurfaceID.removeValue(forKey: surfaceID) MobileDebugLog.anchormux("terminal.output.replay_barrier_cleared_\(reason) surface=\(surfaceID)") return true } @@ -6438,7 +6443,8 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { terminalReplayBarrierTokensBySurfaceID.removeValue(forKey: surfaceID) terminalReplayBarrierAckStreamTokensBySurfaceID.removeValue(forKey: surfaceID) terminalReplayBarrierDroppedOutputSurfaceIDs.remove(surfaceID) - terminalReplayBarrierAckCoversDroppedOutputSurfaceIDs.remove(surfaceID) + terminalReplayBarrierDroppedOutputCountsBySurfaceID.removeValue(forKey: surfaceID) + terminalReplayBarrierAckCoveredDroppedOutputCountsBySurfaceID.removeValue(forKey: surfaceID) terminalScrollQueueTokensBySurfaceID.removeValue(forKey: surfaceID) terminalScrollQueuesBySurfaceID.removeValue(forKey: surfaceID) terminalScrollbackPrefetchStatesBySurfaceID.removeValue(forKey: surfaceID) @@ -6567,7 +6573,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { func requestTerminalReplay( surfaceID: String, replayBarrierToken: UUID? = nil, - coversReplayBarrierDroppedOutput: Bool = false + coveredReplayBarrierDroppedOutputCount: UInt64? = nil ) { guard let client = remoteClient else { clearTerminalReplayBarrierIfCurrent( @@ -6679,10 +6685,11 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { return } if self.terminalReplayBarrierAckStreamTokensBySurfaceID[surfaceID] != nil { - if coversReplayBarrierDroppedOutput { - self.terminalReplayBarrierAckCoversDroppedOutputSurfaceIDs.insert(surfaceID) + if let coveredReplayBarrierDroppedOutputCount { + self.terminalReplayBarrierAckCoveredDroppedOutputCountsBySurfaceID[surfaceID] = + coveredReplayBarrierDroppedOutputCount } else { - self.terminalReplayBarrierAckCoversDroppedOutputSurfaceIDs.remove(surfaceID) + self.terminalReplayBarrierAckCoveredDroppedOutputCountsBySurfaceID.removeValue(forKey: surfaceID) } } self.terminalActiveScreenBySurfaceID[surfaceID] = renderGrid.activeScreen @@ -6707,10 +6714,11 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { ) if accepted, self.terminalReplayBarrierAckStreamTokensBySurfaceID[surfaceID] != nil { - if coversReplayBarrierDroppedOutput { - self.terminalReplayBarrierAckCoversDroppedOutputSurfaceIDs.insert(surfaceID) + if let coveredReplayBarrierDroppedOutputCount { + self.terminalReplayBarrierAckCoveredDroppedOutputCountsBySurfaceID[surfaceID] = + coveredReplayBarrierDroppedOutputCount } else { - self.terminalReplayBarrierAckCoversDroppedOutputSurfaceIDs.remove(surfaceID) + self.terminalReplayBarrierAckCoveredDroppedOutputCountsBySurfaceID.removeValue(forKey: surfaceID) } } if accepted, let replaySeq { @@ -6724,7 +6732,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { ) } } catch { - mobileShellLog.error("CMUX_REPLAY failed surface=\(surfaceID, privacy: .public) error=\(String(describing: error), privacy: .public)") + mobileShellLog.error("CMUX_REPLAY failed surface=\(surfaceID, privacy: .public) error=\(String(describing: error), privacy: .private)") self.clearTerminalReplayBarrierIfCurrent( surfaceID: surfaceID, token: replayBarrierToken, diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift index 5f5ce497e50d..81e5e3f9198f 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift @@ -259,6 +259,62 @@ import Testing #expect(store.terminalReplayBarrierTokensBySurfaceID[surfaceID] == nil) } +@MainActor +@Test func terminalReplayBarrierRequestsFollowUpForDropAfterCoveredRetryResponse() async throws { + let router = LivenessHostRouter() + let box = TransportBox() + let clock = TestClock() + let store = try await makeConnectedStore(router: router, box: box, clock: clock) + let surfaceID = "live-terminal" + + await router.enqueueReplayTexts(["cold-replay", "retry-replay", "follow-up-replay"]) + var iterator = store.terminalOutputStream(surfaceID: surfaceID).makeAsyncIterator() + await router.waitForCount(of: "mobile.terminal.replay", atLeast: 1) + let coldReplayChunk = try #require(await iterator.next()) + #expect(String(decoding: coldReplayChunk.data, as: UTF8.self) == "cold-replay") + store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: coldReplayChunk.streamToken) + let replayCountAfterMount = await router.count(of: "mobile.terminal.replay") + + await router.failNextReplay() + let replayBarrierToken = store.beginTerminalReplayBarrier(surfaceID: surfaceID) + let firstDropAccepted = store.deliverTerminalBytes( + Data("live-during-failed-replay".utf8), + surfaceID: surfaceID + ) + #expect(firstDropAccepted == false) + await router.waitForCount(of: "mobile.terminal.replay", atLeast: replayCountAfterMount + 1) + + let failureSettled = await waitForReplayBarrierFailureToSettle { + !store.terminalReplaySurfaceIDsInFlight.contains(surfaceID) + && store.terminalReplayBarrierTokensBySurfaceID[surfaceID] == replayBarrierToken + } + #expect(failureSettled) + + let retryDropAccepted = store.deliverTerminalBytes( + Data("live-before-retry-request".utf8), + surfaceID: surfaceID + ) + #expect(retryDropAccepted == false) + await router.waitForCount(of: "mobile.terminal.replay", atLeast: replayCountAfterMount + 2) + + let retryReplayChunk = try #require(await iterator.next()) + #expect(String(decoding: retryReplayChunk.data, as: UTF8.self) == "retry-replay") + + let postResponseDropAccepted = store.deliverTerminalBytes( + Data("live-after-retry-response-before-ack".utf8), + surfaceID: surfaceID + ) + #expect(postResponseDropAccepted == false) + + store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: retryReplayChunk.streamToken) + await router.waitForCount(of: "mobile.terminal.replay", atLeast: replayCountAfterMount + 3) + + let followUpChunk = try #require(await iterator.next()) + #expect(String(decoding: followUpChunk.data, as: UTF8.self) == "follow-up-replay") + store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: followUpChunk.streamToken) + #expect(store.terminalReplayBarrierTokensBySurfaceID[surfaceID] == nil) +} + @MainActor private func waitForReplayBarrierFailureToSettle( _ condition: @MainActor () -> Bool From 089fc9ad8a403b563ac7962cf48a68c8e93669ea Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 03:48:38 -0700 Subject: [PATCH 15/56] Fix render recovery teardown lifecycle --- .github/swift-file-length-budget.tsv | 2 +- .../GhosttySurfaceView.swift | 39 +++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/.github/swift-file-length-budget.tsv b/.github/swift-file-length-budget.tsv index d98938b44b3d..365df9291c3d 100644 --- a/.github/swift-file-length-budget.tsv +++ b/.github/swift-file-length-budget.tsv @@ -24,8 +24,8 @@ 4487 Sources/Panels/FilePreviewPanel.swift 4483 Sources/cmuxApp.swift 4367 cmuxTests/BrowserPanelTests.swift +4126 Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift 4121 Sources/BrowserWindowPortal.swift -4087 Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift 3934 Sources/Feed/FeedPanelView.swift 3926 cmuxTests/TabManagerUnitTests.swift 3896 cmuxTests/WindowAndDragTests.swift diff --git a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift index 374ccdaf04c3..be959dae8388 100644 --- a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift +++ b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift @@ -657,6 +657,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { private var outputQueueGeneration: UInt64 = 0 private var pendingSurfaceFreeCount = 0 private var renderPipelineRecoveryBlocked = false + private var deferredRenderPipelineRecovery: DeferredRenderPipelineRecovery? private static let maxPendingSurfaceFrees = 1 private static let renderPipelineStallDeadline: CFTimeInterval = 2.0 private static let outputApplyTimeout: CFTimeInterval = 2.0 @@ -2606,6 +2607,11 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { /// and resume; only ``prepareForDismantle()`` marks the surface dead. private func prepareForReuseAfterDetach() { renderPipelineRecoveryBlocked = false + deferredRenderPipelineRecovery = nil + completePendingSurfaceOperations(returning: false) + renderInFlight = false + renderInFlightSince = nil + needsAnotherRender = false resignInput() stopKeyboardHeightAnimation() stopDisplayLink() @@ -2778,6 +2784,11 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { } guard pendingSurfaceFreeCount < Self.maxPendingSurfaceFrees else { renderPipelineRecoveryBlocked = true + deferredRenderPipelineRecovery = DeferredRenderPipelineRecovery( + reason: reason, + stalledMs: stalledMs, + replay: replay + ) stopDisplayLink() completePendingSurfaceOperations(returning: false) renderInFlight = false @@ -2807,11 +2818,13 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { MobileDebugLog.anchormux( "render.recover.free_drained pendingFrees=\(self.pendingSurfaceFreeCount)" ) + self.retryDeferredRenderPipelineRecoveryIfNeeded() } } oldBridge.detach() surface = nil + deferredRenderPipelineRecovery = nil renderInFlight = false renderInFlightSince = nil needsAnotherRender = false @@ -2833,6 +2846,25 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { return true } + private func retryDeferredRenderPipelineRecoveryIfNeeded() { + guard pendingSurfaceFreeCount < Self.maxPendingSurfaceFrees, + let deferred = deferredRenderPipelineRecovery else { return } + deferredRenderPipelineRecovery = nil + renderPipelineRecoveryBlocked = false + guard !isDismantled, surface != nil else { + completePendingSurfaceOperations(returning: false) + return + } + MobileDebugLog.anchormux( + "render.recover.retry_after_free reason=\(deferred.reason) stalledMs=\(deferred.stalledMs)" + ) + _ = recoverRenderPipeline( + reason: deferred.reason, + stalledMs: deferred.stalledMs, + replay: deferred.replay + ) + } + private var preferredScreenScale: CGFloat { if let screen = window?.windowScene?.screen { return screen.scale @@ -4035,6 +4067,13 @@ nonisolated private enum RenderPipelineRecoveryReplay { case delegateWhenNoCaller } +/// Recovery request deferred only while an older surface free is still draining. +nonisolated private struct DeferredRenderPipelineRecovery { + let reason: String + let stalledMs: Int + let replay: RenderPipelineRecoveryReplay +} + /// One output/geometry operation awaiting either its output-queue completion or /// the display-link deadline that rebuilds the stalled render pipeline. nonisolated private struct PendingSurfaceOperation { From 57e3b8e7f705f84d9529922c0b6dd269876a56d4 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 03:56:19 -0700 Subject: [PATCH 16/56] Add replay failure retry regression test --- .../TerminalOutputDeliveryQueueTests.swift | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift index 81e5e3f9198f..2d8ac0f538b4 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift @@ -220,6 +220,42 @@ import Testing #expect(!store.terminalReplayBarrierDroppedOutputSurfaceIDs.contains(surfaceID)) } +@MainActor +@Test func terminalReplayBarrierRetriesAfterReplayFailureWithoutDroppedOutput() async throws { + let router = LivenessHostRouter() + let box = TransportBox() + let clock = TestClock() + let store = try await makeConnectedStore(router: router, box: box, clock: clock) + let surfaceID = "live-terminal" + + await router.enqueueReplayTexts(["cold-replay", "retry-replay"]) + var iterator = store.terminalOutputStream(surfaceID: surfaceID).makeAsyncIterator() + await router.waitForCount(of: "mobile.terminal.replay", atLeast: 1) + let coldReplayChunk = try #require(await iterator.next()) + #expect(String(decoding: coldReplayChunk.data, as: UTF8.self) == "cold-replay") + store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: coldReplayChunk.streamToken) + let replayCountAfterMount = await router.count(of: "mobile.terminal.replay") + + await router.failNextReplay() + store.deliverTerminalBytes(Data("stalled-first".utf8), surfaceID: surfaceID) + let stalledChunk = try #require(await iterator.next()) + store.terminalOutputDidReset(surfaceID: surfaceID, streamToken: stalledChunk.streamToken) + + let retryRequested = await waitForReplayRequestCount( + router, + atLeast: replayCountAfterMount + 2 + ) + #expect(retryRequested, "failed reset replay must retry even without a later live-output drop") + #expect(store.terminalReplayBarrierTokensBySurfaceID[surfaceID] != nil) + + if retryRequested { + let retryReplayChunk = try #require(await iterator.next()) + #expect(String(decoding: retryReplayChunk.data, as: UTF8.self) == "retry-replay") + store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: retryReplayChunk.streamToken) + #expect(store.terminalReplayBarrierTokensBySurfaceID[surfaceID] == nil) + } +} + @MainActor @Test func terminalReplayBarrierRetriesWhenReplayAckChunkResets() async throws { let router = LivenessHostRouter() @@ -328,6 +364,19 @@ private func waitForReplayBarrierFailureToSettle( return condition() } +private func waitForReplayRequestCount( + _ router: LivenessHostRouter, + atLeast expectedCount: Int +) async -> Bool { + for _ in 0..<1_000 { + if await router.count(of: "mobile.terminal.replay") >= expectedCount { + return true + } + await Task.yield() + } + return await router.count(of: "mobile.terminal.replay") >= expectedCount +} + @Test func terminalOutputQueueCoalescesReplaceableViewportFramesBehindBackpressure() { var queue = TerminalOutputDeliveryQueue() let inFlight = TerminalOutputDelivery(bytes: Data("in-flight".utf8), replaceable: false) From 245f7cb83373be7541fa5a92630abc0f8eb6329b Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 04:02:29 -0700 Subject: [PATCH 17/56] Retry failed terminal replay barriers --- .github/swift-file-length-budget.tsv | 3 +- .../MobileShellComposite.swift | 64 +++++++++++++++++-- .../TerminalOutputDeliveryQueueTests.swift | 19 ++---- 3 files changed, 65 insertions(+), 21 deletions(-) diff --git a/.github/swift-file-length-budget.tsv b/.github/swift-file-length-budget.tsv index 365df9291c3d..a4231d477b7a 100644 --- a/.github/swift-file-length-budget.tsv +++ b/.github/swift-file-length-budget.tsv @@ -13,8 +13,8 @@ 8032 Sources/Panels/BrowserPanelView.swift 8016 CLI/cmux_open.swift 7366 cmuxTests/WorkspaceUnitTests.swift +7224 Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift 7218 cmuxTests/WorkspaceRemoteConnectionTests.swift -7174 Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift 6359 cmuxTests/SessionPersistenceTests.swift 6217 cmuxTests/GhosttyConfigTests.swift 6180 Sources/TabManager.swift @@ -223,6 +223,7 @@ 525 Packages/macOS/CmuxSettings/Sources/CmuxSettings/SocketControl/SocketControlSettings.swift 524 CLI/CMUXCLI+AutoNaming.swift 524 Sources/TerminalPaneDropTargetView.swift +522 Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift 522 Packages/macOS/CmuxControlSocket/Sources/CmuxControlSocket/Coordinator/Surface/ControlCommandCoordinator+Surface.swift 520 CLI/CMUXCLI+AmpExtension.swift 520 cmuxTests/MainWindowVisibilityControllerTests.swift diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift index 244b82e214fc..c721fc03b123 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift @@ -34,6 +34,8 @@ public typealias CMUXMobileShellStore = MobileShellComposite @MainActor @Observable public final class MobileShellComposite: MobileTerminalOutputSinking { + private static let maxTerminalReplayFailureRetries = 2 + private enum TerminalOutputTransport: Equatable { case hybrid case renderGrid @@ -720,6 +722,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { var terminalReplayBarrierDroppedOutputSurfaceIDs: Set var terminalReplayBarrierDroppedOutputCountsBySurfaceID: [String: UInt64] var terminalReplayBarrierAckCoveredDroppedOutputCountsBySurfaceID: [String: UInt64] + private var terminalReplayFailureRetryCountsBySurfaceID: [String: Int] private var terminalOutputTransport: TerminalOutputTransport var terminalByteContinuationsBySurfaceID: [String: AsyncStream.Continuation] var terminalOutputStreamTokensBySurfaceID: [String: UUID] @@ -916,6 +919,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { self.terminalReplayBarrierDroppedOutputSurfaceIDs = [] self.terminalReplayBarrierDroppedOutputCountsBySurfaceID = [:] self.terminalReplayBarrierAckCoveredDroppedOutputCountsBySurfaceID = [:] + self.terminalReplayFailureRetryCountsBySurfaceID = [:] self.terminalOutputTransport = .rawBytes self.terminalByteContinuationsBySurfaceID = [:] self.terminalOutputStreamTokensBySurfaceID = [:] @@ -4948,6 +4952,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { terminalReplayBarrierDroppedOutputSurfaceIDs = [] terminalReplayBarrierDroppedOutputCountsBySurfaceID = [:] terminalReplayBarrierAckCoveredDroppedOutputCountsBySurfaceID = [:] + terminalReplayFailureRetryCountsBySurfaceID = [:] terminalOutputQueuesBySurfaceID = [:] terminalOutputStreamTokensBySurfaceID = terminalOutputStreamTokensBySurfaceID.mapValues { _ in UUID() } terminalScrollQueueTokensBySurfaceID = [:] @@ -6319,6 +6324,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { terminalReplayBarrierDroppedOutputSurfaceIDs.remove(surfaceID) terminalReplayBarrierDroppedOutputCountsBySurfaceID.removeValue(forKey: surfaceID) terminalReplayBarrierAckCoveredDroppedOutputCountsBySurfaceID.removeValue(forKey: surfaceID) + terminalReplayFailureRetryCountsBySurfaceID.removeValue(forKey: surfaceID) return token } @@ -6343,10 +6349,34 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { terminalReplayBarrierDroppedOutputSurfaceIDs.remove(surfaceID) terminalReplayBarrierDroppedOutputCountsBySurfaceID.removeValue(forKey: surfaceID) terminalReplayBarrierAckCoveredDroppedOutputCountsBySurfaceID.removeValue(forKey: surfaceID) + terminalReplayFailureRetryCountsBySurfaceID.removeValue(forKey: surfaceID) MobileDebugLog.anchormux("terminal.output.replay_barrier_cleared_\(reason) surface=\(surfaceID)") return true } + private func prepareTerminalReplayFailureRetry( + surfaceID: String, + replayBarrierToken: UUID? + ) -> UUID? { + guard let replayBarrierToken, + hasTerminalOutputSink(surfaceID: surfaceID), + terminalReplayBarrierTokensBySurfaceID[surfaceID] == replayBarrierToken else { + return nil + } + let retryCount = terminalReplayFailureRetryCountsBySurfaceID[surfaceID] ?? 0 + guard retryCount < Self.maxTerminalReplayFailureRetries else { + MobileDebugLog.anchormux( + "CMUX_REPLAY retry_exhausted surface=\(surfaceID) attempts=\(retryCount)" + ) + return nil + } + terminalReplayFailureRetryCountsBySurfaceID[surfaceID] = retryCount + 1 + MobileDebugLog.anchormux( + "CMUX_REPLAY retry_after_failure surface=\(surfaceID) attempt=\(retryCount + 1)" + ) + return replayBarrierToken + } + private enum RenderGridEventDeliveryDecision { case deliver case advisory(requestReplay: Bool, updateTrackedScreen: Bool, deliverViewportPolicy: Bool) @@ -6445,6 +6475,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { terminalReplayBarrierDroppedOutputSurfaceIDs.remove(surfaceID) terminalReplayBarrierDroppedOutputCountsBySurfaceID.removeValue(forKey: surfaceID) terminalReplayBarrierAckCoveredDroppedOutputCountsBySurfaceID.removeValue(forKey: surfaceID) + terminalReplayFailureRetryCountsBySurfaceID.removeValue(forKey: surfaceID) terminalScrollQueueTokensBySurfaceID.removeValue(forKey: surfaceID) terminalScrollQueuesBySurfaceID.removeValue(forKey: surfaceID) terminalScrollbackPrefetchStatesBySurfaceID.removeValue(forKey: surfaceID) @@ -6611,7 +6642,18 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { terminalReplaySurfaceIDsInFlight.insert(surfaceID) Task { @MainActor [weak self] in guard let self else { return } - defer { self.terminalReplaySurfaceIDsInFlight.remove(surfaceID) } + var retryReplayBarrierToken: UUID? + var retryCoveredDroppedOutputCount: UInt64? + defer { + self.terminalReplaySurfaceIDsInFlight.remove(surfaceID) + if let retryReplayBarrierToken { + self.requestTerminalReplay( + surfaceID: surfaceID, + replayBarrierToken: retryReplayBarrierToken, + coveredReplayBarrierDroppedOutputCount: retryCoveredDroppedOutputCount + ) + } + } do { let request = try MobileCoreRPCClient.requestData( method: "mobile.terminal.replay", @@ -6733,18 +6775,26 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { } } catch { mobileShellLog.error("CMUX_REPLAY failed surface=\(surfaceID, privacy: .public) error=\(String(describing: error), privacy: .private)") + guard self.remoteClient === client else { return } + // The replay request is the view-only/foreground-resume path. A + // definitive auth failure here (after the RPC layer's + // force-refresh-and-retry already gave up) must drive the re-auth + // prompt instead of silently leaving a stale frame. + guard !self.disconnectForAuthorizationFailureIfNeeded(error) else { return } + if let retryToken = self.prepareTerminalReplayFailureRetry( + surfaceID: surfaceID, + replayBarrierToken: replayBarrierToken + ) { + retryReplayBarrierToken = retryToken + retryCoveredDroppedOutputCount = coveredReplayBarrierDroppedOutputCount + return + } self.clearTerminalReplayBarrierIfCurrent( surfaceID: surfaceID, token: replayBarrierToken, reason: "failed", preserveDroppedOutput: true ) - // The replay request is the view-only/foreground-resume path. A - // definitive auth failure here (after the RPC layer's - // force-refresh-and-retry already gave up) must drive the re-auth - // prompt instead of silently leaving a stale frame. - guard self.remoteClient === client else { return } - _ = self.disconnectForAuthorizationFailureIfNeeded(error) } } } diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift index 2d8ac0f538b4..8ca069c2cb27 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift @@ -190,7 +190,7 @@ import Testing let replayCountAfterMount = await router.count(of: "mobile.terminal.replay") await router.failNextReplay() - let replayBarrierToken = store.beginTerminalReplayBarrier(surfaceID: surfaceID) + _ = store.beginTerminalReplayBarrier(surfaceID: surfaceID) let firstDropAccepted = store.deliverTerminalBytes( Data("live-during-failed-replay".utf8), surfaceID: surfaceID @@ -199,19 +199,12 @@ import Testing await router.waitForCount(of: "mobile.terminal.replay", atLeast: replayCountAfterMount + 1) - let failureSettled = await waitForReplayBarrierFailureToSettle { - !store.terminalReplaySurfaceIDsInFlight.contains(surfaceID) - && store.terminalReplayBarrierTokensBySurfaceID[surfaceID] == replayBarrierToken - } - #expect(failureSettled, "failed replay with dropped output must keep the barrier active") - #expect(store.terminalReplayBarrierDroppedOutputSurfaceIDs.contains(surfaceID)) - - let retryDropAccepted = store.deliverTerminalBytes( - Data("live-after-failed-replay".utf8), - surfaceID: surfaceID + let retryRequested = await waitForReplayRequestCount( + router, + atLeast: replayCountAfterMount + 2 ) - #expect(retryDropAccepted == false) - await router.waitForCount(of: "mobile.terminal.replay", atLeast: replayCountAfterMount + 2) + #expect(retryRequested, "failed replay with dropped output must request a replacement replay") + #expect(store.terminalReplayBarrierDroppedOutputSurfaceIDs.contains(surfaceID)) let retryReplayChunk = try #require(await iterator.next()) #expect(String(decoding: retryReplayChunk.data, as: UTF8.self) == "retry-replay") From cd15d203a5a678bf2911dad193b4b3b0a5e0de4b Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 04:05:13 -0700 Subject: [PATCH 18/56] Stabilize replay failure retry scheduling --- .github/swift-file-length-budget.tsv | 4 ++-- .../MobileShellComposite.swift | 21 +++++++++---------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/.github/swift-file-length-budget.tsv b/.github/swift-file-length-budget.tsv index 218ff159d8d9..378d0388fb91 100644 --- a/.github/swift-file-length-budget.tsv +++ b/.github/swift-file-length-budget.tsv @@ -13,7 +13,7 @@ 8032 Sources/Panels/BrowserPanelView.swift 8016 CLI/cmux_open.swift 7366 cmuxTests/WorkspaceUnitTests.swift -7224 Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift +7223 Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift 7218 cmuxTests/WorkspaceRemoteConnectionTests.swift 6359 cmuxTests/SessionPersistenceTests.swift 6217 cmuxTests/GhosttyConfigTests.swift @@ -127,9 +127,9 @@ 762 Packages/iOS/CmuxMobileTransport/Sources/CmuxMobileTransport/CmxNetworkByteTransport.swift 761 Sources/Mobile/AgentChat/AgentChatSessionRegistry.swift 760 Packages/macOS/CMUXAgentLaunch/Tests/CMUXAgentLaunchTests/AgentLaunchSanitizerTests.swift -756 Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/WorkspaceDetailView.swift 756 Sources/Panels/AgentSessionWebRendererCoordinator.swift 754 Sources/TerminalController+ControlWorkspaceContext.swift +753 Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/WorkspaceDetailView.swift 752 cmuxUITests/CloseWorkspaceCmdDUITests.swift 739 cmuxTests/CLICodexHookTimeoutRegressionTests.swift 738 Packages/macOS/CMUXProjectModel/Sources/CMUXProjectModel/XcodeProjectAdapter.swift diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift index c721fc03b123..264fb5ac75ca 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift @@ -6642,16 +6642,10 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { terminalReplaySurfaceIDsInFlight.insert(surfaceID) Task { @MainActor [weak self] in guard let self else { return } - var retryReplayBarrierToken: UUID? - var retryCoveredDroppedOutputCount: UInt64? + var transferredInFlightToRetry = false defer { - self.terminalReplaySurfaceIDsInFlight.remove(surfaceID) - if let retryReplayBarrierToken { - self.requestTerminalReplay( - surfaceID: surfaceID, - replayBarrierToken: retryReplayBarrierToken, - coveredReplayBarrierDroppedOutputCount: retryCoveredDroppedOutputCount - ) + if !transferredInFlightToRetry { + self.terminalReplaySurfaceIDsInFlight.remove(surfaceID) } } do { @@ -6785,8 +6779,13 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { surfaceID: surfaceID, replayBarrierToken: replayBarrierToken ) { - retryReplayBarrierToken = retryToken - retryCoveredDroppedOutputCount = coveredReplayBarrierDroppedOutputCount + self.terminalReplaySurfaceIDsInFlight.remove(surfaceID) + transferredInFlightToRetry = true + self.requestTerminalReplay( + surfaceID: surfaceID, + replayBarrierToken: retryToken, + coveredReplayBarrierDroppedOutputCount: coveredReplayBarrierDroppedOutputCount + ) return } self.clearTerminalReplayBarrierIfCurrent( From 334449b9b309f3c5a1ffa23f01af0bbb09bc553a Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 04:24:51 -0700 Subject: [PATCH 19/56] Address terminal replay recovery review findings --- .github/swift-file-length-budget.tsv | 6 +- .../MobileShellComposite.swift | 67 ++++++++++++++++--- .../TerminalOutputDeliveryQueueTests.swift | 4 ++ .../GhosttySurfaceView.swift | 43 +++++++++--- 4 files changed, 101 insertions(+), 19 deletions(-) diff --git a/.github/swift-file-length-budget.tsv b/.github/swift-file-length-budget.tsv index 378d0388fb91..c4518cc347ed 100644 --- a/.github/swift-file-length-budget.tsv +++ b/.github/swift-file-length-budget.tsv @@ -13,7 +13,7 @@ 8032 Sources/Panels/BrowserPanelView.swift 8016 CLI/cmux_open.swift 7366 cmuxTests/WorkspaceUnitTests.swift -7223 Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift +7274 Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift 7218 cmuxTests/WorkspaceRemoteConnectionTests.swift 6359 cmuxTests/SessionPersistenceTests.swift 6217 cmuxTests/GhosttyConfigTests.swift @@ -24,7 +24,7 @@ 4487 Sources/Panels/FilePreviewPanel.swift 4483 Sources/cmuxApp.swift 4367 cmuxTests/BrowserPanelTests.swift -4126 Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift +4153 Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift 4121 Sources/BrowserWindowPortal.swift 3934 Sources/Feed/FeedPanelView.swift 3926 cmuxTests/TabManagerUnitTests.swift @@ -219,11 +219,11 @@ 528 cmuxUITests/AutomationSocketUITests.swift 527 CLI/CLISocketPathResolver.swift 527 cmuxTests/BrowserHTTPBasicAuthPromptTests.swift +526 Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift 526 Packages/macOS/CMUXAgentLaunch/Sources/CMUXAgentLaunch/Workstream/WorkstreamStore.swift 525 Packages/macOS/CmuxSettings/Sources/CmuxSettings/SocketControl/SocketControlSettings.swift 524 CLI/CMUXCLI+AutoNaming.swift 524 Sources/TerminalPaneDropTargetView.swift -522 Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift 522 Packages/macOS/CmuxControlSocket/Sources/CmuxControlSocket/Coordinator/Surface/ControlCommandCoordinator+Surface.swift 520 CLI/CMUXCLI+AmpExtension.swift 520 cmuxTests/MainWindowVisibilityControllerTests.swift diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift index 264fb5ac75ca..565c5fc869ab 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift @@ -717,6 +717,8 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { private var pendingTerminalByteEndSeqBySurfaceID: [String: UInt64] private var terminalActiveScreenBySurfaceID: [String: MobileTerminalRenderGridFrame.Screen] var terminalReplaySurfaceIDsInFlight: Set + private var terminalReplayRequestIDsInFlightBySurfaceID: [String: UUID] + private var terminalReplayBarrierTokensInFlightBySurfaceID: [String: UUID] var terminalReplayBarrierTokensBySurfaceID: [String: UUID] var terminalReplayBarrierAckStreamTokensBySurfaceID: [String: UUID] var terminalReplayBarrierDroppedOutputSurfaceIDs: Set @@ -914,6 +916,8 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { self.pendingTerminalByteEndSeqBySurfaceID = [:] self.terminalActiveScreenBySurfaceID = [:] self.terminalReplaySurfaceIDsInFlight = [] + self.terminalReplayRequestIDsInFlightBySurfaceID = [:] + self.terminalReplayBarrierTokensInFlightBySurfaceID = [:] self.terminalReplayBarrierTokensBySurfaceID = [:] self.terminalReplayBarrierAckStreamTokensBySurfaceID = [:] self.terminalReplayBarrierDroppedOutputSurfaceIDs = [] @@ -4947,6 +4951,8 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { pendingTerminalByteEndSeqBySurfaceID = [:] terminalActiveScreenBySurfaceID = [:] terminalReplaySurfaceIDsInFlight = [] + terminalReplayRequestIDsInFlightBySurfaceID = [:] + terminalReplayBarrierTokensInFlightBySurfaceID = [:] terminalReplayBarrierTokensBySurfaceID = [:] terminalReplayBarrierAckStreamTokensBySurfaceID = [:] terminalReplayBarrierDroppedOutputSurfaceIDs = [] @@ -6325,6 +6331,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { terminalReplayBarrierDroppedOutputCountsBySurfaceID.removeValue(forKey: surfaceID) terminalReplayBarrierAckCoveredDroppedOutputCountsBySurfaceID.removeValue(forKey: surfaceID) terminalReplayFailureRetryCountsBySurfaceID.removeValue(forKey: surfaceID) + terminalReplayBarrierTokensInFlightBySurfaceID.removeValue(forKey: surfaceID) return token } @@ -6350,6 +6357,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { terminalReplayBarrierDroppedOutputCountsBySurfaceID.removeValue(forKey: surfaceID) terminalReplayBarrierAckCoveredDroppedOutputCountsBySurfaceID.removeValue(forKey: surfaceID) terminalReplayFailureRetryCountsBySurfaceID.removeValue(forKey: surfaceID) + terminalReplayBarrierTokensInFlightBySurfaceID.removeValue(forKey: surfaceID) MobileDebugLog.anchormux("terminal.output.replay_barrier_cleared_\(reason) surface=\(surfaceID)") return true } @@ -6377,6 +6385,27 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { return replayBarrierToken } + private func markTerminalReplayInFlight( + surfaceID: String, + requestID: UUID, + replayBarrierToken: UUID? + ) { + terminalReplaySurfaceIDsInFlight.insert(surfaceID) + terminalReplayRequestIDsInFlightBySurfaceID[surfaceID] = requestID + if let replayBarrierToken { + terminalReplayBarrierTokensInFlightBySurfaceID[surfaceID] = replayBarrierToken + } else { + terminalReplayBarrierTokensInFlightBySurfaceID.removeValue(forKey: surfaceID) + } + } + + private func clearTerminalReplayInFlightIfCurrent(surfaceID: String, requestID: UUID) { + guard terminalReplayRequestIDsInFlightBySurfaceID[surfaceID] == requestID else { return } + terminalReplaySurfaceIDsInFlight.remove(surfaceID) + terminalReplayRequestIDsInFlightBySurfaceID.removeValue(forKey: surfaceID) + terminalReplayBarrierTokensInFlightBySurfaceID.removeValue(forKey: surfaceID) + } + private enum RenderGridEventDeliveryDecision { case deliver case advisory(requestReplay: Bool, updateTrackedScreen: Bool, deliverViewportPolicy: Bool) @@ -6476,6 +6505,8 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { terminalReplayBarrierDroppedOutputCountsBySurfaceID.removeValue(forKey: surfaceID) terminalReplayBarrierAckCoveredDroppedOutputCountsBySurfaceID.removeValue(forKey: surfaceID) terminalReplayFailureRetryCountsBySurfaceID.removeValue(forKey: surfaceID) + terminalReplayRequestIDsInFlightBySurfaceID.removeValue(forKey: surfaceID) + terminalReplayBarrierTokensInFlightBySurfaceID.removeValue(forKey: surfaceID) terminalScrollQueueTokensBySurfaceID.removeValue(forKey: surfaceID) terminalScrollQueuesBySurfaceID.removeValue(forKey: surfaceID) terminalScrollbackPrefetchStatesBySurfaceID.removeValue(forKey: surfaceID) @@ -6633,19 +6664,36 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { MobileDebugLog.anchormux("CMUX_REPLAY skip surface=\(surfaceID) reason=barrier_active") return } - guard replayBarrierToken != nil || !terminalReplaySurfaceIDsInFlight.contains(surfaceID) else { - #if DEBUG - mobileShellLog.info("CMUX_REPLAY skip surface=\(surfaceID, privacy: .public) reason=in_flight") - #endif - return + if let replayBarrierToken { + guard terminalReplayBarrierTokensInFlightBySurfaceID[surfaceID] != replayBarrierToken else { + #if DEBUG + mobileShellLog.info("CMUX_REPLAY skip surface=\(surfaceID, privacy: .public) reason=barrier_in_flight") + #endif + return + } + } else { + guard !terminalReplaySurfaceIDsInFlight.contains(surfaceID) else { + #if DEBUG + mobileShellLog.info("CMUX_REPLAY skip surface=\(surfaceID, privacy: .public) reason=in_flight") + #endif + return + } } - terminalReplaySurfaceIDsInFlight.insert(surfaceID) + let replayRequestID = UUID() + markTerminalReplayInFlight( + surfaceID: surfaceID, + requestID: replayRequestID, + replayBarrierToken: replayBarrierToken + ) Task { @MainActor [weak self] in guard let self else { return } var transferredInFlightToRetry = false defer { if !transferredInFlightToRetry { - self.terminalReplaySurfaceIDsInFlight.remove(surfaceID) + self.clearTerminalReplayInFlightIfCurrent( + surfaceID: surfaceID, + requestID: replayRequestID + ) } } do { @@ -6779,7 +6827,10 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { surfaceID: surfaceID, replayBarrierToken: replayBarrierToken ) { - self.terminalReplaySurfaceIDsInFlight.remove(surfaceID) + self.clearTerminalReplayInFlightIfCurrent( + surfaceID: surfaceID, + requestID: replayRequestID + ) transferredInFlightToRetry = true self.requestTerminalReplay( surfaceID: surfaceID, diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift index 8ca069c2cb27..84533c89ca04 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift @@ -139,6 +139,10 @@ import Testing let coldReplayChunk = try #require(await iterator.next()) #expect(String(decoding: coldReplayChunk.data, as: UTF8.self) == "cold-replay") store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: coldReplayChunk.streamToken) + let coldReplaySettled = await waitForReplayBarrierFailureToSettle { + !store.terminalReplaySurfaceIDsInFlight.contains(surfaceID) + } + #expect(coldReplaySettled, "cold mount replay must settle before starting the held replay") let replayCountAfterMount = await router.count(of: "mobile.terminal.replay") store.deliverTerminalBytes(Data("stalled-first".utf8), surfaceID: surfaceID) diff --git a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift index be959dae8388..ca599a99cc7a 100644 --- a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift +++ b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift @@ -666,6 +666,9 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { private var pendingOutputApply: PendingSurfaceOperation? private var pendingGeometryApply: PendingSurfaceOperation? private var pendingVisibleSnapshot: PendingVisibleSnapshot? + private var hasPendingSurfaceOperationDeadline: Bool { + pendingOutputApply != nil || pendingGeometryApply != nil || pendingVisibleSnapshot != nil + } private static let scrollMechanicsContentHeight: CGFloat = 1_000_000 private var scrollMechanicsIsRecentering = false private var lastScrollMechanicsOffsetY: CGFloat? @@ -2052,7 +2055,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { /// would pump it, so apply immediately. private func scheduleDisplayLinkWork() { needsDraw = true - if displayLink == nil { + if displayLink == nil, !renderPipelineRecoveryBlocked { applyPendingFontSizeIfNeeded() } } @@ -2436,6 +2439,11 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { completion?(true) return } + // A blocked recovery will rebuild this surface once the prior free drains. + // Leave wait-style callers pending so recovery or the deadline pump resumes false. + guard !renderPipelineRecoveryBlocked else { + return + } #if DEBUG if lastInputTimestamp > 0 { let elapsed = (CACurrentMediaTime() - lastInputTimestamp) * 1000.0 @@ -2912,11 +2920,13 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { } private func startDisplayLink() { - guard displayLink == nil, !renderPipelineRecoveryBlocked else { return } + guard displayLink == nil else { return } + guard !renderPipelineRecoveryBlocked || hasPendingSurfaceOperationDeadline else { return } let link = CADisplayLink(target: DisplayLinkProxy(target: self), selector: #selector(DisplayLinkProxy.handleDisplayLink)) link.preferredFrameRateRange = CAFrameRateRange(minimum: 30, maximum: 120, preferred: 120) link.add(to: .main, forMode: .common) displayLink = link + guard !renderPipelineRecoveryBlocked else { return } cursorBlinkState.start(now: CACurrentMediaTime()) needsDraw = true updateCursorOverlay() @@ -2937,8 +2947,22 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { } @objc func handleDisplayLinkFire() { - guard let surface else { return } let now = CACurrentMediaTime() + if checkSurfaceOperationDeadlines(now: now) { + return + } + if renderPipelineRecoveryBlocked { + if !hasPendingSurfaceOperationDeadline { + stopDisplayLink() + } + return + } + guard surface != nil else { + if !hasPendingSurfaceOperationDeadline { + stopDisplayLink() + } + return + } #if DEBUG // Main-thread liveness heartbeat + presented-surface state. Time-gated, // no behavior change. The `contents`/size fields let an IDLE blank be @@ -2965,9 +2989,6 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { ) } #endif - if checkSurfaceOperationDeadlines(now: now) { - return - } if let renderInFlightSince { let stalledMs = Int((now - renderInFlightSince) * 1000) if stalledMs >= Int(Self.renderPipelineStallDeadline * 1000), @@ -3086,7 +3107,8 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { // now bounded in libghostty (so a foreground stall self-heals as a // skipped frame the display link re-drives), but we still gate on // suspension; `resumeRendering` clears it on the next active transition. - guard !renderingSuspended, + guard !renderPipelineRecoveryBlocked, + !renderingSuspended, let surface, !isDismantled else { return } // Coalesce: never let more than one render_now sit on the serial queue. @@ -3131,7 +3153,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { needsDraw = true // A geometry sync (for any reason) satisfies a pending post-zoom resync. zoomSettleFrames = nil - if displayLink == nil, window != nil { + if displayLink == nil, window != nil, !renderPipelineRecoveryBlocked { // No frame pump while detached/backgrounded; apply directly so the // surface still gets sized before the next render path resumes. needsGeometrySync = false @@ -3409,6 +3431,11 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { completion?(true) return } + // The current surface is scheduled for deferred recovery; geometry + // cannot be acknowledged against it without risking a stale replay ack. + guard !renderPipelineRecoveryBlocked else { + return + } // Capture all main-actor inputs as values, then do every libghostty // WRITE (set_content_scale / set_size / fit) and its readback on the From 402f933e27d75a52b97d5049e374ce99366c9f06 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 04:33:45 -0700 Subject: [PATCH 20/56] Add replay retry exhaustion regression test --- .../TerminalOutputDeliveryQueueTests.swift | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift index 84533c89ca04..7f9c3d708dd4 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift @@ -253,6 +253,47 @@ import Testing } } +@MainActor +@Test func terminalReplayBarrierStaysActiveAfterRetryExhaustionWithoutDroppedOutput() async throws { + let router = LivenessHostRouter() + let box = TransportBox() + let clock = TestClock() + let store = try await makeConnectedStore(router: router, box: box, clock: clock) + let surfaceID = "live-terminal" + + await router.enqueueReplayTexts(["cold-replay"]) + var iterator = store.terminalOutputStream(surfaceID: surfaceID).makeAsyncIterator() + await router.waitForCount(of: "mobile.terminal.replay", atLeast: 1) + let coldReplayChunk = try #require(await iterator.next()) + #expect(String(decoding: coldReplayChunk.data, as: UTF8.self) == "cold-replay") + store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: coldReplayChunk.streamToken) + let replayCountAfterMount = await router.count(of: "mobile.terminal.replay") + + await router.failNextReplay(count: 3) + store.deliverTerminalBytes(Data("stalled-first".utf8), surfaceID: surfaceID) + let stalledChunk = try #require(await iterator.next()) + store.terminalOutputDidReset(surfaceID: surfaceID, streamToken: stalledChunk.streamToken) + + let exhaustedRetries = await waitForReplayRequestCount( + router, + atLeast: replayCountAfterMount + 3 + ) + #expect(exhaustedRetries, "reset replay should exhaust the initial request plus two retries") + + let failureSettled = await waitForReplayBarrierFailureToSettle { + !store.terminalReplaySurfaceIDsInFlight.contains(surfaceID) + } + #expect(failureSettled) + #expect(store.terminalReplayBarrierTokensBySurfaceID[surfaceID] != nil) + + let acceptedAfterExhaustion = store.deliverTerminalBytes( + Data("live-after-exhausted-replay".utf8), + surfaceID: surfaceID + ) + #expect(acceptedAfterExhaustion == false) + #expect(store.terminalOutputQueuesBySurfaceID[surfaceID]?.isIdle == true) +} + @MainActor @Test func terminalReplayBarrierRetriesWhenReplayAckChunkResets() async throws { let router = LivenessHostRouter() From 51aa189ae65f48c80372e7b73db876e4eae514d5 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 04:35:28 -0700 Subject: [PATCH 21/56] Preserve replay barrier after retry exhaustion --- .github/swift-file-length-budget.tsv | 4 ++-- .../MobileShellComposite.swift | 21 ++++++++++++++++--- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/.github/swift-file-length-budget.tsv b/.github/swift-file-length-budget.tsv index c4518cc347ed..c6dd35b82c90 100644 --- a/.github/swift-file-length-budget.tsv +++ b/.github/swift-file-length-budget.tsv @@ -13,7 +13,7 @@ 8032 Sources/Panels/BrowserPanelView.swift 8016 CLI/cmux_open.swift 7366 cmuxTests/WorkspaceUnitTests.swift -7274 Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift +7289 Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift 7218 cmuxTests/WorkspaceRemoteConnectionTests.swift 6359 cmuxTests/SessionPersistenceTests.swift 6217 cmuxTests/GhosttyConfigTests.swift @@ -194,6 +194,7 @@ 574 Packages/macOS/CmuxWorkspaces/Tests/CmuxWorkspacesTests/WorkspaceCoordinatorTests.swift 572 Sources/Feed/FeedTextEditorDebugWindowController.swift 568 Packages/Shared/CMUXMobileCore/Sources/CMUXMobileCore/MobileTerminalRenderGrid.swift +567 Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift 567 Packages/macOS/CmuxSettingsUI/Sources/CmuxSettingsUI/Sections/BrowserSection.swift 567 Packages/macOS/CmuxTerminalCore/Sources/CmuxTerminalCore/ConfigDiscovery/GhosttyConfigDiscovery.swift 563 Packages/macOS/CmuxControlSocket/Sources/CmuxControlSocket/Coordinator/Pane/ControlCommandCoordinator+Pane.swift @@ -219,7 +220,6 @@ 528 cmuxUITests/AutomationSocketUITests.swift 527 CLI/CLISocketPathResolver.swift 527 cmuxTests/BrowserHTTPBasicAuthPromptTests.swift -526 Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift 526 Packages/macOS/CMUXAgentLaunch/Sources/CMUXAgentLaunch/Workstream/WorkstreamStore.swift 525 Packages/macOS/CmuxSettings/Sources/CmuxSettings/SocketControl/SocketControlSettings.swift 524 CLI/CMUXCLI+AutoNaming.swift diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift index 565c5fc869ab..dcdc65746aa8 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift @@ -6362,6 +6362,22 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { return true } + @discardableResult + private func preserveTerminalReplayBarrierIfCurrent( + surfaceID: String, + token: UUID?, + reason: String + ) -> Bool { + guard let token, + terminalReplayBarrierTokensBySurfaceID[surfaceID] == token else { + return false + } + terminalReplayBarrierAckStreamTokensBySurfaceID.removeValue(forKey: surfaceID) + terminalReplayBarrierTokensInFlightBySurfaceID.removeValue(forKey: surfaceID) + MobileDebugLog.anchormux("terminal.output.replay_barrier_preserved_\(reason) surface=\(surfaceID)") + return true + } + private func prepareTerminalReplayFailureRetry( surfaceID: String, replayBarrierToken: UUID? @@ -6839,11 +6855,10 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { ) return } - self.clearTerminalReplayBarrierIfCurrent( + self.preserveTerminalReplayBarrierIfCurrent( surfaceID: surfaceID, token: replayBarrierToken, - reason: "failed", - preserveDroppedOutput: true + reason: "failed" ) } } From d4e06a8f9103944c75dcdba4160de24bc0d7fc00 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 04:53:22 -0700 Subject: [PATCH 22/56] Address terminal replay recovery review feedback --- .github/swift-file-length-budget.tsv | 2 +- .../TerminalOutputDeliveryQueueTests.swift | 8 +- .../GhosttySurfaceRegistry.swift | 32 ++------ .../GhosttySurfaceView.swift | 82 ++++++++++++++++++- 4 files changed, 91 insertions(+), 33 deletions(-) diff --git a/.github/swift-file-length-budget.tsv b/.github/swift-file-length-budget.tsv index c6dd35b82c90..a17422968ca3 100644 --- a/.github/swift-file-length-budget.tsv +++ b/.github/swift-file-length-budget.tsv @@ -24,7 +24,7 @@ 4487 Sources/Panels/FilePreviewPanel.swift 4483 Sources/cmuxApp.swift 4367 cmuxTests/BrowserPanelTests.swift -4153 Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift +4233 Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift 4121 Sources/BrowserWindowPortal.swift 3934 Sources/Feed/FeedPanelView.swift 3926 cmuxTests/TabManagerUnitTests.swift diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift index 7f9c3d708dd4..0760452864d9 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift @@ -393,11 +393,11 @@ import Testing private func waitForReplayBarrierFailureToSettle( _ condition: @MainActor () -> Bool ) async -> Bool { - for _ in 0..<1_000 { + for _ in 0..<300 { if condition() { return true } - await Task.yield() + try? await Task.sleep(nanoseconds: 10_000_000) } return condition() } @@ -406,11 +406,11 @@ private func waitForReplayRequestCount( _ router: LivenessHostRouter, atLeast expectedCount: Int ) async -> Bool { - for _ in 0..<1_000 { + for _ in 0..<300 { if await router.count(of: "mobile.terminal.replay") >= expectedCount { return true } - await Task.yield() + try? await Task.sleep(nanoseconds: 10_000_000) } return await router.count(of: "mobile.terminal.replay") >= expectedCount } diff --git a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceRegistry.swift b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceRegistry.swift index 36e51bc9c0b5..aee3702c839f 100644 --- a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceRegistry.swift +++ b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceRegistry.swift @@ -53,16 +53,16 @@ extension GhosttySurfaceView { /// on the serial `outputQueue` because `ghostty_surface_read_text` takes /// the surface lock that `process_output` holds during a render storm, so /// a main-thread read would stall the present and blank the terminal. - /// Unlike that synchronous DEV path there is no bounded semaphore wait - /// here — the caller awaits, so a busy queue just resumes the continuation - /// late while the sheet shows its loading state. + /// The await is bounded by the surface's display-link deadline, like the + /// diagnostic visible snapshot path, so a wedged output queue resumes nil + /// instead of leaving the copy sheet loading forever. /// /// The continuation body enqueues on `outputQueue` synchronously while /// still on the main actor, so the read is FIFO-ordered before any /// later-enqueued `disposeSurface` free of the same pointer — the same /// lifetime argument `visibleTerminalSnapshot()` relies on. /// - /// The read is bounded at the source: iOS surfaces are created with + /// The bytes read are bounded at the source: iOS surfaces are created with /// `scrollback-limit = 2000000` (see `GhosttyRuntime.applyiOSDefaults`), /// so the SCREEN range can never materialize more than ~2MB of text no /// matter how long the session ran. The sheet's 5000-line budget is then @@ -96,28 +96,6 @@ extension GhosttySurfaceView { } guard let matchingView, let surface = matchingView.surface else { return nil } - let handle = CopyableTextSurfaceHandle(surface: surface) - return await withCheckedContinuation { continuation in - matchingView.outputQueue.async { - // SCREEN = scrollback + all written rows. Fall back to the - // viewport-only read if the screen read fails outright. - let text = surfaceText(handle.surface, pointTag: GHOSTTY_POINT_SCREEN) - ?? surfaceText(handle.surface, pointTag: GHOSTTY_POINT_VIEWPORT) - continuation.resume(returning: text) - } - } + return await matchingView.copyableTextForCurrentSurface(surface: surface) } } - -/// Carrier for the "View as Text" sheet's surface pointer across the hop to -/// `GhosttySurfaceView.outputQueue`. Same safety argument as -/// `VisibleSnapshotRequest` in `GhosttySurfaceView.swift`: the pointer is only -/// dereferenced on the queue that owns `process_output` and is FIFO-ordered -/// before any queued free — hence `@unchecked Sendable`. -/// -/// Deliberately `private` to this file: it holds the class's raw -/// `ghostty_surface_t`, which must not escape `GhosttySurfaceView`'s -/// queue/lifetime discipline into the wider module. -private struct CopyableTextSurfaceHandle: @unchecked Sendable { - let surface: ghostty_surface_t -} diff --git a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift index ca599a99cc7a..2f14e8730474 100644 --- a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift +++ b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift @@ -662,12 +662,15 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { private static let renderPipelineStallDeadline: CFTimeInterval = 2.0 private static let outputApplyTimeout: CFTimeInterval = 2.0 private static let visibleSnapshotTimeout: CFTimeInterval = 0.6 + private static let copyableTextTimeout: CFTimeInterval = 2.0 private var nextSurfaceOperationID: UInt64 = 0 private var pendingOutputApply: PendingSurfaceOperation? private var pendingGeometryApply: PendingSurfaceOperation? private var pendingVisibleSnapshot: PendingVisibleSnapshot? + private var pendingCopyableTextRead: PendingCopyableTextRead? private var hasPendingSurfaceOperationDeadline: Bool { pendingOutputApply != nil || pendingGeometryApply != nil || pendingVisibleSnapshot != nil + || pendingCopyableTextRead != nil } private static let scrollMechanicsContentHeight: CGFloat = 1_000_000 private var scrollMechanicsIsRecentering = false @@ -1115,6 +1118,8 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { /// `willResignActive` and `didEnterBackground`. private func suspendRendering() { renderingSuspended = true + skipPendingVisibleSnapshot() + skipPendingCopyableTextRead() stopDisplayLink() guard let surface else { return } ghostty_surface_set_occlusion(surface, false) // false = occluded; drawFrame skips @@ -2397,6 +2402,12 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { pendingVisibleSnapshot = nil pending.continuation.resume(returning: nil) } + + if let pending = pendingCopyableTextRead, + now - pending.startedAt >= Self.copyableTextTimeout { + pendingCopyableTextRead = nil + pending.continuation.resume(returning: nil) + } return false } @@ -2414,6 +2425,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { completed = true } skipPendingVisibleSnapshot() + skipPendingCopyableTextRead() return completed } @@ -2423,6 +2435,12 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { pending.continuation.resume(returning: nil) } + private func skipPendingCopyableTextRead() { + guard let pending = pendingCopyableTextRead else { return } + pendingCopyableTextRead = nil + pending.continuation.resume(returning: nil) + } + @discardableResult private func completePendingVisibleSnapshot(id: UInt64, returning section: String?) -> Bool { guard let pending = pendingVisibleSnapshot, pending.id == id else { return false } @@ -2431,6 +2449,14 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { return true } + @discardableResult + private func completePendingCopyableTextRead(id: UInt64, returning text: String?) -> Bool { + guard let pending = pendingCopyableTextRead, pending.id == id else { return false } + pendingCopyableTextRead = nil + pending.continuation.resume(returning: text) + return true + } + private func processOutput( _ data: Data, completion: (@MainActor @Sendable (Bool) -> Void)? @@ -2732,6 +2758,44 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { return String(decoding: Data(bytes: ptr, count: Int(text.text_len)), as: UTF8.self) } + func copyableTextForCurrentSurface(surface expectedSurface: ghostty_surface_t) async -> String? { + let generation = surfaceGeneration + guard surface == expectedSurface, + !isDismantled, + !renderingSuspended else { + return nil + } + return await withCheckedContinuation { continuation in + let operationID = makeSurfaceOperationID() + if let existing = pendingCopyableTextRead { + pendingCopyableTextRead = nil + existing.continuation.resume(returning: nil) + } + pendingCopyableTextRead = PendingCopyableTextRead( + id: operationID, + startedAt: CACurrentMediaTime(), + continuation: continuation + ) + ensureSurfaceOperationDeadlinePump() + let read = CopyableTextRead(surface: expectedSurface, generation: generation) + outputQueue.async { [weak self] in + // SCREEN = scrollback + all written rows. Fall back to the + // viewport-only read if the screen read fails outright. + let text = Self.surfaceText(read.surface, pointTag: GHOSTTY_POINT_SCREEN) + ?? Self.surfaceText(read.surface, pointTag: GHOSTTY_POINT_VIEWPORT) + Task { @MainActor [weak self] in + guard let self else { return } + guard self.surface == read.surface, + self.surfaceGeneration == read.generation else { + self.completePendingCopyableTextRead(id: operationID, returning: nil) + return + } + self.completePendingCopyableTextRead(id: operationID, returning: text) + } + } + } + } + func renderedHTMLForTesting(pointTag: ghostty_point_tag_e = GHOSTTY_POINT_VIEWPORT) -> String? { _ = pointTag // ghostty_surface_read_text_html not available in this build @@ -2817,6 +2881,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { let oldSurface = surface let oldBridge = bridge let oldQueue = outputQueue + oldBridge.detach() if let oldSurface { GhosttySurfaceView.unregister(surface: oldSurface) pendingSurfaceFreeCount += 1 @@ -2829,7 +2894,6 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { self.retryDeferredRenderPipelineRecoveryIfNeeded() } } - oldBridge.detach() surface = nil deferredRenderPipelineRecovery = nil @@ -4118,6 +4182,13 @@ nonisolated private struct PendingVisibleSnapshot { let continuation: CheckedContinuation } +/// One "View as Text" read awaiting output-queue completion or deadline. +nonisolated private struct PendingCopyableTextRead { + let id: UInt64 + let startedAt: CFTimeInterval + let continuation: CheckedContinuation +} + /// One surface's request for the bounded visible-terminal snapshot. nonisolated private struct VisibleSnapshotRequest { let view: GhosttySurfaceView @@ -4138,6 +4209,15 @@ nonisolated private struct VisibleSnapshotRead: @unchecked Sendable { let font: Int } +/// Raw full-text read payload captured by the off-main output queue. +/// +/// The C surface pointer is dereferenced only on `GhosttySurfaceWorkQueue`, +/// which is the same FIFO queue that owns `process_output` and surface free. +nonisolated private struct CopyableTextRead: @unchecked Sendable { + let surface: ghostty_surface_t + let generation: UInt64 +} + private class DisplayLinkProxy { private weak var target: GhosttySurfaceView? From d8586e4efe439e40f06ca7c43b31e1634550a987 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 04:57:45 -0700 Subject: [PATCH 23/56] Add stale replay client regression test --- ...leShellRenderGridLivenessTestSupport.swift | 45 +++++++++++++++++++ .../TerminalOutputDeliveryQueueTests.swift | 40 +++++++++++++++++ 2 files changed, 85 insertions(+) diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift index 739c5cab75b5..1685a9a9f5a4 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift @@ -71,6 +71,9 @@ actor LivenessHostRouter { private var subscribeRequestCount = 0 private var heldSubscribeRequestNumbers: Set = [] private var holdSubscribe = false + private var replayRequestCount = 0 + private var heldReplayRequestNumbers: Set = [] + private var heldReplayResponsesRemaining = 0 private var hasActiveSubscription = false private var heldContinuations: [CheckedContinuation] = [] private var capabilities = ["events.v1", "terminal.bytes.v1", "terminal.render_grid.v1", "terminal.replay.v1"] @@ -149,6 +152,18 @@ actor LivenessHostRouter { heldSubscribeRequestNumbers.insert(number) } + /// Hold the Nth `mobile.terminal.replay` response (1-based), letting a test + /// swap clients while the old request is still in flight. + func holdReplayRequest(number: Int) { + heldReplayRequestNumbers.insert(number) + } + + /// Hold the next N `mobile.terminal.replay` responses, independent of any + /// replay requests the connect/mount path already used. + func holdNextReplayResponses(count: Int = 1) { + heldReplayResponsesRemaining += count + } + /// Forget the host-side registration, modeling a lost subscription behind /// a live RPC channel: the next subscribe reports /// `already_subscribed: false`. @@ -162,6 +177,8 @@ actor LivenessHostRouter { holdSubscribe = false heldHostStatusRequestNumbers = [] heldSubscribeRequestNumbers = [] + heldReplayRequestNumbers = [] + heldReplayResponsesRemaining = 0 let continuations = heldContinuations heldContinuations = [] for continuation in continuations { @@ -215,6 +232,13 @@ actor LivenessHostRouter { "already_subscribed": alreadySubscribed, ]) case "mobile.terminal.replay": + replayRequestCount += 1 + if heldReplayResponsesRemaining > 0 { + heldReplayResponsesRemaining -= 1 + await park() + } else if heldReplayRequestNumbers.contains(replayRequestCount) { + await park() + } if replayFailuresRemaining > 0 { replayFailuresRemaining -= 1 return try? Self.errorFrame(id: id, message: "replay failed") @@ -511,3 +535,24 @@ func makeConnectedStore( #expect(connected, "scripted connect must succeed") return store } + +@MainActor +func installFreshLivenessRemoteClient( + on store: MobileShellComposite, + router: LivenessHostRouter, + box: TransportBox, + clock: TestClock +) throws { + let runtime = LivenessTestRuntime( + transportFactory: LivenessTransportFactory(router: router, box: box), + now: { clock.now } + ) + let ticket = try makeTicket(clock: clock) + let route = try #require(ticket.routes.first) + store.remoteClient = MobileCoreRPCClient( + runtime: runtime, + route: route, + ticket: ticket, + allowsStackAuthFallback: true + ) +} diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift index 0760452864d9..cb6e6e3f9388 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift @@ -253,6 +253,46 @@ import Testing } } +@MainActor +@Test func terminalReplayBarrierClearsAfterStaleClientReplayFailure() async throws { + let router = LivenessHostRouter() + let box = TransportBox() + let clock = TestClock() + let store = try await makeConnectedStore(router: router, box: box, clock: clock) + let surfaceID = "live-terminal" + + await router.enqueueReplayTexts(["cold-replay"]) + var iterator = store.terminalOutputStream(surfaceID: surfaceID).makeAsyncIterator() + await router.waitForCount(of: "mobile.terminal.replay", atLeast: 1) + let coldReplayChunk = try #require(await iterator.next()) + #expect(String(decoding: coldReplayChunk.data, as: UTF8.self) == "cold-replay") + store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: coldReplayChunk.streamToken) + let replayCountAfterMount = await router.count(of: "mobile.terminal.replay") + await router.holdNextReplayResponses() + + store.deliverTerminalBytes(Data("stalled-first".utf8), surfaceID: surfaceID) + let stalledChunk = try #require(await iterator.next()) + store.terminalOutputDidReset(surfaceID: surfaceID, streamToken: stalledChunk.streamToken) + await router.waitForCount(of: "mobile.terminal.replay", atLeast: replayCountAfterMount + 1) + + let replacementRouter = LivenessHostRouter() + let replacementBox = TransportBox() + try installFreshLivenessRemoteClient( + on: store, + router: replacementRouter, + box: replacementBox, + clock: clock + ) + await router.failNextReplay() + await router.releaseAllHeld() + + let staleFailureClearedBarrier = await waitForReplayBarrierFailureToSettle { + store.terminalReplayBarrierTokensBySurfaceID[surfaceID] == nil + && !store.terminalReplaySurfaceIDsInFlight.contains(surfaceID) + } + #expect(staleFailureClearedBarrier, "stale replay failures must clear the tokened barrier") +} + @MainActor @Test func terminalReplayBarrierStaysActiveAfterRetryExhaustionWithoutDroppedOutput() async throws { let router = LivenessHostRouter() From c00935bae478c3c1c6e4bcf47dfbf861e1290dd4 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 04:58:50 -0700 Subject: [PATCH 24/56] Clear replay barrier after stale client failure --- .github/swift-file-length-budget.tsv | 6 +++--- .../Sources/CmuxMobileShell/MobileShellComposite.swift | 9 ++++++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/swift-file-length-budget.tsv b/.github/swift-file-length-budget.tsv index a17422968ca3..45008f1c4a5c 100644 --- a/.github/swift-file-length-budget.tsv +++ b/.github/swift-file-length-budget.tsv @@ -13,7 +13,7 @@ 8032 Sources/Panels/BrowserPanelView.swift 8016 CLI/cmux_open.swift 7366 cmuxTests/WorkspaceUnitTests.swift -7289 Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift +7296 Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift 7218 cmuxTests/WorkspaceRemoteConnectionTests.swift 6359 cmuxTests/SessionPersistenceTests.swift 6217 cmuxTests/GhosttyConfigTests.swift @@ -171,6 +171,7 @@ 620 cmuxTests/TerminalNotificationQueueTests.swift 615 Sources/SettingsNavigation.swift 608 cmuxUITests/FeedSidebarUITests.swift +607 Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift 607 Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface.swift 607 Sources/SessionIndexModels.swift 607 Sources/SleepyFaceView.swift @@ -194,13 +195,13 @@ 574 Packages/macOS/CmuxWorkspaces/Tests/CmuxWorkspacesTests/WorkspaceCoordinatorTests.swift 572 Sources/Feed/FeedTextEditorDebugWindowController.swift 568 Packages/Shared/CMUXMobileCore/Sources/CMUXMobileCore/MobileTerminalRenderGrid.swift -567 Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift 567 Packages/macOS/CmuxSettingsUI/Sources/CmuxSettingsUI/Sections/BrowserSection.swift 567 Packages/macOS/CmuxTerminalCore/Sources/CmuxTerminalCore/ConfigDiscovery/GhosttyConfigDiscovery.swift 563 Packages/macOS/CmuxControlSocket/Sources/CmuxControlSocket/Coordinator/Pane/ControlCommandCoordinator+Pane.swift 562 cmuxTests/AgentExecutableResolverTests.swift 561 cmuxTests/GhosttyConfigPathResolverTests.swift 560 cmuxTests/CLISSHPTYResizeInputTests.swift +558 Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift 558 Packages/macOS/CmuxGit/Sources/CmuxGit/Parsing/GitMetadataService+Config.swift 553 Sources/RightSidebarPanelView.swift 549 Sources/Panels/BrowserAutomation.swift @@ -235,7 +236,6 @@ 514 Packages/macOS/CmuxSwiftRender/Sources/CmuxSwiftRender/ExpressionEvaluator.swift 514 cmuxUITests/UpdatePillUITests.swift 513 Packages/iOS/CmuxAgentChatUI/Sources/CmuxAgentChatUI/Transcript/ChatTranscriptTableView.swift -513 Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift 509 Packages/macOS/CMUXAgentLaunch/Sources/CMUXAgentLaunch/AgentLaunchSanitizerAdditionalPolicies.swift 507 Sources/TerminalControllerTopSupport.swift 506 Sources/App/MainWindowVisibilityController.swift diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift index dcdc65746aa8..047199a28918 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift @@ -6833,7 +6833,14 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { } } catch { mobileShellLog.error("CMUX_REPLAY failed surface=\(surfaceID, privacy: .public) error=\(String(describing: error), privacy: .private)") - guard self.remoteClient === client else { return } + guard self.remoteClient === client else { + self.clearTerminalReplayBarrierIfCurrent( + surfaceID: surfaceID, + token: replayBarrierToken, + reason: "stale_client" + ) + return + } // The replay request is the view-only/foreground-resume path. A // definitive auth failure here (after the RPC layer's // force-refresh-and-retry already gave up) must drive the re-auth From 73394b600a7a259681c8323b2e99f2f23562fb0d Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 05:05:08 -0700 Subject: [PATCH 25/56] Add replay in-flight recovery regression tests --- .../TerminalOutputDeliveryQueueTests.swift | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift index cb6e6e3f9388..2e42c9869389 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift @@ -334,6 +334,100 @@ import Testing #expect(store.terminalOutputQueuesBySurfaceID[surfaceID]?.isIdle == true) } +@MainActor +@Test func terminalReplayInFlightClearsWhenOutputStreamUnmountsBeforeResponse() async throws { + let router = LivenessHostRouter() + let box = TransportBox() + let clock = TestClock() + let store = try await makeConnectedStore(router: router, box: box, clock: clock) + let surfaceID = "live-terminal" + + await router.holdNextReplayResponses() + let collector = OutputCollector() + collector.mount(store: store, surfaceID: surfaceID) + await router.waitForCount(of: "mobile.terminal.replay", atLeast: 1) + #expect(store.terminalReplaySurfaceIDsInFlight.contains(surfaceID)) + + collector.unmount() + let unregistered = await waitForReplayBarrierFailureToSettle { + store.terminalByteContinuationsBySurfaceID[surfaceID] == nil + } + #expect(unregistered) + #expect(!store.terminalReplaySurfaceIDsInFlight.contains(surfaceID)) + + let replayCountAfterUnmount = await router.count(of: "mobile.terminal.replay") + await router.enqueueReplayTexts(["remount-replay"]) + let remountCollector = OutputCollector() + remountCollector.mount(store: store, surfaceID: surfaceID) + + let remountRequested = await waitForReplayRequestCount( + router, + atLeast: replayCountAfterUnmount + 1 + ) + #expect(remountRequested, "remounting after a cancelled replay must request a fresh replay") + if remountRequested { + let replayDelivered = try await pollUntil { + remountCollector.lines.contains("remount-replay") + } + #expect(replayDelivered) + } + + remountCollector.unmount() + await router.releaseAllHeld() +} + +@MainActor +@Test func genericReplayRequestReusesPreservedBarrierAfterRetryExhaustion() async throws { + let router = LivenessHostRouter() + let box = TransportBox() + let clock = TestClock() + let store = try await makeConnectedStore(router: router, box: box, clock: clock) + let surfaceID = "live-terminal" + + await router.enqueueReplayTexts(["cold-replay"]) + var iterator = store.terminalOutputStream(surfaceID: surfaceID).makeAsyncIterator() + await router.waitForCount(of: "mobile.terminal.replay", atLeast: 1) + let coldReplayChunk = try #require(await iterator.next()) + #expect(String(decoding: coldReplayChunk.data, as: UTF8.self) == "cold-replay") + store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: coldReplayChunk.streamToken) + let replayCountAfterMount = await router.count(of: "mobile.terminal.replay") + + await router.failNextReplay(count: 3) + store.deliverTerminalBytes(Data("stalled-first".utf8), surfaceID: surfaceID) + let stalledChunk = try #require(await iterator.next()) + store.terminalOutputDidReset(surfaceID: surfaceID, streamToken: stalledChunk.streamToken) + + let exhaustedRetries = await waitForReplayRequestCount( + router, + atLeast: replayCountAfterMount + 3 + ) + #expect(exhaustedRetries, "reset replay should exhaust the initial request plus two retries") + + let failureSettled = await waitForReplayBarrierFailureToSettle { + !store.terminalReplaySurfaceIDsInFlight.contains(surfaceID) + } + #expect(failureSettled) + let preservedBarrierToken = try #require(store.terminalReplayBarrierTokensBySurfaceID[surfaceID]) + + let replayCountAfterExhaustion = await router.count(of: "mobile.terminal.replay") + await router.enqueueReplayTexts(["resync-replay"]) + store.requestTerminalReplay(surfaceID: surfaceID) + + let genericReplayRequested = await waitForReplayRequestCount( + router, + atLeast: replayCountAfterExhaustion + 1 + ) + #expect(genericReplayRequested, "generic resync must not be blocked by a preserved barrier") + + if genericReplayRequested { + let replayChunk = try #require(await iterator.next()) + #expect(String(decoding: replayChunk.data, as: UTF8.self) == "resync-replay") + #expect(store.terminalReplayBarrierTokensBySurfaceID[surfaceID] == preservedBarrierToken) + store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: replayChunk.streamToken) + #expect(store.terminalReplayBarrierTokensBySurfaceID[surfaceID] == nil) + } +} + @MainActor @Test func terminalReplayBarrierRetriesWhenReplayAckChunkResets() async throws { let router = LivenessHostRouter() From 56efc846568e8c3e6f1fb2e4b7b9e1145433300b Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 05:06:34 -0700 Subject: [PATCH 26/56] Clear replay in-flight state on remount recovery --- .../MobileShellComposite.swift | 56 ++++++++++--------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift index 047199a28918..0470e7ba4feb 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift @@ -6521,6 +6521,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { terminalReplayBarrierDroppedOutputCountsBySurfaceID.removeValue(forKey: surfaceID) terminalReplayBarrierAckCoveredDroppedOutputCountsBySurfaceID.removeValue(forKey: surfaceID) terminalReplayFailureRetryCountsBySurfaceID.removeValue(forKey: surfaceID) + terminalReplaySurfaceIDsInFlight.remove(surfaceID) terminalReplayRequestIDsInFlightBySurfaceID.removeValue(forKey: surfaceID) terminalReplayBarrierTokensInFlightBySurfaceID.removeValue(forKey: surfaceID) terminalScrollQueueTokensBySurfaceID.removeValue(forKey: surfaceID) @@ -6653,10 +6654,17 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { replayBarrierToken: UUID? = nil, coveredReplayBarrierDroppedOutputCount: UInt64? = nil ) { + let replayBarrierTokenForRequest = replayBarrierToken + ?? terminalReplayBarrierTokensBySurfaceID[surfaceID] + let coveredReplayBarrierDroppedOutputCountForRequest = replayBarrierTokenForRequest == nil + ? nil + : (coveredReplayBarrierDroppedOutputCount + ?? terminalReplayBarrierDroppedOutputCountsBySurfaceID[surfaceID] + ?? 0) guard let client = remoteClient else { clearTerminalReplayBarrierIfCurrent( surfaceID: surfaceID, - token: replayBarrierToken, + token: replayBarrierTokenForRequest, reason: "no_remote_client" ) #if DEBUG @@ -6667,7 +6675,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { guard let workspaceID = workspaceID(forTerminalID: surfaceID) else { clearTerminalReplayBarrierIfCurrent( surfaceID: surfaceID, - token: replayBarrierToken, + token: replayBarrierTokenForRequest, reason: "workspace_not_found" ) #if DEBUG @@ -6676,12 +6684,8 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { return } let remoteWorkspaceID = remoteWorkspaceID(for: workspaceID) - guard replayBarrierToken != nil || terminalReplayBarrierTokensBySurfaceID[surfaceID] == nil else { - MobileDebugLog.anchormux("CMUX_REPLAY skip surface=\(surfaceID) reason=barrier_active") - return - } - if let replayBarrierToken { - guard terminalReplayBarrierTokensInFlightBySurfaceID[surfaceID] != replayBarrierToken else { + if let replayBarrierTokenForRequest { + guard terminalReplayBarrierTokensInFlightBySurfaceID[surfaceID] != replayBarrierTokenForRequest else { #if DEBUG mobileShellLog.info("CMUX_REPLAY skip surface=\(surfaceID, privacy: .public) reason=barrier_in_flight") #endif @@ -6699,7 +6703,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { markTerminalReplayInFlight( surfaceID: surfaceID, requestID: replayRequestID, - replayBarrierToken: replayBarrierToken + replayBarrierToken: replayBarrierTokenForRequest ) Task { @MainActor [weak self] in guard let self else { return } @@ -6724,7 +6728,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { guard self.remoteClient === client else { self.clearTerminalReplayBarrierIfCurrent( surfaceID: surfaceID, - token: replayBarrierToken, + token: replayBarrierTokenForRequest, reason: "stale_client" ) return @@ -6735,8 +6739,8 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { let decodedRenderGrid = payload?.renderGrid let renderGrid = decodedRenderGrid?.surfaceID == surfaceID ? decodedRenderGrid : nil let replaySeq = renderGrid?.stateSeq ?? payload?.sequence - if let replayBarrierToken { - guard self.terminalReplayBarrierTokensBySurfaceID[surfaceID] == replayBarrierToken else { + if let replayBarrierTokenForRequest { + guard self.terminalReplayBarrierTokensBySurfaceID[surfaceID] == replayBarrierTokenForRequest else { MobileDebugLog.anchormux("CMUX_REPLAY barrier_stale surface=\(surfaceID)") return } @@ -6753,7 +6757,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { MobileDebugLog.anchormux("CMUX_REPLAY stale surface=\(surfaceID) delivered=\(deliveredSeq) replay=\(replaySeq)") self.clearTerminalReplayBarrierIfCurrent( surfaceID: surfaceID, - token: replayBarrierToken, + token: replayBarrierTokenForRequest, reason: "stale_sequence" ) return @@ -6773,21 +6777,21 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { let accepted = self.deliverTerminalRenderGrid( renderGrid, surfaceID: surfaceID, - bypassReplayBarrier: replayBarrierToken != nil + bypassReplayBarrier: replayBarrierTokenForRequest != nil ) guard accepted else { self.clearTerminalReplayBarrierIfCurrent( surfaceID: surfaceID, - token: replayBarrierToken, + token: replayBarrierTokenForRequest, reason: "not_delivered", preserveDroppedOutput: true ) return } if self.terminalReplayBarrierAckStreamTokensBySurfaceID[surfaceID] != nil { - if let coveredReplayBarrierDroppedOutputCount { + if let coveredReplayBarrierDroppedOutputCountForRequest { self.terminalReplayBarrierAckCoveredDroppedOutputCountsBySurfaceID[surfaceID] = - coveredReplayBarrierDroppedOutputCount + coveredReplayBarrierDroppedOutputCountForRequest } else { self.terminalReplayBarrierAckCoveredDroppedOutputCountsBySurfaceID.removeValue(forKey: surfaceID) } @@ -6801,7 +6805,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { guard let deliverBytes, !deliverBytes.isEmpty else { self.clearTerminalReplayBarrierIfCurrent( surfaceID: surfaceID, - token: replayBarrierToken, + token: replayBarrierTokenForRequest, reason: "empty", preserveDroppedOutput: true ) @@ -6810,13 +6814,13 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { let accepted = self.deliverTerminalBytes( deliverBytes, surfaceID: surfaceID, - bypassReplayBarrier: replayBarrierToken != nil + bypassReplayBarrier: replayBarrierTokenForRequest != nil ) if accepted, self.terminalReplayBarrierAckStreamTokensBySurfaceID[surfaceID] != nil { - if let coveredReplayBarrierDroppedOutputCount { + if let coveredReplayBarrierDroppedOutputCountForRequest { self.terminalReplayBarrierAckCoveredDroppedOutputCountsBySurfaceID[surfaceID] = - coveredReplayBarrierDroppedOutputCount + coveredReplayBarrierDroppedOutputCountForRequest } else { self.terminalReplayBarrierAckCoveredDroppedOutputCountsBySurfaceID.removeValue(forKey: surfaceID) } @@ -6826,7 +6830,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { } else if !accepted { self.clearTerminalReplayBarrierIfCurrent( surfaceID: surfaceID, - token: replayBarrierToken, + token: replayBarrierTokenForRequest, reason: "not_delivered", preserveDroppedOutput: true ) @@ -6836,7 +6840,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { guard self.remoteClient === client else { self.clearTerminalReplayBarrierIfCurrent( surfaceID: surfaceID, - token: replayBarrierToken, + token: replayBarrierTokenForRequest, reason: "stale_client" ) return @@ -6848,7 +6852,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { guard !self.disconnectForAuthorizationFailureIfNeeded(error) else { return } if let retryToken = self.prepareTerminalReplayFailureRetry( surfaceID: surfaceID, - replayBarrierToken: replayBarrierToken + replayBarrierToken: replayBarrierTokenForRequest ) { self.clearTerminalReplayInFlightIfCurrent( surfaceID: surfaceID, @@ -6858,13 +6862,13 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { self.requestTerminalReplay( surfaceID: surfaceID, replayBarrierToken: retryToken, - coveredReplayBarrierDroppedOutputCount: coveredReplayBarrierDroppedOutputCount + coveredReplayBarrierDroppedOutputCount: coveredReplayBarrierDroppedOutputCountForRequest ) return } self.preserveTerminalReplayBarrierIfCurrent( surfaceID: surfaceID, - token: replayBarrierToken, + token: replayBarrierTokenForRequest, reason: "failed" ) } From 06e24f3a2ec4f2e453eb4e8ea08b6156d8e66739 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 05:07:19 -0700 Subject: [PATCH 27/56] Update Swift file length budget --- .github/swift-file-length-budget.tsv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/swift-file-length-budget.tsv b/.github/swift-file-length-budget.tsv index 45008f1c4a5c..22a537fc0db3 100644 --- a/.github/swift-file-length-budget.tsv +++ b/.github/swift-file-length-budget.tsv @@ -13,7 +13,7 @@ 8032 Sources/Panels/BrowserPanelView.swift 8016 CLI/cmux_open.swift 7366 cmuxTests/WorkspaceUnitTests.swift -7296 Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift +7300 Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift 7218 cmuxTests/WorkspaceRemoteConnectionTests.swift 6359 cmuxTests/SessionPersistenceTests.swift 6217 cmuxTests/GhosttyConfigTests.swift @@ -139,6 +139,7 @@ 710 Sources/TerminalSSHSessionDetector.swift 709 Packages/macOS/CmuxFoundation/Sources/CmuxFoundation/SidebarDrop/SidebarWorkspaceReorderDropResolver.swift 706 CLI/CMUXCLI+Config.swift +701 Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift 699 cmuxTests/TerminalNotificationClearAllTests.swift 698 cmuxTests/RestorableAgentHookProviderResumeTests.swift 696 cmuxTests/KeyboardShortcutContextTests.swift @@ -171,7 +172,6 @@ 620 cmuxTests/TerminalNotificationQueueTests.swift 615 Sources/SettingsNavigation.swift 608 cmuxUITests/FeedSidebarUITests.swift -607 Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift 607 Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface.swift 607 Sources/SessionIndexModels.swift 607 Sources/SleepyFaceView.swift From e9331528a4c6091c837f7f6a0dc8f90aa84df391 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 05:10:17 -0700 Subject: [PATCH 28/56] Cancel stale copyable text reads before surface access --- .github/swift-file-length-budget.tsv | 2 +- .../GhosttySurfaceView.swift | 40 ++++++++++++++++++- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/.github/swift-file-length-budget.tsv b/.github/swift-file-length-budget.tsv index 22a537fc0db3..20c183f213c8 100644 --- a/.github/swift-file-length-budget.tsv +++ b/.github/swift-file-length-budget.tsv @@ -24,7 +24,7 @@ 4487 Sources/Panels/FilePreviewPanel.swift 4483 Sources/cmuxApp.swift 4367 cmuxTests/BrowserPanelTests.swift -4233 Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift +4271 Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift 4121 Sources/BrowserWindowPortal.swift 3934 Sources/Feed/FeedPanelView.swift 3926 cmuxTests/TabManagerUnitTests.swift diff --git a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift index 2f14e8730474..be43911a695d 100644 --- a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift +++ b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift @@ -2406,6 +2406,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { if let pending = pendingCopyableTextRead, now - pending.startedAt >= Self.copyableTextTimeout { pendingCopyableTextRead = nil + pending.cancel() pending.continuation.resume(returning: nil) } return false @@ -2438,6 +2439,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { private func skipPendingCopyableTextRead() { guard let pending = pendingCopyableTextRead else { return } pendingCopyableTextRead = nil + pending.cancel() pending.continuation.resume(returning: nil) } @@ -2453,6 +2455,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { private func completePendingCopyableTextRead(id: UInt64, returning text: String?) -> Bool { guard let pending = pendingCopyableTextRead, pending.id == id else { return false } pendingCopyableTextRead = nil + pending.cancel() pending.continuation.resume(returning: text) return true } @@ -2769,20 +2772,29 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { let operationID = makeSurfaceOperationID() if let existing = pendingCopyableTextRead { pendingCopyableTextRead = nil + existing.cancel() existing.continuation.resume(returning: nil) } + let cancellation = SurfaceOperationCancellationToken() pendingCopyableTextRead = PendingCopyableTextRead( id: operationID, startedAt: CACurrentMediaTime(), + cancellation: cancellation, continuation: continuation ) ensureSurfaceOperationDeadlinePump() - let read = CopyableTextRead(surface: expectedSurface, generation: generation) + let read = CopyableTextRead( + surface: expectedSurface, + generation: generation, + cancellation: cancellation + ) outputQueue.async { [weak self] in + guard !read.cancellation.isCancelled else { return } // SCREEN = scrollback + all written rows. Fall back to the // viewport-only read if the screen read fails outright. let text = Self.surfaceText(read.surface, pointTag: GHOSTTY_POINT_SCREEN) ?? Self.surfaceText(read.surface, pointTag: GHOSTTY_POINT_VIEWPORT) + guard !read.cancellation.isCancelled else { return } Task { @MainActor [weak self] in guard let self else { return } guard self.surface == read.surface, @@ -4186,7 +4198,12 @@ nonisolated private struct PendingVisibleSnapshot { nonisolated private struct PendingCopyableTextRead { let id: UInt64 let startedAt: CFTimeInterval + let cancellation: SurfaceOperationCancellationToken let continuation: CheckedContinuation + + func cancel() { + cancellation.cancel() + } } /// One surface's request for the bounded visible-terminal snapshot. @@ -4216,6 +4233,27 @@ nonisolated private struct VisibleSnapshotRead: @unchecked Sendable { nonisolated private struct CopyableTextRead: @unchecked Sendable { let surface: ghostty_surface_t let generation: UInt64 + let cancellation: SurfaceOperationCancellationToken +} + +nonisolated private final class SurfaceOperationCancellationToken: @unchecked Sendable { + // lint:allow lock - tiny cross-queue cancellation flag for already-enqueued + // libghostty work; actor hops would put the serial surface queue back behind + // the main actor and defeat the stale-read fast path. + private let lock = NSLock() + private var cancelled = false + + var isCancelled: Bool { + lock.lock() + defer { lock.unlock() } + return cancelled + } + + func cancel() { + lock.lock() + cancelled = true + lock.unlock() + } } private class DisplayLinkProxy { From 5fde20a5e56625bce7688b40e95ad31496ad5761 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 05:18:32 -0700 Subject: [PATCH 29/56] Add stale replay response regression test --- .../TerminalOutputDeliveryQueueTests.swift | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift index 2e42c9869389..711ea7d9be0b 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift @@ -376,6 +376,49 @@ import Testing await router.releaseAllHeld() } +@MainActor +@Test func staleReplayResponseAfterRemountDoesNotDeliverIntoCurrentStream() async throws { + let router = LivenessHostRouter() + let box = TransportBox() + let clock = TestClock() + let store = try await makeConnectedStore(router: router, box: box, clock: clock) + let surfaceID = "live-terminal" + + await router.holdNextReplayResponses() + let oldCollector = OutputCollector() + oldCollector.mount(store: store, surfaceID: surfaceID) + await router.waitForCount(of: "mobile.terminal.replay", atLeast: 1) + oldCollector.unmount() + + let unregistered = await waitForReplayBarrierFailureToSettle { + store.terminalByteContinuationsBySurfaceID[surfaceID] == nil + } + #expect(unregistered) + let replayCountAfterUnmount = await router.count(of: "mobile.terminal.replay") + + await router.enqueueReplayTexts(["fresh-replay", "stale-replay"]) + let currentCollector = OutputCollector() + currentCollector.mount(store: store, surfaceID: surfaceID) + let remountRequested = await waitForReplayRequestCount( + router, + atLeast: replayCountAfterUnmount + 1 + ) + #expect(remountRequested) + + let freshDelivered = try await pollUntil { + currentCollector.lines.contains("fresh-replay") + } + #expect(freshDelivered) + + await router.releaseAllHeld() + let staleDelivered = try await pollUntil(attempts: 50) { + currentCollector.lines.contains("stale-replay") + } + #expect(!staleDelivered, "superseded replay responses must not deliver stale bytes") + + currentCollector.unmount() +} + @MainActor @Test func genericReplayRequestReusesPreservedBarrierAfterRetryExhaustion() async throws { let router = LivenessHostRouter() From 5eb9176f6d41705a1e4d250d96bd6aacf68cad22 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 05:19:39 -0700 Subject: [PATCH 30/56] Ignore superseded terminal replay responses --- .github/swift-file-length-budget.tsv | 4 ++-- .../Sources/CmuxMobileShell/MobileShellComposite.swift | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/swift-file-length-budget.tsv b/.github/swift-file-length-budget.tsv index 20c183f213c8..0352bc23cda2 100644 --- a/.github/swift-file-length-budget.tsv +++ b/.github/swift-file-length-budget.tsv @@ -13,7 +13,7 @@ 8032 Sources/Panels/BrowserPanelView.swift 8016 CLI/cmux_open.swift 7366 cmuxTests/WorkspaceUnitTests.swift -7300 Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift +7308 Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift 7218 cmuxTests/WorkspaceRemoteConnectionTests.swift 6359 cmuxTests/SessionPersistenceTests.swift 6217 cmuxTests/GhosttyConfigTests.swift @@ -131,6 +131,7 @@ 754 Sources/TerminalController+ControlWorkspaceContext.swift 753 Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/WorkspaceDetailView.swift 752 cmuxUITests/CloseWorkspaceCmdDUITests.swift +744 Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift 739 cmuxTests/CLICodexHookTimeoutRegressionTests.swift 738 Packages/macOS/CMUXProjectModel/Sources/CMUXProjectModel/XcodeProjectAdapter.swift 718 Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Coordinator/AuthCoordinator.swift @@ -139,7 +140,6 @@ 710 Sources/TerminalSSHSessionDetector.swift 709 Packages/macOS/CmuxFoundation/Sources/CmuxFoundation/SidebarDrop/SidebarWorkspaceReorderDropResolver.swift 706 CLI/CMUXCLI+Config.swift -701 Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift 699 cmuxTests/TerminalNotificationClearAllTests.swift 698 cmuxTests/RestorableAgentHookProviderResumeTests.swift 696 cmuxTests/KeyboardShortcutContextTests.swift diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift index 0470e7ba4feb..6ce6745b5e53 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift @@ -6725,6 +6725,10 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { ] ) let data = try await client.sendRequest(request) + guard self.terminalReplayRequestIDsInFlightBySurfaceID[surfaceID] == replayRequestID else { + MobileDebugLog.anchormux("CMUX_REPLAY stale_request surface=\(surfaceID)") + return + } guard self.remoteClient === client else { self.clearTerminalReplayBarrierIfCurrent( surfaceID: surfaceID, @@ -6836,6 +6840,10 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { ) } } catch { + guard self.terminalReplayRequestIDsInFlightBySurfaceID[surfaceID] == replayRequestID else { + MobileDebugLog.anchormux("CMUX_REPLAY stale_request_failed surface=\(surfaceID)") + return + } mobileShellLog.error("CMUX_REPLAY failed surface=\(surfaceID, privacy: .public) error=\(String(describing: error), privacy: .private)") guard self.remoteClient === client else { self.clearTerminalReplayBarrierIfCurrent( From 2dfd3563b09e43371f8d8573961b6c9726c823d8 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 05:28:46 -0700 Subject: [PATCH 31/56] Use Mutex for copyable text cancellation token --- .github/swift-file-length-budget.tsv | 2 +- .../CmuxMobileTerminal/GhosttySurfaceView.swift | 14 +++++--------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/.github/swift-file-length-budget.tsv b/.github/swift-file-length-budget.tsv index 0352bc23cda2..55a13e921e0d 100644 --- a/.github/swift-file-length-budget.tsv +++ b/.github/swift-file-length-budget.tsv @@ -24,7 +24,7 @@ 4487 Sources/Panels/FilePreviewPanel.swift 4483 Sources/cmuxApp.swift 4367 cmuxTests/BrowserPanelTests.swift -4271 Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift +4267 Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift 4121 Sources/BrowserWindowPortal.swift 3934 Sources/Feed/FeedPanelView.swift 3926 cmuxTests/TabManagerUnitTests.swift diff --git a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift index be43911a695d..ddfa074094a9 100644 --- a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift +++ b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift @@ -5,6 +5,7 @@ import CmuxMobileSupport import CmuxMobileTerminalKit import GhosttyKit import OSLog +import Synchronization import UIKit private let log = Logger(subsystem: "ai.manaflow.cmux.ios", category: "ghostty.surface") @@ -4236,23 +4237,18 @@ nonisolated private struct CopyableTextRead: @unchecked Sendable { let cancellation: SurfaceOperationCancellationToken } -nonisolated private final class SurfaceOperationCancellationToken: @unchecked Sendable { +nonisolated private final class SurfaceOperationCancellationToken: Sendable { // lint:allow lock - tiny cross-queue cancellation flag for already-enqueued // libghostty work; actor hops would put the serial surface queue back behind // the main actor and defeat the stale-read fast path. - private let lock = NSLock() - private var cancelled = false + private let cancelled = Mutex(false) var isCancelled: Bool { - lock.lock() - defer { lock.unlock() } - return cancelled + cancelled.withLock { $0 } } func cancel() { - lock.lock() - cancelled = true - lock.unlock() + cancelled.withLock { $0 = true } } } From 093467ae650370c2368b2612ded0af796baff9e5 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 05:30:00 -0700 Subject: [PATCH 32/56] Check copyable text cancellation before viewport fallback --- .github/swift-file-length-budget.tsv | 2 +- .../Sources/CmuxMobileTerminal/GhosttySurfaceView.swift | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/swift-file-length-budget.tsv b/.github/swift-file-length-budget.tsv index 55a13e921e0d..d008ab27c731 100644 --- a/.github/swift-file-length-budget.tsv +++ b/.github/swift-file-length-budget.tsv @@ -24,7 +24,7 @@ 4487 Sources/Panels/FilePreviewPanel.swift 4483 Sources/cmuxApp.swift 4367 cmuxTests/BrowserPanelTests.swift -4267 Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift +4268 Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift 4121 Sources/BrowserWindowPortal.swift 3934 Sources/Feed/FeedPanelView.swift 3926 cmuxTests/TabManagerUnitTests.swift diff --git a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift index ddfa074094a9..08f2dbc1a9a1 100644 --- a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift +++ b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift @@ -2793,8 +2793,9 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { guard !read.cancellation.isCancelled else { return } // SCREEN = scrollback + all written rows. Fall back to the // viewport-only read if the screen read fails outright. - let text = Self.surfaceText(read.surface, pointTag: GHOSTTY_POINT_SCREEN) - ?? Self.surfaceText(read.surface, pointTag: GHOSTTY_POINT_VIEWPORT) + let screenText = Self.surfaceText(read.surface, pointTag: GHOSTTY_POINT_SCREEN) + guard !read.cancellation.isCancelled else { return } + let text = screenText ?? Self.surfaceText(read.surface, pointTag: GHOSTTY_POINT_VIEWPORT) guard !read.cancellation.isCancelled else { return } Task { @MainActor [weak self] in guard let self else { return } From c6e87013be545e0444ef55034f3170fb82301a8c Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 05:41:29 -0700 Subject: [PATCH 33/56] Add replay barrier stale resync regression tests --- .github/swift-file-length-budget.tsv | 4 +- ...leShellRenderGridLivenessTestSupport.swift | 9 ++ .../TerminalOutputDeliveryQueueTests.swift | 107 +++++++++++++++++- 3 files changed, 113 insertions(+), 7 deletions(-) diff --git a/.github/swift-file-length-budget.tsv b/.github/swift-file-length-budget.tsv index d008ab27c731..0eb1bde28b3c 100644 --- a/.github/swift-file-length-budget.tsv +++ b/.github/swift-file-length-budget.tsv @@ -111,6 +111,7 @@ 859 Packages/macOS/CmuxControlSocket/Sources/CmuxControlSocket/Coordinator/Workspace/ControlCommandCoordinator+Workspace.swift 847 cmuxTests/AgentSessionAutoResumeSettingsTests.swift 845 cmuxTests/SSHStartupSignalLifecycleTests.swift +841 Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift 834 Sources/MainWindowFocusController.swift 830 Sources/TaskManagerTypes.swift 825 Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/TerminalComposerView.swift @@ -131,7 +132,6 @@ 754 Sources/TerminalController+ControlWorkspaceContext.swift 753 Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/WorkspaceDetailView.swift 752 cmuxUITests/CloseWorkspaceCmdDUITests.swift -744 Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift 739 cmuxTests/CLICodexHookTimeoutRegressionTests.swift 738 Packages/macOS/CMUXProjectModel/Sources/CMUXProjectModel/XcodeProjectAdapter.swift 718 Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Coordinator/AuthCoordinator.swift @@ -195,13 +195,13 @@ 574 Packages/macOS/CmuxWorkspaces/Tests/CmuxWorkspacesTests/WorkspaceCoordinatorTests.swift 572 Sources/Feed/FeedTextEditorDebugWindowController.swift 568 Packages/Shared/CMUXMobileCore/Sources/CMUXMobileCore/MobileTerminalRenderGrid.swift +567 Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift 567 Packages/macOS/CmuxSettingsUI/Sources/CmuxSettingsUI/Sections/BrowserSection.swift 567 Packages/macOS/CmuxTerminalCore/Sources/CmuxTerminalCore/ConfigDiscovery/GhosttyConfigDiscovery.swift 563 Packages/macOS/CmuxControlSocket/Sources/CmuxControlSocket/Coordinator/Pane/ControlCommandCoordinator+Pane.swift 562 cmuxTests/AgentExecutableResolverTests.swift 561 cmuxTests/GhosttyConfigPathResolverTests.swift 560 cmuxTests/CLISSHPTYResizeInputTests.swift -558 Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift 558 Packages/macOS/CmuxGit/Sources/CmuxGit/Parsing/GitMetadataService+Config.swift 553 Sources/RightSidebarPanelView.swift 549 Sources/Panels/BrowserAutomation.swift diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift index 1685a9a9f5a4..1690760a1e31 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift @@ -79,6 +79,7 @@ actor LivenessHostRouter { private var capabilities = ["events.v1", "terminal.bytes.v1", "terminal.render_grid.v1", "terminal.replay.v1"] private var replayTexts: [String] = [] private var replayFailuresRemaining = 0 + private var emptyReplayResponsesRemaining = 0 func record(method: String?, topics: [String]?) { recorded.append(RecordedRequest(method: method, topics: topics)) @@ -135,6 +136,10 @@ actor LivenessHostRouter { replayFailuresRemaining += count } + func enqueueEmptyReplayResponses(count: Int = 1) { + emptyReplayResponsesRemaining += count + } + /// Hold every `mobile.events.subscribe` response until released. func setHoldSubscribe(_ hold: Bool) { holdSubscribe = hold @@ -243,6 +248,10 @@ actor LivenessHostRouter { replayFailuresRemaining -= 1 return try? Self.errorFrame(id: id, message: "replay failed") } + if emptyReplayResponsesRemaining > 0 { + emptyReplayResponsesRemaining -= 1 + return try? Self.resultFrame(id: id, result: [:]) + } guard !replayTexts.isEmpty else { return try? Self.resultFrame(id: id, result: [:]) } diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift index 711ea7d9be0b..bd76b6fb06f6 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift @@ -254,7 +254,7 @@ import Testing } @MainActor -@Test func terminalReplayBarrierClearsAfterStaleClientReplayFailure() async throws { +@Test func terminalReplayBarrierReplaysOnReplacementClientAfterStaleResponse() async throws { let router = LivenessHostRouter() let box = TransportBox() let clock = TestClock() @@ -277,6 +277,55 @@ import Testing let replacementRouter = LivenessHostRouter() let replacementBox = TransportBox() + await replacementRouter.enqueueReplayTexts(["replacement-replay"]) + try installFreshLivenessRemoteClient( + on: store, + router: replacementRouter, + box: replacementBox, + clock: clock + ) + await router.enqueueReplayTexts(["stale-replay"]) + await router.releaseAllHeld() + + let replacementReplayRequested = await waitForReplayRequestCount( + replacementRouter, + atLeast: 1 + ) + #expect(replacementReplayRequested, "stale replay responses must resync on the replacement client") + + if replacementReplayRequested { + let replacementReplayChunk = try #require(await iterator.next()) + #expect(String(data: replacementReplayChunk.data, encoding: .utf8) == "replacement-replay") + store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: replacementReplayChunk.streamToken) + #expect(store.terminalReplayBarrierTokensBySurfaceID[surfaceID] == nil) + } +} + +@MainActor +@Test func terminalReplayBarrierReplaysOnReplacementClientAfterStaleFailure() async throws { + let router = LivenessHostRouter() + let box = TransportBox() + let clock = TestClock() + let store = try await makeConnectedStore(router: router, box: box, clock: clock) + let surfaceID = "live-terminal" + + await router.enqueueReplayTexts(["cold-replay"]) + var iterator = store.terminalOutputStream(surfaceID: surfaceID).makeAsyncIterator() + await router.waitForCount(of: "mobile.terminal.replay", atLeast: 1) + let coldReplayChunk = try #require(await iterator.next()) + #expect(String(decoding: coldReplayChunk.data, as: UTF8.self) == "cold-replay") + store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: coldReplayChunk.streamToken) + let replayCountAfterMount = await router.count(of: "mobile.terminal.replay") + await router.holdNextReplayResponses() + + store.deliverTerminalBytes(Data("stalled-first".utf8), surfaceID: surfaceID) + let stalledChunk = try #require(await iterator.next()) + store.terminalOutputDidReset(surfaceID: surfaceID, streamToken: stalledChunk.streamToken) + await router.waitForCount(of: "mobile.terminal.replay", atLeast: replayCountAfterMount + 1) + + let replacementRouter = LivenessHostRouter() + let replacementBox = TransportBox() + await replacementRouter.enqueueReplayTexts(["replacement-replay"]) try installFreshLivenessRemoteClient( on: store, router: replacementRouter, @@ -286,11 +335,18 @@ import Testing await router.failNextReplay() await router.releaseAllHeld() - let staleFailureClearedBarrier = await waitForReplayBarrierFailureToSettle { - store.terminalReplayBarrierTokensBySurfaceID[surfaceID] == nil - && !store.terminalReplaySurfaceIDsInFlight.contains(surfaceID) + let replacementReplayRequested = await waitForReplayRequestCount( + replacementRouter, + atLeast: 1 + ) + #expect(replacementReplayRequested, "stale replay failures must resync on the replacement client") + + if replacementReplayRequested { + let replacementReplayChunk = try #require(await iterator.next()) + #expect(String(data: replacementReplayChunk.data, encoding: .utf8) == "replacement-replay") + store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: replacementReplayChunk.streamToken) + #expect(store.terminalReplayBarrierTokensBySurfaceID[surfaceID] == nil) } - #expect(staleFailureClearedBarrier, "stale replay failures must clear the tokened barrier") } @MainActor @@ -566,6 +622,47 @@ import Testing #expect(store.terminalReplayBarrierTokensBySurfaceID[surfaceID] == nil) } +@MainActor +@Test func terminalReplayBarrierRetriesEmptyReplayAfterDroppedOutput() async throws { + let router = LivenessHostRouter() + let box = TransportBox() + let clock = TestClock() + let store = try await makeConnectedStore(router: router, box: box, clock: clock) + let surfaceID = "live-terminal" + + await router.enqueueReplayTexts(["cold-replay"]) + var iterator = store.terminalOutputStream(surfaceID: surfaceID).makeAsyncIterator() + await router.waitForCount(of: "mobile.terminal.replay", atLeast: 1) + let coldReplayChunk = try #require(await iterator.next()) + #expect(String(decoding: coldReplayChunk.data, as: UTF8.self) == "cold-replay") + store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: coldReplayChunk.streamToken) + let replayCountAfterMount = await router.count(of: "mobile.terminal.replay") + + await router.enqueueEmptyReplayResponses() + await router.enqueueReplayTexts(["follow-up-replay"]) + let replayBarrierToken = store.beginTerminalReplayBarrier(surfaceID: surfaceID) + let droppedOutputAccepted = store.deliverTerminalBytes( + Data("live-during-empty-replay".utf8), + surfaceID: surfaceID + ) + #expect(droppedOutputAccepted == false) + #expect(store.terminalReplayBarrierTokensBySurfaceID[surfaceID] == replayBarrierToken) + + await router.waitForCount(of: "mobile.terminal.replay", atLeast: replayCountAfterMount + 1) + let followUpReplayRequested = await waitForReplayRequestCount( + router, + atLeast: replayCountAfterMount + 2 + ) + #expect(followUpReplayRequested, "empty replay with dropped output must immediately request a replacement replay") + + if followUpReplayRequested { + let followUpChunk = try #require(await iterator.next()) + #expect(String(data: followUpChunk.data, encoding: .utf8) == "follow-up-replay") + store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: followUpChunk.streamToken) + #expect(store.terminalReplayBarrierTokensBySurfaceID[surfaceID] == nil) + } +} + @MainActor private func waitForReplayBarrierFailureToSettle( _ condition: @MainActor () -> Bool From 6e5604c1a7c21ae3d2ae299648c8a2747cf27959 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 05:43:43 -0700 Subject: [PATCH 34/56] Retry replay barriers after stale client responses --- .github/swift-file-length-budget.tsv | 2 +- .../MobileShellComposite.swift | 81 +++++++++++++++++-- 2 files changed, 74 insertions(+), 9 deletions(-) diff --git a/.github/swift-file-length-budget.tsv b/.github/swift-file-length-budget.tsv index 0eb1bde28b3c..6a1f9e3ab4a9 100644 --- a/.github/swift-file-length-budget.tsv +++ b/.github/swift-file-length-budget.tsv @@ -12,8 +12,8 @@ 9497 cmuxTests/CLINotifyProcessIntegrationRegressionTests.swift 8032 Sources/Panels/BrowserPanelView.swift 8016 CLI/cmux_open.swift +7373 Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift 7366 cmuxTests/WorkspaceUnitTests.swift -7308 Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift 7218 cmuxTests/WorkspaceRemoteConnectionTests.swift 6359 cmuxTests/SessionPersistenceTests.swift 6217 cmuxTests/GhosttyConfigTests.swift diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift index 6ce6745b5e53..3f8a68af7020 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift @@ -6401,6 +6401,28 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { return replayBarrierToken } + @discardableResult + private func requestTerminalReplayForCurrentBarrier( + surfaceID: String, + replayBarrierToken: UUID?, + coveredReplayBarrierDroppedOutputCount: UInt64?, + reason: String + ) -> Bool { + guard let replayBarrierToken, + hasTerminalOutputSink(surfaceID: surfaceID), + terminalReplayBarrierTokensBySurfaceID[surfaceID] == replayBarrierToken, + remoteClient != nil else { + return false + } + MobileDebugLog.anchormux("CMUX_REPLAY retry_\(reason) surface=\(surfaceID)") + requestTerminalReplay( + surfaceID: surfaceID, + replayBarrierToken: replayBarrierToken, + coveredReplayBarrierDroppedOutputCount: coveredReplayBarrierDroppedOutputCount + ) + return true + } + private func markTerminalReplayInFlight( surfaceID: String, requestID: UUID, @@ -6730,11 +6752,24 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { return } guard self.remoteClient === client else { - self.clearTerminalReplayBarrierIfCurrent( + self.clearTerminalReplayInFlightIfCurrent( surfaceID: surfaceID, - token: replayBarrierTokenForRequest, - reason: "stale_client" + requestID: replayRequestID ) + transferredInFlightToRetry = true + guard self.requestTerminalReplayForCurrentBarrier( + surfaceID: surfaceID, + replayBarrierToken: replayBarrierTokenForRequest, + coveredReplayBarrierDroppedOutputCount: nil, + reason: "stale_client" + ) else { + self.clearTerminalReplayBarrierIfCurrent( + surfaceID: surfaceID, + token: replayBarrierTokenForRequest, + reason: "stale_client" + ) + return + } return } let payload = try? MobileTerminalReplayResponse.decode(data) @@ -6807,11 +6842,28 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { return } guard let deliverBytes, !deliverBytes.isEmpty else { + if self.terminalReplayBarrierDroppedOutputSurfaceIDs.contains(surfaceID), + let retryToken = self.prepareTerminalReplayFailureRetry( + surfaceID: surfaceID, + replayBarrierToken: replayBarrierTokenForRequest + ) { + self.clearTerminalReplayInFlightIfCurrent( + surfaceID: surfaceID, + requestID: replayRequestID + ) + transferredInFlightToRetry = true + self.requestTerminalReplay( + surfaceID: surfaceID, + replayBarrierToken: retryToken, + coveredReplayBarrierDroppedOutputCount: + self.terminalReplayBarrierDroppedOutputCountsBySurfaceID[surfaceID] + ) + return + } self.clearTerminalReplayBarrierIfCurrent( surfaceID: surfaceID, token: replayBarrierTokenForRequest, - reason: "empty", - preserveDroppedOutput: true + reason: "empty" ) return } @@ -6846,11 +6898,24 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { } mobileShellLog.error("CMUX_REPLAY failed surface=\(surfaceID, privacy: .public) error=\(String(describing: error), privacy: .private)") guard self.remoteClient === client else { - self.clearTerminalReplayBarrierIfCurrent( + self.clearTerminalReplayInFlightIfCurrent( surfaceID: surfaceID, - token: replayBarrierTokenForRequest, - reason: "stale_client" + requestID: replayRequestID ) + transferredInFlightToRetry = true + guard self.requestTerminalReplayForCurrentBarrier( + surfaceID: surfaceID, + replayBarrierToken: replayBarrierTokenForRequest, + coveredReplayBarrierDroppedOutputCount: nil, + reason: "stale_client" + ) else { + self.clearTerminalReplayBarrierIfCurrent( + surfaceID: surfaceID, + token: replayBarrierTokenForRequest, + reason: "stale_client" + ) + return + } return } // The replay request is the view-only/foreground-resume path. A From 566d6715be88f1dc388911e41c5b88fef98a7cdc Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 05:45:41 -0700 Subject: [PATCH 35/56] Address terminal replay test review feedback --- .github/swift-file-length-budget.tsv | 2 +- .../TerminalOutputDeliveryQueueTests.swift | 66 +++++++++---------- 2 files changed, 32 insertions(+), 36 deletions(-) diff --git a/.github/swift-file-length-budget.tsv b/.github/swift-file-length-budget.tsv index 6a1f9e3ab4a9..bfc08e0946cd 100644 --- a/.github/swift-file-length-budget.tsv +++ b/.github/swift-file-length-budget.tsv @@ -111,7 +111,7 @@ 859 Packages/macOS/CmuxControlSocket/Sources/CmuxControlSocket/Coordinator/Workspace/ControlCommandCoordinator+Workspace.swift 847 cmuxTests/AgentSessionAutoResumeSettingsTests.swift 845 cmuxTests/SSHStartupSignalLifecycleTests.swift -841 Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift +837 Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift 834 Sources/MainWindowFocusController.swift 830 Sources/TaskManagerTypes.swift 825 Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/TerminalComposerView.swift diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift index bd76b6fb06f6..7579d12c4d9c 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift @@ -38,7 +38,7 @@ import Testing store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: currentChunk.streamToken) let secondChunk = try #require(await currentIterator.next()) - #expect(String(decoding: secondChunk.data, as: UTF8.self) == "new-second") + #expect(String(data: secondChunk.data, encoding: .utf8) == "new-second") } @MainActor @@ -73,7 +73,7 @@ import Testing bypassReplayBarrier: true ) let replayChunk = try #require(await iterator.next()) - #expect(String(decoding: replayChunk.data, as: UTF8.self) == "authoritative-replay") + #expect(String(data: replayChunk.data, encoding: .utf8) == "authoritative-replay") #expect(replayChunk.streamToken != stalledChunk.streamToken) let liveBeforeReplayAckAccepted = store.deliverTerminalBytes( @@ -96,7 +96,7 @@ import Testing store.deliverTerminalBytes(Data("after-replay-ack".utf8), surfaceID: surfaceID) let afterReplayAck = try #require(await iterator.next()) - #expect(String(decoding: afterReplayAck.data, as: UTF8.self) == "after-replay-ack") + #expect(String(data: afterReplayAck.data, encoding: .utf8) == "after-replay-ack") } @MainActor @@ -122,7 +122,7 @@ import Testing #expect(accepted == true) let afterAbort = try #require(await iterator.next()) - #expect(String(decoding: afterAbort.data, as: UTF8.self) == "after-aborted-replay") + #expect(String(data: afterAbort.data, encoding: .utf8) == "after-aborted-replay") } @MainActor @@ -137,7 +137,7 @@ import Testing var iterator = store.terminalOutputStream(surfaceID: surfaceID).makeAsyncIterator() await router.waitForCount(of: "mobile.terminal.replay", atLeast: 1) let coldReplayChunk = try #require(await iterator.next()) - #expect(String(decoding: coldReplayChunk.data, as: UTF8.self) == "cold-replay") + #expect(String(data: coldReplayChunk.data, encoding: .utf8) == "cold-replay") store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: coldReplayChunk.streamToken) let coldReplaySettled = await waitForReplayBarrierFailureToSettle { !store.terminalReplaySurfaceIDsInFlight.contains(surfaceID) @@ -153,7 +153,7 @@ import Testing await router.waitForCount(of: "mobile.terminal.replay", atLeast: replayCountAfterMount + 1) let replayChunk = try #require(await iterator.next()) - #expect(String(decoding: replayChunk.data, as: UTF8.self) == "first-replay") + #expect(String(data: replayChunk.data, encoding: .utf8) == "first-replay") let acceptedDuringBarrier = store.deliverTerminalBytes( Data("live-during-barrier".utf8), @@ -166,7 +166,7 @@ import Testing await router.waitForCount(of: "mobile.terminal.replay", atLeast: replayCountAfterMount + 2) let followUpChunk = try #require(await iterator.next()) - #expect(String(decoding: followUpChunk.data, as: UTF8.self) == "follow-up-replay") + #expect(String(data: followUpChunk.data, encoding: .utf8) == "follow-up-replay") #expect(!store.terminalReplayBarrierDroppedOutputSurfaceIDs.contains(surfaceID)) store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: followUpChunk.streamToken) @@ -174,7 +174,7 @@ import Testing store.deliverTerminalBytes(Data("after-follow-up".utf8), surfaceID: surfaceID) let afterFollowUp = try #require(await iterator.next()) - #expect(String(decoding: afterFollowUp.data, as: UTF8.self) == "after-follow-up") + #expect(String(data: afterFollowUp.data, encoding: .utf8) == "after-follow-up") } @MainActor @@ -189,7 +189,7 @@ import Testing var iterator = store.terminalOutputStream(surfaceID: surfaceID).makeAsyncIterator() await router.waitForCount(of: "mobile.terminal.replay", atLeast: 1) let coldReplayChunk = try #require(await iterator.next()) - #expect(String(decoding: coldReplayChunk.data, as: UTF8.self) == "cold-replay") + #expect(String(data: coldReplayChunk.data, encoding: .utf8) == "cold-replay") store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: coldReplayChunk.streamToken) let replayCountAfterMount = await router.count(of: "mobile.terminal.replay") @@ -211,7 +211,7 @@ import Testing #expect(store.terminalReplayBarrierDroppedOutputSurfaceIDs.contains(surfaceID)) let retryReplayChunk = try #require(await iterator.next()) - #expect(String(decoding: retryReplayChunk.data, as: UTF8.self) == "retry-replay") + #expect(String(data: retryReplayChunk.data, encoding: .utf8) == "retry-replay") store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: retryReplayChunk.streamToken) #expect(store.terminalReplayBarrierTokensBySurfaceID[surfaceID] == nil) #expect(!store.terminalReplayBarrierDroppedOutputSurfaceIDs.contains(surfaceID)) @@ -229,7 +229,7 @@ import Testing var iterator = store.terminalOutputStream(surfaceID: surfaceID).makeAsyncIterator() await router.waitForCount(of: "mobile.terminal.replay", atLeast: 1) let coldReplayChunk = try #require(await iterator.next()) - #expect(String(decoding: coldReplayChunk.data, as: UTF8.self) == "cold-replay") + #expect(String(data: coldReplayChunk.data, encoding: .utf8) == "cold-replay") store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: coldReplayChunk.streamToken) let replayCountAfterMount = await router.count(of: "mobile.terminal.replay") @@ -247,7 +247,7 @@ import Testing if retryRequested { let retryReplayChunk = try #require(await iterator.next()) - #expect(String(decoding: retryReplayChunk.data, as: UTF8.self) == "retry-replay") + #expect(String(data: retryReplayChunk.data, encoding: .utf8) == "retry-replay") store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: retryReplayChunk.streamToken) #expect(store.terminalReplayBarrierTokensBySurfaceID[surfaceID] == nil) } @@ -265,7 +265,7 @@ import Testing var iterator = store.terminalOutputStream(surfaceID: surfaceID).makeAsyncIterator() await router.waitForCount(of: "mobile.terminal.replay", atLeast: 1) let coldReplayChunk = try #require(await iterator.next()) - #expect(String(decoding: coldReplayChunk.data, as: UTF8.self) == "cold-replay") + #expect(String(data: coldReplayChunk.data, encoding: .utf8) == "cold-replay") store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: coldReplayChunk.streamToken) let replayCountAfterMount = await router.count(of: "mobile.terminal.replay") await router.holdNextReplayResponses() @@ -313,7 +313,7 @@ import Testing var iterator = store.terminalOutputStream(surfaceID: surfaceID).makeAsyncIterator() await router.waitForCount(of: "mobile.terminal.replay", atLeast: 1) let coldReplayChunk = try #require(await iterator.next()) - #expect(String(decoding: coldReplayChunk.data, as: UTF8.self) == "cold-replay") + #expect(String(data: coldReplayChunk.data, encoding: .utf8) == "cold-replay") store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: coldReplayChunk.streamToken) let replayCountAfterMount = await router.count(of: "mobile.terminal.replay") await router.holdNextReplayResponses() @@ -361,7 +361,7 @@ import Testing var iterator = store.terminalOutputStream(surfaceID: surfaceID).makeAsyncIterator() await router.waitForCount(of: "mobile.terminal.replay", atLeast: 1) let coldReplayChunk = try #require(await iterator.next()) - #expect(String(decoding: coldReplayChunk.data, as: UTF8.self) == "cold-replay") + #expect(String(data: coldReplayChunk.data, encoding: .utf8) == "cold-replay") store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: coldReplayChunk.streamToken) let replayCountAfterMount = await router.count(of: "mobile.terminal.replay") @@ -416,17 +416,14 @@ import Testing let remountCollector = OutputCollector() remountCollector.mount(store: store, surfaceID: surfaceID) - let remountRequested = await waitForReplayRequestCount( - router, + await router.waitForCount( + of: "mobile.terminal.replay", atLeast: replayCountAfterUnmount + 1 ) - #expect(remountRequested, "remounting after a cancelled replay must request a fresh replay") - if remountRequested { - let replayDelivered = try await pollUntil { - remountCollector.lines.contains("remount-replay") - } - #expect(replayDelivered) + let replayDelivered = try await pollUntil { + remountCollector.lines.contains("remount-replay") } + #expect(replayDelivered) remountCollector.unmount() await router.releaseAllHeld() @@ -455,11 +452,10 @@ import Testing await router.enqueueReplayTexts(["fresh-replay", "stale-replay"]) let currentCollector = OutputCollector() currentCollector.mount(store: store, surfaceID: surfaceID) - let remountRequested = await waitForReplayRequestCount( - router, + await router.waitForCount( + of: "mobile.terminal.replay", atLeast: replayCountAfterUnmount + 1 ) - #expect(remountRequested) let freshDelivered = try await pollUntil { currentCollector.lines.contains("fresh-replay") @@ -487,7 +483,7 @@ import Testing var iterator = store.terminalOutputStream(surfaceID: surfaceID).makeAsyncIterator() await router.waitForCount(of: "mobile.terminal.replay", atLeast: 1) let coldReplayChunk = try #require(await iterator.next()) - #expect(String(decoding: coldReplayChunk.data, as: UTF8.self) == "cold-replay") + #expect(String(data: coldReplayChunk.data, encoding: .utf8) == "cold-replay") store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: coldReplayChunk.streamToken) let replayCountAfterMount = await router.count(of: "mobile.terminal.replay") @@ -520,7 +516,7 @@ import Testing if genericReplayRequested { let replayChunk = try #require(await iterator.next()) - #expect(String(decoding: replayChunk.data, as: UTF8.self) == "resync-replay") + #expect(String(data: replayChunk.data, encoding: .utf8) == "resync-replay") #expect(store.terminalReplayBarrierTokensBySurfaceID[surfaceID] == preservedBarrierToken) store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: replayChunk.streamToken) #expect(store.terminalReplayBarrierTokensBySurfaceID[surfaceID] == nil) @@ -539,7 +535,7 @@ import Testing var iterator = store.terminalOutputStream(surfaceID: surfaceID).makeAsyncIterator() await router.waitForCount(of: "mobile.terminal.replay", atLeast: 1) let coldReplayChunk = try #require(await iterator.next()) - #expect(String(decoding: coldReplayChunk.data, as: UTF8.self) == "cold-replay") + #expect(String(data: coldReplayChunk.data, encoding: .utf8) == "cold-replay") store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: coldReplayChunk.streamToken) let replayCountAfterMount = await router.count(of: "mobile.terminal.replay") @@ -551,14 +547,14 @@ import Testing await router.waitForCount(of: "mobile.terminal.replay", atLeast: replayCountAfterMount + 1) let replayChunk = try #require(await iterator.next()) - #expect(String(decoding: replayChunk.data, as: UTF8.self) == "first-replay") + #expect(String(data: replayChunk.data, encoding: .utf8) == "first-replay") let firstBarrierToken = try #require(store.terminalReplayBarrierTokensBySurfaceID[surfaceID]) store.terminalOutputDidReset(surfaceID: surfaceID, streamToken: replayChunk.streamToken) await router.waitForCount(of: "mobile.terminal.replay", atLeast: replayCountAfterMount + 2) let retryReplayChunk = try #require(await iterator.next()) - #expect(String(decoding: retryReplayChunk.data, as: UTF8.self) == "retry-replay") + #expect(String(data: retryReplayChunk.data, encoding: .utf8) == "retry-replay") let retryBarrierToken = try #require(store.terminalReplayBarrierTokensBySurfaceID[surfaceID]) #expect(retryBarrierToken != firstBarrierToken) @@ -578,7 +574,7 @@ import Testing var iterator = store.terminalOutputStream(surfaceID: surfaceID).makeAsyncIterator() await router.waitForCount(of: "mobile.terminal.replay", atLeast: 1) let coldReplayChunk = try #require(await iterator.next()) - #expect(String(decoding: coldReplayChunk.data, as: UTF8.self) == "cold-replay") + #expect(String(data: coldReplayChunk.data, encoding: .utf8) == "cold-replay") store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: coldReplayChunk.streamToken) let replayCountAfterMount = await router.count(of: "mobile.terminal.replay") @@ -605,7 +601,7 @@ import Testing await router.waitForCount(of: "mobile.terminal.replay", atLeast: replayCountAfterMount + 2) let retryReplayChunk = try #require(await iterator.next()) - #expect(String(decoding: retryReplayChunk.data, as: UTF8.self) == "retry-replay") + #expect(String(data: retryReplayChunk.data, encoding: .utf8) == "retry-replay") let postResponseDropAccepted = store.deliverTerminalBytes( Data("live-after-retry-response-before-ack".utf8), @@ -617,7 +613,7 @@ import Testing await router.waitForCount(of: "mobile.terminal.replay", atLeast: replayCountAfterMount + 3) let followUpChunk = try #require(await iterator.next()) - #expect(String(decoding: followUpChunk.data, as: UTF8.self) == "follow-up-replay") + #expect(String(data: followUpChunk.data, encoding: .utf8) == "follow-up-replay") store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: followUpChunk.streamToken) #expect(store.terminalReplayBarrierTokensBySurfaceID[surfaceID] == nil) } @@ -634,7 +630,7 @@ import Testing var iterator = store.terminalOutputStream(surfaceID: surfaceID).makeAsyncIterator() await router.waitForCount(of: "mobile.terminal.replay", atLeast: 1) let coldReplayChunk = try #require(await iterator.next()) - #expect(String(decoding: coldReplayChunk.data, as: UTF8.self) == "cold-replay") + #expect(String(data: coldReplayChunk.data, encoding: .utf8) == "cold-replay") store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: coldReplayChunk.streamToken) let replayCountAfterMount = await router.count(of: "mobile.terminal.replay") From e1a86a8f5b9942ff86c7d9e3a80367d10b5140a6 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 05:56:00 -0700 Subject: [PATCH 36/56] Add replay retry exhaustion drop regression test --- .github/swift-file-length-budget.tsv | 2 +- .../TerminalOutputDeliveryQueueTests.swift | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/swift-file-length-budget.tsv b/.github/swift-file-length-budget.tsv index bfc08e0946cd..bb1e4dfc253c 100644 --- a/.github/swift-file-length-budget.tsv +++ b/.github/swift-file-length-budget.tsv @@ -111,7 +111,7 @@ 859 Packages/macOS/CmuxControlSocket/Sources/CmuxControlSocket/Coordinator/Workspace/ControlCommandCoordinator+Workspace.swift 847 cmuxTests/AgentSessionAutoResumeSettingsTests.swift 845 cmuxTests/SSHStartupSignalLifecycleTests.swift -837 Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift +843 Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift 834 Sources/MainWindowFocusController.swift 830 Sources/TaskManagerTypes.swift 825 Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/TerminalComposerView.swift diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift index 7579d12c4d9c..e328c06580df 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift @@ -382,12 +382,18 @@ import Testing #expect(failureSettled) #expect(store.terminalReplayBarrierTokensBySurfaceID[surfaceID] != nil) + let replayCountAfterExhaustion = await router.count(of: "mobile.terminal.replay") let acceptedAfterExhaustion = store.deliverTerminalBytes( Data("live-after-exhausted-replay".utf8), surfaceID: surfaceID ) #expect(acceptedAfterExhaustion == false) #expect(store.terminalOutputQueuesBySurfaceID[surfaceID]?.isIdle == true) + let replayRestartedAfterExhaustion = await waitForReplayRequestCount( + router, + atLeast: replayCountAfterExhaustion + 1 + ) + #expect(!replayRestartedAfterExhaustion, "dropped live output must not bypass the replay retry cap") } @MainActor From cd3bdef4a31d431e65ec85a34bb4d13a8c1f0679 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 05:57:28 -0700 Subject: [PATCH 37/56] Respect replay retry exhaustion on live output drops --- .github/swift-file-length-budget.tsv | 2 +- .../MobileShellComposite+TerminalOutputDelivery.swift | 3 ++- .../Sources/CmuxMobileShell/MobileShellComposite.swift | 4 ++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/swift-file-length-budget.tsv b/.github/swift-file-length-budget.tsv index bb1e4dfc253c..685290d9fb77 100644 --- a/.github/swift-file-length-budget.tsv +++ b/.github/swift-file-length-budget.tsv @@ -12,7 +12,7 @@ 9497 cmuxTests/CLINotifyProcessIntegrationRegressionTests.swift 8032 Sources/Panels/BrowserPanelView.swift 8016 CLI/cmux_open.swift -7373 Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift +7377 Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift 7366 cmuxTests/WorkspaceUnitTests.swift 7218 cmuxTests/WorkspaceRemoteConnectionTests.swift 6359 cmuxTests/SessionPersistenceTests.swift diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift index 46459fa73c2d..4ba1186521cb 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift @@ -66,7 +66,8 @@ extension MobileShellComposite { MobileDebugLog.anchormux("terminal.output.drop_replay_barrier surface=\(surfaceID)") if remoteClient != nil, terminalReplayBarrierAckStreamTokensBySurfaceID[surfaceID] == nil, - !terminalReplaySurfaceIDsInFlight.contains(surfaceID) { + !terminalReplaySurfaceIDsInFlight.contains(surfaceID), + !terminalReplayFailureRetryExhausted(surfaceID: surfaceID) { MobileDebugLog.anchormux("terminal.output.replay_retry_after_drop surface=\(surfaceID)") requestTerminalReplay( surfaceID: surfaceID, diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift index 3f8a68af7020..ca25a6774252 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift @@ -6401,6 +6401,10 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { return replayBarrierToken } + func terminalReplayFailureRetryExhausted(surfaceID: String) -> Bool { + (terminalReplayFailureRetryCountsBySurfaceID[surfaceID] ?? 0) >= Self.maxTerminalReplayFailureRetries + } + @discardableResult private func requestTerminalReplayForCurrentBarrier( surfaceID: String, From 3c80ddf71fa39c8911f7d46ed66a59fb1dbc862f Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 06:15:10 -0700 Subject: [PATCH 38/56] Defer replay until render recovery can run --- .github/swift-file-length-budget.tsv | 4 +-- ...leShellRenderGridLivenessTestSupport.swift | 35 +++---------------- .../GhosttySurfaceView.swift | 15 ++++++-- 3 files changed, 19 insertions(+), 35 deletions(-) diff --git a/.github/swift-file-length-budget.tsv b/.github/swift-file-length-budget.tsv index 685290d9fb77..9d4d778c2636 100644 --- a/.github/swift-file-length-budget.tsv +++ b/.github/swift-file-length-budget.tsv @@ -24,7 +24,7 @@ 4487 Sources/Panels/FilePreviewPanel.swift 4483 Sources/cmuxApp.swift 4367 cmuxTests/BrowserPanelTests.swift -4268 Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift +4277 Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift 4121 Sources/BrowserWindowPortal.swift 3934 Sources/Feed/FeedPanelView.swift 3926 cmuxTests/TabManagerUnitTests.swift @@ -195,7 +195,6 @@ 574 Packages/macOS/CmuxWorkspaces/Tests/CmuxWorkspacesTests/WorkspaceCoordinatorTests.swift 572 Sources/Feed/FeedTextEditorDebugWindowController.swift 568 Packages/Shared/CMUXMobileCore/Sources/CMUXMobileCore/MobileTerminalRenderGrid.swift -567 Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift 567 Packages/macOS/CmuxSettingsUI/Sources/CmuxSettingsUI/Sections/BrowserSection.swift 567 Packages/macOS/CmuxTerminalCore/Sources/CmuxTerminalCore/ConfigDiscovery/GhosttyConfigDiscovery.swift 563 Packages/macOS/CmuxControlSocket/Sources/CmuxControlSocket/Coordinator/Pane/ControlCommandCoordinator+Pane.swift @@ -208,6 +207,7 @@ 547 Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/BackingUpPairedMacStore.swift 546 Packages/macOS/CmuxSettingsUI/Sources/CmuxSettingsUI/Sections/AutomationSection.swift 544 cmuxUITests/DisplayResolutionRegressionUITests.swift +542 Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift 539 CLI/CMUXCLI+Themes.swift 539 CLI/CodexTeamsApprovalBridge.swift 538 Packages/macOS/CmuxRemoteWorkspace/Sources/CmuxRemoteWorkspace/PTYBridge/RemotePTYBridgeSession.swift diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift index 1690760a1e31..78453cc7aefd 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift @@ -58,14 +58,7 @@ actor LivenessHostRouter { var topics: [String]? } - private struct CountWaiter { - var method: String - var expectedCount: Int - var continuation: CheckedContinuation - } - private var recorded: [RecordedRequest] = [] - private var countWaiters: [CountWaiter] = [] private var hostStatusRequestCount = 0 private var heldHostStatusRequestNumbers: Set = [] private var subscribeRequestCount = 0 @@ -83,7 +76,6 @@ actor LivenessHostRouter { func record(method: String?, topics: [String]?) { recorded.append(RecordedRequest(method: method, topics: topics)) - resumeSatisfiedCountWaiters() } func count(of method: String) -> Int { @@ -91,30 +83,13 @@ actor LivenessHostRouter { } func waitForCount(of method: String, atLeast expectedCount: Int) async { - guard count(of: method) < expectedCount else { return } - await withCheckedContinuation { continuation in - countWaiters.append(CountWaiter( - method: method, - expectedCount: expectedCount, - continuation: continuation - )) - } - } - - private func resumeSatisfiedCountWaiters() { - var remaining: [CountWaiter] = [] - var ready: [CheckedContinuation] = [] - for waiter in countWaiters { - if count(of: waiter.method) >= waiter.expectedCount { - ready.append(waiter.continuation) - } else { - remaining.append(waiter) + for _ in 0..<300 { + if count(of: method) >= expectedCount { + return } + try? await Task.sleep(nanoseconds: 10_000_000) } - countWaiters = remaining - for continuation in ready { - continuation.resume(returning: ()) - } + Issue.record("timed out waiting for \(method) count >= \(expectedCount)") } func topics(for method: String) -> [[String]] { diff --git a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift index 08f2dbc1a9a1..d4e51811710e 100644 --- a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift +++ b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift @@ -2380,7 +2380,11 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { stalledMs: elapsedMs, replay: .callerWillRequestReplay ) - pending.continuation.resume(returning: false) + if recovered || !renderPipelineRecoveryBlocked { + pending.continuation.resume(returning: false) + } else { + pendingOutputApply = pending + } return recovered } @@ -2394,7 +2398,11 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { stalledMs: elapsedMs, replay: .callerWillRequestReplay ) - pending.continuation.resume(returning: false) + if recovered || !renderPipelineRecoveryBlocked { + pending.continuation.resume(returning: false) + } else { + pendingGeometryApply = pending + } return recovered } @@ -2876,7 +2884,8 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { replay: replay ) stopDisplayLink() - completePendingSurfaceOperations(returning: false) + skipPendingVisibleSnapshot() + skipPendingCopyableTextRead() renderInFlight = false renderInFlightSince = nil needsAnotherRender = false From 043676453a8b77853c14351af1004672ff84ba3e Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 06:21:03 -0700 Subject: [PATCH 39/56] Cancel superseded terminal replay tasks --- .github/swift-file-length-budget.tsv | 2 +- .../MobileShellComposite.swift | 48 +++++++++++++++++-- 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/.github/swift-file-length-budget.tsv b/.github/swift-file-length-budget.tsv index 9d4d778c2636..5aba846bea9b 100644 --- a/.github/swift-file-length-budget.tsv +++ b/.github/swift-file-length-budget.tsv @@ -12,7 +12,7 @@ 9497 cmuxTests/CLINotifyProcessIntegrationRegressionTests.swift 8032 Sources/Panels/BrowserPanelView.swift 8016 CLI/cmux_open.swift -7377 Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift +7417 Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift 7366 cmuxTests/WorkspaceUnitTests.swift 7218 cmuxTests/WorkspaceRemoteConnectionTests.swift 6359 cmuxTests/SessionPersistenceTests.swift diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift index ca25a6774252..c381a121bf22 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift @@ -718,6 +718,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { private var terminalActiveScreenBySurfaceID: [String: MobileTerminalRenderGridFrame.Screen] var terminalReplaySurfaceIDsInFlight: Set private var terminalReplayRequestIDsInFlightBySurfaceID: [String: UUID] + private var terminalReplayTasksBySurfaceID: [String: Task] private var terminalReplayBarrierTokensInFlightBySurfaceID: [String: UUID] var terminalReplayBarrierTokensBySurfaceID: [String: UUID] var terminalReplayBarrierAckStreamTokensBySurfaceID: [String: UUID] @@ -917,6 +918,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { self.terminalActiveScreenBySurfaceID = [:] self.terminalReplaySurfaceIDsInFlight = [] self.terminalReplayRequestIDsInFlightBySurfaceID = [:] + self.terminalReplayTasksBySurfaceID = [:] self.terminalReplayBarrierTokensInFlightBySurfaceID = [:] self.terminalReplayBarrierTokensBySurfaceID = [:] self.terminalReplayBarrierAckStreamTokensBySurfaceID = [:] @@ -949,6 +951,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { createTerminalTask?.cancel() workspaceListRefreshTask?.cancel() pullToRefreshTask?.cancel() + cancelAllTerminalReplayTasks() teardownSecondaryMacSubscriptions() if let remoteClient { Task { await remoteClient.disconnect() } @@ -4944,9 +4947,11 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { workspaceListRefreshTask = nil pullToRefreshTask?.cancel() pullToRefreshTask = nil + cancelAllTerminalReplayTasks() } private func resetTerminalOutputTracking() { + cancelAllTerminalReplayTasks() deliveredTerminalByteEndSeqBySurfaceID = [:] pendingTerminalByteEndSeqBySurfaceID = [:] terminalActiveScreenBySurfaceID = [:] @@ -6319,6 +6324,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { } func beginTerminalReplayBarrier(surfaceID: String) -> UUID { + cancelTerminalReplayInFlight(surfaceID: surfaceID) terminalOutputQueuesBySurfaceID[surfaceID] = TerminalOutputDeliveryQueue() terminalOutputStreamTokensBySurfaceID[surfaceID] = UUID() deliveredTerminalByteEndSeqBySurfaceID.removeValue(forKey: surfaceID) @@ -6432,6 +6438,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { requestID: UUID, replayBarrierToken: UUID? ) { + cancelTerminalReplayInFlight(surfaceID: surfaceID) terminalReplaySurfaceIDsInFlight.insert(surfaceID) terminalReplayRequestIDsInFlightBySurfaceID[surfaceID] = requestID if let replayBarrierToken { @@ -6441,13 +6448,43 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { } } + private func storeTerminalReplayTask( + surfaceID: String, + requestID: UUID, + task: Task + ) { + guard terminalReplayRequestIDsInFlightBySurfaceID[surfaceID] == requestID else { + task.cancel() + return + } + terminalReplayTasksBySurfaceID[surfaceID] = task + } + private func clearTerminalReplayInFlightIfCurrent(surfaceID: String, requestID: UUID) { guard terminalReplayRequestIDsInFlightBySurfaceID[surfaceID] == requestID else { return } terminalReplaySurfaceIDsInFlight.remove(surfaceID) terminalReplayRequestIDsInFlightBySurfaceID.removeValue(forKey: surfaceID) + terminalReplayTasksBySurfaceID.removeValue(forKey: surfaceID) terminalReplayBarrierTokensInFlightBySurfaceID.removeValue(forKey: surfaceID) } + private func cancelTerminalReplayInFlight(surfaceID: String) { + terminalReplayTasksBySurfaceID.removeValue(forKey: surfaceID)?.cancel() + terminalReplaySurfaceIDsInFlight.remove(surfaceID) + terminalReplayRequestIDsInFlightBySurfaceID.removeValue(forKey: surfaceID) + terminalReplayBarrierTokensInFlightBySurfaceID.removeValue(forKey: surfaceID) + } + + private func cancelAllTerminalReplayTasks() { + for task in terminalReplayTasksBySurfaceID.values { + task.cancel() + } + terminalReplayTasksBySurfaceID = [:] + terminalReplaySurfaceIDsInFlight = [] + terminalReplayRequestIDsInFlightBySurfaceID = [:] + terminalReplayBarrierTokensInFlightBySurfaceID = [:] + } + private enum RenderGridEventDeliveryDecision { case deliver case advisory(requestReplay: Bool, updateTrackedScreen: Bool, deliverViewportPolicy: Bool) @@ -6538,6 +6575,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { } private func unregisterTerminalOutput(surfaceID: String) { + cancelTerminalReplayInFlight(surfaceID: surfaceID) terminalByteContinuationsBySurfaceID.removeValue(forKey: surfaceID) terminalOutputStreamTokensBySurfaceID.removeValue(forKey: surfaceID) terminalOutputQueuesBySurfaceID.removeValue(forKey: surfaceID) @@ -6547,9 +6585,6 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { terminalReplayBarrierDroppedOutputCountsBySurfaceID.removeValue(forKey: surfaceID) terminalReplayBarrierAckCoveredDroppedOutputCountsBySurfaceID.removeValue(forKey: surfaceID) terminalReplayFailureRetryCountsBySurfaceID.removeValue(forKey: surfaceID) - terminalReplaySurfaceIDsInFlight.remove(surfaceID) - terminalReplayRequestIDsInFlightBySurfaceID.removeValue(forKey: surfaceID) - terminalReplayBarrierTokensInFlightBySurfaceID.removeValue(forKey: surfaceID) terminalScrollQueueTokensBySurfaceID.removeValue(forKey: surfaceID) terminalScrollQueuesBySurfaceID.removeValue(forKey: surfaceID) terminalScrollbackPrefetchStatesBySurfaceID.removeValue(forKey: surfaceID) @@ -6731,7 +6766,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { requestID: replayRequestID, replayBarrierToken: replayBarrierTokenForRequest ) - Task { @MainActor [weak self] in + let replayTask = Task { @MainActor [weak self] in guard let self else { return } var transferredInFlightToRetry = false defer { @@ -6950,6 +6985,11 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { ) } } + storeTerminalReplayTask( + surfaceID: surfaceID, + requestID: replayRequestID, + task: replayTask + ) } private func handleTerminalRenderGridEvent(_ event: MobileEventEnvelope) { From a1af4c07d0068c92e961b7ae27c64eb07c841620 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 06:31:23 -0700 Subject: [PATCH 40/56] Fail blocked terminal surface waiters --- .github/swift-file-length-budget.tsv | 2 +- .../GhosttySurfaceView.swift | 28 ++++++++----------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/.github/swift-file-length-budget.tsv b/.github/swift-file-length-budget.tsv index 5aba846bea9b..7a660efb671d 100644 --- a/.github/swift-file-length-budget.tsv +++ b/.github/swift-file-length-budget.tsv @@ -24,7 +24,7 @@ 4487 Sources/Panels/FilePreviewPanel.swift 4483 Sources/cmuxApp.swift 4367 cmuxTests/BrowserPanelTests.swift -4277 Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift +4273 Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift 4121 Sources/BrowserWindowPortal.swift 3934 Sources/Feed/FeedPanelView.swift 3926 cmuxTests/TabManagerUnitTests.swift diff --git a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift index d4e51811710e..47275248e33f 100644 --- a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift +++ b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift @@ -2293,8 +2293,8 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { byteCount: data.count, continuation: continuation ) - processOutput(data) { applied in - self.completePendingOutputApply(id: operationID, returning: applied) + processOutput(data) { [weak self] applied in + self?.completePendingOutputApply(id: operationID, returning: applied) } } } @@ -2380,11 +2380,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { stalledMs: elapsedMs, replay: .callerWillRequestReplay ) - if recovered || !renderPipelineRecoveryBlocked { - pending.continuation.resume(returning: false) - } else { - pendingOutputApply = pending - } + pending.continuation.resume(returning: false) return recovered } @@ -2398,11 +2394,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { stalledMs: elapsedMs, replay: .callerWillRequestReplay ) - if recovered || !renderPipelineRecoveryBlocked { - pending.continuation.resume(returning: false) - } else { - pendingGeometryApply = pending - } + pending.continuation.resume(returning: false) return recovered } @@ -2477,9 +2469,11 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { completion?(true) return } - // A blocked recovery will rebuild this surface once the prior free drains. - // Leave wait-style callers pending so recovery or the deadline pump resumes false. + // A blocked recovery will rebuild this surface if the prior free drains. + // Do not park wait-style callers behind that free: the free is queued on + // the same surface queue that may already be wedged. guard !renderPipelineRecoveryBlocked else { + completion?(false) return } #if DEBUG @@ -2886,6 +2880,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { stopDisplayLink() skipPendingVisibleSnapshot() skipPendingCopyableTextRead() + completePendingSurfaceOperations(returning: false) renderInFlight = false renderInFlightSince = nil needsAnotherRender = false @@ -3504,8 +3499,8 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { pendingGeometryReassert = false return await withCheckedContinuation { continuation in let operationID = registerPendingGeometryApply(continuation: continuation) - syncSurfaceGeometry(shouldReassertNaturalSize: shouldReassertNaturalSize) { applied in - self.completePendingGeometryApply(id: operationID, returning: applied) + syncSurfaceGeometry(shouldReassertNaturalSize: shouldReassertNaturalSize) { [weak self] applied in + self?.completePendingGeometryApply(id: operationID, returning: applied) } } } @@ -3521,6 +3516,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { // The current surface is scheduled for deferred recovery; geometry // cannot be acknowledged against it without risking a stale replay ack. guard !renderPipelineRecoveryBlocked else { + completion?(false) return } From 590684b203483c0419d9ed23a3529677f7bd7b8f Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 06:37:37 -0700 Subject: [PATCH 41/56] Address terminal replay review feedback --- .github/swift-file-length-budget.tsv | 4 +- .../MobileShellComposite.swift | 26 ++++--- ...leShellRenderGridLivenessTestSupport.swift | 77 +++++++++++++++++-- 3 files changed, 89 insertions(+), 18 deletions(-) diff --git a/.github/swift-file-length-budget.tsv b/.github/swift-file-length-budget.tsv index 7a660efb671d..9684b525878f 100644 --- a/.github/swift-file-length-budget.tsv +++ b/.github/swift-file-length-budget.tsv @@ -12,7 +12,7 @@ 9497 cmuxTests/CLINotifyProcessIntegrationRegressionTests.swift 8032 Sources/Panels/BrowserPanelView.swift 8016 CLI/cmux_open.swift -7417 Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift +7423 Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift 7366 cmuxTests/WorkspaceUnitTests.swift 7218 cmuxTests/WorkspaceRemoteConnectionTests.swift 6359 cmuxTests/SessionPersistenceTests.swift @@ -172,6 +172,7 @@ 620 cmuxTests/TerminalNotificationQueueTests.swift 615 Sources/SettingsNavigation.swift 608 cmuxUITests/FeedSidebarUITests.swift +607 Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift 607 Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface.swift 607 Sources/SessionIndexModels.swift 607 Sources/SleepyFaceView.swift @@ -207,7 +208,6 @@ 547 Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/BackingUpPairedMacStore.swift 546 Packages/macOS/CmuxSettingsUI/Sources/CmuxSettingsUI/Sections/AutomationSection.swift 544 cmuxUITests/DisplayResolutionRegressionUITests.swift -542 Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift 539 CLI/CMUXCLI+Themes.swift 539 CLI/CodexTeamsApprovalBridge.swift 538 Packages/macOS/CmuxRemoteWorkspace/Sources/CmuxRemoteWorkspace/PTYBridge/RemotePTYBridgeSession.swift diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift index c381a121bf22..04027e0512da 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift @@ -6767,6 +6767,19 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { replayBarrierToken: replayBarrierTokenForRequest ) let replayTask = Task { @MainActor [weak self] in + let replayResult: Result + do { + let request = try MobileCoreRPCClient.requestData( + method: "mobile.terminal.replay", + params: [ + "workspace_id": remoteWorkspaceID.rawValue, + "surface_id": surfaceID, + ] + ) + replayResult = .success(try await client.sendRequest(request)) + } catch { + replayResult = .failure(error) + } guard let self else { return } var transferredInFlightToRetry = false defer { @@ -6777,15 +6790,8 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { ) } } - do { - let request = try MobileCoreRPCClient.requestData( - method: "mobile.terminal.replay", - params: [ - "workspace_id": remoteWorkspaceID.rawValue, - "surface_id": surfaceID, - ] - ) - let data = try await client.sendRequest(request) + switch replayResult { + case .success(let data): guard self.terminalReplayRequestIDsInFlightBySurfaceID[surfaceID] == replayRequestID else { MobileDebugLog.anchormux("CMUX_REPLAY stale_request surface=\(surfaceID)") return @@ -6930,7 +6936,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { preserveDroppedOutput: true ) } - } catch { + case .failure(let error): guard self.terminalReplayRequestIDsInFlightBySurfaceID[surfaceID] == replayRequestID else { MobileDebugLog.anchormux("CMUX_REPLAY stale_request_failed surface=\(surfaceID)") return diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift index 78453cc7aefd..75f2207d8591 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift @@ -58,7 +58,15 @@ actor LivenessHostRouter { var topics: [String]? } + private struct CountWaiter { + let id: UUID + let method: String + let expectedCount: Int + let continuation: CheckedContinuation + } + private var recorded: [RecordedRequest] = [] + private var countWaiters: [CountWaiter] = [] private var hostStatusRequestCount = 0 private var heldHostStatusRequestNumbers: Set = [] private var subscribeRequestCount = 0 @@ -76,20 +84,77 @@ actor LivenessHostRouter { func record(method: String?, topics: [String]?) { recorded.append(RecordedRequest(method: method, topics: topics)) + resumeSatisfiedCountWaiters() } func count(of method: String) -> Int { recorded.filter { $0.method == method }.count } - func waitForCount(of method: String, atLeast expectedCount: Int) async { - for _ in 0..<300 { - if count(of: method) >= expectedCount { - return + func waitForCount( + of method: String, + atLeast expectedCount: Int, + timeoutNanoseconds: UInt64 = 3_000_000_000 + ) async { + let reached = await withTaskGroup(of: Bool.self) { group in + group.addTask { + await self.waitUntilCountReached(of: method, atLeast: expectedCount) + return true + } + group.addTask { + // Test assertion deadline only; request arrival is signaled by record(). + try? await Task.sleep(nanoseconds: timeoutNanoseconds) + return false + } + let reached = await group.next() ?? false + group.cancelAll() + return reached + } + if !reached { + Issue.record("timed out waiting for \(method) count >= \(expectedCount)") + } + } + + private func waitUntilCountReached(of method: String, atLeast expectedCount: Int) async { + guard count(of: method) < expectedCount else { return } + let waiterID = UUID() + await withTaskCancellationHandler { + await withCheckedContinuation { continuation in + countWaiters.append(CountWaiter( + id: waiterID, + method: method, + expectedCount: expectedCount, + continuation: continuation + )) + resumeSatisfiedCountWaiters() + } + } onCancel: { + Task { await self.cancelCountWaiter(id: waiterID) } + } + } + + private func resumeSatisfiedCountWaiters() { + var remaining: [CountWaiter] = [] + var satisfied: [CheckedContinuation] = [] + for waiter in countWaiters { + if count(of: waiter.method) >= waiter.expectedCount { + satisfied.append(waiter.continuation) + } else { + remaining.append(waiter) } - try? await Task.sleep(nanoseconds: 10_000_000) } - Issue.record("timed out waiting for \(method) count >= \(expectedCount)") + countWaiters = remaining + for continuation in satisfied { + continuation.resume() + } + } + + private func cancelCountWaiter(id: UUID) { + guard let index = countWaiters.firstIndex(where: { $0.id == id }) else { + return + } + let waiter = countWaiters.remove(at: index) + waiter.continuation.resume() } func topics(for method: String) -> [[String]] { From c6f5fd71500b81f1b6577c64d2802debf420cf69 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 06:40:19 -0700 Subject: [PATCH 42/56] Route replay count waits through causal signal --- .github/swift-file-length-budget.tsv | 4 ++-- .../MobileShellRenderGridLivenessTestSupport.swift | 9 ++++++--- .../TerminalOutputDeliveryQueueTests.swift | 12 +++++------- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/.github/swift-file-length-budget.tsv b/.github/swift-file-length-budget.tsv index 9684b525878f..a576457c1eb4 100644 --- a/.github/swift-file-length-budget.tsv +++ b/.github/swift-file-length-budget.tsv @@ -111,7 +111,7 @@ 859 Packages/macOS/CmuxControlSocket/Sources/CmuxControlSocket/Coordinator/Workspace/ControlCommandCoordinator+Workspace.swift 847 cmuxTests/AgentSessionAutoResumeSettingsTests.swift 845 cmuxTests/SSHStartupSignalLifecycleTests.swift -843 Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift +841 Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift 834 Sources/MainWindowFocusController.swift 830 Sources/TaskManagerTypes.swift 825 Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/TerminalComposerView.swift @@ -171,8 +171,8 @@ 620 cmuxTests/FinderFileDropRegressionTests.swift 620 cmuxTests/TerminalNotificationQueueTests.swift 615 Sources/SettingsNavigation.swift +610 Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift 608 cmuxUITests/FeedSidebarUITests.swift -607 Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift 607 Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface.swift 607 Sources/SessionIndexModels.swift 607 Sources/SleepyFaceView.swift diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift index 75f2207d8591..b53cc0eae476 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift @@ -91,11 +91,13 @@ actor LivenessHostRouter { recorded.filter { $0.method == method }.count } + @discardableResult func waitForCount( of method: String, atLeast expectedCount: Int, - timeoutNanoseconds: UInt64 = 3_000_000_000 - ) async { + timeoutNanoseconds: UInt64 = 3_000_000_000, + recordIssueOnTimeout: Bool = true + ) async -> Bool { let reached = await withTaskGroup(of: Bool.self) { group in group.addTask { await self.waitUntilCountReached(of: method, atLeast: expectedCount) @@ -110,9 +112,10 @@ actor LivenessHostRouter { group.cancelAll() return reached } - if !reached { + if !reached, recordIssueOnTimeout { Issue.record("timed out waiting for \(method) count >= \(expectedCount)") } + return reached } private func waitUntilCountReached(of method: String, atLeast expectedCount: Int) async { diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift index e328c06580df..93308773f485 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift @@ -682,13 +682,11 @@ private func waitForReplayRequestCount( _ router: LivenessHostRouter, atLeast expectedCount: Int ) async -> Bool { - for _ in 0..<300 { - if await router.count(of: "mobile.terminal.replay") >= expectedCount { - return true - } - try? await Task.sleep(nanoseconds: 10_000_000) - } - return await router.count(of: "mobile.terminal.replay") >= expectedCount + await router.waitForCount( + of: "mobile.terminal.replay", + atLeast: expectedCount, + recordIssueOnTimeout: false + ) } @Test func terminalOutputQueueCoalescesReplaceableViewportFramesBehindBackpressure() { From a193055af317d0d0b505b01f3f9c28f596fd4db2 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 06:53:58 -0700 Subject: [PATCH 43/56] Defer replay until blocked render recovery runs --- .github/swift-file-length-budget.tsv | 2 +- .../GhosttySurfaceView.swift | 54 +++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/.github/swift-file-length-budget.tsv b/.github/swift-file-length-budget.tsv index a576457c1eb4..9ed376186dad 100644 --- a/.github/swift-file-length-budget.tsv +++ b/.github/swift-file-length-budget.tsv @@ -24,7 +24,7 @@ 4487 Sources/Panels/FilePreviewPanel.swift 4483 Sources/cmuxApp.swift 4367 cmuxTests/BrowserPanelTests.swift -4273 Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift +4327 Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift 4121 Sources/BrowserWindowPortal.swift 3934 Sources/Feed/FeedPanelView.swift 3926 cmuxTests/TabManagerUnitTests.swift diff --git a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift index 47275248e33f..ad0401b96dc7 100644 --- a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift +++ b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift @@ -669,6 +669,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { private var pendingGeometryApply: PendingSurfaceOperation? private var pendingVisibleSnapshot: PendingVisibleSnapshot? private var pendingCopyableTextRead: PendingCopyableTextRead? + private var pendingRenderRecoveryWaiters: [PendingRenderRecoveryWaiter] = [] private var hasPendingSurfaceOperationDeadline: Bool { pendingOutputApply != nil || pendingGeometryApply != nil || pendingVisibleSnapshot != nil || pendingCopyableTextRead != nil @@ -2288,6 +2289,9 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { /// or `false` when the caller should reset its delivery queue and replay. @discardableResult public func processOutputAndWait(_ data: Data) async -> Bool { + if await waitForDeferredRenderPipelineRecoveryIfBlocked() { + return false + } return await withCheckedContinuation { continuation in let operationID = registerPendingOutputApply( byteCount: data.count, @@ -2648,6 +2652,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { private func prepareForReuseAfterDetach() { renderPipelineRecoveryBlocked = false deferredRenderPipelineRecovery = nil + resumePendingRenderRecoveryWaiters() completePendingSurfaceOperations(returning: false) renderInFlight = false renderInFlightSince = nil @@ -2942,6 +2947,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { deferredRenderPipelineRecovery = nil renderPipelineRecoveryBlocked = false guard !isDismantled, surface != nil else { + resumePendingRenderRecoveryWaiters() completePendingSurfaceOperations(returning: false) return } @@ -2953,6 +2959,45 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { stalledMs: deferred.stalledMs, replay: deferred.replay ) + resumePendingRenderRecoveryWaiters() + } + + private func waitForDeferredRenderPipelineRecoveryIfBlocked() async -> Bool { + guard renderPipelineRecoveryBlocked else { return false } + let waiterID = UUID() + await withTaskCancellationHandler { + await withCheckedContinuation { continuation in + guard renderPipelineRecoveryBlocked else { + continuation.resume() + return + } + pendingRenderRecoveryWaiters.append(PendingRenderRecoveryWaiter( + id: waiterID, + continuation: continuation + )) + } + } onCancel: { + Task { @MainActor [weak self] in + self?.cancelPendingRenderRecoveryWaiter(id: waiterID) + } + } + return true + } + + private func cancelPendingRenderRecoveryWaiter(id: UUID) { + guard let index = pendingRenderRecoveryWaiters.firstIndex(where: { $0.id == id }) else { + return + } + let waiter = pendingRenderRecoveryWaiters.remove(at: index) + waiter.continuation.resume() + } + + private func resumePendingRenderRecoveryWaiters() { + let waiters = pendingRenderRecoveryWaiters + pendingRenderRecoveryWaiters = [] + for waiter in waiters { + waiter.continuation.resume() + } } private var preferredScreenScale: CGFloat { @@ -3495,6 +3540,9 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { } private func syncSurfaceGeometryAndWait(shouldReassertNaturalSize: Bool = true) async -> Bool { + if await waitForDeferredRenderPipelineRecoveryIfBlocked() { + return false + } needsGeometrySync = false pendingGeometryReassert = false return await withCheckedContinuation { continuation in @@ -4193,6 +4241,12 @@ nonisolated private struct PendingSurfaceOperation { let continuation: CheckedContinuation } +/// One apply caller waiting for a blocked deferred recovery to rebuild first. +nonisolated private struct PendingRenderRecoveryWaiter { + let id: UUID + let continuation: CheckedContinuation +} + /// One visible-terminal snapshot read awaiting output-queue completion or its /// display-link deadline. A timeout skips only the diagnostic snapshot. nonisolated private struct PendingVisibleSnapshot { From 2bd52e6d716391da69a15955194aaa3b61e056c1 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 07:05:34 -0700 Subject: [PATCH 44/56] Satisfy mobile terminal policy gate --- .github/swift-file-length-budget.tsv | 4 +- ...leShellRenderGridLivenessTestSupport.swift | 23 ++--- .../GhosttySurfaceView.swift | 83 ++++++++++--------- 3 files changed, 58 insertions(+), 52 deletions(-) diff --git a/.github/swift-file-length-budget.tsv b/.github/swift-file-length-budget.tsv index 9ed376186dad..d3b11ababeec 100644 --- a/.github/swift-file-length-budget.tsv +++ b/.github/swift-file-length-budget.tsv @@ -24,7 +24,7 @@ 4487 Sources/Panels/FilePreviewPanel.swift 4483 Sources/cmuxApp.swift 4367 cmuxTests/BrowserPanelTests.swift -4327 Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift +4330 Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift 4121 Sources/BrowserWindowPortal.swift 3934 Sources/Feed/FeedPanelView.swift 3926 cmuxTests/TabManagerUnitTests.swift @@ -171,7 +171,7 @@ 620 cmuxTests/FinderFileDropRegressionTests.swift 620 cmuxTests/TerminalNotificationQueueTests.swift 615 Sources/SettingsNavigation.swift -610 Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift +613 Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift 608 cmuxUITests/FeedSidebarUITests.swift 607 Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface.swift 607 Sources/SessionIndexModels.swift diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift index b53cc0eae476..a0436e19a5ce 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellRenderGridLivenessTestSupport.swift @@ -58,15 +58,13 @@ actor LivenessHostRouter { var topics: [String]? } - private struct CountWaiter { - let id: UUID - let method: String - let expectedCount: Int - let continuation: CheckedContinuation - } - private var recorded: [RecordedRequest] = [] - private var countWaiters: [CountWaiter] = [] + private var countWaiters: [( + id: UUID, + method: String, + expectedCount: Int, + continuation: CheckedContinuation + )] = [] private var hostStatusRequestCount = 0 private var heldHostStatusRequestNumbers: Set = [] private var subscribeRequestCount = 0 @@ -123,7 +121,7 @@ actor LivenessHostRouter { let waiterID = UUID() await withTaskCancellationHandler { await withCheckedContinuation { continuation in - countWaiters.append(CountWaiter( + countWaiters.append(( id: waiterID, method: method, expectedCount: expectedCount, @@ -137,7 +135,12 @@ actor LivenessHostRouter { } private func resumeSatisfiedCountWaiters() { - var remaining: [CountWaiter] = [] + var remaining: [( + id: UUID, + method: String, + expectedCount: Int, + continuation: CheckedContinuation + )] = [] var satisfied: [CheckedContinuation] = [] for waiter in countWaiters { if count(of: waiter.method) >= waiter.expectedCount { diff --git a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift index ad0401b96dc7..7836604e9185 100644 --- a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift +++ b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift @@ -4077,46 +4077,48 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { /// terminal surface, for the DEV "Copy Debug Logs" action so a bug report /// pairs the on-screen content with the debug log. Reads the VIEWPORT /// (visible grid only, not scrollback) via libghostty. - public static func visibleTerminalSnapshot() async -> String { - registeredSurfaceViews = registeredSurfaceViews.filter { $0.value.value != nil } - // Collect the main-actor state + surface pointers first, then read the - // viewport text on the serial output queue. `ghostty_surface_read_text` - // takes the same surface lock as `process_output` (which runs off-main); - // reading it on the MAIN thread here contends that lock during a render - // storm and stalls the present — tapping Copy Debug Logs would itself - // blank the terminal. The output queue is never concurrent with - // `process_output`, so the read can't wedge. The await is bounded by - // the surface's display-link deadline so this diagnostic path does not - // add a sleeping timer task or block the main actor. - var pending: [VisibleSnapshotRequest] = [] - for view in registeredSurfaceViews.values.compactMap(\.value) { - guard view.window != nil, !view.isHidden, view.alpha > 0.01, - let surface = view.surface else { continue } - let grid = view.effectiveGrid.map { "\($0.cols)x\($0.rows)" } ?? "?" - pending.append(VisibleSnapshotRequest( - view: view, - grid: grid, - font: Int(view.liveFontSize), - surface: surface, - generation: view.surfaceGeneration - )) - } - if pending.isEmpty { - return "===== visible terminal: (no on-screen surface) =====" - } - var sections: [String] = [] - for item in pending { - guard let section = await item.view.visibleSnapshotSection( - surface: item.surface, - generation: item.generation, - grid: item.grid, - font: item.font - ) else { - return "===== visible terminal: (snapshot skipped — render busy) =====" + public static var visibleTerminalSnapshot: @MainActor () async -> String { + { + registeredSurfaceViews = registeredSurfaceViews.filter { $0.value.value != nil } + // Collect the main-actor state + surface pointers first, then read the + // viewport text on the serial output queue. `ghostty_surface_read_text` + // takes the same surface lock as `process_output` (which runs off-main); + // reading it on the MAIN thread here contends that lock during a render + // storm and stalls the present — tapping Copy Debug Logs would itself + // blank the terminal. The output queue is never concurrent with + // `process_output`, so the read can't wedge. The await is bounded by + // the surface's display-link deadline so this diagnostic path does not + // add a sleeping timer task or block the main actor. + var pending: [VisibleSnapshotRequest] = [] + for view in registeredSurfaceViews.values.compactMap(\.value) { + guard view.window != nil, !view.isHidden, view.alpha > 0.01, + let surface = view.surface else { continue } + let grid = view.effectiveGrid.map { "\($0.cols)x\($0.rows)" } ?? "?" + pending.append(VisibleSnapshotRequest( + view: view, + grid: grid, + font: Int(view.liveFontSize), + surface: surface, + generation: view.surfaceGeneration + )) + } + if pending.isEmpty { + return "===== visible terminal: (no on-screen surface) =====" } - sections.append(section) + var sections: [String] = [] + for item in pending { + guard let section = await item.view.visibleSnapshotSection( + surface: item.surface, + generation: item.generation, + grid: item.grid, + font: item.font + ) else { + return "===== visible terminal: (snapshot skipped — render busy) =====" + } + sections.append(section) + } + return sections.joined(separator: "\n\n") } - return sections.joined(separator: "\n\n") } private func visibleSnapshotSection( @@ -4147,7 +4149,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { let text = Self.surfaceText(read.surface, pointTag: GHOSTTY_POINT_VIEWPORT) ?? "(unavailable)" let section = "===== visible terminal · grid=\(read.grid) · font=\(read.font) =====\n" + text - DispatchQueue.main.async { [weak self] in + Task { @MainActor [weak self] in guard let view = self else { return } guard view.surface == read.surface, view.surfaceGeneration == read.generation else { @@ -4301,7 +4303,8 @@ nonisolated private final class SurfaceOperationCancellationToken: Sendable { // lint:allow lock - tiny cross-queue cancellation flag for already-enqueued // libghostty work; actor hops would put the serial surface queue back behind // the main actor and defeat the stale-read fast path. - private let cancelled = Mutex(false) + private let cancelled: Mutex + = .init(false) var isCancelled: Bool { cancelled.withLock { $0 } From b46e8680d6c81d4797225ee95d44dbe01c2b95dd Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 07:21:56 -0700 Subject: [PATCH 45/56] Allow repeated terminal render recovery --- .../GhosttySurfaceView.swift | 138 +----------------- 1 file changed, 4 insertions(+), 134 deletions(-) diff --git a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift index 7836604e9185..8ef339b6fc23 100644 --- a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift +++ b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift @@ -657,9 +657,6 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { var outputQueue = GhosttySurfaceWorkQueue(generation: 0) private var outputQueueGeneration: UInt64 = 0 private var pendingSurfaceFreeCount = 0 - private var renderPipelineRecoveryBlocked = false - private var deferredRenderPipelineRecovery: DeferredRenderPipelineRecovery? - private static let maxPendingSurfaceFrees = 1 private static let renderPipelineStallDeadline: CFTimeInterval = 2.0 private static let outputApplyTimeout: CFTimeInterval = 2.0 private static let visibleSnapshotTimeout: CFTimeInterval = 0.6 @@ -669,7 +666,6 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { private var pendingGeometryApply: PendingSurfaceOperation? private var pendingVisibleSnapshot: PendingVisibleSnapshot? private var pendingCopyableTextRead: PendingCopyableTextRead? - private var pendingRenderRecoveryWaiters: [PendingRenderRecoveryWaiter] = [] private var hasPendingSurfaceOperationDeadline: Bool { pendingOutputApply != nil || pendingGeometryApply != nil || pendingVisibleSnapshot != nil || pendingCopyableTextRead != nil @@ -2062,7 +2058,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { /// would pump it, so apply immediately. private func scheduleDisplayLinkWork() { needsDraw = true - if displayLink == nil, !renderPipelineRecoveryBlocked { + if displayLink == nil { applyPendingFontSizeIfNeeded() } } @@ -2289,9 +2285,6 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { /// or `false` when the caller should reset its delivery queue and replay. @discardableResult public func processOutputAndWait(_ data: Data) async -> Bool { - if await waitForDeferredRenderPipelineRecoveryIfBlocked() { - return false - } return await withCheckedContinuation { continuation in let operationID = registerPendingOutputApply( byteCount: data.count, @@ -2473,13 +2466,6 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { completion?(true) return } - // A blocked recovery will rebuild this surface if the prior free drains. - // Do not park wait-style callers behind that free: the free is queued on - // the same surface queue that may already be wedged. - guard !renderPipelineRecoveryBlocked else { - completion?(false) - return - } #if DEBUG if lastInputTimestamp > 0 { let elapsed = (CACurrentMediaTime() - lastInputTimestamp) * 1000.0 @@ -2650,9 +2636,6 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { /// tree. Does not set ``isDismantled`` so a transient detach can re-attach /// and resume; only ``prepareForDismantle()`` marks the surface dead. private func prepareForReuseAfterDetach() { - renderPipelineRecoveryBlocked = false - deferredRenderPipelineRecovery = nil - resumePendingRenderRecoveryWaiters() completePendingSurfaceOperations(returning: false) renderInFlight = false renderInFlightSince = nil @@ -2875,28 +2858,8 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { surface != nil else { return false } - guard pendingSurfaceFreeCount < Self.maxPendingSurfaceFrees else { - renderPipelineRecoveryBlocked = true - deferredRenderPipelineRecovery = DeferredRenderPipelineRecovery( - reason: reason, - stalledMs: stalledMs, - replay: replay - ) - stopDisplayLink() - skipPendingVisibleSnapshot() - skipPendingCopyableTextRead() - completePendingSurfaceOperations(returning: false) - renderInFlight = false - renderInFlightSince = nil - needsAnotherRender = false - MobileDebugLog.anchormux( - "render.recover.blocked reason=\(reason) stalledMs=\(stalledMs) pendingFrees=\(pendingSurfaceFreeCount)" - ) - return false - } - MobileDebugLog.anchormux( - "render.recover reason=\(reason) stalledMs=\(stalledMs) generation=\(surfaceGeneration)" + "render.recover reason=\(reason) stalledMs=\(stalledMs) generation=\(surfaceGeneration) pendingFrees=\(pendingSurfaceFreeCount)" ) let completedFailedOperation = completePendingSurfaceOperations(returning: false) @@ -2914,12 +2877,10 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { MobileDebugLog.anchormux( "render.recover.free_drained pendingFrees=\(self.pendingSurfaceFreeCount)" ) - self.retryDeferredRenderPipelineRecoveryIfNeeded() } } surface = nil - deferredRenderPipelineRecovery = nil renderInFlight = false renderInFlightSince = nil needsAnotherRender = false @@ -2941,65 +2902,6 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { return true } - private func retryDeferredRenderPipelineRecoveryIfNeeded() { - guard pendingSurfaceFreeCount < Self.maxPendingSurfaceFrees, - let deferred = deferredRenderPipelineRecovery else { return } - deferredRenderPipelineRecovery = nil - renderPipelineRecoveryBlocked = false - guard !isDismantled, surface != nil else { - resumePendingRenderRecoveryWaiters() - completePendingSurfaceOperations(returning: false) - return - } - MobileDebugLog.anchormux( - "render.recover.retry_after_free reason=\(deferred.reason) stalledMs=\(deferred.stalledMs)" - ) - _ = recoverRenderPipeline( - reason: deferred.reason, - stalledMs: deferred.stalledMs, - replay: deferred.replay - ) - resumePendingRenderRecoveryWaiters() - } - - private func waitForDeferredRenderPipelineRecoveryIfBlocked() async -> Bool { - guard renderPipelineRecoveryBlocked else { return false } - let waiterID = UUID() - await withTaskCancellationHandler { - await withCheckedContinuation { continuation in - guard renderPipelineRecoveryBlocked else { - continuation.resume() - return - } - pendingRenderRecoveryWaiters.append(PendingRenderRecoveryWaiter( - id: waiterID, - continuation: continuation - )) - } - } onCancel: { - Task { @MainActor [weak self] in - self?.cancelPendingRenderRecoveryWaiter(id: waiterID) - } - } - return true - } - - private func cancelPendingRenderRecoveryWaiter(id: UUID) { - guard let index = pendingRenderRecoveryWaiters.firstIndex(where: { $0.id == id }) else { - return - } - let waiter = pendingRenderRecoveryWaiters.remove(at: index) - waiter.continuation.resume() - } - - private func resumePendingRenderRecoveryWaiters() { - let waiters = pendingRenderRecoveryWaiters - pendingRenderRecoveryWaiters = [] - for waiter in waiters { - waiter.continuation.resume() - } - } - private var preferredScreenScale: CGFloat { if let screen = window?.windowScene?.screen { return screen.scale @@ -3032,7 +2934,6 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { guard let app = runtime?.app else { return } surface = makeSurface(app: app) if let surface { - renderPipelineRecoveryBlocked = false GhosttySurfaceView.register(surface: surface, for: self) if let config = runtime?.config { applyBackgroundColorFromConfig(config) @@ -3048,12 +2949,10 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { private func startDisplayLink() { guard displayLink == nil else { return } - guard !renderPipelineRecoveryBlocked || hasPendingSurfaceOperationDeadline else { return } let link = CADisplayLink(target: DisplayLinkProxy(target: self), selector: #selector(DisplayLinkProxy.handleDisplayLink)) link.preferredFrameRateRange = CAFrameRateRange(minimum: 30, maximum: 120, preferred: 120) link.add(to: .main, forMode: .common) displayLink = link - guard !renderPipelineRecoveryBlocked else { return } cursorBlinkState.start(now: CACurrentMediaTime()) needsDraw = true updateCursorOverlay() @@ -3078,12 +2977,6 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { if checkSurfaceOperationDeadlines(now: now) { return } - if renderPipelineRecoveryBlocked { - if !hasPendingSurfaceOperationDeadline { - stopDisplayLink() - } - return - } guard surface != nil else { if !hasPendingSurfaceOperationDeadline { stopDisplayLink() @@ -3234,8 +3127,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { // now bounded in libghostty (so a foreground stall self-heals as a // skipped frame the display link re-drives), but we still gate on // suspension; `resumeRendering` clears it on the next active transition. - guard !renderPipelineRecoveryBlocked, - !renderingSuspended, + guard !renderingSuspended, let surface, !isDismantled else { return } // Coalesce: never let more than one render_now sit on the serial queue. @@ -3280,7 +3172,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { needsDraw = true // A geometry sync (for any reason) satisfies a pending post-zoom resync. zoomSettleFrames = nil - if displayLink == nil, window != nil, !renderPipelineRecoveryBlocked { + if displayLink == nil, window != nil { // No frame pump while detached/backgrounded; apply directly so the // surface still gets sized before the next render path resumes. needsGeometrySync = false @@ -3540,9 +3432,6 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { } private func syncSurfaceGeometryAndWait(shouldReassertNaturalSize: Bool = true) async -> Bool { - if await waitForDeferredRenderPipelineRecoveryIfBlocked() { - return false - } needsGeometrySync = false pendingGeometryReassert = false return await withCheckedContinuation { continuation in @@ -3561,12 +3450,6 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { completion?(true) return } - // The current surface is scheduled for deferred recovery; geometry - // cannot be acknowledged against it without risking a stale replay ack. - guard !renderPipelineRecoveryBlocked else { - completion?(false) - return - } // Capture all main-actor inputs as values, then do every libghostty // WRITE (set_content_scale / set_size / fit) and its readback on the @@ -4227,13 +4110,6 @@ nonisolated private enum RenderPipelineRecoveryReplay { case delegateWhenNoCaller } -/// Recovery request deferred only while an older surface free is still draining. -nonisolated private struct DeferredRenderPipelineRecovery { - let reason: String - let stalledMs: Int - let replay: RenderPipelineRecoveryReplay -} - /// One output/geometry operation awaiting either its output-queue completion or /// the display-link deadline that rebuilds the stalled render pipeline. nonisolated private struct PendingSurfaceOperation { @@ -4243,12 +4119,6 @@ nonisolated private struct PendingSurfaceOperation { let continuation: CheckedContinuation } -/// One apply caller waiting for a blocked deferred recovery to rebuild first. -nonisolated private struct PendingRenderRecoveryWaiter { - let id: UUID - let continuation: CheckedContinuation -} - /// One visible-terminal snapshot read awaiting output-queue completion or its /// display-link deadline. A timeout skips only the diagnostic snapshot. nonisolated private struct PendingVisibleSnapshot { From 4cb70a1bd4534715639f1881992e90ffd47ba79c Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 07:25:07 -0700 Subject: [PATCH 46/56] Localize mobile terminal fallback labels --- .../GhosttySurfaceRepresentable.swift | 8 ++++- .../WorkspaceDetailView.swift | 6 ++-- ios/cmux/Resources/Localizable.xcstrings | 34 +++++++++++++++++++ 3 files changed, 45 insertions(+), 3 deletions(-) diff --git a/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/GhosttySurfaceRepresentable.swift b/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/GhosttySurfaceRepresentable.swift index 30f6e1389cc1..953941349cc2 100644 --- a/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/GhosttySurfaceRepresentable.swift +++ b/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/GhosttySurfaceRepresentable.swift @@ -49,7 +49,13 @@ struct GhosttySurfaceRepresentable: UIViewRepresentable { fallback.numberOfLines = 0 fallback.textColor = .white fallback.backgroundColor = UIColor(red: 0x27/255.0, green: 0x28/255.0, blue: 0x22/255.0, alpha: 1) - fallback.text = "Ghostty runtime failed to initialise:\n\(error.localizedDescription)" + fallback.text = String( + format: L10n.string( + "mobile.terminal.ghosttyRuntimeFailed", + defaultValue: "Ghostty runtime failed to initialise:\n%@" + ), + error.localizedDescription + ) return fallback } let view = GhosttySurfaceView( diff --git a/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/WorkspaceDetailView.swift b/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/WorkspaceDetailView.swift index ef94e6a5e9cc..f6089b86285b 100644 --- a/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/WorkspaceDetailView.swift +++ b/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/WorkspaceDetailView.swift @@ -484,8 +484,10 @@ struct WorkspaceDetailView: View { #if DEBUG Button(action: copyDebugLogsFromMenu) { - // DEV-only debug tooling; not shipped, so not localized. - Label("Copy Debug Logs", systemImage: "doc.on.clipboard") + Label( + L10n.string("mobile.debug.copyLogs", defaultValue: "Copy Debug Logs"), + systemImage: "doc.on.clipboard" + ) } .accessibilityIdentifier("MobileCopyDebugLogsMenuItem") #endif diff --git a/ios/cmux/Resources/Localizable.xcstrings b/ios/cmux/Resources/Localizable.xcstrings index a7e8b26cc5e7..ca39f72f6030 100644 --- a/ios/cmux/Resources/Localizable.xcstrings +++ b/ios/cmux/Resources/Localizable.xcstrings @@ -2296,6 +2296,23 @@ } } }, + "mobile.debug.copyLogs": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "Copy Debug Logs" + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "デバッグログをコピー" + } + } + } + }, "mobile.feedback.cancel": { "extractionState": "manual", "localizations": { @@ -5968,6 +5985,23 @@ } } }, + "mobile.terminal.ghosttyRuntimeFailed": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "Ghostty runtime failed to initialise:\n%@" + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "Ghosttyランタイムの初期化に失敗しました:\n%@" + } + } + } + }, "mobile.terminal.viewAsText": { "extractionState": "manual", "localizations": { From cbb4b81fdc093abc25ffbd20fa7058ad028af848 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 07:33:47 -0700 Subject: [PATCH 47/56] Fix mobile shell UI localization import --- .../Sources/CmuxMobileShellUI/GhosttySurfaceRepresentable.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/GhosttySurfaceRepresentable.swift b/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/GhosttySurfaceRepresentable.swift index 953941349cc2..5b30b2c7aa52 100644 --- a/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/GhosttySurfaceRepresentable.swift +++ b/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/GhosttySurfaceRepresentable.swift @@ -3,6 +3,7 @@ import CMUXMobileCore import CmuxMobileDiagnostics import CmuxMobileShell import CmuxMobileShellModel +import CmuxMobileSupport import CmuxMobileTerminal import SwiftUI import UIKit From c496d75b5f65e514f7c4532f34da763dae6dce2f Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 07:38:47 -0700 Subject: [PATCH 48/56] Keep mobile shell UI within file budget --- .../Sources/CmuxMobileShellUI/WorkspaceDetailView.swift | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/WorkspaceDetailView.swift b/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/WorkspaceDetailView.swift index f6089b86285b..159048f6ad1b 100644 --- a/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/WorkspaceDetailView.swift +++ b/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/WorkspaceDetailView.swift @@ -484,10 +484,7 @@ struct WorkspaceDetailView: View { #if DEBUG Button(action: copyDebugLogsFromMenu) { - Label( - L10n.string("mobile.debug.copyLogs", defaultValue: "Copy Debug Logs"), - systemImage: "doc.on.clipboard" - ) + Label(L10n.string("mobile.debug.copyLogs", defaultValue: "Copy Debug Logs"), systemImage: "doc.on.clipboard") } .accessibilityIdentifier("MobileCopyDebugLogsMenuItem") #endif From f54500f5e40687909089c91e8c2fc322c7f0653f Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 07:50:10 -0700 Subject: [PATCH 49/56] Bound repeated terminal render recovery --- .../GhosttySurfaceView.swift | 72 ++++++++++++++++++- 1 file changed, 69 insertions(+), 3 deletions(-) diff --git a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift index 8ef339b6fc23..bdcd984cf2a3 100644 --- a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift +++ b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift @@ -657,10 +657,13 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { var outputQueue = GhosttySurfaceWorkQueue(generation: 0) private var outputQueueGeneration: UInt64 = 0 private var pendingSurfaceFreeCount = 0 + private var renderPipelineRecoveryPaused = false + private var lastRecoveryPausedDropLogTime: CFTimeInterval = 0 private static let renderPipelineStallDeadline: CFTimeInterval = 2.0 private static let outputApplyTimeout: CFTimeInterval = 2.0 private static let visibleSnapshotTimeout: CFTimeInterval = 0.6 private static let copyableTextTimeout: CFTimeInterval = 2.0 + private static let maxPendingSurfaceFrees = 1 private var nextSurfaceOperationID: UInt64 = 0 private var pendingOutputApply: PendingSurfaceOperation? private var pendingGeometryApply: PendingSurfaceOperation? @@ -2302,7 +2305,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { } private func ensureSurfaceOperationDeadlinePump() { - guard window != nil, displayLink == nil, !renderingSuspended else { return } + guard window != nil, displayLink == nil, !renderingSuspended, !renderPipelineRecoveryPaused else { return } startDisplayLink() } @@ -2462,6 +2465,11 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { _ data: Data, completion: (@MainActor @Sendable (Bool) -> Void)? ) { + guard !renderPipelineRecoveryPaused else { + logRecoveryPausedDrop(kind: "output", byteCount: data.count) + completion?(true) + return + } guard let surface, !isDismantled else { completion?(true) return @@ -2756,6 +2764,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { let generation = surfaceGeneration guard surface == expectedSurface, !isDismantled, + !renderPipelineRecoveryPaused, !renderingSuspended else { return nil } @@ -2848,6 +2857,49 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { } } + private func logRecoveryPausedDrop(kind: String, byteCount: Int? = nil) { + let now = CACurrentMediaTime() + guard now - lastRecoveryPausedDropLogTime >= 1 else { return } + lastRecoveryPausedDropLogTime = now + MobileDebugLog.anchormux( + "render.recover.paused_drop kind=\(kind) bytes=\(byteCount ?? 0) pendingFrees=\(pendingSurfaceFreeCount)" + ) + } + + @discardableResult + private func pauseRenderPipelineRecovery( + reason: String, + stalledMs: Int + ) -> Bool { + MobileDebugLog.anchormux( + "render.recover.paused reason=\(reason) stalledMs=\(stalledMs) pendingFrees=\(pendingSurfaceFreeCount)" + ) + renderPipelineRecoveryPaused = true + stopDisplayLink() + _ = completePendingSurfaceOperations(returning: false) + renderInFlight = false + renderInFlightSince = nil + needsAnotherRender = false + needsDraw = false + return true + } + + private func resumePausedRenderPipelineRecoveryIfPossible() { + guard renderPipelineRecoveryPaused, + !isDismantled, + surface != nil, + pendingSurfaceFreeCount < Self.maxPendingSurfaceFrees else { return } + MobileDebugLog.anchormux( + "render.recover.resuming pendingFrees=\(pendingSurfaceFreeCount)" + ) + renderPipelineRecoveryPaused = false + _ = recoverRenderPipeline( + reason: "free_drained", + stalledMs: 0, + replay: .delegateWhenNoCaller + ) + } + @discardableResult private func recoverRenderPipeline( reason: String, @@ -2858,6 +2910,12 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { surface != nil else { return false } + guard !renderPipelineRecoveryPaused else { + return pauseRenderPipelineRecovery(reason: reason, stalledMs: stalledMs) + } + guard pendingSurfaceFreeCount < Self.maxPendingSurfaceFrees else { + return pauseRenderPipelineRecovery(reason: reason, stalledMs: stalledMs) + } MobileDebugLog.anchormux( "render.recover reason=\(reason) stalledMs=\(stalledMs) generation=\(surfaceGeneration) pendingFrees=\(pendingSurfaceFreeCount)" ) @@ -2877,6 +2935,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { MobileDebugLog.anchormux( "render.recover.free_drained pendingFrees=\(self.pendingSurfaceFreeCount)" ) + self.resumePausedRenderPipelineRecoveryIfPossible() } } @@ -3127,7 +3186,8 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { // now bounded in libghostty (so a foreground stall self-heals as a // skipped frame the display link re-drives), but we still gate on // suspension; `resumeRendering` clears it on the next active transition. - guard !renderingSuspended, + guard !renderPipelineRecoveryPaused, + !renderingSuspended, let surface, !isDismantled else { return } // Coalesce: never let more than one render_now sit on the serial queue. @@ -3446,6 +3506,11 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { shouldReassertNaturalSize: Bool = true, completion: (@MainActor @Sendable (Bool) -> Void)? = nil ) { + guard !renderPipelineRecoveryPaused else { + logRecoveryPausedDrop(kind: "geometry") + completion?(true) + return + } guard let surface else { completion?(true) return @@ -4011,7 +4076,8 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { font: Int ) async -> String? { guard self.surface == surface, - surfaceGeneration == generation else { + surfaceGeneration == generation, + !renderPipelineRecoveryPaused else { return nil } return await withCheckedContinuation { continuation in From 7b75e17e015cf7bbeddd648501c3e4976e709765 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 08:23:48 -0700 Subject: [PATCH 50/56] Address terminal recovery review feedback --- .../CmuxMobileShellUI/GhosttySurfaceRepresentable.swift | 9 +++------ .../Sources/CmuxMobileTerminal/GhosttySurfaceView.swift | 4 ++-- ios/cmux/Resources/Localizable.xcstrings | 6 +++--- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/GhosttySurfaceRepresentable.swift b/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/GhosttySurfaceRepresentable.swift index 5b30b2c7aa52..16c04614b5bf 100644 --- a/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/GhosttySurfaceRepresentable.swift +++ b/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/GhosttySurfaceRepresentable.swift @@ -50,12 +50,9 @@ struct GhosttySurfaceRepresentable: UIViewRepresentable { fallback.numberOfLines = 0 fallback.textColor = .white fallback.backgroundColor = UIColor(red: 0x27/255.0, green: 0x28/255.0, blue: 0x22/255.0, alpha: 1) - fallback.text = String( - format: L10n.string( - "mobile.terminal.ghosttyRuntimeFailed", - defaultValue: "Ghostty runtime failed to initialise:\n%@" - ), - error.localizedDescription + fallback.text = L10n.string( + "mobile.terminal.rendererFailed", + defaultValue: "Terminal renderer failed to start." ) return fallback } diff --git a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift index bdcd984cf2a3..f084d18011eb 100644 --- a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift +++ b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift @@ -2467,7 +2467,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { ) { guard !renderPipelineRecoveryPaused else { logRecoveryPausedDrop(kind: "output", byteCount: data.count) - completion?(true) + completion?(false) return } guard let surface, !isDismantled else { @@ -3508,7 +3508,7 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { ) { guard !renderPipelineRecoveryPaused else { logRecoveryPausedDrop(kind: "geometry") - completion?(true) + completion?(false) return } guard let surface else { diff --git a/ios/cmux/Resources/Localizable.xcstrings b/ios/cmux/Resources/Localizable.xcstrings index ca39f72f6030..938f3f49ee74 100644 --- a/ios/cmux/Resources/Localizable.xcstrings +++ b/ios/cmux/Resources/Localizable.xcstrings @@ -5985,19 +5985,19 @@ } } }, - "mobile.terminal.ghosttyRuntimeFailed": { + "mobile.terminal.rendererFailed": { "extractionState": "manual", "localizations": { "en": { "stringUnit": { "state": "translated", - "value": "Ghostty runtime failed to initialise:\n%@" + "value": "Terminal renderer failed to start." } }, "ja": { "stringUnit": { "state": "translated", - "value": "Ghosttyランタイムの初期化に失敗しました:\n%@" + "value": "ターミナルレンダラーを起動できませんでした。" } } } From 527c0ef03e76440060f51492f79d0605ffde4c52 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 08:37:45 -0700 Subject: [PATCH 51/56] Rate-limit replay barrier drop logging --- .../MobileShellComposite+TerminalOutputDelivery.swift | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift index 4ba1186521cb..d3e20f3d1232 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift @@ -63,7 +63,11 @@ extension MobileShellComposite { terminalReplayBarrierDroppedOutputSurfaceIDs.insert(surfaceID) let droppedOutputCount = (terminalReplayBarrierDroppedOutputCountsBySurfaceID[surfaceID] ?? 0) &+ 1 terminalReplayBarrierDroppedOutputCountsBySurfaceID[surfaceID] = droppedOutputCount - MobileDebugLog.anchormux("terminal.output.drop_replay_barrier surface=\(surfaceID)") + if droppedOutputCount == 1 || droppedOutputCount.isMultiple(of: 32) { + MobileDebugLog.anchormux( + "terminal.output.drop_replay_barrier surface=\(surfaceID) count=\(droppedOutputCount)" + ) + } if remoteClient != nil, terminalReplayBarrierAckStreamTokensBySurfaceID[surfaceID] == nil, !terminalReplaySurfaceIDsInFlight.contains(surfaceID), From 4f3b3d0f07630e2df4953361800164e75b4f02de Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 08:47:53 -0700 Subject: [PATCH 52/56] Add replay barrier regression coverage --- ...MobileShellReplayFallbackScreenTests.swift | 81 +++++++++++++++++++ .../TerminalOutputDeliveryQueueTests.swift | 2 +- .../TerminalReplayBarrierAckResetTests.swift | 56 +++++++++++++ 3 files changed, 138 insertions(+), 1 deletion(-) create mode 100644 Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellReplayFallbackScreenTests.swift create mode 100644 Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalReplayBarrierAckResetTests.swift diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellReplayFallbackScreenTests.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellReplayFallbackScreenTests.swift new file mode 100644 index 000000000000..b3bc2265d588 --- /dev/null +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileShellReplayFallbackScreenTests.swift @@ -0,0 +1,81 @@ +import Foundation +import Testing +@testable import CmuxMobileShell + +@MainActor +@Test func replayByteFallbackPreservesAlternateScreenSuppression() async throws { + let clock = TestClock() + let router = LivenessHostRouter() + let box = TransportBox() + let store = try await makeConnectedStore(router: router, box: box, clock: clock) + + await router.enqueueReplayTexts(["cold-replay", "fallback-replay"]) + let collector = OutputCollector() + collector.mount(store: store, surfaceID: "live-terminal") + await router.waitForCount(of: "mobile.terminal.replay", atLeast: 1) + let coldReplayDelivered = try await pollUntil { + collector.lines.contains { $0.contains("cold-replay") } + } + #expect(coldReplayDelivered) + let transport = try #require(box.get()) + + await transport.deliver(try renderGridEventFrame( + surfaceID: "live-terminal", + seq: 3, + text: "alt", + activeScreen: .alternate + )) + let altDelivered = try await pollUntil { + collector.viewportPolicies.last == .remoteGrid(columns: 16, rows: 4) + } + #expect(altDelivered) + + let replayCountAfterAlternate = await router.count(of: "mobile.terminal.replay") + let replayBarrierToken = store.beginTerminalReplayBarrier(surfaceID: "live-terminal") + store.requestTerminalReplay( + surfaceID: "live-terminal", + replayBarrierToken: replayBarrierToken + ) + await router.waitForCount( + of: "mobile.terminal.replay", + atLeast: replayCountAfterAlternate + 1 + ) + let fallbackReplayDelivered = try await pollUntil { + collector.lines.contains { $0.contains("fallback-replay") } + } + #expect(fallbackReplayDelivered) + #expect(store.terminalReplayBarrierTokensBySurfaceID["live-terminal"] == nil) + + await transport.deliver(try terminalBytesEventFrame( + surfaceID: "live-terminal", + seq: 4, + text: "raw-after-fallback" + )) + let rawAfterFallbackDelivered = try await pollUntil(attempts: 50) { + collector.lines.contains { $0.contains("raw-after-fallback") } + } + #expect( + !rawAfterFallbackDelivered, + "byte/snapshot replay fallbacks must not clear alternate-screen raw-byte suppression" + ) + + await transport.deliver(try renderGridEventFrame( + surfaceID: "live-terminal", + seq: 10, + text: "primary-full" + )) + let primaryDelivered = try await pollUntil { + collector.lines.contains { $0.contains("primary-full") } + } + #expect(primaryDelivered) + await transport.deliver(try terminalBytesEventFrame( + surfaceID: "live-terminal", + seq: 10, + text: "raw-after-primary" + )) + let rawAfterPrimaryDelivered = try await pollUntil { + collector.lines.contains { $0.contains("raw-after-primary") } + } + #expect(rawAfterPrimaryDelivered) + collector.unmount() +} diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift index 93308773f485..3a338ecffbbf 100644 --- a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalOutputDeliveryQueueTests.swift @@ -562,7 +562,7 @@ import Testing let retryReplayChunk = try #require(await iterator.next()) #expect(String(data: retryReplayChunk.data, encoding: .utf8) == "retry-replay") let retryBarrierToken = try #require(store.terminalReplayBarrierTokensBySurfaceID[surfaceID]) - #expect(retryBarrierToken != firstBarrierToken) + #expect(retryBarrierToken == firstBarrierToken) store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: retryReplayChunk.streamToken) #expect(store.terminalReplayBarrierTokensBySurfaceID[surfaceID] == nil) diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalReplayBarrierAckResetTests.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalReplayBarrierAckResetTests.swift new file mode 100644 index 000000000000..389cc7d939a1 --- /dev/null +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalReplayBarrierAckResetTests.swift @@ -0,0 +1,56 @@ +import Foundation +import Testing +@testable import CmuxMobileShell + +@MainActor +@Test func terminalReplayBarrierCapsRepeatedReplayAckResets() async throws { + let router = LivenessHostRouter() + let box = TransportBox() + let clock = TestClock() + let store = try await makeConnectedStore(router: router, box: box, clock: clock) + let surfaceID = "live-terminal" + + await router.enqueueReplayTexts([ + "cold-replay", + "first-replay", + "second-replay", + "third-replay", + "unexpected-replay", + ]) + var iterator = store.terminalOutputStream(surfaceID: surfaceID).makeAsyncIterator() + await router.waitForCount(of: "mobile.terminal.replay", atLeast: 1) + let coldReplayChunk = try #require(await iterator.next()) + #expect(String(data: coldReplayChunk.data, encoding: .utf8) == "cold-replay") + store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: coldReplayChunk.streamToken) + let replayCountAfterMount = await router.count(of: "mobile.terminal.replay") + + store.deliverTerminalBytes(Data("stalled-first".utf8), surfaceID: surfaceID) + let stalledChunk = try #require(await iterator.next()) + store.terminalOutputDidReset(surfaceID: surfaceID, streamToken: stalledChunk.streamToken) + await router.waitForCount(of: "mobile.terminal.replay", atLeast: replayCountAfterMount + 1) + + let firstReplayChunk = try #require(await iterator.next()) + #expect(String(data: firstReplayChunk.data, encoding: .utf8) == "first-replay") + let barrierToken = try #require(store.terminalReplayBarrierTokensBySurfaceID[surfaceID]) + + store.terminalOutputDidReset(surfaceID: surfaceID, streamToken: firstReplayChunk.streamToken) + await router.waitForCount(of: "mobile.terminal.replay", atLeast: replayCountAfterMount + 2) + let secondReplayChunk = try #require(await iterator.next()) + #expect(String(data: secondReplayChunk.data, encoding: .utf8) == "second-replay") + #expect(store.terminalReplayBarrierTokensBySurfaceID[surfaceID] == barrierToken) + + store.terminalOutputDidReset(surfaceID: surfaceID, streamToken: secondReplayChunk.streamToken) + await router.waitForCount(of: "mobile.terminal.replay", atLeast: replayCountAfterMount + 3) + let thirdReplayChunk = try #require(await iterator.next()) + #expect(String(data: thirdReplayChunk.data, encoding: .utf8) == "third-replay") + #expect(store.terminalReplayBarrierTokensBySurfaceID[surfaceID] == barrierToken) + + store.terminalOutputDidReset(surfaceID: surfaceID, streamToken: thirdReplayChunk.streamToken) + let replayRestartedAfterExhaustion = await router.waitForCount( + of: "mobile.terminal.replay", + atLeast: replayCountAfterMount + 4, + recordIssueOnTimeout: false + ) + #expect(!replayRestartedAfterExhaustion, "replay ack resets must not bypass the replay retry cap") + #expect(store.terminalReplayBarrierTokensBySurfaceID[surfaceID] == barrierToken) +} From 99ed20c14f9812333459ae383419ee5e241121b3 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 08:52:25 -0700 Subject: [PATCH 53/56] Preserve replay barrier state across ack resets --- ...hellComposite+TerminalOutputDelivery.swift | 43 +++++++++++++++++-- .../MobileShellComposite.swift | 13 +++--- 2 files changed, 45 insertions(+), 11 deletions(-) diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift index d3e20f3d1232..30b618ad05fe 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift @@ -153,14 +153,15 @@ extension MobileShellComposite { public func terminalOutputDidReset(surfaceID: String, streamToken: UUID) { guard terminalOutputStreamTokensBySurfaceID[surfaceID] == streamToken, terminalOutputQueuesBySurfaceID[surfaceID] != nil else { return } - if terminalReplayBarrierTokensBySurfaceID[surfaceID] != nil { + if let replayBarrierToken = terminalReplayBarrierTokensBySurfaceID[surfaceID] { guard terminalReplayBarrierAckStreamTokensBySurfaceID[surfaceID] == streamToken else { MobileDebugLog.anchormux("terminal.output.reset_barrier_active surface=\(surfaceID)") return } - let replayBarrierToken = beginTerminalReplayBarrier(surfaceID: surfaceID) - MobileDebugLog.anchormux("terminal.output.reset_replay_ack surface=\(surfaceID)") - requestTerminalReplay(surfaceID: surfaceID, replayBarrierToken: replayBarrierToken) + retryTerminalReplayAfterAckReset( + surfaceID: surfaceID, + replayBarrierToken: replayBarrierToken + ) return } let replayBarrierToken = beginTerminalReplayBarrier(surfaceID: surfaceID) @@ -168,6 +169,40 @@ extension MobileShellComposite { requestTerminalReplay(surfaceID: surfaceID, replayBarrierToken: replayBarrierToken) } + private func retryTerminalReplayAfterAckReset( + surfaceID: String, + replayBarrierToken: UUID + ) { + guard terminalReplayBarrierTokensBySurfaceID[surfaceID] == replayBarrierToken else { + return + } + terminalOutputQueuesBySurfaceID[surfaceID] = TerminalOutputDeliveryQueue() + terminalOutputStreamTokensBySurfaceID[surfaceID] = UUID() + deliveredTerminalByteEndSeqBySurfaceID.removeValue(forKey: surfaceID) + pendingTerminalByteEndSeqBySurfaceID.removeValue(forKey: surfaceID) + terminalReplayBarrierAckStreamTokensBySurfaceID.removeValue(forKey: surfaceID) + terminalReplayBarrierAckCoveredDroppedOutputCountsBySurfaceID.removeValue(forKey: surfaceID) + terminalReplayBarrierTokensInFlightBySurfaceID.removeValue(forKey: surfaceID) + guard let retryToken = prepareTerminalReplayFailureRetry( + surfaceID: surfaceID, + replayBarrierToken: replayBarrierToken + ) else { + preserveTerminalReplayBarrierIfCurrent( + surfaceID: surfaceID, + token: replayBarrierToken, + reason: "reset_replay_ack" + ) + return + } + MobileDebugLog.anchormux("terminal.output.reset_replay_ack surface=\(surfaceID)") + requestTerminalReplay( + surfaceID: surfaceID, + replayBarrierToken: retryToken, + coveredReplayBarrierDroppedOutputCount: + terminalReplayBarrierDroppedOutputCountsBySurfaceID[surfaceID] + ) + } + /// Ask the Mac to replay the authoritative terminal state for a surface. public func terminalOutputNeedsReplay(surfaceID: String) { guard terminalByteContinuationsBySurfaceID[surfaceID] != nil else { return } diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift index 04027e0512da..2b6bb2477bc7 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift @@ -713,19 +713,19 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { /// ``secondaryAggregationTask``, can reject old-team results after awaits. var secondaryAggregationScopeGeneration = 0 private var reportedViewportSizesByTerminalKey: [MobileTerminalViewportKey: MobileTerminalViewportSize] - private var deliveredTerminalByteEndSeqBySurfaceID: [String: UInt64] - private var pendingTerminalByteEndSeqBySurfaceID: [String: UInt64] + var deliveredTerminalByteEndSeqBySurfaceID: [String: UInt64] + var pendingTerminalByteEndSeqBySurfaceID: [String: UInt64] private var terminalActiveScreenBySurfaceID: [String: MobileTerminalRenderGridFrame.Screen] var terminalReplaySurfaceIDsInFlight: Set private var terminalReplayRequestIDsInFlightBySurfaceID: [String: UUID] private var terminalReplayTasksBySurfaceID: [String: Task] - private var terminalReplayBarrierTokensInFlightBySurfaceID: [String: UUID] + var terminalReplayBarrierTokensInFlightBySurfaceID: [String: UUID] var terminalReplayBarrierTokensBySurfaceID: [String: UUID] var terminalReplayBarrierAckStreamTokensBySurfaceID: [String: UUID] var terminalReplayBarrierDroppedOutputSurfaceIDs: Set var terminalReplayBarrierDroppedOutputCountsBySurfaceID: [String: UInt64] var terminalReplayBarrierAckCoveredDroppedOutputCountsBySurfaceID: [String: UInt64] - private var terminalReplayFailureRetryCountsBySurfaceID: [String: Int] + var terminalReplayFailureRetryCountsBySurfaceID: [String: Int] private var terminalOutputTransport: TerminalOutputTransport var terminalByteContinuationsBySurfaceID: [String: AsyncStream.Continuation] var terminalOutputStreamTokensBySurfaceID: [String: UUID] @@ -6329,7 +6329,6 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { terminalOutputStreamTokensBySurfaceID[surfaceID] = UUID() deliveredTerminalByteEndSeqBySurfaceID.removeValue(forKey: surfaceID) pendingTerminalByteEndSeqBySurfaceID.removeValue(forKey: surfaceID) - terminalActiveScreenBySurfaceID.removeValue(forKey: surfaceID) let token = UUID() terminalReplayBarrierTokensBySurfaceID[surfaceID] = token terminalReplayBarrierAckStreamTokensBySurfaceID.removeValue(forKey: surfaceID) @@ -6369,7 +6368,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { } @discardableResult - private func preserveTerminalReplayBarrierIfCurrent( + func preserveTerminalReplayBarrierIfCurrent( surfaceID: String, token: UUID?, reason: String @@ -6384,7 +6383,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { return true } - private func prepareTerminalReplayFailureRetry( + func prepareTerminalReplayFailureRetry( surfaceID: String, replayBarrierToken: UUID? ) -> UUID? { From e1e8e456428a9b774efb9948e02f4ba7ff9fb501 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 09:04:44 -0700 Subject: [PATCH 54/56] Keep visible snapshot provider immutable --- .../GhosttySurfaceView.swift | 78 +++++++++---------- 1 file changed, 38 insertions(+), 40 deletions(-) diff --git a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift index f084d18011eb..67567dc949f2 100644 --- a/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift +++ b/Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift @@ -4025,48 +4025,46 @@ public final class GhosttySurfaceView: UIView, TerminalSurfaceHosting { /// terminal surface, for the DEV "Copy Debug Logs" action so a bug report /// pairs the on-screen content with the debug log. Reads the VIEWPORT /// (visible grid only, not scrollback) via libghostty. - public static var visibleTerminalSnapshot: @MainActor () async -> String { - { - registeredSurfaceViews = registeredSurfaceViews.filter { $0.value.value != nil } - // Collect the main-actor state + surface pointers first, then read the - // viewport text on the serial output queue. `ghostty_surface_read_text` - // takes the same surface lock as `process_output` (which runs off-main); - // reading it on the MAIN thread here contends that lock during a render - // storm and stalls the present — tapping Copy Debug Logs would itself - // blank the terminal. The output queue is never concurrent with - // `process_output`, so the read can't wedge. The await is bounded by - // the surface's display-link deadline so this diagnostic path does not - // add a sleeping timer task or block the main actor. - var pending: [VisibleSnapshotRequest] = [] - for view in registeredSurfaceViews.values.compactMap(\.value) { - guard view.window != nil, !view.isHidden, view.alpha > 0.01, - let surface = view.surface else { continue } - let grid = view.effectiveGrid.map { "\($0.cols)x\($0.rows)" } ?? "?" - pending.append(VisibleSnapshotRequest( - view: view, - grid: grid, - font: Int(view.liveFontSize), - surface: surface, - generation: view.surfaceGeneration - )) - } - if pending.isEmpty { - return "===== visible terminal: (no on-screen surface) =====" - } - var sections: [String] = [] - for item in pending { - guard let section = await item.view.visibleSnapshotSection( - surface: item.surface, - generation: item.generation, - grid: item.grid, - font: item.font - ) else { - return "===== visible terminal: (snapshot skipped — render busy) =====" - } - sections.append(section) + public static let visibleTerminalSnapshot: @MainActor () async -> String = { + registeredSurfaceViews = registeredSurfaceViews.filter { $0.value.value != nil } + // Collect the main-actor state + surface pointers first, then read the + // viewport text on the serial output queue. `ghostty_surface_read_text` + // takes the same surface lock as `process_output` (which runs off-main); + // reading it on the MAIN thread here contends that lock during a render + // storm and stalls the present — tapping Copy Debug Logs would itself + // blank the terminal. The output queue is never concurrent with + // `process_output`, so the read can't wedge. The await is bounded by + // the surface's display-link deadline so this diagnostic path does not + // add a sleeping timer task or block the main actor. + var pending: [VisibleSnapshotRequest] = [] + for view in registeredSurfaceViews.values.compactMap(\.value) { + guard view.window != nil, !view.isHidden, view.alpha > 0.01, + let surface = view.surface else { continue } + let grid = view.effectiveGrid.map { "\($0.cols)x\($0.rows)" } ?? "?" + pending.append(VisibleSnapshotRequest( + view: view, + grid: grid, + font: Int(view.liveFontSize), + surface: surface, + generation: view.surfaceGeneration + )) + } + if pending.isEmpty { + return "===== visible terminal: (no on-screen surface) =====" + } + var sections: [String] = [] + for item in pending { + guard let section = await item.view.visibleSnapshotSection( + surface: item.surface, + generation: item.generation, + grid: item.grid, + font: item.font + ) else { + return "===== visible terminal: (snapshot skipped — render busy) =====" } - return sections.joined(separator: "\n\n") + sections.append(section) } + return sections.joined(separator: "\n\n") } private func visibleSnapshotSection( From fe0bb8d7237c8a04b269f0bd3ad2aba228b457ae Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 09:14:06 -0700 Subject: [PATCH 55/56] Add replay follow-up cap regression test --- .../TerminalReplayBarrierFollowUpTests.swift | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalReplayBarrierFollowUpTests.swift diff --git a/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalReplayBarrierFollowUpTests.swift b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalReplayBarrierFollowUpTests.swift new file mode 100644 index 000000000000..f6925369f929 --- /dev/null +++ b/Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/TerminalReplayBarrierFollowUpTests.swift @@ -0,0 +1,63 @@ +import Foundation +import Testing +@testable import CmuxMobileShell + +@MainActor +@Test func terminalReplayBarrierCapsFollowUpReplaysUnderContinuousOutput() async throws { + let router = LivenessHostRouter() + let box = TransportBox() + let clock = TestClock() + let store = try await makeConnectedStore(router: router, box: box, clock: clock) + let surfaceID = "live-terminal" + + await router.enqueueReplayTexts([ + "cold-replay", + "first-replay", + "follow-up-replay", + "unexpected-second-follow-up", + ]) + var iterator = store.terminalOutputStream(surfaceID: surfaceID).makeAsyncIterator() + await router.waitForCount(of: "mobile.terminal.replay", atLeast: 1) + let coldReplayChunk = try #require(await iterator.next()) + #expect(String(data: coldReplayChunk.data, encoding: .utf8) == "cold-replay") + store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: coldReplayChunk.streamToken) + let replayCountAfterMount = await router.count(of: "mobile.terminal.replay") + + store.deliverTerminalBytes(Data("stalled-first".utf8), surfaceID: surfaceID) + let stalledChunk = try #require(await iterator.next()) + store.terminalOutputDidReset(surfaceID: surfaceID, streamToken: stalledChunk.streamToken) + await router.waitForCount(of: "mobile.terminal.replay", atLeast: replayCountAfterMount + 1) + + let replayChunk = try #require(await iterator.next()) + #expect(String(data: replayChunk.data, encoding: .utf8) == "first-replay") + let firstDropAccepted = store.deliverTerminalBytes( + Data("live-during-first-barrier".utf8), + surfaceID: surfaceID + ) + #expect(firstDropAccepted == false) + + store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: replayChunk.streamToken) + await router.waitForCount(of: "mobile.terminal.replay", atLeast: replayCountAfterMount + 2) + + let followUpChunk = try #require(await iterator.next()) + #expect(String(data: followUpChunk.data, encoding: .utf8) == "follow-up-replay") + let followUpDropAccepted = store.deliverTerminalBytes( + Data("live-during-follow-up-barrier".utf8), + surfaceID: surfaceID + ) + #expect(followUpDropAccepted == false) + + store.terminalOutputDidProcess(surfaceID: surfaceID, streamToken: followUpChunk.streamToken) + let secondFollowUpRequested = await router.waitForCount( + of: "mobile.terminal.replay", + atLeast: replayCountAfterMount + 3, + timeoutNanoseconds: 200_000_000, + recordIssueOnTimeout: false + ) + #expect(!secondFollowUpRequested, "continuous output must not keep replaying indefinitely") + #expect(store.terminalReplayBarrierTokensBySurfaceID[surfaceID] == nil) + + store.deliverTerminalBytes(Data("after-bounded-replay".utf8), surfaceID: surfaceID) + let afterBoundedReplay = try #require(await iterator.next()) + #expect(String(data: afterBoundedReplay.data, encoding: .utf8) == "after-bounded-replay") +} From 8c1d045e151338a7f4301531118ed1bffb66a1ee Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 30 Jun 2026 09:18:33 -0700 Subject: [PATCH 56/56] Cap terminal replay follow-up loops --- .github/swift-file-length-budget.tsv | 6 ++--- ...hellComposite+TerminalOutputDelivery.swift | 23 +++++++++++++++++-- .../MobileShellComposite.swift | 14 ++++++++++- 3 files changed, 37 insertions(+), 6 deletions(-) diff --git a/.github/swift-file-length-budget.tsv b/.github/swift-file-length-budget.tsv index d3b11ababeec..1c3a174286d2 100644 --- a/.github/swift-file-length-budget.tsv +++ b/.github/swift-file-length-budget.tsv @@ -12,7 +12,7 @@ 9497 cmuxTests/CLINotifyProcessIntegrationRegressionTests.swift 8032 Sources/Panels/BrowserPanelView.swift 8016 CLI/cmux_open.swift -7423 Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift +7434 Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift 7366 cmuxTests/WorkspaceUnitTests.swift 7218 cmuxTests/WorkspaceRemoteConnectionTests.swift 6359 cmuxTests/SessionPersistenceTests.swift @@ -24,7 +24,7 @@ 4487 Sources/Panels/FilePreviewPanel.swift 4483 Sources/cmuxApp.swift 4367 cmuxTests/BrowserPanelTests.swift -4330 Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift +4264 Packages/iOS/CmuxMobileTerminal/Sources/CmuxMobileTerminal/GhosttySurfaceView.swift 4121 Sources/BrowserWindowPortal.swift 3934 Sources/Feed/FeedPanelView.swift 3926 cmuxTests/TabManagerUnitTests.swift @@ -130,7 +130,7 @@ 760 Packages/macOS/CMUXAgentLaunch/Tests/CMUXAgentLaunchTests/AgentLaunchSanitizerTests.swift 756 Sources/Panels/AgentSessionWebRendererCoordinator.swift 754 Sources/TerminalController+ControlWorkspaceContext.swift -753 Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/WorkspaceDetailView.swift +752 Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/WorkspaceDetailView.swift 752 cmuxUITests/CloseWorkspaceCmdDUITests.swift 739 cmuxTests/CLICodexHookTimeoutRegressionTests.swift 738 Packages/macOS/CMUXProjectModel/Sources/CMUXProjectModel/XcodeProjectAdapter.swift diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift index 30b618ad05fe..6aa0fb42f5cd 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+TerminalOutputDelivery.swift @@ -4,6 +4,19 @@ import CmuxMobileShellModel public import Foundation extension MobileShellComposite { + func claimTerminalReplayBarrierFollowUp(surfaceID: String) -> Bool { + let followUpCount = terminalReplayBarrierFollowUpCountsBySurfaceID[surfaceID] ?? 0 + guard followUpCount < Self.maxTerminalReplayBarrierFollowUps else { + MobileDebugLog.anchormux( + "terminal.output.replay_followup_cap_reached surface=\(surfaceID) attempts=\(followUpCount)" + ) + terminalReplayBarrierFollowUpCountsBySurfaceID.removeValue(forKey: surfaceID) + return false + } + terminalReplayBarrierFollowUpCountsBySurfaceID[surfaceID] = followUpCount + 1 + return true + } + /// Yield a raw PTY byte chunk to the surface stream, if one is attached. @discardableResult func deliverTerminalBytes( @@ -125,12 +138,18 @@ extension MobileShellComposite { MobileDebugLog.anchormux("terminal.output.replay_barrier_cleared surface=\(surfaceID)") let droppedOutputDuringBarrier = terminalReplayBarrierDroppedOutputSurfaceIDs.remove(surfaceID) != nil terminalReplayBarrierDroppedOutputCountsBySurfaceID.removeValue(forKey: surfaceID) - if droppedOutputDuringBarrier, needsFollowUpReplay { - let replayBarrierToken = beginTerminalReplayBarrier(surfaceID: surfaceID) + if droppedOutputDuringBarrier, + needsFollowUpReplay, + claimTerminalReplayBarrierFollowUp(surfaceID: surfaceID) { + let replayBarrierToken = beginTerminalReplayBarrier( + surfaceID: surfaceID, + preservingFollowUpCount: true + ) MobileDebugLog.anchormux("terminal.output.replay_followup surface=\(surfaceID)") requestTerminalReplay(surfaceID: surfaceID, replayBarrierToken: replayBarrierToken) return } + terminalReplayBarrierFollowUpCountsBySurfaceID.removeValue(forKey: surfaceID) } guard let next, let continuation = terminalByteContinuationsBySurfaceID[surfaceID], diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift index 2b6bb2477bc7..86d2ed739348 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift @@ -35,6 +35,7 @@ public typealias CMUXMobileShellStore = MobileShellComposite @Observable public final class MobileShellComposite: MobileTerminalOutputSinking { private static let maxTerminalReplayFailureRetries = 2 + static let maxTerminalReplayBarrierFollowUps = 1 private enum TerminalOutputTransport: Equatable { case hybrid @@ -726,6 +727,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { var terminalReplayBarrierDroppedOutputCountsBySurfaceID: [String: UInt64] var terminalReplayBarrierAckCoveredDroppedOutputCountsBySurfaceID: [String: UInt64] var terminalReplayFailureRetryCountsBySurfaceID: [String: Int] + var terminalReplayBarrierFollowUpCountsBySurfaceID: [String: Int] private var terminalOutputTransport: TerminalOutputTransport var terminalByteContinuationsBySurfaceID: [String: AsyncStream.Continuation] var terminalOutputStreamTokensBySurfaceID: [String: UUID] @@ -926,6 +928,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { self.terminalReplayBarrierDroppedOutputCountsBySurfaceID = [:] self.terminalReplayBarrierAckCoveredDroppedOutputCountsBySurfaceID = [:] self.terminalReplayFailureRetryCountsBySurfaceID = [:] + self.terminalReplayBarrierFollowUpCountsBySurfaceID = [:] self.terminalOutputTransport = .rawBytes self.terminalByteContinuationsBySurfaceID = [:] self.terminalOutputStreamTokensBySurfaceID = [:] @@ -4964,6 +4967,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { terminalReplayBarrierDroppedOutputCountsBySurfaceID = [:] terminalReplayBarrierAckCoveredDroppedOutputCountsBySurfaceID = [:] terminalReplayFailureRetryCountsBySurfaceID = [:] + terminalReplayBarrierFollowUpCountsBySurfaceID = [:] terminalOutputQueuesBySurfaceID = [:] terminalOutputStreamTokensBySurfaceID = terminalOutputStreamTokensBySurfaceID.mapValues { _ in UUID() } terminalScrollQueueTokensBySurfaceID = [:] @@ -6323,7 +6327,10 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { } } - func beginTerminalReplayBarrier(surfaceID: String) -> UUID { + func beginTerminalReplayBarrier( + surfaceID: String, + preservingFollowUpCount: Bool = false + ) -> UUID { cancelTerminalReplayInFlight(surfaceID: surfaceID) terminalOutputQueuesBySurfaceID[surfaceID] = TerminalOutputDeliveryQueue() terminalOutputStreamTokensBySurfaceID[surfaceID] = UUID() @@ -6336,6 +6343,9 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { terminalReplayBarrierDroppedOutputCountsBySurfaceID.removeValue(forKey: surfaceID) terminalReplayBarrierAckCoveredDroppedOutputCountsBySurfaceID.removeValue(forKey: surfaceID) terminalReplayFailureRetryCountsBySurfaceID.removeValue(forKey: surfaceID) + if !preservingFollowUpCount { + terminalReplayBarrierFollowUpCountsBySurfaceID.removeValue(forKey: surfaceID) + } terminalReplayBarrierTokensInFlightBySurfaceID.removeValue(forKey: surfaceID) return token } @@ -6362,6 +6372,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { terminalReplayBarrierDroppedOutputCountsBySurfaceID.removeValue(forKey: surfaceID) terminalReplayBarrierAckCoveredDroppedOutputCountsBySurfaceID.removeValue(forKey: surfaceID) terminalReplayFailureRetryCountsBySurfaceID.removeValue(forKey: surfaceID) + terminalReplayBarrierFollowUpCountsBySurfaceID.removeValue(forKey: surfaceID) terminalReplayBarrierTokensInFlightBySurfaceID.removeValue(forKey: surfaceID) MobileDebugLog.anchormux("terminal.output.replay_barrier_cleared_\(reason) surface=\(surfaceID)") return true @@ -6584,6 +6595,7 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { terminalReplayBarrierDroppedOutputCountsBySurfaceID.removeValue(forKey: surfaceID) terminalReplayBarrierAckCoveredDroppedOutputCountsBySurfaceID.removeValue(forKey: surfaceID) terminalReplayFailureRetryCountsBySurfaceID.removeValue(forKey: surfaceID) + terminalReplayBarrierFollowUpCountsBySurfaceID.removeValue(forKey: surfaceID) terminalScrollQueueTokensBySurfaceID.removeValue(forKey: surfaceID) terminalScrollQueuesBySurfaceID.removeValue(forKey: surfaceID) terminalScrollbackPrefetchStatesBySurfaceID.removeValue(forKey: surfaceID)