diff --git a/CLI/cmux.swift b/CLI/cmux.swift index 31e7edfd13c1..0b352555bb09 100644 --- a/CLI/cmux.swift +++ b/CLI/cmux.swift @@ -15397,7 +15397,7 @@ struct CMUXCLI { return """ Usage: cmux agent-hibernation [--json] - Enable or disable Agent Hibernation. + Enable or disable routine Agent Hibernation. Configure idle and live-terminal limits from Settings or cmux settings JSON. """ case "restore-session": diff --git a/Packages/macOS/CmuxSettingsUI/Sources/CmuxSettingsUI/Sections/TerminalSection.swift b/Packages/macOS/CmuxSettingsUI/Sources/CmuxSettingsUI/Sections/TerminalSection.swift index e853e3a73df8..b1b61a625503 100644 --- a/Packages/macOS/CmuxSettingsUI/Sources/CmuxSettingsUI/Sections/TerminalSection.swift +++ b/Packages/macOS/CmuxSettingsUI/Sources/CmuxSettingsUI/Sections/TerminalSection.swift @@ -378,7 +378,7 @@ public struct TerminalSection: View { String(localized: "settings.terminal.agentHibernation", defaultValue: "Agent Hibernation"), subtitle: hibernation.current ? String(localized: "settings.terminal.agentHibernation.subtitleOn", defaultValue: "Idle background agent terminals can be suspended when the live-terminal limit is exceeded.") - : String(localized: "settings.terminal.agentHibernation.subtitleOff", defaultValue: "Agent terminals stay live until you close them or quit cmux.") + : String(localized: "settings.terminal.agentHibernation.subtitleOff", defaultValue: "Scheduled hibernation is off. During critical memory pressure, cmux may still hibernate safe idle background agents.") ) { Toggle("", isOn: Binding(get: { hibernation.current }, set: { hibernation.set($0) })) .labelsHidden() diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownAdmission.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownAdmission.swift new file mode 100644 index 000000000000..403ba88a3186 --- /dev/null +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownAdmission.swift @@ -0,0 +1,40 @@ +import Foundation + +/// Main-actor admission state for one coordinator's bounded native-free slots. +@MainActor +final class TerminalSurfaceRuntimeTeardownAdmission { + private var availableExecutionSlots: Set + private var executionSlotByReservationID: [UUID: Int] = [:] + + nonisolated init() { + availableExecutionSlots = Set( + 0.. TerminalSurfaceRuntimeTeardownReservation? { + guard let executionSlot = availableExecutionSlots.min() else { + return nil + } + let reservation = TerminalSurfaceRuntimeTeardownReservation() + availableExecutionSlots.remove(executionSlot) + executionSlotByReservationID[reservation.id] = executionSlot + return reservation + } + + func executionSlot( + for reservation: TerminalSurfaceRuntimeTeardownReservation + ) -> Int? { + executionSlotByReservationID[reservation.id] + } + + func release(_ reservation: TerminalSurfaceRuntimeTeardownReservation) { + guard let executionSlot = executionSlotByReservationID.removeValue( + forKey: reservation.id + ) else { + return + } + availableExecutionSlots.insert(executionSlot) + } +} diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCompletion.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCompletion.swift new file mode 100644 index 000000000000..01599ac057bd --- /dev/null +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCompletion.swift @@ -0,0 +1,42 @@ +import Foundation + +/// One native-free completion shared by the enqueueing surface and teardown worker. +actor TerminalSurfaceRuntimeTeardownCompletion { + private var didFinish = false + private var waiters: [UUID: CheckedContinuation] = [:] + + func wait() async -> Bool { + if didFinish { return true } + if Task.isCancelled { return false } + let waiterID = UUID() + return await withTaskCancellationHandler { + await withCheckedContinuation { continuation in + if didFinish { + continuation.resume(returning: true) + } else if Task.isCancelled { + continuation.resume(returning: false) + } else { + waiters[waiterID] = continuation + } + } + } onCancel: { + Task { + await self.cancel(waiterID: waiterID) + } + } + } + + func finish() { + guard !didFinish else { return } + didFinish = true + let pendingWaiters = waiters.values + waiters.removeAll(keepingCapacity: false) + for waiter in pendingWaiters { + waiter.resume(returning: true) + } + } + + private func cancel(waiterID: UUID) { + waiters.removeValue(forKey: waiterID)?.resume(returning: false) + } +} diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift index 024a955b692f..af2ba25d4dbf 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift @@ -1,17 +1,23 @@ public import Foundation public import GhosttyKit public import CmuxTerminalCore +internal import Dispatch #if DEBUG internal import CMUXDebugLog #endif -/// Serializes native `ghostty_surface_free` calls off the close/deinit paths. +/// Coordinates native `ghostty_surface_free` calls off the close/deinit paths. /// -/// Frees run one at a time on a utility worker so re-entrant close/deinit -/// loops cannot form, with a deadline observer that reports (but never -/// blocks on) a stuck native free. The app constructs exactly one instance -/// and injects it through ``TerminalSurfaceRuntimeDependencies``. +/// Close/deinit frees run one at a time on a utility worker so re-entrant +/// teardown loops cannot form. Each admitted hibernation owns one independently +/// startable utility slot, so one stuck native join cannot strand another pane. +/// Deadline observers report, but never block on, stuck frees. The app constructs +/// exactly one instance and injects it through +/// ``TerminalSurfaceRuntimeDependencies``. public actor TerminalSurfaceRuntimeTeardownCoordinator { + /// Largest batch that can own independently startable native-free slots. + public static let maximumIsolatedHibernationTeardownCount = 2 + private let timeout: Duration = .seconds(5) #if DEBUG // Readable at internal scope in DEBUG so the debug-only extension in @@ -23,9 +29,34 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { #endif private var queuedRequests: [TerminalSurfaceRuntimeTeardownRequest] = [] private var isWorkerRunning = false + private let isolatedHibernationQueues: [DispatchQueue] + private nonisolated let isolatedHibernationAdmission = + TerminalSurfaceRuntimeTeardownAdmission() /// Creates the process's teardown coordinator. - public init() {} + public init() { + isolatedHibernationQueues = ( + 0.. TerminalSurfaceRuntimeTeardownReservation? { + isolatedHibernationAdmission.reserve() + } + + @MainActor + func cancelIsolatedHibernationTeardown( + _ reservation: TerminalSurfaceRuntimeTeardownReservation + ) { + isolatedHibernationAdmission.release(reservation) + } /// Reads a bounded screen tail away from the main actor and before any /// subsequently enqueued native free for the same surface. @@ -50,6 +81,8 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { /// main actor after the free completes. /// - freeSurface: The free operation; defaults to /// `ghostty_surface_free`. + /// - Returns: A ticket that completes after the native free and userdata releases. + @discardableResult public nonisolated func enqueueRuntimeTeardown( id: UUID, workspaceId: UUID, @@ -59,7 +92,7 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { freeSurface: @escaping @Sendable (ghostty_surface_t) -> Void = { surface in ghostty_surface_free(surface) } - ) { + ) -> TerminalSurfaceRuntimeTeardownTicket { enqueueRuntimeTeardown( id: id, workspaceId: workspaceId, @@ -97,6 +130,8 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { /// actor after the free completes. /// - freeSurface: The free operation; defaults to /// `ghostty_surface_free`. + /// - Returns: A ticket that completes after the native free and userdata releases. + @discardableResult nonisolated func enqueueRuntimeTeardown( id: UUID, workspaceId: UUID, @@ -105,10 +140,15 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { callbackContext: Unmanaged?, manualIOContext: Unmanaged?, byteTeeLease: (any TerminalByteTeeLease)?, + executionLane: TerminalSurfaceRuntimeTeardownExecutionLane = .serializedClose, + isolatedHibernationReservation: + TerminalSurfaceRuntimeTeardownReservation? = nil, freeSurface: @escaping @Sendable (ghostty_surface_t) -> Void = { surface in ghostty_surface_free(surface) } - ) { + ) -> TerminalSurfaceRuntimeTeardownTicket { + let completion = TerminalSurfaceRuntimeTeardownCompletion() + let ticket = TerminalSurfaceRuntimeTeardownTicket(completion: completion) let request = TerminalSurfaceRuntimeTeardownRequest( id: id, workspaceId: workspaceId, @@ -117,15 +157,61 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { callbackContext: callbackContext, manualIOContext: manualIOContext, byteTeeLease: byteTeeLease, - freeSurface: freeSurface + freeSurface: freeSurface, + completion: completion ) Task { - await self.enqueue(request) + await self.enqueue( + request, + executionLane: executionLane, + isolatedHibernationReservation: isolatedHibernationReservation + ) } + return ticket } - func enqueue(_ request: TerminalSurfaceRuntimeTeardownRequest) { + func enqueue( + _ request: TerminalSurfaceRuntimeTeardownRequest, + executionLane: TerminalSurfaceRuntimeTeardownExecutionLane = .serializedClose, + isolatedHibernationReservation: + TerminalSurfaceRuntimeTeardownReservation? = nil + ) async { pendingReasonsById[request.id] = request.reason + switch executionLane { + case .isolatedHibernation: + if let isolatedHibernationReservation, + let executionSlot = await isolatedHibernationAdmission.executionSlot( + for: isolatedHibernationReservation + ), + isolatedHibernationQueues.indices.contains(executionSlot) { + // Each reservation exclusively owns one queue until its native free + // returns. Ghostty locks its shared surface registry, while renderer + // and IO joins are surface-owned, so separate surfaces may tear down + // concurrently. This bounds blocked native workers at two without + // letting one stuck pane strand another admitted pane. + Task { + await self.observeTimeout(id: request.id) + } + isolatedHibernationQueues[executionSlot].async { + self.freeNativeSurface(request) + Task { + await self.isolatedHibernationAdmission.release( + isolatedHibernationReservation + ) + await self.finishFree(request) + await self.complete(id: request.id) + } + } + return + } + if let isolatedHibernationReservation { + await isolatedHibernationAdmission.release( + isolatedHibernationReservation + ) + } + case .serializedClose: + break + } queuedRequests.append(request) if !isWorkerRunning { isWorkerRunning = true @@ -134,7 +220,8 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { Task { await self.observeTimeout(id: request.id) } - await Self.free(request) + self.freeNativeSurface(request) + await self.finishFree(request) await self.complete(id: request.id) } } @@ -149,7 +236,9 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { return queuedRequests.removeFirst() } - private nonisolated static func free(_ request: TerminalSurfaceRuntimeTeardownRequest) async { + private nonisolated func freeNativeSurface( + _ request: TerminalSurfaceRuntimeTeardownRequest + ) { #if DEBUG logDebugEvent( "surface.lifecycle.nativeFree.begin surface=\(request.surfaceToken) " + @@ -157,6 +246,11 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { ) #endif request.freeSurface(request.surface) + } + + private nonisolated func finishFree( + _ request: TerminalSurfaceRuntimeTeardownRequest + ) async { if request.callbackContext != nil || request.manualIOContext != nil || request.byteTeeLease != nil { // The request is the @unchecked Sendable transport for the // Unmanaged contexts; release through the request so the @Sendable @@ -171,6 +265,7 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { request.byteTeeLease?.release() } } + await request.completion.finish() #if DEBUG logDebugEvent( "surface.lifecycle.nativeFree.end surface=\(request.surfaceToken) " + diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownExecutionLane.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownExecutionLane.swift new file mode 100644 index 000000000000..0a2a800691d5 --- /dev/null +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownExecutionLane.swift @@ -0,0 +1,8 @@ +/// Selects the ownership boundary for a native surface free. +enum TerminalSurfaceRuntimeTeardownExecutionLane: Sendable { + /// Preserves ordering for close/deinit flows that can re-enter teardown. + case serializedClose + + /// Gives an explicitly owned hibernation join an independent bounded slot. + case isolatedHibernation +} diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownRequest.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownRequest.swift index 3395648179c5..18d82e8f78b1 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownRequest.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownRequest.swift @@ -25,6 +25,7 @@ struct TerminalSurfaceRuntimeTeardownRequest: @unchecked Sendable { let manualIOContext: Unmanaged? let byteTeeLease: (any TerminalByteTeeLease)? let freeSurface: @Sendable (ghostty_surface_t) -> Void + let completion: TerminalSurfaceRuntimeTeardownCompletion #if DEBUG let surfaceToken: String let workspaceToken: String @@ -38,7 +39,8 @@ struct TerminalSurfaceRuntimeTeardownRequest: @unchecked Sendable { callbackContext: Unmanaged?, manualIOContext: Unmanaged?, byteTeeLease: (any TerminalByteTeeLease)?, - freeSurface: @escaping @Sendable (ghostty_surface_t) -> Void + freeSurface: @escaping @Sendable (ghostty_surface_t) -> Void, + completion: TerminalSurfaceRuntimeTeardownCompletion ) { self.id = id self.workspaceId = workspaceId @@ -48,6 +50,7 @@ struct TerminalSurfaceRuntimeTeardownRequest: @unchecked Sendable { self.manualIOContext = manualIOContext self.byteTeeLease = byteTeeLease self.freeSurface = freeSurface + self.completion = completion #if DEBUG self.surfaceToken = String(id.uuidString.prefix(5)) self.workspaceToken = String(workspaceId.uuidString.prefix(5)) diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownReservation.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownReservation.swift new file mode 100644 index 000000000000..c79fd74320ee --- /dev/null +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownReservation.swift @@ -0,0 +1,10 @@ +import Foundation + +/// Exclusive admission to one failure-isolated hibernation teardown slot. +struct TerminalSurfaceRuntimeTeardownReservation: Equatable, Sendable { + let id: UUID + + init(id: UUID = UUID()) { + self.id = id + } +} diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownTicket.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownTicket.swift new file mode 100644 index 000000000000..1ed5df6ac631 --- /dev/null +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownTicket.swift @@ -0,0 +1,46 @@ +public import Foundation + +/// An awaitable handle for one serialized native surface teardown. +public struct TerminalSurfaceRuntimeTeardownTicket: Sendable { + /// Stable identity used by the surface to clear only its current teardown. + public let id: UUID + + private let completion: TerminalSurfaceRuntimeTeardownCompletion + + init( + id: UUID = UUID(), + completion: TerminalSurfaceRuntimeTeardownCompletion + ) { + self.id = id + self.completion = completion + } + + /// Waits until the native free and callback-userdata releases finish. + /// + /// A `nil` deadline is the event-driven recovery wait used after a bounded + /// foreground attempt has already reported failure. + /// + /// - Parameter timeout: Maximum wait, or `nil` to wait until completion. + /// - Returns: `true` only when teardown completed. + public func wait(timeout: Duration?) async -> Bool { + guard let timeout else { + return await completion.wait() + } + return await withTaskGroup(of: Bool.self) { group in + group.addTask { + await completion.wait() + } + group.addTask { + do { + try await ContinuousClock().sleep(for: timeout) + return false + } catch { + return false + } + } + let completed = await group.next() ?? false + group.cancelAll() + return completed + } + } +} diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Runtime/AgentHibernationRecording.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Runtime/AgentHibernationRecording.swift index bfef80ce3b09..bc0d00e2f39f 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Runtime/AgentHibernationRecording.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Runtime/AgentHibernationRecording.swift @@ -3,14 +3,14 @@ public import Foundation /// Records terminal input liveness for agent-hibernation tracking. /// /// Implemented in the app over `AgentHibernationController`; the recorder is -/// called synchronously on the input hot path and is responsible for its own -/// enablement gate and main-actor hop, exactly like the legacy -/// `recordAgentHibernationTerminalInput` helper it replaces. +/// called synchronously on the main-actor input hot path and is responsible for +/// its own lock-free enablement gate. public protocol AgentHibernationRecording: AnyObject, Sendable { /// Records that a terminal surface received input. /// /// - Parameters: /// - workspaceId: The owning workspace id. /// - panelId: The surface/panel id that received input. + @MainActor func recordTerminalInput(workspaceId: UUID, panelId: UUID) } diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift index 9e6b464c787e..20e6d81afdec 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift @@ -299,8 +299,18 @@ extension TerminalSurface { /// Frees the runtime surface while keeping the model alive for an /// agent-hibernation resume. + /// + /// - Returns: `false` without changing the surface when the bounded + /// hibernation teardown lane has no capacity. + @discardableResult @MainActor - public func suspendRuntimeSurfaceForAgentHibernation(reason: String) { + public func suspendRuntimeSurfaceForAgentHibernation(reason: String) -> Bool { + guard let teardownReservation = + agentHibernationRuntimeTeardownReservation ?? + runtimeTeardown.reserveIsolatedHibernationTeardown() else { + return false + } + agentHibernationRuntimeTeardownReservation = nil _ = fontSizeLineageSnapshot() mobileViewportFontFitState = nil runtimeSurfaceSuspendedForAgentHibernation = true @@ -330,10 +340,13 @@ extension TerminalSurface { desiredFocusState = false guard let surfaceToFree else { + runtimeTeardown.cancelIsolatedHibernationTeardown( + teardownReservation + ) callbackContext?.release() manualIOContext?.release() teeLease?.release() - return + return true } #if DEBUG @@ -348,7 +361,7 @@ extension TerminalSurface { // Transport manualIOContext and teeLease through the request too: // the coordinator releases all callback userdata only after the // native free, which is what joins ghostty's IO threads. - runtimeTeardown.enqueueRuntimeTeardown( + agentHibernationRuntimeTeardownTicket = runtimeTeardown.enqueueRuntimeTeardown( id: id, workspaceId: tabId, reason: reason, @@ -356,26 +369,76 @@ extension TerminalSurface { callbackContext: callbackContext, manualIOContext: manualIOContext, byteTeeLease: teeLease, + executionLane: .isolatedHibernation, + isolatedHibernationReservation: teardownReservation, freeSurface: freeSurface ) - return + return true } #endif - Task { @MainActor in - ghostty_surface_free(surfaceToFree) - callbackContext?.release() - manualIOContext?.release() - teeLease?.release() + agentHibernationRuntimeTeardownTicket = runtimeTeardown.enqueueRuntimeTeardown( + id: id, + workspaceId: tabId, + reason: reason, + surface: surfaceToFree, + callbackContext: callbackContext, + manualIOContext: manualIOContext, + byteTeeLease: teeLease, + executionLane: .isolatedHibernation, + isolatedHibernationReservation: teardownReservation + ) + return true + } + + /// Reserves the bounded native-free lane at the final pre-signal gate. + @MainActor + public func reserveAgentHibernationRuntimeTeardown() -> Bool { + guard agentHibernationRuntimeTeardownTicket == nil else { return false } + if agentHibernationRuntimeTeardownReservation != nil { return true } + guard let reservation = + runtimeTeardown.reserveIsolatedHibernationTeardown() else { + return false } + agentHibernationRuntimeTeardownReservation = reservation + return true + } + + /// Releases an unused reservation when the signal batch fails before commit. + @MainActor + public func cancelAgentHibernationRuntimeTeardownReservation() { + guard let reservation = agentHibernationRuntimeTeardownReservation else { + return + } + agentHibernationRuntimeTeardownReservation = nil + runtimeTeardown.cancelIsolatedHibernationTeardown(reservation) + } + + /// Waits for the old hibernated runtime generation to finish native teardown. + /// + /// - Parameter timeout: Maximum wait, or `nil` for event-driven recovery. + /// - Returns: `true` only after the old native surface and callback contexts are gone. + @MainActor + public func waitForAgentHibernationRuntimeTeardown(timeout: Duration?) async -> Bool { + guard let ticket = agentHibernationRuntimeTeardownTicket else { return true } + let completed = await ticket.wait(timeout: timeout) + if completed, agentHibernationRuntimeTeardownTicket?.id == ticket.id { + agentHibernationRuntimeTeardownTicket = nil + } + return completed } /// Marks the resume side of agent hibernation and primes the next runtime /// spawn's initial input. + /// + /// - Returns: `true` when the old runtime is fully gone and resume was armed. + @discardableResult @MainActor - public func prepareAgentHibernationResume(initialInput: String?) { + public func prepareAgentHibernationResume(initialInput: String?) -> Bool { + guard agentHibernationRuntimeTeardownTicket == nil else { return false } runtimeSurfaceSuspendedForAgentHibernation = false prepareNextRuntimeInitialInput(initialInput) + return true } /// Primes the initial input for the next runtime spawn only. diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface.swift index b17b21a14834..3dca6ca9265d 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface.swift @@ -269,6 +269,9 @@ public final class TerminalSurface: Identifiable, ObservableObject { (any TerminalSurfaceNativeViewing)? var requiresRestoreSpawnPacing = false var runtimeSurfaceSuspendedForAgentHibernation = false + var agentHibernationRuntimeTeardownTicket: TerminalSurfaceRuntimeTeardownTicket? + var agentHibernationRuntimeTeardownReservation: + TerminalSurfaceRuntimeTeardownReservation? var headlessStartupWindow: NSWindow? var surfaceCallbackContext: Unmanaged? var claudeCommandShim: ClaudeCommandShim? diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/FakeHibernationRecorder.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/FakeHibernationRecorder.swift index 11616275d93f..c6e64610eb63 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/FakeHibernationRecorder.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/FakeHibernationRecorder.swift @@ -2,5 +2,6 @@ import Foundation @testable import CmuxTerminal final class FakeHibernationRecorder: AgentHibernationRecording { + @MainActor func recordTerminalInput(workspaceId: UUID, panelId: UUID) {} } diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift index e82833142d3b..25973959cc17 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift @@ -1,3 +1,4 @@ +import Dispatch import Foundation import os import Testing @@ -59,13 +60,24 @@ private final class LifetimeRecordingByteTeeLease: TerminalByteTeeLease, @unchec } @Suite struct TerminalSurfaceRuntimeTeardownCoordinatorTests { + @Test func ticketDistinguishesDeadlineFromEventualCompletion() async { + let completion = TerminalSurfaceRuntimeTeardownCompletion() + let ticket = TerminalSurfaceRuntimeTeardownTicket(completion: completion) + + #expect(await ticket.wait(timeout: .zero) == false) + + await completion.finish() + + #expect(await ticket.wait(timeout: nil)) + } + @Test func enqueuedTeardownInvokesInjectedFreeWithTheSamePointer() async { let coordinator = TerminalSurfaceRuntimeTeardownCoordinator() let recorder = FreedSurfaceRecorder() let surface = UnsafeMutableRawPointer.allocate(byteCount: 8, alignment: 8) defer { surface.deallocate() } - coordinator.enqueueRuntimeTeardown( + let ticket = coordinator.enqueueRuntimeTeardown( id: UUID(), workspaceId: UUID(), reason: "test", @@ -78,6 +90,7 @@ private final class LifetimeRecordingByteTeeLease: TerminalByteTeeLease, @unchec ) await recorder.waitForFreeCount(1) + #expect(await ticket.wait(timeout: .seconds(1))) #expect(await recorder.freed == [UInt(bitPattern: surface)]) } @@ -107,6 +120,126 @@ private final class LifetimeRecordingByteTeeLease: TerminalByteTeeLease, @unchec #expect(await Set(recorder.freed) == Set(surfaces.map { UInt(bitPattern: $0) })) } + @Test func stuckHibernationFreeDoesNotStrandAnotherAdmissionOrClose() async throws { + let coordinator = TerminalSurfaceRuntimeTeardownCoordinator() + let isolatedSurface = UnsafeMutableRawPointer.allocate(byteCount: 8, alignment: 8) + let queuedIsolatedSurface = UnsafeMutableRawPointer.allocate( + byteCount: 8, + alignment: 8 + ) + let serializedSurface = UnsafeMutableRawPointer.allocate(byteCount: 8, alignment: 8) + defer { + isolatedSurface.deallocate() + queuedIsolatedSurface.deallocate() + serializedSurface.deallocate() + } + let isolatedFreeStarted = AsyncStream.makeStream() + let releaseIsolatedFree = DispatchSemaphore(value: 0) + let secondIsolatedFreeCount = OSAllocatedUnfairLock(initialState: 0) + let serializedFreeCount = OSAllocatedUnfairLock(initialState: 0) + defer { + releaseIsolatedFree.signal() + isolatedFreeStarted.continuation.finish() + } + + let isolatedReservation = try #require( + await coordinator.reserveIsolatedHibernationTeardown() + ) + let isolatedTicket = coordinator.enqueueRuntimeTeardown( + id: UUID(), + workspaceId: UUID(), + reason: "test.isolatedHibernation", + surface: isolatedSurface, + callbackContext: nil, + manualIOContext: nil, + byteTeeLease: nil, + executionLane: .isolatedHibernation, + isolatedHibernationReservation: isolatedReservation, + freeSurface: { _ in + isolatedFreeStarted.continuation.yield() + _ = releaseIsolatedFree.wait(timeout: .distantFuture) + } + ) + var isolatedFreeIterator = isolatedFreeStarted.stream.makeAsyncIterator() + _ = await isolatedFreeIterator.next() + + let secondReservation = try #require( + await coordinator.reserveIsolatedHibernationTeardown() + ) + #expect(await coordinator.reserveIsolatedHibernationTeardown() == nil) + let secondIsolatedTicket = coordinator.enqueueRuntimeTeardown( + id: UUID(), + workspaceId: UUID(), + reason: "test.secondIsolatedHibernation", + surface: queuedIsolatedSurface, + callbackContext: nil, + manualIOContext: nil, + byteTeeLease: nil, + executionLane: .isolatedHibernation, + isolatedHibernationReservation: secondReservation, + freeSurface: { _ in + secondIsolatedFreeCount.withLock { $0 += 1 } + } + ) + let serializedTicket = coordinator.enqueueRuntimeTeardown( + id: UUID(), + workspaceId: UUID(), + reason: "test.serializedClose", + surface: serializedSurface, + callbackContext: nil, + freeSurface: { _ in + serializedFreeCount.withLock { $0 += 1 } + } + ) + + #expect(await serializedTicket.wait(timeout: .seconds(1))) + #expect(await secondIsolatedTicket.wait(timeout: .seconds(1))) + #expect(serializedFreeCount.withLock { $0 } == 1) + #expect(await isolatedTicket.wait(timeout: .zero) == false) + #expect(secondIsolatedFreeCount.withLock { $0 } == 1) + + let replacementReservation = try #require( + await coordinator.reserveIsolatedHibernationTeardown() + ) + #expect(await coordinator.reserveIsolatedHibernationTeardown() == nil) + await coordinator.cancelIsolatedHibernationTeardown(replacementReservation) + releaseIsolatedFree.signal() + #expect(await isolatedTicket.wait(timeout: .seconds(1))) + let nextReservation = try #require( + await coordinator.reserveIsolatedHibernationTeardown() + ) + await coordinator.cancelIsolatedHibernationTeardown(nextReservation) + } + + @Test func staleIsolatedReservationFallsBackToSerializedFree() async throws { + let coordinator = TerminalSurfaceRuntimeTeardownCoordinator() + let surface = UnsafeMutableRawPointer.allocate(byteCount: 8, alignment: 8) + defer { surface.deallocate() } + let freeCount = OSAllocatedUnfairLock(initialState: 0) + let staleReservation = try #require( + await coordinator.reserveIsolatedHibernationTeardown() + ) + await coordinator.cancelIsolatedHibernationTeardown(staleReservation) + + let ticket = coordinator.enqueueRuntimeTeardown( + id: UUID(), + workspaceId: UUID(), + reason: "test.staleIsolatedReservation", + surface: surface, + callbackContext: nil, + manualIOContext: nil, + byteTeeLease: nil, + executionLane: .isolatedHibernation, + isolatedHibernationReservation: staleReservation, + freeSurface: { _ in + freeCount.withLock { $0 += 1 } + } + ) + + #expect(await ticket.wait(timeout: .seconds(1))) + #expect(freeCount.withLock { $0 } == 1) + } + @Test func byteTeeCallbackOwnerIsReleasedOnlyAfterNativeFreeReturns() async { let coordinator = TerminalSurfaceRuntimeTeardownCoordinator() let recorder = TeardownLifetimeRecorder() diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceTeardownCallbackLifetimeTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceTeardownCallbackLifetimeTests.swift index 2ceb5cb14b44..a6bac1c55886 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceTeardownCallbackLifetimeTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceTeardownCallbackLifetimeTests.swift @@ -77,6 +77,64 @@ import Testing #expect(recorder.events == [.nativeFree, .teeLeaseRelease]) } + @Test func agentHibernationResumeWaitsForNativeFreeCompletion() async { + let registry = TerminalSurfaceRegistry() + let surface = makeSurface(registry: registry) + let runtimeSurface = UnsafeMutableRawPointer.allocate(byteCount: 8, alignment: 8) + registry.registerRuntimeSurface(runtimeSurface, ownerId: surface.id) + surface.installRuntimeSurfaceForTesting(runtimeSurface) + defer { runtimeSurface.deallocate() } + let freeStarted = AsyncStream.makeStream() + let allowFree = DispatchSemaphore(value: 0) + TerminalSurface.runtimeSurfaceFreeOverrideForTesting = { _ in + freeStarted.continuation.yield() + allowFree.wait() + } + defer { + allowFree.signal() + TerminalSurface.runtimeSurfaceFreeOverrideForTesting = nil + } + + surface.suspendRuntimeSurfaceForAgentHibernation(reason: "test.hibernate") + var freeStartedIterator = freeStarted.stream.makeAsyncIterator() + _ = await freeStartedIterator.next() + + #expect(!surface.prepareAgentHibernationResume(initialInput: nil)) + + allowFree.signal() + #expect(await surface.waitForAgentHibernationRuntimeTeardown(timeout: .seconds(1))) + #expect(surface.prepareAgentHibernationResume(initialInput: nil)) + freeStarted.continuation.finish() + } + + @Test func hibernationAdmissionFailurePreservesLiveRuntimeSurface() throws { + let coordinator = TerminalSurfaceRuntimeTeardownCoordinator() + let firstReservation = try #require( + coordinator.reserveIsolatedHibernationTeardown() + ) + let secondReservation = try #require( + coordinator.reserveIsolatedHibernationTeardown() + ) + defer { + coordinator.cancelIsolatedHibernationTeardown(firstReservation) + coordinator.cancelIsolatedHibernationTeardown(secondReservation) + } + + let surface = makeSurface(runtimeTeardown: coordinator) + surface.installRuntimeSurfaceForTesting(fakeRuntimeSurface()) + TerminalSurface.runtimeSurfaceFreeOverrideForTesting = { _ in } + defer { TerminalSurface.runtimeSurfaceFreeOverrideForTesting = nil } + + #expect( + surface.suspendRuntimeSurfaceForAgentHibernation( + reason: "test.admissionFailure" + ) == false + ) + #expect(surface.hasLiveSurface) + + surface.teardownSurface() + } + @Test func deinitKeepsTeeLeaseUntilCoordinatorFree() async { let recorder = TeardownOrderRecorder() var surface: TerminalSurface? = makeSurface() @@ -161,7 +219,9 @@ import Testing } private func makeSurface( - registry: any TerminalSurfaceRegistering = FakeSurfaceRegistry() + registry: any TerminalSurfaceRegistering = FakeSurfaceRegistry(), + runtimeTeardown: TerminalSurfaceRuntimeTeardownCoordinator = + TerminalSurfaceRuntimeTeardownCoordinator() ) -> TerminalSurface { let nativeView = FakeTerminalSurfaceNativeView(frame: NSRect(x: 0, y: 0, width: 800, height: 600)) let paneHost = FakeTerminalSurfacePaneHost(surfaceView: nativeView) @@ -177,7 +237,7 @@ import Testing byteTee: FakeTerminalByteTee(), rendererRealization: FakeRendererRealizationScheduler(), hibernationRecorder: FakeHibernationRecorder(), - runtimeTeardown: TerminalSurfaceRuntimeTeardownCoordinator(), + runtimeTeardown: runtimeTeardown, restoreSpawnScheduler: TerminalSurfaceRestoreSpawnScheduler(interSpawnDelay: .zero), runtimeFilesystem: TerminalSurfaceRuntimeFilesystem( claudeCommandShimTemporaryDirectory: URL(fileURLWithPath: "/tmp/cmux-terminal-tests", isDirectory: true), diff --git a/Resources/Localizable.xcstrings b/Resources/Localizable.xcstrings index c9d0cf1d987a..deff352e3ffb 100644 --- a/Resources/Localizable.xcstrings +++ b/Resources/Localizable.xcstrings @@ -194452,13 +194452,13 @@ "en": { "stringUnit": { "state": "translated", - "value": "Agent terminals stay live until you close them or quit cmux." + "value": "Scheduled hibernation is off. During critical memory pressure, cmux may still hibernate safe idle background agents." } }, "ja": { "stringUnit": { "state": "translated", - "value": "エージェント端末は、閉じるか cmux を終了するまでライブのままです。" + "value": "通常の休止はオフです。メモリ負荷が危険な状態では、安全なアイドル中のバックグラウンドエージェントを cmux が休止する場合があります。" } } } @@ -234758,6 +234758,40 @@ } } }, + "terminal.agentHibernation.failed": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "Agent shutdown needs attention" + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "エージェントの終了を確認してください" + } + } + } + }, + "terminal.agentHibernation.finishing": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "Finishing agent shutdown" + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "エージェントを終了しています" + } + } + } + }, "terminal.agentHibernation.lastActivity": { "extractionState": "manual", "localizations": { @@ -234792,6 +234826,23 @@ } } }, + "terminal.agentHibernation.retry": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "Retry shutdown" + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "終了を再試行" + } + } + } + }, "terminal.agentHibernation.title": { "extractionState": "manual", "localizations": { diff --git a/Sources/AgentHibernation/AgentHibernationPanelPhase.swift b/Sources/AgentHibernation/AgentHibernationPanelPhase.swift new file mode 100644 index 000000000000..dc8dca798c9c --- /dev/null +++ b/Sources/AgentHibernation/AgentHibernationPanelPhase.swift @@ -0,0 +1,43 @@ +enum AgentHibernationPanelPhase { + case live + case terminating(AgentHibernationPanelState) + case recovering(AgentHibernationPanelState) + case terminationFailed(AgentHibernationPanelState) + case hibernated(AgentHibernationPanelState) + + var state: AgentHibernationPanelState? { + switch self { + case .live: + nil + case .terminating(let state), + .recovering(let state), + .terminationFailed(let state), + .hibernated(let state): + state + } + } + + var isCommitted: Bool { + if case .live = self { return false } + return true + } + + var isTerminating: Bool { + if case .terminating = self { return true } + return false + } + + var terminationFailed: Bool { + if case .terminationFailed = self { return true } + return false + } + + var isAwaitingCommit: Bool { + switch self { + case .terminating, .recovering, .terminationFailed: + true + case .live, .hibernated: + false + } + } +} diff --git a/Sources/AgentHibernation/AgentHibernationPanelState.swift b/Sources/AgentHibernation/AgentHibernationPanelState.swift new file mode 100644 index 000000000000..e5baf4bba86d --- /dev/null +++ b/Sources/AgentHibernation/AgentHibernationPanelState.swift @@ -0,0 +1,11 @@ +import Foundation + +struct AgentHibernationPanelState { + let agent: SessionRestorableAgentSnapshot + let hibernatedAt: Date + let lastActivityAt: Date + + var agentDisplayName: String { + agent.agentDisplayName + } +} diff --git a/Sources/AgentHibernation/AgentHibernationResumePreparation.swift b/Sources/AgentHibernation/AgentHibernationResumePreparation.swift new file mode 100644 index 000000000000..090d9a6ea42a --- /dev/null +++ b/Sources/AgentHibernation/AgentHibernationResumePreparation.swift @@ -0,0 +1,14 @@ +enum AgentHibernationResumePreparation: Equatable { + case unavailable + case resumed(queuedStartupInput: Bool) + + var didResume: Bool { + if case .resumed = self { return true } + return false + } + + var queuedStartupInput: Bool { + if case .resumed(let queuedStartupInput) = self { return queuedStartupInput } + return false + } +} diff --git a/Sources/App/AgentHibernationController+CommittedTerminationCleanup.swift b/Sources/App/AgentHibernationController+CommittedTerminationCleanup.swift new file mode 100644 index 000000000000..b16af8846f40 --- /dev/null +++ b/Sources/App/AgentHibernationController+CommittedTerminationCleanup.swift @@ -0,0 +1,8 @@ +import Foundation + +extension AgentHibernationController { + struct CommittedTerminationCleanup { + let requestID: UUID + let task: Task + } +} diff --git a/Sources/App/AgentHibernationController+CommittedTerminationObservation.swift b/Sources/App/AgentHibernationController+CommittedTerminationObservation.swift new file mode 100644 index 000000000000..6a876b8baa43 --- /dev/null +++ b/Sources/App/AgentHibernationController+CommittedTerminationObservation.swift @@ -0,0 +1,81 @@ +import Foundation + +extension AgentHibernationController { + final class CommittedTerminationObservation { + let requestID: UUID + let processExitCompletion: AgentHibernationProcessExitCompletion + var task: Task? + var retryRecovery: (@MainActor () -> Void)? + + init( + requestID: UUID, + processExitCompletion: AgentHibernationProcessExitCompletion + ) { + self.requestID = requestID + self.processExitCompletion = processExitCompletion + } + } + + @discardableResult + func registerCommittedTerminationObservation( + panelID: UUID, + requestID: UUID = UUID(), + processExitCompletion: AgentHibernationProcessExitCompletion + ) -> UUID { + if let current = committedTerminationObservationsByPanelID[panelID], + current.requestID == requestID { + return requestID + } + committedTerminationCleanupByPanelID + .removeValue(forKey: panelID)? + .task + .cancel() + if let previous = committedTerminationObservationsByPanelID.removeValue( + forKey: panelID + ) { + previous.task?.cancel() + if previous.processExitCompletion !== processExitCompletion { + Task { + await previous.processExitCompletion.finish(false) + } + } + } + committedTerminationObservationsByPanelID[panelID] = + CommittedTerminationObservation( + requestID: requestID, + processExitCompletion: processExitCompletion + ) + return requestID + } + + @discardableResult + func replaceCommittedTerminationTask( + panelID: UUID, + requestID: UUID, + with task: Task + ) -> Bool { + guard let observation = committedTerminationObservationsByPanelID[panelID], + observation.requestID == requestID else { + task.cancel() + return false + } + let previousTask = observation.task + observation.task = task + previousTask?.cancel() + return true + } + + func removeCommittedTerminationObservation( + panelID: UUID, + requestID: UUID + ) { + guard committedTerminationObservationsByPanelID[panelID]?.requestID == requestID, + let observation = committedTerminationObservationsByPanelID.removeValue( + forKey: panelID + ) else { + return + } + observation.task?.cancel() + committedTerminationCleanupByPanelID.removeValue(forKey: panelID)?.task.cancel() + } +} diff --git a/Sources/App/AgentHibernationController+Confirmation.swift b/Sources/App/AgentHibernationController+Confirmation.swift index 9892bf9221a2..5453efc7189c 100644 --- a/Sources/App/AgentHibernationController+Confirmation.swift +++ b/Sources/App/AgentHibernationController+Confirmation.swift @@ -2,7 +2,9 @@ import Foundation extension AgentHibernationController { struct Confirmation: Sendable { + let trigger: AgentHibernationReclaimTrigger let fingerprint: String + let processIdentities: [Int: AgentPIDProcessIdentity] let sampledAt: TimeInterval let dueAt: TimeInterval } diff --git a/Sources/App/AgentHibernationController+InFlightTeardown.swift b/Sources/App/AgentHibernationController+InFlightTeardown.swift index aec7c0429bf3..8bc4ded312ec 100644 --- a/Sources/App/AgentHibernationController+InFlightTeardown.swift +++ b/Sources/App/AgentHibernationController+InFlightTeardown.swift @@ -3,5 +3,7 @@ import Foundation extension AgentHibernationController { struct InFlightTeardown: Sendable { let requestID: UUID + let trigger: AgentHibernationReclaimTrigger } + } diff --git a/Sources/App/AgentHibernationController+MemoryPressure.swift b/Sources/App/AgentHibernationController+MemoryPressure.swift new file mode 100644 index 000000000000..a0404c10da71 --- /dev/null +++ b/Sources/App/AgentHibernationController+MemoryPressure.swift @@ -0,0 +1,79 @@ +import Foundation + +extension AgentHibernationController { + /// Starts one asynchronous critical-pressure evaluation. + /// + /// The existing hibernation lifecycle remains the sole teardown owner: + /// pressure only changes which safe idle agents it selects. Transcript + /// protection, confirmation, activity revalidation, and scoped process + /// termination are unchanged. + @discardableResult + func reclaimIdleAgentsForSystemMemoryPressure( + now: Date, + isPressureStillCritical: @escaping @MainActor () -> Bool, + onHibernationCompleted: @escaping @MainActor (Int) -> Void + ) -> Bool { + guard memoryPressureEvaluation == nil, + isPressureStillCritical() else { + return false + } + + let requestID = UUID() + let task = Task { @MainActor [weak self] in + guard let self else { return } + var awaitsTeardownCompletion = false + defer { + if !awaitsTeardownCompletion { + self.finishMemoryPressureEvaluation(requestID: requestID) + } + } + + let settings = AgentHibernationSettings.values() + let index = await RestorableAgentSessionIndex.loadIncludingProcessDetectedSnapshots() + guard !Task.isCancelled, + isPressureStillCritical() else { + return + } + let initialEvaluation = self.evaluate( + index: index, + settings: settings, + now: now, + trigger: .systemMemoryPressure + ) + guard initialEvaluation.hasCandidates else { return } + + do { + try await ContinuousClock().sleep(for: .seconds(settings.confirmationSeconds)) + } catch { + return + } + guard isPressureStillCritical() else { return } + let confirmationIndex = await RestorableAgentSessionIndex + .loadIncludingProcessDetectedSnapshots() + guard !Task.isCancelled, + isPressureStillCritical() else { + return + } + let confirmationEvaluation = self.evaluate( + index: confirmationIndex, + settings: AgentHibernationSettings.values(), + now: .now, + trigger: .systemMemoryPressure, + teardownShouldProceed: isPressureStillCritical, + onHibernationCompleted: { [weak self] hibernatedCount in + self?.finishMemoryPressureEvaluation(requestID: requestID) + onHibernationCompleted(hibernatedCount) + } + ) + awaitsTeardownCompletion = confirmationEvaluation.beganTeardowns + } + memoryPressureEvaluation = (requestID, task) + return true + } + + private func finishMemoryPressureEvaluation(requestID: UUID) { + guard memoryPressureEvaluation?.id == requestID else { return } + memoryPressureEvaluation = nil + clearMemoryPressureConfirmations() + } +} diff --git a/Sources/App/AgentHibernationController+PanelLifecycle.swift b/Sources/App/AgentHibernationController+PanelLifecycle.swift new file mode 100644 index 000000000000..e324e7fda7b9 --- /dev/null +++ b/Sources/App/AgentHibernationController+PanelLifecycle.swift @@ -0,0 +1,120 @@ +import Foundation + +extension AgentHibernationController { + func discardTrackingStateForClosedPanel(workspaceId: UUID, panelId: UUID) { + limitCommittedTerminationObservationAfterPanelClose(panelID: panelId) + let key = AgentHibernationPanelKey(workspaceId: workspaceId, panelId: panelId) + activityByPanel.removeValue(forKey: key) + terminalInputByPanel.removeValue(forKey: key) + lifecycleChangeByPanel.removeValue(forKey: key) + teardownValidationEpochByPanel.removeValue(forKey: key) + discardTransientTrackingState(for: key) + } + + func transferTrackingStateForMovedPanel( + panelId: UUID, + from sourceWorkspaceId: UUID, + to destinationWorkspaceId: UUID + ) { + guard sourceWorkspaceId != destinationWorkspaceId else { return } + let sourceKey = AgentHibernationPanelKey( + workspaceId: sourceWorkspaceId, + panelId: panelId + ) + let destinationKey = AgentHibernationPanelKey( + workspaceId: destinationWorkspaceId, + panelId: panelId + ) + let hadTrackingState = + activityByPanel[sourceKey] != nil || + terminalInputByPanel[sourceKey] != nil || + lifecycleChangeByPanel[sourceKey] != nil || + teardownValidationEpochByPanel[sourceKey] != nil || + activityByPanel[destinationKey] != nil || + terminalInputByPanel[destinationKey] != nil || + lifecycleChangeByPanel[destinationKey] != nil || + teardownValidationEpochByPanel[destinationKey] != nil + + transferTimestamp(&activityByPanel, from: sourceKey, to: destinationKey) + transferTimestamp(&terminalInputByPanel, from: sourceKey, to: destinationKey) + transferTimestamp(&lifecycleChangeByPanel, from: sourceKey, to: destinationKey) + let sourceEpoch = teardownValidationEpochByPanel.removeValue(forKey: sourceKey) + let destinationEpoch = teardownValidationEpochByPanel.removeValue(forKey: destinationKey) + if hadTrackingState { + teardownValidationEpochByPanel[destinationKey] = + max(sourceEpoch ?? 0, destinationEpoch ?? 0) &+ 1 + } + // Confirmation, fingerprints, and retries are ownership-specific. A move + // preserves raw input/lifecycle safety but requires a fresh qualification. + discardTransientTrackingState(for: sourceKey) + discardTransientTrackingState(for: destinationKey) + } + + func limitCommittedTerminationObservationAfterPanelClose( + panelID: UUID, + cleanupDelay: Duration = .seconds(30), + sleepUntilDeadline: @escaping @Sendable (Duration) async -> Bool = { duration in + do { + try await ContinuousClock().sleep(for: duration) + return true + } catch { + return false + } + } + ) { + guard let observation = committedTerminationObservationsByPanelID[panelID] else { + return + } + let observationRequestID = observation.requestID + committedTerminationCleanupByPanelID + .removeValue(forKey: panelID)? + .task + .cancel() + let cleanupID = UUID() + let cleanupTask = Task { @MainActor [weak self] in + let didReachDeadline = await sleepUntilDeadline(cleanupDelay) + guard let self else { return } + defer { + if self.committedTerminationCleanupByPanelID[panelID]?.requestID == + cleanupID { + self.committedTerminationCleanupByPanelID.removeValue( + forKey: panelID + ) + } + } + guard didReachDeadline, + !Task.isCancelled, + self.committedTerminationCleanupByPanelID[panelID]?.requestID == + cleanupID, + self.committedTerminationObservationsByPanelID[panelID]?.requestID == + observationRequestID, + let removed = self.committedTerminationObservationsByPanelID.removeValue( + forKey: panelID + ) else { + return + } + removed.task?.cancel() + await removed.processExitCompletion.finish(false) + } + committedTerminationCleanupByPanelID[panelID] = CommittedTerminationCleanup( + requestID: cleanupID, + task: cleanupTask + ) + } + + private func transferTimestamp( + _ timestamps: inout [AgentHibernationPanelKey: TimeInterval], + from sourceKey: AgentHibernationPanelKey, + to destinationKey: AgentHibernationPanelKey + ) { + guard let sourceTimestamp = timestamps.removeValue(forKey: sourceKey) else { return } + timestamps[destinationKey] = max(sourceTimestamp, timestamps[destinationKey] ?? 0) + } + + private func discardTransientTrackingState(for key: AgentHibernationPanelKey) { + unableToProtectByPanel.removeValue(forKey: key) + teardownInFlightByPanel.removeValue(forKey: key) + confirmations.removeValue(forKey: key) + tailFingerprintSamples.removeValue(forKey: key) + } +} diff --git a/Sources/App/AgentHibernationController+ProcessExitObservation.swift b/Sources/App/AgentHibernationController+ProcessExitObservation.swift new file mode 100644 index 000000000000..18faa21fe1dc --- /dev/null +++ b/Sources/App/AgentHibernationController+ProcessExitObservation.swift @@ -0,0 +1,450 @@ +import Foundation + +extension AgentHibernationController { + typealias CommittedTerminationRetry = + @Sendable () async -> ScopedProcessTerminationResult + + func observeCommittedTermination( + panelID: UUID, + requestID: UUID? = nil, + terminations: [ScopedProcessTermination], + processScopeKey: AgentHibernationPanelKey? = nil, + processExitCompletion: AgentHibernationProcessExitCompletion = + AgentHibernationProcessExitCompletion(), + waitForExit: (@Sendable ([ScopedProcessTermination]) async -> Bool)? = nil, + waitForRecoveryExit: (@Sendable ([ScopedProcessTermination]) async -> Bool)? = nil, + retryTermination: CommittedTerminationRetry? = nil, + waitForRecoveryReadiness: @escaping @MainActor @Sendable () async -> Bool = { true }, + recoveryDeadline: Duration = .seconds(30), + sleepUntilRecoveryDeadline: @escaping @Sendable (Duration) async -> Bool = { + duration in + do { + // Genuine recovery deadline; cancellation aborts the wait. + try await ContinuousClock().sleep(for: duration) + return true + } catch { + return false + } + }, + onExit: @escaping @MainActor () async -> Bool, + onFailure: @escaping @MainActor () async -> Void = {}, + onRecovery: @escaping @MainActor () async -> Void, + onRecoveryFailure: @escaping @MainActor () async -> Void = {}, + onRecoveryRetry: @escaping @MainActor () -> Void = {} + ) { + let observationRequestID: UUID + if let requestID { + guard let observation = committedTerminationObservationsByPanelID[panelID], + observation.requestID == requestID, + observation.processExitCompletion === processExitCompletion else { + return + } + observationRequestID = requestID + } else { + observationRequestID = registerCommittedTerminationObservation( + panelID: panelID, + processExitCompletion: processExitCompletion + ) + } + let nextEpochProvider = processExitEpochProvider() + let task = Task { @MainActor [weak self] in + let didExit = await Self.waitForScopedProcessGenerationsToExitAfterEscalation( + terminations, + processScopeKey: processScopeKey, + waitForExit: waitForExit, + nextEpochProvider: nextEpochProvider + ) + guard let self, + self.committedTerminationObservationsByPanelID[panelID]?.requestID == + observationRequestID, + !Task.isCancelled else { + return + } + if didExit { + await processExitCompletion.finish(true) + } + guard self.committedTerminationObservationsByPanelID[panelID]?.requestID == + observationRequestID, + !Task.isCancelled else { + return + } + let didFinish = didExit ? await onExit() : false + guard self.committedTerminationObservationsByPanelID[panelID]?.requestID == + observationRequestID, + !Task.isCancelled else { + return + } + if didFinish { + self.removeCommittedTerminationObservation( + panelID: panelID, + requestID: observationRequestID + ) + return + } + await onFailure() + guard self.committedTerminationObservationsByPanelID[panelID]?.requestID == + observationRequestID, + !Task.isCancelled else { + return + } + self.replaceCommittedTerminationWithRecoveryObservation( + panelID: panelID, + requestID: observationRequestID, + terminations: terminations, + processScopeKey: processScopeKey, + processExitCompletion: processExitCompletion, + waitForRecoveryExit: waitForRecoveryExit, + retryTermination: retryTermination, + waitForRecoveryReadiness: waitForRecoveryReadiness, + recoveryDeadline: recoveryDeadline, + sleepUntilRecoveryDeadline: sleepUntilRecoveryDeadline, + nextEpochProvider: nextEpochProvider, + onRecovery: onRecovery, + onRecoveryFailure: onRecoveryFailure, + onRecoveryRetry: onRecoveryRetry + ) + } + replaceCommittedTerminationTask( + panelID: panelID, + requestID: observationRequestID, + with: task + ) + } + + func observeCommittedTerminationRecovery( + panelID: UUID, + requestID: UUID? = nil, + terminations: [ScopedProcessTermination], + processScopeKey: AgentHibernationPanelKey? = nil, + processExitCompletion: AgentHibernationProcessExitCompletion = + AgentHibernationProcessExitCompletion(), + waitForRecoveryExit: (@Sendable ([ScopedProcessTermination]) async -> Bool)? = nil, + retryTermination: CommittedTerminationRetry? = nil, + waitForRecoveryReadiness: @escaping @MainActor @Sendable () async -> Bool = { true }, + recoveryDeadline: Duration = .seconds(30), + sleepUntilRecoveryDeadline: @escaping @Sendable (Duration) async -> Bool = { + duration in + do { + // Genuine recovery deadline; cancellation aborts the wait. + try await ContinuousClock().sleep(for: duration) + return true + } catch { + return false + } + }, + onRecovery: @escaping @MainActor () async -> Void, + onRecoveryFailure: @escaping @MainActor () async -> Void = {}, + onRecoveryRetry: @escaping @MainActor () -> Void = {} + ) { + let observationRequestID: UUID + if let requestID { + guard let observation = committedTerminationObservationsByPanelID[panelID], + observation.requestID == requestID, + observation.processExitCompletion === processExitCompletion else { + return + } + observationRequestID = requestID + } else { + observationRequestID = registerCommittedTerminationObservation( + panelID: panelID, + processExitCompletion: processExitCompletion + ) + } + replaceCommittedTerminationWithRecoveryObservation( + panelID: panelID, + requestID: observationRequestID, + terminations: terminations, + processScopeKey: processScopeKey, + processExitCompletion: processExitCompletion, + waitForRecoveryExit: waitForRecoveryExit, + retryTermination: retryTermination, + waitForRecoveryReadiness: waitForRecoveryReadiness, + recoveryDeadline: recoveryDeadline, + sleepUntilRecoveryDeadline: sleepUntilRecoveryDeadline, + nextEpochProvider: processExitEpochProvider(), + onRecovery: onRecovery, + onRecoveryFailure: onRecoveryFailure, + onRecoveryRetry: onRecoveryRetry + ) + } + + func retryCommittedTerminationRecovery(panelID: UUID) { + committedTerminationObservationsByPanelID[panelID]?.retryRecovery?() + } + + private func replaceCommittedTerminationWithRecoveryObservation( + panelID: UUID, + requestID: UUID, + terminations: [ScopedProcessTermination], + processScopeKey: AgentHibernationPanelKey?, + processExitCompletion: AgentHibernationProcessExitCompletion, + waitForRecoveryExit: (@Sendable ([ScopedProcessTermination]) async -> Bool)?, + retryTermination: CommittedTerminationRetry?, + waitForRecoveryReadiness: @escaping @MainActor @Sendable () async -> Bool, + recoveryDeadline: Duration, + sleepUntilRecoveryDeadline: @escaping @Sendable (Duration) async -> Bool, + nextEpochProvider: @escaping ProcessExitEpochProvider, + onRecovery: @escaping @MainActor () async -> Void, + onRecoveryFailure: @escaping @MainActor () async -> Void, + onRecoveryRetry: @escaping @MainActor () -> Void + ) { + guard let observation = committedTerminationObservationsByPanelID[panelID], + observation.requestID == requestID else { + return + } + let recoveryTask = Task { @MainActor [weak self] in + let didRecover = await Self.waitForCommittedTerminationRecovery( + terminations: terminations, + processScopeKey: processScopeKey, + waitForRecoveryExit: waitForRecoveryExit, + waitForRecoveryReadiness: waitForRecoveryReadiness, + recoveryDeadline: recoveryDeadline, + sleepUntilRecoveryDeadline: sleepUntilRecoveryDeadline, + nextEpochProvider: nextEpochProvider + ) + guard let self, + self.committedTerminationObservationsByPanelID[panelID]?.requestID == + requestID, + !Task.isCancelled else { + return + } + guard didRecover else { + await onRecoveryFailure() + return + } + await processExitCompletion.finish(true) + guard self.committedTerminationObservationsByPanelID[panelID]?.requestID == + requestID, + !Task.isCancelled else { + return + } + await onRecovery() + guard self.committedTerminationObservationsByPanelID[panelID]?.requestID == + requestID, + !Task.isCancelled else { + return + } + self.removeCommittedTerminationObservation( + panelID: panelID, + requestID: requestID + ) + } + observation.retryRecovery = { @MainActor [weak self] in + guard let self, + self.committedTerminationObservationsByPanelID[panelID]?.requestID == + requestID else { + return + } + onRecoveryRetry() + if let retryTermination { + self.replaceCommittedTerminationWithRetryObservation( + panelID: panelID, + requestID: requestID, + terminations: terminations, + processScopeKey: processScopeKey, + processExitCompletion: processExitCompletion, + waitForRecoveryExit: waitForRecoveryExit, + retryTermination: retryTermination, + waitForRecoveryReadiness: waitForRecoveryReadiness, + recoveryDeadline: recoveryDeadline, + sleepUntilRecoveryDeadline: sleepUntilRecoveryDeadline, + nextEpochProvider: nextEpochProvider, + onRecovery: onRecovery, + onRecoveryFailure: onRecoveryFailure, + onRecoveryRetry: onRecoveryRetry + ) + return + } + self.replaceCommittedTerminationWithRecoveryObservation( + panelID: panelID, + requestID: requestID, + terminations: terminations, + processScopeKey: processScopeKey, + processExitCompletion: processExitCompletion, + waitForRecoveryExit: waitForRecoveryExit, + retryTermination: nil, + waitForRecoveryReadiness: waitForRecoveryReadiness, + recoveryDeadline: recoveryDeadline, + sleepUntilRecoveryDeadline: sleepUntilRecoveryDeadline, + nextEpochProvider: nextEpochProvider, + onRecovery: onRecovery, + onRecoveryFailure: onRecoveryFailure, + onRecoveryRetry: onRecoveryRetry + ) + } + replaceCommittedTerminationTask( + panelID: panelID, + requestID: requestID, + with: recoveryTask + ) + } + + private func replaceCommittedTerminationWithRetryObservation( + panelID: UUID, + requestID: UUID, + terminations: [ScopedProcessTermination], + processScopeKey: AgentHibernationPanelKey?, + processExitCompletion: AgentHibernationProcessExitCompletion, + waitForRecoveryExit: (@Sendable ([ScopedProcessTermination]) async -> Bool)?, + retryTermination: @escaping CommittedTerminationRetry, + waitForRecoveryReadiness: @escaping @MainActor @Sendable () async -> Bool, + recoveryDeadline: Duration, + sleepUntilRecoveryDeadline: @escaping @Sendable (Duration) async -> Bool, + nextEpochProvider: @escaping ProcessExitEpochProvider, + onRecovery: @escaping @MainActor () async -> Void, + onRecoveryFailure: @escaping @MainActor () async -> Void, + onRecoveryRetry: @escaping @MainActor () -> Void + ) { + guard let observation = committedTerminationObservationsByPanelID[panelID], + observation.requestID == requestID else { + return + } + let retryTask = Task { @MainActor [weak self] in + let terminationResult = await retryTermination() + guard let self, + self.committedTerminationObservationsByPanelID[panelID]?.requestID == + requestID, + !Task.isCancelled else { + return + } + + let didExit: Bool + switch terminationResult { + case .rejected: + await onRecoveryFailure() + return + case .exited: + didExit = true + case .committedAwaitingExit: + didExit = await Self.waitForScopedProcessGenerationsToExitAfterEscalation( + terminations, + processScopeKey: processScopeKey, + waitForExit: waitForRecoveryExit, + nextEpochProvider: nextEpochProvider + ) + } + if didExit { + await processExitCompletion.finish(true) + } + guard self.committedTerminationObservationsByPanelID[panelID]?.requestID == + requestID, + !Task.isCancelled else { + return + } + + let didRecover = await Self.waitForCommittedTerminationRecovery( + terminations: didExit ? [] : terminations, + processScopeKey: processScopeKey, + waitForRecoveryExit: didExit + ? Self.completedRecoveryExitWait + : waitForRecoveryExit, + waitForRecoveryReadiness: waitForRecoveryReadiness, + recoveryDeadline: recoveryDeadline, + sleepUntilRecoveryDeadline: sleepUntilRecoveryDeadline, + nextEpochProvider: nextEpochProvider + ) + guard self.committedTerminationObservationsByPanelID[panelID]?.requestID == + requestID, + !Task.isCancelled else { + return + } + guard didRecover else { + await onRecoveryFailure() + return + } + await processExitCompletion.finish(true) + await onRecovery() + guard self.committedTerminationObservationsByPanelID[panelID]?.requestID == + requestID, + !Task.isCancelled else { + return + } + self.removeCommittedTerminationObservation( + panelID: panelID, + requestID: requestID + ) + } + observation.retryRecovery = { @MainActor [weak self] in + guard let self, + self.committedTerminationObservationsByPanelID[panelID]?.requestID == + requestID else { + return + } + onRecoveryRetry() + self.replaceCommittedTerminationWithRetryObservation( + panelID: panelID, + requestID: requestID, + terminations: terminations, + processScopeKey: processScopeKey, + processExitCompletion: processExitCompletion, + waitForRecoveryExit: waitForRecoveryExit, + retryTermination: retryTermination, + waitForRecoveryReadiness: waitForRecoveryReadiness, + recoveryDeadline: recoveryDeadline, + sleepUntilRecoveryDeadline: sleepUntilRecoveryDeadline, + nextEpochProvider: nextEpochProvider, + onRecovery: onRecovery, + onRecoveryFailure: onRecoveryFailure, + onRecoveryRetry: onRecoveryRetry + ) + } + replaceCommittedTerminationTask( + panelID: panelID, + requestID: requestID, + with: retryTask + ) + } + + private nonisolated static func waitForCommittedTerminationRecovery( + terminations: [ScopedProcessTermination], + processScopeKey: AgentHibernationPanelKey?, + waitForRecoveryExit: (@Sendable ([ScopedProcessTermination]) async -> Bool)?, + waitForRecoveryReadiness: @escaping @MainActor @Sendable () async -> Bool, + recoveryDeadline: Duration, + sleepUntilRecoveryDeadline: @escaping @Sendable (Duration) async -> Bool, + nextEpochProvider: @escaping ProcessExitEpochProvider + ) async -> Bool { + await withTaskGroup(of: Bool.self) { group in + group.addTask(priority: .utility) { + guard await waitForRecoveryReadiness() else { return false } + if let waitForRecoveryExit { + return await waitForRecoveryExit(terminations) + } + return await waitForScopedProcessGenerationsToExitWithoutTimeout( + terminations, + processScopeKey: processScopeKey, + nextEpochProvider: nextEpochProvider + ) + } + group.addTask(priority: .utility) { + guard await sleepUntilRecoveryDeadline(recoveryDeadline) else { + return false + } + return false + } + let recovered = await group.next() ?? false + group.cancelAll() + return recovered + } + } + + private func processExitEpochProvider() -> ProcessExitEpochProvider { + { + [processSnapshotCoordinator] + processGroupLeaders, + processScopeKey, + ttyDevice, + exitedIdentities in + await processSnapshotCoordinator.refreshedExitEpoch( + processGroupLeaders: processGroupLeaders, + processScopeKey: processScopeKey, + ttyDevice: ttyDevice, + excluding: exitedIdentities + ) + } + } + + private nonisolated static let completedRecoveryExitWait: + @Sendable ([ScopedProcessTermination]) async -> Bool = { _ in true } + +} diff --git a/Sources/App/AgentHibernationController+ProcessExitWaiting.swift b/Sources/App/AgentHibernationController+ProcessExitWaiting.swift new file mode 100644 index 000000000000..be45e154f29e --- /dev/null +++ b/Sources/App/AgentHibernationController+ProcessExitWaiting.swift @@ -0,0 +1,301 @@ +import Darwin +import Foundation + +extension AgentHibernationController { + typealias ProcessExitEpochProvider = @Sendable ( + [pid_t: AgentPIDProcessIdentity], + AgentHibernationPanelKey?, + Int64?, + Set + ) async -> AgentHibernationProcessExitEpoch? + + #if compiler(>=6.2) + @concurrent + #endif + nonisolated static func waitForScopedProcessGenerationsToExitAfterEscalation( + _ terminations: [ScopedProcessTermination], + processScopeKey: AgentHibernationPanelKey? = nil, + gracePeriod: Duration = .seconds(5), + postKillExitPeriod: Duration = .seconds(5), + waitForExit: (@Sendable ([ScopedProcessTermination]) async -> Bool)? = nil, + sleepUntilDeadline: @escaping @Sendable (Duration) async -> Bool = { duration in + do { + // Genuine termination deadline; cancellation aborts the wait. + try await ContinuousClock().sleep(for: duration) + return true + } catch { + return false + } + }, + processIdentityProvider: @escaping @Sendable (pid_t) -> AgentPIDProcessIdentity? = { + AgentPIDProcessIdentity(pid: $0) + }, + processGroupProvider: @escaping @Sendable (pid_t) -> pid_t = { + getpgid($0) + }, + processArgumentsProvider: @escaping @Sendable (Int) -> CmuxTopProcessArguments? = { + CmuxTopProcessSnapshot.processArgumentsAndEnvironment(for: $0) + }, + processTTYDeviceProvider: @escaping @Sendable (pid_t) -> Int64? = { + agentLiveProcessIdentity(pid: $0)?.ttyDevice + }, + signalErrorProvider: @escaping @Sendable (pid_t, Int32) -> Int32? = { target, signal in + kill(target, signal) == 0 ? nil : errno + }, + nextEpochProvider: @escaping ProcessExitEpochProvider = { _, _, _, _ in nil } + ) async -> Bool { + let ttyDevice = commonTTYDevice(in: terminations) + if processScopeKey != nil, ttyDevice == nil { + return false + } + return await withTaskGroup( + of: (didExit: Bool?, graceElapsed: Bool?).self + ) { group in + group.addTask(priority: .utility) { + if let waitForExit { + return (await waitForExit(terminations), nil) + } + return ( + await waitForScopedProcessGenerationsToExitWithoutTimeout( + terminations, + processScopeKey: processScopeKey, + nextEpochProvider: nextEpochProvider + ), + nil + ) + } + group.addTask(priority: .utility) { + (nil, await sleepUntilDeadline(gracePeriod)) + } + guard let firstEvent = await group.next() else { return false } + if let didExit = firstEvent.didExit { + group.cancelAll() + return didExit + } + guard firstEvent.graceElapsed == true else { + group.cancelAll() + return false + } + + let processGroupLeaders = processGroupLeaders(in: terminations) + guard processGroupLeaders.count == + Set(terminations.map(\.processGroupID)).filter({ $0 > 1 }).count, + let refreshedEpoch = await nextEpochProvider( + processGroupLeaders, + processScopeKey, + ttyDevice, + [] + ) else { + group.cancelAll() + return false + } + + let refreshedTerminations = refreshedEpoch.terminations + if refreshedTerminations.isEmpty { + let originalGenerationsExited = terminations.allSatisfy { termination in + processIdentityProvider(pid_t(termination.processID)) != + termination.processIdentity + } + group.cancelAll() + return originalGenerationsExited + } + guard refreshedTerminations.allSatisfy({ termination in + let processID = pid_t(termination.processID) + return processIdentityProvider(processID) == + termination.processIdentity && + processGroupProvider(processID) == termination.processGroupID + }) else { + group.cancelAll() + return false + } + if let processScopeKey { + guard refreshedTerminations.allSatisfy({ termination in + guard let ttyDevice = termination.ttyDevice else { + return false + } + return processTTYDeviceProvider( + pid_t(termination.processID) + ) == ttyDevice + }), + refreshedTerminations.allSatisfy({ termination in + processArgumentsProvider(termination.processID)? + .matchesCMUXScope( + workspaceId: processScopeKey.workspaceId, + surfaceId: processScopeKey.panelId + ) == true + }), + // Close same-generation exec and process-group races + // introduced by the uncached scope/TTY probes. + refreshedTerminations.allSatisfy({ termination in + let processID = pid_t(termination.processID) + return processIdentityProvider(processID) == + termination.processIdentity && + processGroupProvider(processID) == + termination.processGroupID + }) else { + group.cancelAll() + return false + } + } + for (processGroupID, leaderIdentity) in refreshedEpoch.processGroupLeaders { + let liveLeader = processIdentityProvider(processGroupID) + guard liveLeader == nil || liveLeader == leaderIdentity else { + continue + } + if let error = signalErrorProvider(-processGroupID, SIGKILL), + error != ESRCH { + group.cancelAll() + return false + } + } + group.addTask(priority: .utility) { + (nil, await sleepUntilDeadline(postKillExitPeriod)) + } + guard let postKillEvent = await group.next() else { return false } + if let didExit = postKillEvent.didExit { + group.cancelAll() + return didExit + } + group.cancelAll() + return false + } + } + + #if compiler(>=6.2) + @concurrent + #endif + nonisolated static func waitForScopedProcessGenerationsToExitWithoutTimeout( + _ terminations: [ScopedProcessTermination], + processScopeKey: AgentHibernationPanelKey? = nil, + processIdentityProvider: @escaping @Sendable (pid_t) -> AgentPIDProcessIdentity? = { + AgentPIDProcessIdentity(pid: $0) + }, + waitForExactEpoch: (@Sendable ([ScopedProcessTermination]) async -> Bool)? = nil, + nextEpochProvider: @escaping ProcessExitEpochProvider = { _, _, _, _ in nil } + ) async -> Bool { + let ttyDevice = commonTTYDevice(in: terminations) + if processScopeKey != nil, ttyDevice == nil { + return false + } + var processGroupLeaders = processGroupLeaders(in: terminations) + guard processGroupLeaders.count == + Set(terminations.map(\.processGroupID)).filter({ $0 > 1 }).count else { + return false + } + var epoch = terminations + var exitedIdentities: Set = [] + var refreshCount = 0 + while true { + let didExit: Bool + if let waitForExactEpoch { + didExit = await waitForExactEpoch(epoch) + } else { + didExit = await waitForExactProcessGenerationsToExitWithoutTimeout( + epoch, + processIdentityProvider: processIdentityProvider + ) + } + guard didExit, !Task.isCancelled else { return false } + exitedIdentities.formUnion(epoch.map(\.processIdentity)) + guard refreshCount < maximumProcessExitEpochRefreshCount else { + return false + } + refreshCount += 1 + guard let nextEpoch = await nextEpochProvider( + processGroupLeaders, + processScopeKey, + ttyDevice, + exitedIdentities + ) else { + return false + } + guard !nextEpoch.terminations.isEmpty else { return true } + processGroupLeaders = nextEpoch.processGroupLeaders + epoch = nextEpoch.terminations + } + } + + nonisolated static func processGroupLeaders( + in terminations: [ScopedProcessTermination] + ) -> [pid_t: AgentPIDProcessIdentity] { + Dictionary( + uniqueKeysWithValues: terminations.compactMap { termination in + let processGroupID = termination.processGroupID + guard processGroupID > 1, + termination.processID == Int(processGroupID) else { + return nil + } + return (processGroupID, termination.processIdentity) + } + ) + } + + nonisolated static func commonTTYDevice( + in terminations: [ScopedProcessTermination] + ) -> Int64? { + let ttyDevices = Set(terminations.compactMap(\.ttyDevice)) + guard ttyDevices.count == 1 else { return nil } + return ttyDevices.first + } + + #if compiler(>=6.2) + @concurrent + #endif + nonisolated static func waitForExactProcessGenerationsToExitWithoutTimeout( + _ terminations: [ScopedProcessTermination], + processIdentityProvider: @escaping @Sendable (pid_t) -> AgentPIDProcessIdentity? = { + AgentPIDProcessIdentity(pid: $0) + } + ) async -> Bool { + await withTaskGroup(of: Bool.self) { group in + for termination in terminations { + group.addTask(priority: .utility) { + await waitForExactProcessGenerationToExit( + termination, + processIdentityProvider: processIdentityProvider + ) + } + } + for await didExit in group where !didExit { + group.cancelAll() + return false + } + return true + } + } + + private nonisolated static func waitForExactProcessGenerationToExit( + _ termination: ScopedProcessTermination, + processIdentityProvider: @escaping @Sendable (pid_t) -> AgentPIDProcessIdentity? + ) async -> Bool { + let processID = pid_t(termination.processID) + guard processIdentityProvider(processID) == termination.processIdentity else { + return true + } + + let exitEvents = AsyncStream(bufferingPolicy: .bufferingNewest(1)) { continuation in + let source = DispatchSource.makeProcessSource( + identifier: processID, + eventMask: .exit, + queue: .global(qos: .utility) + ) + source.setEventHandler { + continuation.yield() + continuation.finish() + } + continuation.onTermination = { @Sendable _ in + source.cancel() + } + source.resume() + } + // Close the registration race without ever accepting a reused PID. + guard processIdentityProvider(processID) == termination.processIdentity else { + return true + } + + for await _ in exitEvents { + return true + } + return processIdentityProvider(processID) != termination.processIdentity + } +} diff --git a/Sources/App/AgentHibernationController+ProcessSignaling.swift b/Sources/App/AgentHibernationController+ProcessSignaling.swift new file mode 100644 index 000000000000..5adec24025cd --- /dev/null +++ b/Sources/App/AgentHibernationController+ProcessSignaling.swift @@ -0,0 +1,190 @@ +import Darwin +import Foundation + +extension AgentHibernationController { + @discardableResult + #if compiler(>=6.2) + @concurrent + #else + @Sendable + #endif + nonisolated static func terminateScopedProcessesForHibernation( + _ terminations: [ScopedProcessTermination], + processScopeKey: AgentHibernationPanelKey, + shouldCommit: @escaping @MainActor @Sendable () -> Bool = { true }, + onTeardownCommit: @escaping @MainActor @Sendable () -> Void = {}, + currentProcessID: pid_t = getpid(), + currentProcessGroupID: pid_t = getpgrp(), + processIdentityProvider: @escaping @Sendable (pid_t) -> AgentPIDProcessIdentity? = { + AgentPIDProcessIdentity(pid: $0) + }, + processGroupProvider: @escaping @Sendable (pid_t) -> pid_t = { getpgid($0) }, + processArgumentsProvider: @escaping @Sendable (Int) -> CmuxTopProcessArguments? = { + CmuxTopProcessSnapshot.processArgumentsAndEnvironment(for: $0) + }, + processTTYDeviceProvider: @escaping @Sendable (pid_t) -> Int64? = { + agentLiveProcessIdentity(pid: $0)?.ttyDevice + }, + signalErrorProvider: @escaping @Sendable (pid_t, Int32) -> Int32? = { + target, signal in + kill(target, signal) == 0 ? nil : errno + } + ) async -> ScopedProcessTerminationResult { + let processGroupLeaders = processGroupLeaders(in: terminations) + guard processGroupLeaders.count == + Set(terminations.map(\.processGroupID)).filter({ $0 > 1 }).count else { + return .rejected + } + return await terminateScopedProcessEpochForHibernation( + .init( + terminations: terminations, + processGroupLeaders: processGroupLeaders, + signalableProcessIdentities: Set(terminations.map(\.processIdentity)) + ), + processScopeKey: processScopeKey, + shouldCommit: shouldCommit, + onTeardownCommit: onTeardownCommit, + currentProcessID: currentProcessID, + currentProcessGroupID: currentProcessGroupID, + processIdentityProvider: processIdentityProvider, + processGroupProvider: processGroupProvider, + processArgumentsProvider: processArgumentsProvider, + processTTYDeviceProvider: processTTYDeviceProvider, + signalErrorProvider: signalErrorProvider + ) + } + + @discardableResult + #if compiler(>=6.2) + @concurrent + #else + @Sendable + #endif + nonisolated static func terminateScopedProcessEpochForHibernation( + _ epoch: AgentHibernationProcessExitEpoch, + processScopeKey: AgentHibernationPanelKey, + shouldCommit: @escaping @MainActor @Sendable () -> Bool = { true }, + onTeardownCommit: @escaping @MainActor @Sendable () -> Void = {}, + currentProcessID: pid_t = getpid(), + currentProcessGroupID: pid_t = getpgrp(), + processIdentityProvider: @escaping @Sendable (pid_t) -> AgentPIDProcessIdentity? = { + AgentPIDProcessIdentity(pid: $0) + }, + processGroupProvider: @escaping @Sendable (pid_t) -> pid_t = { getpgid($0) }, + processArgumentsProvider: @escaping @Sendable (Int) -> CmuxTopProcessArguments? = { + CmuxTopProcessSnapshot.processArgumentsAndEnvironment(for: $0) + }, + processTTYDeviceProvider: @escaping @Sendable (pid_t) -> Int64? = { + agentLiveProcessIdentity(pid: $0)?.ttyDevice + }, + signalErrorProvider: @escaping @Sendable (pid_t, Int32) -> Int32? = { + target, signal in + kill(target, signal) == 0 ? nil : errno + } + ) async -> ScopedProcessTerminationResult { + let terminations = epoch.terminations + guard !terminations.isEmpty else { + return await MainActor.run { + guard shouldCommit() else { return .rejected } + onTeardownCommit() + return .exited + } + } + guard terminations.count <= maximumScopedProcessTerminationCount, + commonTTYDevice(in: terminations) != nil else { + return .rejected + } + + let signalableTerminations = terminations.filter { + epoch.signalableProcessIdentities.contains($0.processIdentity) + } + let signalableProcessGroupIDs = Set( + signalableTerminations.map(\.processGroupID) + ) + guard !signalableProcessGroupIDs.isEmpty, + signalableProcessGroupIDs.allSatisfy({ + $0 > 1 && + $0 != currentProcessGroupID && + epoch.processGroupLeaders[$0] != nil + }), + terminations.allSatisfy({ termination in + let processID = pid_t(termination.processID) + return processID != currentProcessID && + processIdentityProvider(processID) == + termination.processIdentity && + processGroupProvider(processID) == termination.processGroupID + }) else { + return .rejected + } + + guard terminations.allSatisfy({ termination in + guard let ttyDevice = termination.ttyDevice else { + return false + } + return processTTYDeviceProvider( + pid_t(termination.processID) + ) == ttyDevice + }), + terminations.allSatisfy({ termination in + processArgumentsProvider(termination.processID)?.matchesCMUXScope( + workspaceId: processScopeKey.workspaceId, + surfaceId: processScopeKey.panelId + ) == true + }), + terminations.allSatisfy({ termination in + let processID = pid_t(termination.processID) + return processIdentityProvider(processID) == + termination.processIdentity && + processGroupProvider(processID) == termination.processGroupID + }) else { + return .rejected + } + + // The probes above can block, so they deliberately run off the main actor. + // The mutable panel gate, minimal exact kernel trust recheck, signal batch, + // and UI commit below are one synchronous main-actor boundary. The full + // process snapshot and argv/environment probes remain off-main, while no + // input/focus event or actor hop can interleave after the final decision. + return await MainActor.run { + guard terminations.allSatisfy({ termination in + let processID = pid_t(termination.processID) + guard let ttyDevice = termination.ttyDevice else { + return false + } + return processIdentityProvider(processID) == + termination.processIdentity && + processGroupProvider(processID) == termination.processGroupID && + processTTYDeviceProvider(processID) == ttyDevice + }), + signalableProcessGroupIDs.allSatisfy({ processGroupID in + guard let expectedLeader = + epoch.processGroupLeaders[processGroupID] else { + return false + } + // A process group can outlive its leader. A reused leader + // PID, however, invalidates the original group authority. + let liveLeader = processIdentityProvider(processGroupID) + return liveLeader == nil || liveLeader == expectedLeader + }), + shouldCommit() else { + return .rejected + } + + var teardownIsCommitted = false + for processGroupID in signalableProcessGroupIDs.sorted() { + if let error = signalErrorProvider(-processGroupID, SIGTERM) { + if error != ESRCH, !teardownIsCommitted { + return .rejected + } + } else { + teardownIsCommitted = true + } + } + // Every target may have exited between validation and signaling. + // That is still an irreversible commit: no authorized generation + // remains, and exact-exit observation resolves the final state. + onTeardownCommit() + return .committedAwaitingExit + } + } +} diff --git a/Sources/App/AgentHibernationController+ProcessTermination.swift b/Sources/App/AgentHibernationController+ProcessTermination.swift new file mode 100644 index 000000000000..0db0d2eb295a --- /dev/null +++ b/Sources/App/AgentHibernationController+ProcessTermination.swift @@ -0,0 +1,435 @@ +import Darwin +import Foundation + +extension AgentHibernationRecord { + var processTerminationScope: AgentHibernationController.ProcessTerminationScope { + AgentHibernationController.ProcessTerminationScope( + key: key, + processIDs: processIDs, + processIdentities: processIdentities + ) + } +} + +extension AgentHibernationController { + /// Bounds synchronous exact-kernel validation in the final signal commit boundary. + nonisolated static let maximumScopedProcessTerminationCount = 32 + /// Bounds full process-table refreshes while discovering late process generations. + nonisolated static let maximumProcessExitEpochRefreshCount = 8 + + struct ProcessTerminationScope: Sendable { + let key: AgentHibernationPanelKey + let processIDs: Set + let processIdentities: [Int: AgentPIDProcessIdentity] + } + + struct ScopedProcessTermination: Equatable, Sendable { + let processID: Int + let processIdentity: AgentPIDProcessIdentity + let processGroupID: pid_t + let ttyDevice: Int64? + + init( + processID: Int, + processIdentity: AgentPIDProcessIdentity, + processGroupID: pid_t, + ttyDevice: Int64? = nil + ) { + self.processID = processID + self.processIdentity = processIdentity + self.processGroupID = processGroupID + self.ttyDevice = ttyDevice + } + } + + nonisolated static func processIdentities( + for processIDs: Set + ) -> [Int: AgentPIDProcessIdentity] { + Dictionary(uniqueKeysWithValues: processIDs.compactMap { processID in + guard processID > 0, + processID <= Int(Int32.max), + let identity = AgentPIDProcessIdentity(pid: pid_t(processID)) else { + return nil + } + return (processID, identity) + }) + } + + #if compiler(>=6.2) + @concurrent + #else + @Sendable + #endif + nonisolated static func scopedProcessTerminations( + for scopes: [ProcessTerminationScope] + ) async -> [AgentHibernationPanelKey: [ScopedProcessTermination]] { + await withTaskGroup( + of: (AgentHibernationPanelKey, [ScopedProcessTermination]?).self, + returning: [AgentHibernationPanelKey: [ScopedProcessTermination]].self + ) { group in + var terminationsByPanel: [AgentHibernationPanelKey: [ScopedProcessTermination]] = Dictionary( + uniqueKeysWithValues: scopes.compactMap { scope in + scope.processIDs.isEmpty ? (scope.key, []) : nil + } + ) + for scope in scopes where !scope.processIDs.isEmpty { + group.addTask(priority: .utility) { + ( + scope.key, + validatedScopedProcessTerminations( + for: scope, + processIdentityProvider: { + AgentPIDProcessIdentity(pid: pid_t($0)) + }, + processGroupProvider: { getpgid(pid_t($0)) }, + processTTYDeviceProvider: { + agentLiveProcessIdentity(pid: pid_t($0))?.ttyDevice + } + ) + ) + } + } + for await (key, terminations) in group { + if let terminations { + terminationsByPanel[key] = terminations + } + } + return terminationsByPanel + } + } + + nonisolated static func validatedScopedProcessTerminations( + for scope: ProcessTerminationScope, + processIdentityProvider: (Int) -> AgentPIDProcessIdentity?, + processGroupProvider: (Int) -> pid_t, + processTTYDeviceProvider: (Int) -> Int64? = { + agentLiveProcessIdentity(pid: pid_t($0))?.ttyDevice + } + ) -> [ScopedProcessTermination]? { + guard scope.processIDs.count <= maximumScopedProcessTerminationCount, + Set(scope.processIdentities.keys) == scope.processIDs else { + return nil + } + var terminations: [ScopedProcessTermination] = [] + for processID in scope.processIDs.sorted(by: >) { + guard processID > 0, + processID <= Int(Int32.max), + let expectedIdentity = scope.processIdentities[processID], + processIdentityProvider(processID) == expectedIdentity else { + return nil + } + terminations.append( + ScopedProcessTermination( + processID: processID, + processIdentity: expectedIdentity, + processGroupID: processGroupProvider(processID), + ttyDevice: processTTYDeviceProvider(processID) + ) + ) + } + return terminations + } + + func commitConfirmedTeardown( + _ request: ConfirmedTeardownRequest, + snapshotOutcome: AgentHibernationTranscriptGuard.TeardownSnapshotOutcome, + scopedProcessTerminations: [ScopedProcessTermination], + shouldProceed: (@MainActor () -> Bool)?, + restoreOwnedSnapshotPaths: inout Set + ) async -> Bool { + let record = request.record + let snapshot: AgentHibernationTranscriptGuard.TeardownTranscriptSnapshot? + switch snapshotOutcome { + case .snapshot(let value): + snapshot = value + case .nothingToProtect: + snapshot = nil + case .unableToProtect: + // Forfeit hibernation rather than risk issue #6565 transcript loss. + unableToProtectByPanel[record.key] = UnableToProtectMarker( + fingerprint: request.confirmationFingerprint, + lastActivityAt: request.effectiveLastActivityAt, + retryAfter: Date.now.timeIntervalSince1970 + Self.unableToProtectRetrySeconds + ) + return false + } + let processExitCompletion = AgentHibernationProcessExitCompletion() + let committedTerminationRequestID = UUID() + + if let snapshot { + // Hand off any older monitor only after every other precondition passed. + await cancelPostTeardownRestoreTaskForReplacement( + transcriptPath: snapshot.transcriptPath + ) + guard shouldProceed?() ?? true else { + preserveSnapshotAfterAbortedTeardown( + snapshot, + record: record, + restoreOwnedSnapshotPaths: &restoreOwnedSnapshotPaths + ) + return false + } + guard AgentHibernationTranscriptGuard.liveFileVersionStillMatches(snapshot) else { + unableToProtectByPanel[record.key] = UnableToProtectMarker( + fingerprint: request.confirmationFingerprint, + lastActivityAt: request.effectiveLastActivityAt, + retryAfter: Date.now.timeIntervalSince1970 + Self.unableToProtectRetrySeconds + ) + preserveSnapshotAfterAbortedTeardown( + snapshot, + record: record, + restoreOwnedSnapshotPaths: &restoreOwnedSnapshotPaths + ) + return false + } + // Protection must already be active when SIGTERM or PTY closure can + // trigger an interrupted-exit transcript rewrite. + guard armPostTeardownRestoreMonitor( + snapshot: snapshot, + processIDs: record.processIDs, + awaitProcessExit: { + await processExitCompletion.wait() + } + ) else { + preserveSnapshotAfterAbortedTeardown( + snapshot, + record: record, + restoreOwnedSnapshotPaths: &restoreOwnedSnapshotPaths + ) + return false + } + restoreOwnedSnapshotPaths.insert(snapshot.snapshotPath) + } + + // The monitor handoff above awaits an older task and makes the main actor + // reentrant. Refresh the process generation snapshot, then re-check every + // mutable safety condition at the last synchronous boundary before SIGTERM. + let preSignalIndex = await RestorableAgentSessionIndex + .loadIncludingProcessDetectedSnapshots() + guard teardownIsStillSafe( + request, + index: preSignalIndex, + shouldProceed: shouldProceed + ), + snapshot.map({ + AgentHibernationTranscriptGuard.liveFileVersionStillMatches($0) + }) ?? true else { + if let snapshot { + await releaseArmedRestoreMonitorAfterAbortedTeardown( + snapshot, + sessionId: record.agent.sessionId, + restoreOwnedSnapshotPaths: &restoreOwnedSnapshotPaths + ) + } + return false + } + + let lastActivityAt = Date( + timeIntervalSince1970: request.effectiveLastActivityAt + ) + let panelID = record.key.panelId + let workspaceID = record.key.workspaceId + let agent = record.agent + let finalizeTeardown: @MainActor () -> Bool = { + [weak workspace = record.workspace, weak terminalPanel = record.terminalPanel] in + guard let terminalPanel else { return true } + if let workspace, + let currentPanel = workspace.panels[panelID] as? TerminalPanel, + currentPanel === terminalPanel, + terminalPanel.workspaceId == workspaceID, + terminalPanel.isAgentHibernationCommitPending { + return workspace.enterAgentHibernation( + panelId: panelID, + agent: agent, + lastActivityAt: lastActivityAt + ) + } + // A live move carries the phase on TerminalPanel. Its new owner + // gets the same resumable state without consulting the source. + terminalPanel.completeAgentHibernationTermination() + return true + } + let finishTeardown: @MainActor () async -> Bool = { + [weak terminalPanel = record.terminalPanel] in + guard let terminalPanel else { return true } + guard await terminalPanel.surface.waitForAgentHibernationRuntimeTeardown( + timeout: .seconds(5) + ) else { + return false + } + return finalizeTeardown() + } + let recoverTeardown: @MainActor () async -> Void = { + [weak terminalPanel = record.terminalPanel] in + guard terminalPanel != nil else { return } + _ = finalizeTeardown() + } + let waitForRecoveryReadiness: @MainActor @Sendable () async -> Bool = { + [weak terminalPanel = record.terminalPanel] in + guard let terminalPanel else { return true } + return await terminalPanel.surface.waitForAgentHibernationRuntimeTeardown( + timeout: .seconds(30) + ) + } + let beginTerminationRecovery: @MainActor () async -> Void = { + [weak terminalPanel = record.terminalPanel] in + terminalPanel?.beginAgentHibernationTerminationRecovery() + } + let handleRecoveryFailure: @MainActor () async -> Void = { + [weak self, weak terminalPanel = record.terminalPanel] in + guard let self, let terminalPanel else { return } + terminalPanel.failAgentHibernationTermination() + terminalPanel.onRequestAgentHibernationTerminationRetry = { + [weak self, weak terminalPanel] in + guard let self, let terminalPanel else { return } + self.retryCommittedTerminationRecovery(panelID: terminalPanel.id) + } + } + let beginRecoveryRetry: @MainActor () -> Void = { + [weak terminalPanel = record.terminalPanel] in + terminalPanel?.beginAgentHibernationTerminationRecovery() + } + let finalCommitIsSafe: @MainActor @Sendable () -> Bool = { + [weak self, weak terminalPanel = record.terminalPanel] in + guard let self, let terminalPanel, + self.teardownIsStillSafe( + request, + index: preSignalIndex, + shouldProceed: shouldProceed + ), + (snapshot.map { + AgentHibernationTranscriptGuard.liveFileVersionStillMatches($0) + } ?? true) else { + return false + } + return terminalPanel.surface.reserveAgentHibernationRuntimeTeardown() + } + let processGroupLeaders = Self.processGroupLeaders( + in: scopedProcessTerminations + ) + let processScopeKey = record.key + let processSnapshotCoordinator = processSnapshotCoordinator + let retryTerminationIsSafe: @MainActor @Sendable () -> Bool = { + [weak self, weak terminalPanel = record.terminalPanel] in + guard let self, let terminalPanel, + terminalPanel.isAgentHibernationCommitPending, + self.committedTerminationObservationsByPanelID[panelID]? + .requestID == committedTerminationRequestID else { + return false + } + return true + } + let retryTermination: CommittedTerminationRetry = { + guard !scopedProcessTerminations.isEmpty else { + return .exited + } + guard let epoch = await processSnapshotCoordinator.refreshedExitEpoch( + processGroupLeaders: processGroupLeaders, + processScopeKey: processScopeKey, + ttyDevice: Self.commonTTYDevice(in: scopedProcessTerminations), + excluding: [] + ) else { + return .rejected + } + return await Self.terminateScopedProcessEpochForHibernation( + epoch, + processScopeKey: processScopeKey, + shouldCommit: retryTerminationIsSafe + ) + } + let terminationResult = await Self.terminateScopedProcessesForHibernation( + scopedProcessTerminations, + processScopeKey: record.key, + shouldCommit: finalCommitIsSafe, + onTeardownCommit: { + [self, weak terminalPanel = record.terminalPanel] in + registerCommittedTerminationObservation( + panelID: panelID, + requestID: committedTerminationRequestID, + processExitCompletion: processExitCompletion + ) + terminalPanel?.beginAgentHibernationTermination( + agent: agent, + lastActivityAt: lastActivityAt + ) + } + ) + switch terminationResult { + case .rejected: + record.terminalPanel.surface + .cancelAgentHibernationRuntimeTeardownReservation() + if let snapshot { + await releaseArmedRestoreMonitorAfterAbortedTeardown( + snapshot, + sessionId: record.agent.sessionId, + restoreOwnedSnapshotPaths: &restoreOwnedSnapshotPaths + ) + } + return false + case .exited: + guard committedTerminationObservationsByPanelID[panelID]?.requestID == + committedTerminationRequestID else { + return false + } + await processExitCompletion.finish(true) + guard committedTerminationObservationsByPanelID[panelID]?.requestID == + committedTerminationRequestID else { + return false + } + guard await finishTeardown() else { + guard committedTerminationObservationsByPanelID[panelID]?.requestID == + committedTerminationRequestID else { + return false + } + await beginTerminationRecovery() + guard committedTerminationObservationsByPanelID[panelID]?.requestID == + committedTerminationRequestID else { + return false + } + observeCommittedTerminationRecovery( + panelID: panelID, + requestID: committedTerminationRequestID, + terminations: [], + processScopeKey: nil, + processExitCompletion: processExitCompletion, + retryTermination: retryTermination, + waitForRecoveryReadiness: waitForRecoveryReadiness, + recoveryDeadline: .seconds(60), + onRecovery: recoverTeardown, + onRecoveryFailure: handleRecoveryFailure, + onRecoveryRetry: beginRecoveryRetry + ) + return false + } + guard committedTerminationObservationsByPanelID[panelID]?.requestID == + committedTerminationRequestID else { + return false + } + removeCommittedTerminationObservation( + panelID: panelID, + requestID: committedTerminationRequestID + ) + case .committedAwaitingExit: + observeCommittedTermination( + panelID: panelID, + requestID: committedTerminationRequestID, + terminations: scopedProcessTerminations, + processScopeKey: record.key, + processExitCompletion: processExitCompletion, + retryTermination: retryTermination, + waitForRecoveryReadiness: waitForRecoveryReadiness, + recoveryDeadline: .seconds(60), + onExit: finishTeardown, + onFailure: beginTerminationRecovery, + onRecovery: recoverTeardown, + onRecoveryFailure: handleRecoveryFailure, + onRecoveryRetry: beginRecoveryRetry + ) + return false + } + + return record.terminalPanel.isAgentHibernated && + !record.terminalPanel.isAgentHibernationTerminating + } + +} diff --git a/Sources/App/AgentHibernationController+Records.swift b/Sources/App/AgentHibernationController+Records.swift index aea21c9b7e3d..ce01215ab47e 100644 --- a/Sources/App/AgentHibernationController+Records.swift +++ b/Sources/App/AgentHibernationController+Records.swift @@ -1,5 +1,14 @@ import Foundation +extension AgentHibernationRecord { + var hasPressureSafeProcessEvidence: Bool { + hasLiveProcess && + !containsUnrelatedProcess && + !processIDs.isEmpty && + processIdentities.count == processIDs.count + } +} + extension AppDelegate { @MainActor func agentHibernationPanelIsProtected(workspace: Workspace, panelId: UUID) -> Bool { @@ -49,6 +58,11 @@ extension AppDelegate { panelId: panelId, fallback: index.lifecycle(workspaceId: workspace.id, panelId: panelId) ) + let processEntry = index.entry( + workspaceId: workspace.id, + panelId: panelId + ) + let panelProcessIDs = processEntry?.processIDs ?? [] records.append( AgentHibernationRecord( key: key, @@ -59,8 +73,11 @@ extension AppDelegate { hasUnconfirmedTerminalInput: terminalInputAt > lifecycleChangeAt, lastActivityAt: max(indexActivity, localActivity, createdAt), isProtected: workspaceIsVisible && visiblePanelIds.contains(panelId), - hasLiveProcess: index.hasLiveProcess(workspaceId: workspace.id, panelId: panelId), - processIDs: index.processIDs(workspaceId: workspace.id, panelId: panelId) + hasLiveProcess: !panelProcessIDs.isEmpty, + containsUnrelatedProcess: processEntry?.containsUnrelatedProcess ?? false, + panelProcessIDs: processEntry?.hibernationPanelProcessIDs ?? [], + processIDs: processEntry?.terminationProcessIDs ?? [], + processIdentities: processEntry?.terminationProcessIdentities ?? [:] ) ) } diff --git a/Sources/App/AgentHibernationController+ScopedProcessTerminationResult.swift b/Sources/App/AgentHibernationController+ScopedProcessTerminationResult.swift new file mode 100644 index 000000000000..1eba038e53da --- /dev/null +++ b/Sources/App/AgentHibernationController+ScopedProcessTerminationResult.swift @@ -0,0 +1,9 @@ +import Foundation + +extension AgentHibernationController { + enum ScopedProcessTerminationResult: Equatable, Sendable { + case rejected + case exited + case committedAwaitingExit + } +} diff --git a/Sources/App/AgentHibernationController+Teardown.swift b/Sources/App/AgentHibernationController+Teardown.swift index 3022da25907f..1109862a591f 100644 --- a/Sources/App/AgentHibernationController+Teardown.swift +++ b/Sources/App/AgentHibernationController+Teardown.swift @@ -17,6 +17,7 @@ extension AgentHibernationController { let requestID: UUID let epoch: UInt64 let generation: UInt64 + let trigger: AgentHibernationReclaimTrigger } /// Runs the transcript snapshot off the main actor, then resumes teardown on the @@ -24,13 +25,19 @@ extension AgentHibernationController { /// SIGTERM / pty-close can trigger Claude's interrupted-exit transcript rewrite, /// so the teardown is sequenced after it rather than racing it; the re-validation /// below covers disable/stop and anything else that changed during the brief I/O hop. - func beginConfirmedTeardowns(_ requests: [ConfirmedTeardownRequest]) { + func beginConfirmedTeardowns( + _ requests: [ConfirmedTeardownRequest], + shouldProceed: (@MainActor () -> Bool)? = nil, + onCompletion: (@MainActor (Int) -> Void)? = nil + ) { guard !requests.isEmpty else { return } Task { @MainActor in + var hibernatedCount = 0 defer { for request in requests { self.clearInFlightTeardown(request.record.key, requestID: request.requestID) } + onCompletion?(hibernatedCount) } var snapshotOutcomes = await Self.snapshotOutcomes(for: requests) @@ -80,6 +87,9 @@ extension AgentHibernationController { } } + let scopedProcessTerminationsByPanel = await Self.scopedProcessTerminations( + for: requests.map { $0.record.processTerminationScope } + ) let postSnapshotSequence = markPostSnapshotValidationPoint() let postSnapshotIndex = await sharedPostSnapshotValidationIndexTask( minimumStartSequence: postSnapshotSequence @@ -87,105 +97,73 @@ extension AgentHibernationController { for request in requests { let record = request.record - guard let snapshotOutcome = snapshotOutcomes[record.key] else { continue } - let currentAgent = record.workspace.restorableAgentForHibernation( - panelId: record.key.panelId, - index: postSnapshotIndex - ) - let currentLifecycle = postSnapshotLifecycle(for: record, index: postSnapshotIndex) - let currentEffectiveLastActivityAt = postSnapshotEffectiveLastActivityAt( - for: record, - index: postSnapshotIndex - ) - // Re-validate: the pane must still be exactly as confirmed. Any activity, - // scrollback change, visibility/protection change, hibernation disable, - // hibernation, or surface loss during the hop aborts; the regular 30s - // tick will re-arm if still idle. - guard AgentHibernationTrackingGate.isEnabled(), - record.isStillOwnedByOriginalWorkspace, - !postSnapshotIndex.hasLiveProcess(workspaceId: record.key.workspaceId, panelId: record.key.panelId), - TabManager.restorableAgentSnapshotFingerprint(currentAgent) == - TabManager.restorableAgentSnapshotFingerprint(record.agent), - !record.terminalPanel.isAgentHibernated, - record.terminalPanel.surface.hasLiveSurface, - AppDelegate.shared?.agentHibernationPanelIsProtected( - workspace: record.workspace, - panelId: record.key.panelId - ) == false, - currentLifecycle.allowsHibernation, - (self.terminalInputByPanel[record.key] ?? 0) <= - (self.lifecycleChangeByPanel[record.key] ?? 0), - self.teardownValidationGeneration == request.generation, - (self.teardownValidationEpochByPanel[record.key] ?? 0) == request.epoch, - let currentFingerprint = self.hibernationFingerprint(for: record), - currentFingerprint == request.confirmationFingerprint, - currentEffectiveLastActivityAt <= request.effectiveLastActivityAt else { + guard let snapshotOutcome = snapshotOutcomes[record.key], + let scopedProcessTerminations = scopedProcessTerminationsByPanel[record.key] else { continue } - - let snapshot: AgentHibernationTranscriptGuard.TeardownTranscriptSnapshot? - switch snapshotOutcome { - case .snapshot(let value): - snapshot = value - case .nothingToProtect: - snapshot = nil - case .unableToProtect: - // Forfeit hibernation rather than risk issue #6565 transcript loss. - self.unableToProtectByPanel[record.key] = UnableToProtectMarker( - fingerprint: request.confirmationFingerprint, - lastActivityAt: request.effectiveLastActivityAt, - retryAfter: Date().timeIntervalSince1970 + Self.unableToProtectRetrySeconds - ) + // Re-validate: the pane must still be exactly as confirmed. Any activity, + // process-set or scrollback change, visibility/protection change, + // hibernation, or surface loss during the hop aborts; a later evaluation + // can re-arm it if it is still idle. + guard teardownIsStillSafe( + request, + index: postSnapshotIndex, + shouldProceed: shouldProceed + ) else { continue } - if let snapshot { - // An armed monitor for this transcript (a prior hibernation's, - // or one stored earlier in this batch) hands off here: quiesce - // it only now that this request is otherwise committed, and - // re-check the path version. Nothing may suspend between the - // check and SIGTERM, or a rewrite can invalidate the snapshot. - await cancelPostTeardownRestoreTaskForReplacement( - transcriptPath: snapshot.transcriptPath - ) - guard AgentHibernationTranscriptGuard.liveFileVersionStillMatches(snapshot) else { - self.unableToProtectByPanel[record.key] = UnableToProtectMarker( - fingerprint: request.confirmationFingerprint, - lastActivityAt: request.effectiveLastActivityAt, - retryAfter: Date().timeIntervalSince1970 + Self.unableToProtectRetrySeconds - ) - // The quiesce above disarmed the path's previous monitor, so - // forfeiting must re-arm protection with the fresh snapshot; - // its restore checks fail closed if the live file has turns. - if self.armPostTeardownRestoreMonitor( - snapshot: snapshot, - processIDs: record.processIDs, - snapshotDisposal: .retainForRecovery(sessionId: record.agent.sessionId) - ) { - restoreOwnedSnapshotPaths.insert(snapshot.snapshotPath) - } else { - AgentHibernationTranscriptGuard.retainSnapshotForRecovery( - snapshot, - sessionId: record.agent.sessionId - ) - } - continue - } - } - self.terminateScopedProcessesForHibernation(record: record) - record.workspace.enterAgentHibernation( - panelId: record.key.panelId, - agent: record.agent, - lastActivityAt: Date(timeIntervalSince1970: request.effectiveLastActivityAt) - ) - guard let snapshot else { continue } - if self.armPostTeardownRestoreMonitor(snapshot: snapshot, processIDs: record.processIDs) { - restoreOwnedSnapshotPaths.insert(snapshot.snapshotPath) + if await commitConfirmedTeardown( + request, + snapshotOutcome: snapshotOutcome, + scopedProcessTerminations: scopedProcessTerminations, + shouldProceed: shouldProceed, + restoreOwnedSnapshotPaths: &restoreOwnedSnapshotPaths + ) { + hibernatedCount += 1 } } } } + func preserveSnapshotAfterAbortedTeardown( + _ snapshot: AgentHibernationTranscriptGuard.TeardownTranscriptSnapshot, + record: AgentHibernationRecord, + restoreOwnedSnapshotPaths: inout Set + ) { + if armPostTeardownRestoreMonitor( + snapshot: snapshot, + processIDs: record.processIDs, + snapshotDisposal: .retainForRecovery(sessionId: record.agent.sessionId) + ) { + restoreOwnedSnapshotPaths.insert(snapshot.snapshotPath) + } else { + AgentHibernationTranscriptGuard.retainSnapshotForRecovery( + snapshot, + sessionId: record.agent.sessionId + ) + } + } + + func releaseArmedRestoreMonitorAfterAbortedTeardown( + _ snapshot: AgentHibernationTranscriptGuard.TeardownTranscriptSnapshot, + sessionId: String?, + restoreOwnedSnapshotPaths: inout Set + ) async { + await cancelPostTeardownRestoreTaskForReplacement( + transcriptPath: snapshot.transcriptPath + ) + restoreOwnedSnapshotPaths.remove(snapshot.snapshotPath) + guard AgentHibernationTranscriptGuard.liveFileVersionStillMatches(snapshot) else { + AgentHibernationTranscriptGuard.retainSnapshotForRecovery( + snapshot, + sessionId: sessionId + ) + return + } + try? FileManager.default.removeItem(atPath: snapshot.snapshotPath) + } + private static func snapshotOutcomes( for requests: [ConfirmedTeardownRequest] ) async -> [AgentHibernationPanelKey: AgentHibernationTranscriptGuard.TeardownSnapshotOutcome] { @@ -294,7 +272,8 @@ extension AgentHibernationController { func armPostTeardownRestoreMonitor( snapshot: AgentHibernationTranscriptGuard.TeardownTranscriptSnapshot, processIDs: Set, - snapshotDisposal: AgentHibernationTranscriptGuard.PostTeardownSnapshotDisposal = .deleteWhenSafe + snapshotDisposal: AgentHibernationTranscriptGuard.PostTeardownSnapshotDisposal = .deleteWhenSafe, + awaitProcessExit: (@Sendable () async -> Bool)? = nil ) -> Bool { let transcriptPath = snapshot.transcriptPath let requestID = UUID() @@ -304,6 +283,7 @@ extension AgentHibernationController { snapshot: snapshot, processIDs: processIDs, snapshotDisposal: snapshotDisposal, + awaitProcessExit: awaitProcessExit, shouldContinue: { await MainActor.run { AgentHibernationController.shared.postTeardownRestoreTaskIsCurrent( diff --git a/Sources/App/AgentHibernationController+TeardownValidation.swift b/Sources/App/AgentHibernationController+TeardownValidation.swift new file mode 100644 index 000000000000..230a826d04ae --- /dev/null +++ b/Sources/App/AgentHibernationController+TeardownValidation.swift @@ -0,0 +1,61 @@ +import Foundation + +extension AgentHibernationController { + func teardownIsStillSafe( + _ request: ConfirmedTeardownRequest, + index: RestorableAgentSessionIndex, + shouldProceed: (@MainActor () -> Bool)? + ) -> Bool { + let record = request.record + let currentAgent = record.workspace.restorableAgentForHibernation( + panelId: record.key.panelId, + index: index + ) + let currentLifecycle = postSnapshotLifecycle(for: record, index: index) + let currentEffectiveLastActivityAt = postSnapshotEffectiveLastActivityAt( + for: record, + index: index + ) + let currentProcessEntry = index.entry( + workspaceId: record.key.workspaceId, + panelId: record.key.panelId + ) + let currentHibernationPanelProcessIDs = + currentProcessEntry?.hibernationPanelProcessIDs ?? [] + let currentTerminationProcessIDs = currentProcessEntry?.terminationProcessIDs ?? [] + let currentTerminationProcessIdentities = + currentProcessEntry?.terminationProcessIdentities ?? [:] + return (shouldProceed?() ?? true) && + AgentHibernationTrackingGate.isEnabled() && + record.isStillOwnedByOriginalWorkspace && + ( + request.trigger == .systemMemoryPressure || + !index.hasLiveProcess( + workspaceId: record.key.workspaceId, + panelId: record.key.panelId + ) + ) && + ( + request.trigger != .systemMemoryPressure || + currentProcessEntry?.containsUnrelatedProcess == false + ) && + currentHibernationPanelProcessIDs == record.panelProcessIDs && + currentTerminationProcessIDs == record.processIDs && + currentTerminationProcessIdentities == record.processIdentities && + TabManager.restorableAgentSnapshotFingerprint(currentAgent) == + TabManager.restorableAgentSnapshotFingerprint(record.agent) && + !record.terminalPanel.isAgentHibernated && + record.terminalPanel.surface.hasLiveSurface && + AppDelegate.shared?.agentHibernationPanelIsProtected( + workspace: record.workspace, + panelId: record.key.panelId + ) == false && + currentLifecycle.allowsHibernation && + (terminalInputByPanel[record.key] ?? 0) <= + (lifecycleChangeByPanel[record.key] ?? 0) && + teardownValidationGeneration == request.generation && + (teardownValidationEpochByPanel[record.key] ?? 0) == request.epoch && + hibernationFingerprint(for: record) == request.confirmationFingerprint && + currentEffectiveLastActivityAt <= request.effectiveLastActivityAt + } +} diff --git a/Sources/App/AgentHibernationController.swift b/Sources/App/AgentHibernationController.swift index e9e66810d380..0632197944c3 100644 --- a/Sources/App/AgentHibernationController.swift +++ b/Sources/App/AgentHibernationController.swift @@ -1,5 +1,3 @@ -import AppKit -import Darwin import Foundation struct AgentHibernationPanelKey: Hashable, Sendable { @@ -18,7 +16,10 @@ struct AgentHibernationRecord { let lastActivityAt: TimeInterval let isProtected: Bool let hasLiveProcess: Bool + let containsUnrelatedProcess: Bool + let panelProcessIDs: Set let processIDs: Set + let processIdentities: [Int: AgentPIDProcessIdentity] } @MainActor @@ -40,13 +41,18 @@ final class AgentHibernationController { var postTeardownRestoreDrainTask: Task? var postSnapshotValidationIndexSequence: UInt64 = 0 var postSnapshotValidationIndexTask: PostSnapshotValidationIndexTask? - private var teardownInFlightByPanel: [AgentHibernationPanelKey: InFlightTeardown] = [:] - private var confirmations: [AgentHibernationPanelKey: Confirmation] = [:] - private var tailFingerprintSamples: [AgentHibernationPanelKey: TailFingerprintSample] = [:] + var teardownInFlightByPanel: [AgentHibernationPanelKey: InFlightTeardown] = [:] + var committedTerminationObservationsByPanelID: [UUID: CommittedTerminationObservation] = [:] + var committedTerminationCleanupByPanelID: [UUID: CommittedTerminationCleanup] = [:] + let processSnapshotCoordinator = AgentHibernationProcessSnapshotCoordinator() + var confirmations: [AgentHibernationPanelKey: Confirmation] = [:] + var tailFingerprintSamples: [AgentHibernationPanelKey: TailFingerprintSample] = [:] + var memoryPressureEvaluation: (id: UUID, task: Task)? private init() {} func start() { + AgentHibernationTrackingGate.setEnabled(true) guard settingsObserver == nil else { updateTimerForCurrentSettings() return @@ -66,6 +72,8 @@ final class AgentHibernationController { func stop() { timer?.cancel() timer = nil + memoryPressureEvaluation?.task.cancel() + memoryPressureEvaluation = nil AgentHibernationTrackingGate.setEnabled(false) clearTrackingState() if let settingsObserver { @@ -76,8 +84,13 @@ final class AgentHibernationController { func recordTerminalInput(workspaceId: UUID, panelId: UUID, recordedAt: Date? = nil) { guard AgentHibernationTrackingGate.isEnabled() else { return } + let key = AgentHibernationPanelKey(workspaceId: workspaceId, panelId: panelId) + if let terminalInputAt = terminalInputByPanel[key], + terminalInputAt > (lifecycleChangeByPanel[key] ?? 0) { + return + } let recordedAt = recordedAt ?? Date() - let key = recordActivity(workspaceId: workspaceId, panelId: panelId, recordedAt: recordedAt) + recordActivity(workspaceId: workspaceId, panelId: panelId, recordedAt: recordedAt) terminalInputByPanel[key] = recordedAt.timeIntervalSince1970 } @@ -116,18 +129,16 @@ final class AgentHibernationController { private func recordSettingsChange() { teardownValidationGeneration = teardownValidationGeneration &+ 1 - confirmations.removeAll(keepingCapacity: false) + confirmations = confirmations.filter { $0.value.trigger == .systemMemoryPressure } unableToProtectByPanel.removeAll(keepingCapacity: false) updateTimerForCurrentSettings() } private func updateTimerForCurrentSettings() { let enabled = AgentHibernationSettings.isEnabled() - AgentHibernationTrackingGate.setEnabled(enabled) guard enabled else { timer?.cancel() timer = nil - clearTrackingState() return } guard timer == nil else { return } @@ -148,17 +159,21 @@ final class AgentHibernationController { self.timer = timer } - private func evaluate( + @discardableResult + func evaluate( index: RestorableAgentSessionIndex, settings: AgentHibernationSettings.Values, - now: Date - ) { - guard settings.enabled else { - AgentHibernationTrackingGate.setEnabled(false) - clearTrackingState() - return + now: Date, + trigger: AgentHibernationReclaimTrigger = .scheduled, + teardownShouldProceed: (@MainActor () -> Bool)? = nil, + onHibernationCompleted: (@MainActor (Int) -> Void)? = nil + ) -> (hasCandidates: Bool, beganTeardowns: Bool) { + guard trigger != .scheduled || settings.enabled else { + return (false, false) + } + guard let appDelegate = AppDelegate.shared else { + return (false, false) } - guard let appDelegate = AppDelegate.shared else { return } let records = appDelegate.agentHibernationRecords( index: index, @@ -181,10 +196,16 @@ final class AgentHibernationController { let isLive = isLiveByKey[record.key] ?? false var effectiveLastActivityAt = record.lastActivityAt if record.hasLiveProcess { - bumpTeardownValidationEpoch(record.key) tailFingerprintSamples.removeValue(forKey: record.key) - confirmations.removeValue(forKey: record.key) - unableToProtectByPanel.removeValue(forKey: record.key) + if confirmations[record.key]?.trigger == .scheduled { + confirmations.removeValue(forKey: record.key) + } + if teardownInFlightByPanel[record.key]?.trigger == .scheduled { + bumpTeardownValidationEpoch(record.key) + } + if trigger == .scheduled { + unableToProtectByPanel.removeValue(forKey: record.key) + } } if shouldMaintainTailSamples, isLive, @@ -216,10 +237,15 @@ final class AgentHibernationController { let selectedKeys = AgentHibernationPlanner.selectedPanelKeys( inputs: plannerInputs, settings: settings, - now: nowTime + now: nowTime, + trigger: trigger ) let currentKeys = Set(records.map(\.key)) - pruneTrackingState(currentKeys: currentKeys, selectedKeys: selectedKeys) + pruneTrackingState( + currentKeys: currentKeys, + selectedKeys: selectedKeys, + trigger: trigger + ) let confirmedTeardowns = records.compactMap { record -> ConfirmedTeardownRequest? in guard selectedKeys.contains(record.key) else { return nil } @@ -227,22 +253,32 @@ final class AgentHibernationController { record: record, effectiveLastActivityAt: effectiveActivityByKey[record.key] ?? record.lastActivityAt, settings: settings, - now: nowTime + now: nowTime, + trigger: trigger + ) + } + if !confirmedTeardowns.isEmpty { + beginConfirmedTeardowns( + confirmedTeardowns, + shouldProceed: teardownShouldProceed, + onCompletion: onHibernationCompleted ) } - if !confirmedTeardowns.isEmpty { beginConfirmedTeardowns(confirmedTeardowns) } + return (!selectedKeys.isEmpty, !confirmedTeardowns.isEmpty) } private func evaluateConfirmation( record: AgentHibernationRecord, effectiveLastActivityAt: TimeInterval, settings: AgentHibernationSettings.Values, - now: TimeInterval + now: TimeInterval, + trigger: AgentHibernationReclaimTrigger ) -> ConfirmedTeardownRequest? { guard record.lifecycle.allowsHibernation, !record.hasUnconfirmedTerminalInput, !record.isProtected, - !record.hasLiveProcess, + (trigger == .systemMemoryPressure || !record.hasLiveProcess), + trigger != .systemMemoryPressure || record.hasPressureSafeProcessEvidence, record.terminalPanel.surface.hasLiveSurface, !record.terminalPanel.isAgentHibernated else { confirmations.removeValue(forKey: record.key) @@ -251,19 +287,24 @@ final class AgentHibernationController { } if teardownInFlightByPanel[record.key] != nil { confirmations.removeValue(forKey: record.key); return nil } - if let confirmation = confirmations[record.key] { + if let confirmation = confirmations[record.key], + confirmation.trigger == trigger { guard now >= confirmation.dueAt else { return nil } guard effectiveLastActivityAt <= confirmation.sampledAt else { confirmations.removeValue(forKey: record.key) return nil } guard let fingerprint = hibernationFingerprint(for: record), - fingerprint == confirmation.fingerprint else { + fingerprint == confirmation.fingerprint, + record.processIdentities == confirmation.processIdentities else { confirmations.removeValue(forKey: record.key) return nil } let requestID = UUID() - teardownInFlightByPanel[record.key] = InFlightTeardown(requestID: requestID) + teardownInFlightByPanel[record.key] = InFlightTeardown( + requestID: requestID, + trigger: trigger + ) confirmations.removeValue(forKey: record.key) return ConfirmedTeardownRequest( record: record, @@ -271,9 +312,13 @@ final class AgentHibernationController { effectiveLastActivityAt: effectiveLastActivityAt, requestID: requestID, epoch: teardownValidationEpochByPanel[record.key] ?? 0, - generation: teardownValidationGeneration + generation: teardownValidationGeneration, + trigger: trigger ) } + if confirmations[record.key]?.trigger == .systemMemoryPressure { + return nil + } guard let fingerprint = hibernationFingerprint(for: record) else { return nil } if let marker = unableToProtectByPanel[record.key], @@ -287,7 +332,9 @@ final class AgentHibernationController { } unableToProtectByPanel.removeValue(forKey: record.key) confirmations[record.key] = Confirmation( + trigger: trigger, fingerprint: fingerprint, + processIdentities: record.processIdentities, sampledAt: now, dueAt: now + settings.confirmationSeconds ) @@ -391,29 +438,6 @@ final class AgentHibernationController { ) } - func terminateScopedProcessesForHibernation(record: AgentHibernationRecord) { - guard !record.processIDs.isEmpty else { return } - let currentProcessID = getpid() - let currentProcessGroupID = getpgrp() - var signaledProcessGroups: Set = [] - for rawPID in record.processIDs.sorted(by: >) { - guard rawPID > 0, rawPID <= Int(Int32.max) else { continue } - let pid = pid_t(rawPID) - guard pid != currentProcessID else { continue } - guard let process = CmuxTopProcessSnapshot.processArgumentsAndEnvironment(for: rawPID), - process.matchesCMUXScope(workspaceId: record.key.workspaceId, surfaceId: record.key.panelId) else { - continue - } - let processGroupID = getpgid(pid) - if processGroupID > 1, - processGroupID != currentProcessGroupID, - signaledProcessGroups.insert(processGroupID).inserted { - _ = kill(-processGroupID, SIGTERM) - } - _ = kill(pid, SIGTERM) - } - } - private func clearTrackingState() { cancelPostTeardownRestoreTasks() teardownValidationGeneration = teardownValidationGeneration &+ 1 @@ -429,7 +453,8 @@ final class AgentHibernationController { private func pruneTrackingState( currentKeys: Set, - selectedKeys: Set + selectedKeys: Set, + trigger: AgentHibernationReclaimTrigger ) { activityByPanel = activityByPanel.filter { currentKeys.contains($0.key) } terminalInputByPanel = terminalInputByPanel.filter { currentKeys.contains($0.key) } @@ -437,12 +462,17 @@ final class AgentHibernationController { teardownValidationEpochByPanel = teardownValidationEpochByPanel.filter { currentKeys.contains($0.key) } unableToProtectByPanel = unableToProtectByPanel.filter { currentKeys.contains($0.key) } teardownInFlightByPanel = teardownInFlightByPanel.filter { currentKeys.contains($0.key) } - confirmations = confirmations.filter { key, _ in - currentKeys.contains(key) && selectedKeys.contains(key) + confirmations = confirmations.filter { key, confirmation in + currentKeys.contains(key) && + (confirmation.trigger != trigger || selectedKeys.contains(key)) } tailFingerprintSamples = tailFingerprintSamples.filter { currentKeys.contains($0.key) } } + func clearMemoryPressureConfirmations() { + confirmations = confirmations.filter { $0.value.trigger != .systemMemoryPressure } + } + func clearInFlightTeardown(_ key: AgentHibernationPanelKey, requestID: UUID) { guard teardownInFlightByPanel[key]?.requestID == requestID else { return } teardownInFlightByPanel.removeValue(forKey: key) diff --git a/Sources/App/AgentHibernationMemoryPressureResponder.swift b/Sources/App/AgentHibernationMemoryPressureResponder.swift new file mode 100644 index 000000000000..fa145d8612e0 --- /dev/null +++ b/Sources/App/AgentHibernationMemoryPressureResponder.swift @@ -0,0 +1,46 @@ +import Foundation + +@MainActor +final class AgentHibernationMemoryPressureResponder: MemoryPressureResponder { + let memoryPressureResponderID = "idle-agent-hibernation" + let memoryPressureMinimumSeverity: MemoryPressureSeverity = .critical + let memoryPressurePriority = 80 + + private let controller: AgentHibernationController + private let isPressureCritical: @MainActor () -> Bool + + init( + controller: AgentHibernationController, + isPressureCritical: @escaping @MainActor () -> Bool + ) { + self.controller = controller + self.isPressureCritical = isPressureCritical + } + + func shedMemory(for snapshot: MemoryPressureSnapshot) -> MemoryPressureShedResult { + let responderID = memoryPressureResponderID + let severity = snapshot.severity + let didSchedule = controller.reclaimIdleAgentsForSystemMemoryPressure( + now: snapshot.sampledAt, + isPressureStillCritical: isPressureCritical + ) { hibernatedCount in + guard hibernatedCount > 0 else { return } + MemoryPressureResponderRegistry.logShedAction( + MemoryPressureShedAction( + responderID: responderID, + severity: severity, + reclaimedItemCount: hibernatedCount, + estimatedBytes: nil, + detail: "hidden-idle-agents", + performedAt: .now + ) + ) + } + return MemoryPressureShedResult( + reclaimedItemCount: 0, + detail: didSchedule + ? "hidden-idle-agent-evaluation" + : "hidden-idle-agent-evaluation-in-flight" + ) + } +} diff --git a/Sources/App/AgentHibernationPlanner.swift b/Sources/App/AgentHibernationPlanner.swift index d1d14d8e2b3f..350244bbff51 100644 --- a/Sources/App/AgentHibernationPlanner.swift +++ b/Sources/App/AgentHibernationPlanner.swift @@ -1,26 +1,52 @@ +import CmuxTerminal import Foundation +enum AgentHibernationReclaimTrigger: Equatable, Sendable { + case scheduled + case systemMemoryPressure +} + enum AgentHibernationPlanner { static func selectedPanelKeys( inputs: [AgentHibernationPlannerInput], settings: AgentHibernationSettings.Values, - now: TimeInterval + now: TimeInterval, + trigger: AgentHibernationReclaimTrigger = .scheduled ) -> Set { - guard settings.enabled else { return [] } let liveRestorable = inputs.filter { $0.hasRestorableAgent && $0.isLive } - let excess = liveRestorable.count - settings.maxLiveTerminals + let excess: Int + switch trigger { + case .scheduled: + guard settings.enabled else { return [] } + excess = liveRestorable.count - settings.maxLiveTerminals + case .systemMemoryPressure: + // Snapshot and reclaim a small batch before the next pressure pass. + excess = min( + liveRestorable.count, + TerminalSurfaceRuntimeTeardownCoordinator + .maximumIsolatedHibernationTeardownCount + ) + } guard excess > 0 else { return [] } - // Live scoped processes still create cap pressure, but they are not - // eligible for teardown; reclaim safe idle panes first instead. + // Scheduled hibernation never reaps a live scoped process. Critical + // pressure may select one only while its lifecycle is explicitly idle; + // the controller still confirms stability, protects the transcript, + // and revalidates the exact process set before teardown. let eligible = liveRestorable .filter { input in !input.isProtected && - !input.hasLiveProcess && + ( + trigger == .systemMemoryPressure || + !input.hasLiveProcess + ) && input.lifecycle.allowsHibernation && !input.isTemporarilyUnableToProtect && !input.hasUnconfirmedTerminalInput && - now - input.lastActivityAt >= settings.idleSeconds + ( + trigger == .systemMemoryPressure || + now - input.lastActivityAt >= settings.idleSeconds + ) } .sorted { lhs, rhs in if lhs.lastActivityAt == rhs.lastActivityAt { diff --git a/Sources/App/AgentHibernationProcessExitCompletion.swift b/Sources/App/AgentHibernationProcessExitCompletion.swift new file mode 100644 index 000000000000..2107290eeb72 --- /dev/null +++ b/Sources/App/AgentHibernationProcessExitCompletion.swift @@ -0,0 +1,42 @@ +import Foundation + +/// One process-exit result shared by the panel observation and transcript guard. +actor AgentHibernationProcessExitCompletion { + private var result: Bool? + private var waiters: [UUID: CheckedContinuation] = [:] + + func wait() async -> Bool { + if let result { return result } + if Task.isCancelled { return false } + let waiterID = UUID() + return await withTaskCancellationHandler { + await withCheckedContinuation { continuation in + if let result { + continuation.resume(returning: result) + } else if Task.isCancelled { + continuation.resume(returning: false) + } else { + waiters[waiterID] = continuation + } + } + } onCancel: { + Task { + await self.cancel(waiterID: waiterID) + } + } + } + + func finish(_ result: Bool) { + guard self.result == nil else { return } + self.result = result + let pendingWaiters = waiters.values + waiters.removeAll(keepingCapacity: false) + for waiter in pendingWaiters { + waiter.resume(returning: result) + } + } + + private func cancel(waiterID: UUID) { + waiters.removeValue(forKey: waiterID)?.resume(returning: false) + } +} diff --git a/Sources/App/AgentHibernationProcessExitEpoch.swift b/Sources/App/AgentHibernationProcessExitEpoch.swift new file mode 100644 index 000000000000..96503a73814b --- /dev/null +++ b/Sources/App/AgentHibernationProcessExitEpoch.swift @@ -0,0 +1,19 @@ +import Darwin +import Foundation + +/// One refreshed process generation observed after hibernation commits. +struct AgentHibernationProcessExitEpoch: Sendable { + let terminations: [AgentHibernationController.ScopedProcessTermination] + let processGroupLeaders: [pid_t: AgentPIDProcessIdentity] + let signalableProcessIdentities: Set + + init( + terminations: [AgentHibernationController.ScopedProcessTermination], + processGroupLeaders: [pid_t: AgentPIDProcessIdentity], + signalableProcessIdentities: Set = [] + ) { + self.terminations = terminations + self.processGroupLeaders = processGroupLeaders + self.signalableProcessIdentities = signalableProcessIdentities + } +} diff --git a/Sources/App/AgentHibernationProcessSnapshotCoordinator.swift b/Sources/App/AgentHibernationProcessSnapshotCoordinator.swift new file mode 100644 index 000000000000..4eff3906cb4c --- /dev/null +++ b/Sources/App/AgentHibernationProcessSnapshotCoordinator.swift @@ -0,0 +1,235 @@ +import Darwin +import Foundation + +/// Coalesces authoritative process-table refreshes across hibernating panels. +actor AgentHibernationProcessSnapshotCoordinator { + typealias SnapshotCapture = @Sendable () async -> CmuxTopProcessSnapshot + typealias CaptureScheduler = @Sendable () async -> Void + typealias ProcessArgumentsProvider = + @Sendable (Int) -> CmuxTopProcessArguments? + typealias ProcessIdentityProvider = + @Sendable (pid_t) -> AgentPIDProcessIdentity? + typealias ProcessGroupProvider = @Sendable (pid_t) -> pid_t + + private let captureSnapshot: SnapshotCapture + private let beforeCapture: CaptureScheduler + private let processArgumentsProvider: ProcessArgumentsProvider + private let processIdentityProvider: ProcessIdentityProvider + private let processGroupProvider: ProcessGroupProvider + private var activeSnapshotWaiters: + [UUID: CheckedContinuation] = [:] + private var queuedSnapshotWaiters: + [UUID: CheckedContinuation] = [:] + private var captureTask: Task? + private var captureHasStarted = false + + var queuedSnapshotWaiterCount: Int { + queuedSnapshotWaiters.count + } + + init( + beforeCapture: @escaping CaptureScheduler = { + await Task.yield() + }, + captureSnapshot: @escaping SnapshotCapture = { + await AgentHibernationProcessSnapshotCoordinator.captureFreshSnapshot() + }, + processArgumentsProvider: @escaping ProcessArgumentsProvider = { + CmuxTopProcessSnapshot.processArgumentsAndEnvironment(for: $0) + }, + processIdentityProvider: @escaping ProcessIdentityProvider = { + AgentPIDProcessIdentity(pid: $0) + }, + processGroupProvider: @escaping ProcessGroupProvider = { + getpgid($0) + } + ) { + self.beforeCapture = beforeCapture + self.captureSnapshot = captureSnapshot + self.processArgumentsProvider = processArgumentsProvider + self.processIdentityProvider = processIdentityProvider + self.processGroupProvider = processGroupProvider + } + + /// Returns one fresh snapshot shared by requests already queued for this capture epoch. + func nextSnapshot() async -> CmuxTopProcessSnapshot? { + let waiterID = UUID() + return await withTaskCancellationHandler { + await withCheckedContinuation { continuation in + guard !Task.isCancelled else { + continuation.resume(returning: nil) + return + } + queuedSnapshotWaiters[waiterID] = continuation + scheduleCaptureIfNeeded() + } + } onCancel: { + Task { + await self.cancelSnapshotWaiter(waiterID) + } + } + } + + func refreshedExitEpoch( + processGroupLeaders: [pid_t: AgentPIDProcessIdentity], + processScopeKey: AgentHibernationPanelKey?, + ttyDevice: Int64?, + excluding exitedIdentities: Set + ) async -> AgentHibernationProcessExitEpoch? { + guard let snapshot = await nextSnapshot() else { return nil } + return Self.exitEpoch( + in: snapshot, + processGroupLeaders: processGroupLeaders, + processScopeKey: processScopeKey, + ttyDevice: ttyDevice, + processArgumentsProvider: processArgumentsProvider, + processIdentityProvider: processIdentityProvider, + processGroupProvider: processGroupProvider, + excluding: exitedIdentities + ) + } + + private func scheduleCaptureIfNeeded() { + guard captureTask == nil, !queuedSnapshotWaiters.isEmpty else { return } + captureTask = Task { [weak self, beforeCapture] in + // Let all exit events already queued on the actor join one fresh capture. + await beforeCapture() + guard !Task.isCancelled else { + await self?.finishCancelledCapture() + return + } + await self?.runCapture() + } + } + + private func runCapture() async { + captureHasStarted = true + activeSnapshotWaiters = queuedSnapshotWaiters + queuedSnapshotWaiters.removeAll(keepingCapacity: true) + let snapshot = await captureSnapshot() + let waiters = Array(activeSnapshotWaiters.values) + activeSnapshotWaiters.removeAll(keepingCapacity: true) + for waiter in waiters { + waiter.resume(returning: snapshot) + } + captureHasStarted = false + captureTask = nil + scheduleCaptureIfNeeded() + } + + private func cancelSnapshotWaiter(_ waiterID: UUID) { + let waiter = activeSnapshotWaiters.removeValue(forKey: waiterID) ?? + queuedSnapshotWaiters.removeValue(forKey: waiterID) + waiter?.resume(returning: nil) + let captureHasNoWaiters = captureHasStarted + ? activeSnapshotWaiters.isEmpty + : queuedSnapshotWaiters.isEmpty + if captureHasNoWaiters { + captureTask?.cancel() + } + } + + private func finishCancelledCapture() { + captureHasStarted = false + captureTask = nil + scheduleCaptureIfNeeded() + } + + #if compiler(>=6.2) + @concurrent + #endif + private nonisolated static func captureFreshSnapshot() async -> CmuxTopProcessSnapshot { + CmuxTopProcessSnapshot.capture( + includeProcessDetails: false, + includeCMUXScope: false + ) + } + + private nonisolated static func exitEpoch( + in snapshot: CmuxTopProcessSnapshot, + processGroupLeaders: [pid_t: AgentPIDProcessIdentity], + processScopeKey: AgentHibernationPanelKey?, + ttyDevice: Int64?, + processArgumentsProvider: ProcessArgumentsProvider, + processIdentityProvider: ProcessIdentityProvider, + processGroupProvider: ProcessGroupProvider, + excluding exitedIdentities: Set + ) -> AgentHibernationProcessExitEpoch? { + var authorizedLeaders: [pid_t: AgentPIDProcessIdentity] = [:] + for (processGroupID, leaderIdentity) in processGroupLeaders { + let memberIDs = snapshot.pids(forProcessGroupID: Int(processGroupID)) + guard !memberIDs.isEmpty else { continue } + if snapshot.processesByPID[Int(processGroupID)] != nil, + processIdentityProvider(processGroupID) != leaderIdentity { + continue + } + authorizedLeaders[processGroupID] = leaderIdentity + } + + var observedProcessIDs: Set = [] + for processGroupID in authorizedLeaders.keys { + observedProcessIDs.formUnion( + snapshot.pids(forProcessGroupID: Int(processGroupID)) + ) + } + if processScopeKey != nil { + guard let ttyDevice else { return nil } + observedProcessIDs.formUnion( + snapshot.pids(forTTYDevice: ttyDevice) + ) + } + guard observedProcessIDs.count <= + AgentHibernationController.maximumScopedProcessTerminationCount else { + return nil + } + if let processScopeKey { + guard observedProcessIDs.allSatisfy({ processID in + processArgumentsProvider(processID)?.matchesCMUXScope( + workspaceId: processScopeKey.workspaceId, + surfaceId: processScopeKey.panelId + ) == true + }) else { + return nil + } + } + + var nextTerminations: [AgentPIDProcessIdentity: + AgentHibernationController.ScopedProcessTermination] = [:] + var liveAuthorizedLeaders: [pid_t: AgentPIDProcessIdentity] = [:] + var signalableProcessIdentities: Set = [] + for processID in observedProcessIDs { + guard processID > 0, + let process = snapshot.processesByPID[processID], + let rawProcessGroupID = process.processGroupID, + rawProcessGroupID > 1 else { + return nil + } + let processGroupID = pid_t(rawProcessGroupID) + guard let identity = processIdentityProvider(pid_t(processID)) else { + return nil + } + guard !exitedIdentities.contains(identity) else { continue } + guard processGroupProvider(pid_t(processID)) == processGroupID else { + return nil + } + nextTerminations[identity] = .init( + processID: processID, + processIdentity: identity, + processGroupID: processGroupID, + ttyDevice: process.ttyDevice + ) + if let leaderIdentity = authorizedLeaders[processGroupID] { + liveAuthorizedLeaders[processGroupID] = leaderIdentity + signalableProcessIdentities.insert(identity) + } + } + + return AgentHibernationProcessExitEpoch( + terminations: nextTerminations.values.sorted { + $0.processID > $1.processID + }, + processGroupLeaders: liveAuthorizedLeaders, + signalableProcessIdentities: signalableProcessIdentities + ) + } +} diff --git a/Sources/App/AgentHibernationTranscriptGuard+PostTeardownRestore.swift b/Sources/App/AgentHibernationTranscriptGuard+PostTeardownRestore.swift index 8db77762e8de..86b579db7413 100644 --- a/Sources/App/AgentHibernationTranscriptGuard+PostTeardownRestore.swift +++ b/Sources/App/AgentHibernationTranscriptGuard+PostTeardownRestore.swift @@ -21,6 +21,7 @@ extension AgentHibernationTranscriptGuard { clock: ContinuousClock = ContinuousClock(), fileManager: FileManager = .default, snapshotDisposal: PostTeardownSnapshotDisposal = .deleteWhenSafe, + awaitProcessExit: (@Sendable () async -> Bool)? = nil, shouldContinue: @Sendable () async -> Bool = { true }, shouldRestoreOnCancellation: @Sendable () async -> Bool = { true } ) async { @@ -66,7 +67,16 @@ extension AgentHibernationTranscriptGuard { } } - if !processIDs.isEmpty { + if let awaitProcessExit { + let didExit = await awaitProcessExit() + if Task.isCancelled { + await restoreBeforeStoppedReturn() + return + } + if didExit { + if await stopIfNoLongerCurrent() { return } + } + } else if !processIDs.isEmpty { let deadline = clock.now.advanced(by: .seconds(30)) while clock.now < deadline { let anyAlive = processIDs.contains { pid in diff --git a/Sources/App/WorkspaceRuntimeSettings.swift b/Sources/App/WorkspaceRuntimeSettings.swift index d984e3f68a20..784696e6c73c 100644 --- a/Sources/App/WorkspaceRuntimeSettings.swift +++ b/Sources/App/WorkspaceRuntimeSettings.swift @@ -1,4 +1,5 @@ import Darwin +import CmuxFoundation import Foundation enum WorkspaceTitlebarSettings { static let showTitlebarKey = "workspaceTitlebarVisible" @@ -269,7 +270,7 @@ enum AgentHibernationSettings { static let confirmationSecondsKey = "terminal.agentHibernation.confirmationSeconds" static let defaultEnabled = false - // Hibernation is opt-in. Once enabled, reclaim idle background agents quickly: + // Routine hibernation is opt-in. Once enabled, reclaim idle background agents quickly: // the maxLiveTerminals cap and the confirmationSeconds settle window keep this safe. static let defaultIdleSeconds: TimeInterval = 5 static let defaultMaxLiveTerminals = 12 @@ -364,8 +365,8 @@ enum AgentHibernationSettings { } /// Settings for non-destructive offscreen renderer reclamation. Unlike -/// `AgentHibernationSettings` (which kills a resumable agent's PTY and is opt-in), -/// this only releases an offscreen terminal's GPU renderer (Metal swap chain / +/// routine `AgentHibernationSettings` (which kills a resumable agent's PTY and is +/// opt-in), this only releases an offscreen terminal's GPU renderer (Metal swap chain / /// IOSurface) while keeping its PTY and terminal state alive, rebuilding it on /// re-show. It is therefore safe to default ON. The cap keeps recently-used tabs /// warm so switching stays instant; the idle window avoids reclaiming a tab the @@ -462,19 +463,14 @@ enum RendererRealizationSettings { } enum AgentHibernationTrackingGate { - private static let lock = NSLock() - private static var enabled = AgentHibernationSettings.isEnabled() + private static let gate = AtomicBooleanGate(false) static func isEnabled() -> Bool { - lock.lock() - defer { lock.unlock() } - return enabled + gate.loadRelaxed() } static func setEnabled(_ nextEnabled: Bool) { - lock.lock() - enabled = nextEnabled - lock.unlock() + gate.storeRelease(nextEnabled) } } diff --git a/Sources/AppDelegate+PaneMemoryGuardrail.swift b/Sources/AppDelegate+PaneMemoryGuardrail.swift index cd24f1f9bd56..f2fc17729773 100644 --- a/Sources/AppDelegate+PaneMemoryGuardrail.swift +++ b/Sources/AppDelegate+PaneMemoryGuardrail.swift @@ -34,6 +34,14 @@ extension AppDelegate { self?.paneMemoryGuardrailTabManagers() ?? [] } ) + monitor.registry.register( + AgentHibernationMemoryPressureResponder( + controller: AgentHibernationController.shared, + isPressureCritical: { [weak monitor] in + monitor?.currentSeverity == .critical + } + ) + ) if let notificationStore { monitor.registry.register( NotificationCacheMemoryPressureResponder(store: notificationStore) diff --git a/Sources/AppDelegate.swift b/Sources/AppDelegate.swift index 215e44fb2365..f6ac33a4366e 100644 --- a/Sources/AppDelegate.swift +++ b/Sources/AppDelegate.swift @@ -12771,7 +12771,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate, UNUserNotificationCent queue: .main ) { [weak self] _ in GhosttyConfig.invalidateLoadCache() - MainActor.assumeIsolated { + _ = MainActor.assumeIsolated { self?.reloadConfiguration( source: "globalFontMagnificationDidChange", reloadSettingsFromFile: false diff --git a/Sources/CmuxTopSnapshot.swift b/Sources/CmuxTopSnapshot.swift index 61ebc43325e0..eeb90bbcb6eb 100644 --- a/Sources/CmuxTopSnapshot.swift +++ b/Sources/CmuxTopSnapshot.swift @@ -137,6 +137,7 @@ final class CmuxTopProcessSnapshot: @unchecked Sendable { private let childrenByParentPID: [Int: [Int]] private let pidsByTTYDevice: [Int64: [Int]] private let pidsByCMUXSurfaceID: [UUID: [Int]] + private let pidsByProcessGroupID: [Int: [Int]] private let residentMemorySources: [CmuxTopProcessMemorySource] static func capture( @@ -176,6 +177,7 @@ final class CmuxTopProcessSnapshot: @unchecked Sendable { var children: [Int: [Int]] = [:] var ttyMap: [Int64: [Int]] = [:] var cmuxSurfaceMap: [UUID: [Int]] = [:] + var processGroupMap: [Int: [Int]] = [:] for process in processMap.values { if process.parentPID > 0 { children[process.parentPID, default: []].append(process.pid) @@ -186,10 +188,14 @@ final class CmuxTopProcessSnapshot: @unchecked Sendable { if let cmuxSurfaceID = process.cmuxSurfaceID { cmuxSurfaceMap[cmuxSurfaceID, default: []].append(process.pid) } + if let processGroupID = process.processGroupID { + processGroupMap[processGroupID, default: []].append(process.pid) + } } self.childrenByParentPID = children.mapValues { $0.sorted() } self.pidsByTTYDevice = ttyMap.mapValues { $0.sorted() } self.pidsByCMUXSurfaceID = cmuxSurfaceMap.mapValues { $0.sorted() } + self.pidsByProcessGroupID = processGroupMap.mapValues { $0.sorted() } } func samplePayload() -> [String: Any] { @@ -240,13 +246,21 @@ final class CmuxTopProcessSnapshot: @unchecked Sendable { guard let device = Self.deviceIdentifier(forTTYName: ttyName) else { return [] } - return Set(pidsByTTYDevice[device] ?? []) + return pids(forTTYDevice: device) + } + + func pids(forTTYDevice ttyDevice: Int64) -> Set { + Set(pidsByTTYDevice[ttyDevice] ?? []) } func pids(forCMUXSurfaceID surfaceID: UUID) -> Set { Set(pidsByCMUXSurfaceID[surfaceID] ?? []) } + func pids(forProcessGroupID processGroupID: Int) -> Set { + Set(pidsByProcessGroupID[processGroupID] ?? []) + } + func cmuxScopedProcesses() -> [CmuxTopProcessInfo] { processesByPID.values .filter { $0.cmuxWorkspaceID != nil && $0.cmuxSurfaceID != nil } @@ -269,6 +283,59 @@ final class CmuxTopProcessSnapshot: @unchecked Sendable { return result } + func agentHibernationProcessScope( + panelProcessIDs: Set, + agentProcessIDs: Set + ) -> RestorableAgentSessionIndex.HibernationProcessScope { + let terminalDevices = Set(agentProcessIDs.compactMap { + processesByPID[$0]?.ttyDevice + }) + let terminalProcessIDs = Set(terminalDevices.flatMap { + pidsByTTYDevice[$0] ?? [] + }) + let observedPanelProcessIDs = panelProcessIDs.union(terminalProcessIDs) + let descendantProcessIDs = expandedPIDs(rootPIDs: agentProcessIDs) + var allowedProcessIDs = descendantProcessIDs + + for rootProcessID in agentProcessIDs { + var currentProcessID = rootProcessID + var visitedProcessIDs: Set = [] + while visitedProcessIDs.insert(currentProcessID).inserted, + let parentProcessID = processesByPID[currentProcessID]?.parentPID, + observedPanelProcessIDs.contains(parentProcessID) { + allowedProcessIDs.insert(parentProcessID) + currentProcessID = parentProcessID + } + } + let processGroupIDs = Set(descendantProcessIDs.compactMap { + processesByPID[$0]?.processGroupID + }).filter { $0 > 1 } + let processGroupMemberIDs = Set(processGroupIDs.flatMap { + pidsByProcessGroupID[$0] ?? [] + }) + let terminationProcessIDs = descendantProcessIDs.union(processGroupMemberIDs) + + let hasCompleteAgentRoots = + !agentProcessIDs.isEmpty && + agentProcessIDs.isSubset(of: terminationProcessIDs) + let hasTerminalEvidence = agentProcessIDs.allSatisfy { + processesByPID[$0]?.ttyDevice != nil + } + let hasCompleteProcessGroups = processGroupIDs.allSatisfy { processGroupID in + processesByPID[processGroupID]?.processGroupID == processGroupID + } + return ( + observedPanelProcessIDs, + terminationProcessIDs, + !hasCompleteAgentRoots || + !hasTerminalEvidence || + processGroupIDs.isEmpty || + !hasCompleteProcessGroups || + !processGroupMemberIDs.isSubset(of: allowedProcessIDs) || + !observedPanelProcessIDs.isSubset(of: allowedProcessIDs) + ) + } + func descendantPIDs(rootPID: Int, includeRoot: Bool = false) -> Set { guard rootPID > 0 else { return [] } diff --git a/Sources/DockSplitStore+PanelDestruction.swift b/Sources/DockSplitStore+PanelDestruction.swift new file mode 100644 index 000000000000..05eaf9af3727 --- /dev/null +++ b/Sources/DockSplitStore+PanelDestruction.swift @@ -0,0 +1,38 @@ +import Bonsplit +import Foundation + +extension DockSplitStore { + @discardableResult + func discardPanelOwnershipAndClose(panelId: UUID) -> (any Panel)? { + let tabIDs = surfaceIdToPanelId.compactMap { tabID, ownedPanelID in + ownedPanelID == panelId ? tabID : nil + } + for tabID in tabIDs { + surfaceIdToPanelId.removeValue(forKey: tabID) + } + return discardPanelStateAndClose(panelId: panelId) + } + + @discardableResult + func discardPanelStateAndClose(panelId: UUID) -> (any Panel)? { + panelCancellables[panelId]?.cancel() + panelCancellables.removeValue(forKey: panelId) + AppDelegate.shared?.notificationStore?.clearNotifications( + forTabId: workspaceId, + surfaceId: panelId + ) + detachedSurfaceTransfersByPanelId.removeValue(forKey: panelId) + clearSessionRestoreState(panelId: panelId) + + guard let panel = panels.removeValue(forKey: panelId) else { return nil } + if let terminalPanel = panel as? TerminalPanel { + terminalFontSizeChangeCoordinator? + .terminalDidLeaveDock( + terminalPanel, + dock: self + ) + } + panel.close() + return panel + } +} diff --git a/Sources/DockSplitStore+Reset.swift b/Sources/DockSplitStore+Reset.swift index 7a1c257cf682..838072b31a8f 100644 --- a/Sources/DockSplitStore+Reset.swift +++ b/Sources/DockSplitStore+Reset.swift @@ -10,18 +10,10 @@ extension DockSplitStore { for tabId in tabIds { _ = bonsplitController.closeTab(tabId) } collapseToSingleEmptyPane() reconcilePanels() - for panel in panels.values { - if let terminalPanel = panel as? TerminalPanel { - terminalFontSizeChangeCoordinator? - .terminalDidLeaveDock( - terminalPanel, - dock: self - ) - } - panel.close() - } - panels.removeAll() surfaceIdToPanelId.removeAll() + for panelId in Array(panels.keys) { + discardPanelStateAndClose(panelId: panelId) + } detachedSurfaceTransfersByPanelId.removeAll() restoredTerminalScrollbackByPanelId.removeAll() restoredAgentLifecycle.snapshotsByPanelId.removeAll() diff --git a/Sources/DockSplitStore+SessionRestore.swift b/Sources/DockSplitStore+SessionRestore.swift index fc223d099046..6d4440884ab2 100644 --- a/Sources/DockSplitStore+SessionRestore.swift +++ b/Sources/DockSplitStore+SessionRestore.swift @@ -91,9 +91,15 @@ extension DockSplitStore { snapshot, snapshotWorkspaceId: sourceWorkspaceId, excludingStableIdentities: excludingStableIdentities - ) { + ) { let restoredPanelId = attachDetachedSurface(detached, inPane: paneId, focus: false) - if restoredPanelId == nil { detached.panel.close() } + if restoredPanelId == nil { + AgentHibernationController.shared.discardTrackingStateForClosedPanel( + workspaceId: detached.sourceWorkspaceId, + panelId: detached.panelId + ) + detached.panel.close() + } return restoredPanelId } if sourceWorkspaceId != nil, snapshot.terminal?.isRemoteTerminal == true { @@ -334,8 +340,7 @@ extension DockSplitStore { isPinned: false, inPane: paneId ) else { - panels.removeValue(forKey: panel.id) - panel.close() + discardPanelOwnershipAndClose(panelId: panel.id) return nil } surfaceIdToPanelId[tabId] = panel.id diff --git a/Sources/DockSplitStore+SurfaceTransfer.swift b/Sources/DockSplitStore+SurfaceTransfer.swift index 6919c7798e31..5157f37a88e0 100644 --- a/Sources/DockSplitStore+SurfaceTransfer.swift +++ b/Sources/DockSplitStore+SurfaceTransfer.swift @@ -285,6 +285,11 @@ extension DockSplitStore { return nil } surfaceIdToPanelId[newTabId] = detached.panelId + AgentHibernationController.shared.transferTrackingStateForMovedPanel( + panelId: detached.panelId, + from: detached.sourceWorkspaceId, + to: workspaceId + ) if let index { _ = bonsplitController.reorderTab(newTabId, toIndex: index) } @@ -373,6 +378,11 @@ extension DockSplitStore { clearSessionRestoreState(panelId: detached.panelId) return nil } + AgentHibernationController.shared.transferTrackingStateForMovedPanel( + panelId: detached.panelId, + from: detached.sourceWorkspaceId, + to: workspaceId + ) repairPlaceholderOnlyDockPane(paneId) finishAttachingDetachedSurface( diff --git a/Sources/DockSplitStore.swift b/Sources/DockSplitStore.swift index b23ac066e633..7a20f4400b82 100644 --- a/Sources/DockSplitStore.swift +++ b/Sources/DockSplitStore.swift @@ -387,9 +387,7 @@ final class DockSplitStore: BonsplitDelegate { ) } guard splitResult != nil else { - surfaceIdToPanelId.removeValue(forKey: newTab.id) - panels.removeValue(forKey: panel.id) - panel.close() + discardPanelOwnershipAndClose(panelId: panel.id) return nil } installSubscription(for: panel, tracksTerminalTitle: true) @@ -701,8 +699,7 @@ final class DockSplitStore: BonsplitDelegate { isPinned: false, inPane: paneId ) else { - panels.removeValue(forKey: panel.id) - panel.close() + discardPanelOwnershipAndClose(panelId: panel.id) return nil } surfaceIdToPanelId[tabId] = panel.id @@ -776,23 +773,11 @@ final class DockSplitStore: BonsplitDelegate { func reconcilePanels() { let live = Set(bonsplitController.allTabIds) let staleTabIds = surfaceIdToPanelId.keys.filter { !live.contains($0) } - for tabId in staleTabIds { - guard let panelId = surfaceIdToPanelId.removeValue(forKey: tabId) else { continue } - panelCancellables[panelId]?.cancel() - panelCancellables.removeValue(forKey: panelId) - AppDelegate.shared?.notificationStore?.clearNotifications(forTabId: workspaceId, surfaceId: panelId) - detachedSurfaceTransfersByPanelId.removeValue(forKey: panelId) - clearSessionRestoreState(panelId: panelId) - if let panel = panels.removeValue(forKey: panelId) { - if let terminalPanel = panel as? TerminalPanel { - terminalFontSizeChangeCoordinator? - .terminalDidLeaveDock( - terminalPanel, - dock: self - ) - } - panel.close() - } + let stalePanelIds = Set(staleTabIds.compactMap { surfaceIdToPanelId[$0] }) + surfaceIdToPanelId = surfaceIdToPanelId.filter { live.contains($0.key) } + let livePanelIds = Set(surfaceIdToPanelId.values) + for panelId in stalePanelIds.subtracting(livePanelIds) { + discardPanelStateAndClose(panelId: panelId) } } @@ -979,9 +964,7 @@ final class DockSplitStore: BonsplitDelegate { ) } guard seedSplitResult != nil else { - surfaceIdToPanelId.removeValue(forKey: newTab.id) - panels.removeValue(forKey: panel.id) - panel.close() + discardPanelOwnershipAndClose(panelId: panel.id) continue } installSubscription(for: panel, tracksTerminalTitle: tracksTitle) diff --git a/Sources/GhosttyTerminalView.swift b/Sources/GhosttyTerminalView.swift index 82b99ba0ecfb..e7489c3538f1 100644 --- a/Sources/GhosttyTerminalView.swift +++ b/Sources/GhosttyTerminalView.swift @@ -1156,7 +1156,9 @@ class GhosttyApp { object: nil, queue: .main ) { [weak self] _ in - self?.reloadConfiguration(source: "settings.terminal.copyOnSelect") + _ = MainActor.assumeIsolated { + self?.reloadConfiguration(source: "settings.terminal.copyOnSelect") + } }) #endif @@ -3583,16 +3585,13 @@ private final class TerminalSharedBackdropCutoutFilter: CIFilter { // TerminalSurfaceFocusPlacement moved to CmuxTerminalCore (SurfaceRegistry/). +@MainActor private func recordAgentHibernationTerminalInput(workspaceId: UUID, panelId: UUID) { guard AgentHibernationTrackingGate.isEnabled() else { return } - let recordedAt = Date() - Task { @MainActor in - AgentHibernationController.shared.recordTerminalInput( - workspaceId: workspaceId, - panelId: panelId, - recordedAt: recordedAt - ) - } + AgentHibernationController.shared.recordTerminalInput( + workspaceId: workspaceId, + panelId: panelId + ) } // TerminalSurface and its SearchState moved to the CmuxTerminal package diff --git a/Sources/Panels/AgentHibernationPlaceholderMode.swift b/Sources/Panels/AgentHibernationPlaceholderMode.swift new file mode 100644 index 000000000000..1c980b6b89d6 --- /dev/null +++ b/Sources/Panels/AgentHibernationPlaceholderMode.swift @@ -0,0 +1,5 @@ +enum AgentHibernationPlaceholderMode: Equatable { + case hibernated + case recovering + case failed +} diff --git a/Sources/Panels/TerminalPanel+AgentHibernation.swift b/Sources/Panels/TerminalPanel+AgentHibernation.swift new file mode 100644 index 000000000000..8aeae162e554 --- /dev/null +++ b/Sources/Panels/TerminalPanel+AgentHibernation.swift @@ -0,0 +1,134 @@ +import Foundation + +extension TerminalPanel { + var isAgentHibernated: Bool { + agentHibernationPhase.isCommitted + } + + var isAgentHibernationTerminating: Bool { + agentHibernationPhase.isTerminating + } + + var agentHibernationTerminationFailed: Bool { + agentHibernationPhase.terminationFailed + } + + var isAgentHibernationCommitPending: Bool { + agentHibernationPhase.isAwaitingCommit + } + + var agentHibernationState: AgentHibernationPanelState? { + agentHibernationPhase.state + } + + @discardableResult + func enterAgentHibernation( + agent: SessionRestorableAgentSnapshot, + lastActivityAt: Date, + hibernatedAt: Date = .now + ) -> Bool { + if isAgentHibernationCommitPending { + completeAgentHibernationTermination() + return true + } + let state = AgentHibernationPanelState( + agent: agent, + hibernatedAt: hibernatedAt, + lastActivityAt: lastActivityAt + ) + guard suspendRuntimeForAgentHibernation(reason: "agentHibernation") else { + return false + } + agentHibernationPhase = .hibernated(state) + return true + } + + @discardableResult + func beginAgentHibernationTermination( + agent: SessionRestorableAgentSnapshot, + lastActivityAt: Date, + committedAt: Date = .now + ) -> Bool { + guard case .live = agentHibernationPhase else { return false } + onRequestAgentHibernationTerminationRetry = nil + let state = AgentHibernationPanelState( + agent: agent, + hibernatedAt: committedAt, + lastActivityAt: lastActivityAt + ) + guard suspendRuntimeForAgentHibernation( + reason: "agentHibernation.terminating" + ) else { + return false + } + agentHibernationPhase = .terminating(state) + return true + } + + func completeAgentHibernationTermination() { + let state: AgentHibernationPanelState + switch agentHibernationPhase { + case .terminating(let value), + .recovering(let value), + .terminationFailed(let value): + state = value + case .live, .hibernated: + return + } + onRequestAgentHibernationTerminationRetry = nil + agentHibernationPhase = .hibernated(state) + } + + func beginAgentHibernationTerminationRecovery() { + let state: AgentHibernationPanelState + switch agentHibernationPhase { + case .terminating(let value), .terminationFailed(let value): + state = value + case .live, .recovering, .hibernated: + return + } + agentHibernationPhase = .recovering(state) + } + + func failAgentHibernationTermination() { + guard case .recovering(let state) = agentHibernationPhase else { return } + agentHibernationPhase = .terminationFailed(state) + } + + func discardAgentHibernationPhaseForPermanentClose() { + onRequestAgentHibernationTerminationRetry = nil + agentHibernationPhase = .live + } + + func retryAgentHibernationTermination() { + guard agentHibernationTerminationFailed else { return } + onRequestAgentHibernationTerminationRetry?() + } + + private func suspendRuntimeForAgentHibernation(reason: String) -> Bool { + guard surface.suspendRuntimeSurfaceForAgentHibernation(reason: reason) else { + return false + } + unfocus() + searchState = nil + hostedView.setVisibleInUI(false) + TerminalWindowPortalRegistry.detach(hostedView: hostedView) + requestViewReattach() + return true + } + + @discardableResult + func prepareAgentHibernationResume() -> AgentHibernationResumePreparation { + guard case .hibernated(let state) = agentHibernationPhase else { + return .unavailable + } + let resumeStartupInput = state.agent.resumeStartupInput() + guard surface.prepareAgentHibernationResume(initialInput: resumeStartupInput) else { + return .unavailable + } + agentHibernationPhase = .live + requestViewReattach() + surface.requestBackgroundSurfaceStartIfNeeded() + return .resumed(queuedStartupInput: resumeStartupInput != nil) + } +} diff --git a/Sources/Panels/TerminalPanel.swift b/Sources/Panels/TerminalPanel.swift index 887dea71bf5d..4d6a46102934 100644 --- a/Sources/Panels/TerminalPanel.swift +++ b/Sources/Panels/TerminalPanel.swift @@ -6,31 +6,6 @@ import Bonsplit import CmuxTerminal import CmuxWorkspaces -struct AgentHibernationPanelState { - let agent: SessionRestorableAgentSnapshot - let hibernatedAt: Date - let lastActivityAt: Date - - var agentDisplayName: String { - agent.agentDisplayName - } -} - -enum AgentHibernationResumePreparation: Equatable { - case unavailable - case resumed(queuedStartupInput: Bool) - - var didResume: Bool { - if case .resumed = self { return true } - return false - } - - var queuedStartupInput: Bool { - if case .resumed(let queuedStartupInput) = self { return queuedStartupInput } - return false - } -} - /// TerminalPanel wraps an existing TerminalSurface and conforms to the Panel protocol. /// This allows TerminalSurface to be used within the bonsplit-based layout system. @MainActor @@ -117,10 +92,11 @@ final class TerminalPanel: Panel, ObservableObject { /// (hostedView.window == nil) until the user switches workspaces. @Published var viewReattachToken: UInt64 = 0 - @Published private(set) var agentHibernationState: AgentHibernationPanelState? + @Published var agentHibernationPhase: AgentHibernationPanelPhase = .live var onRequestWorkspacePaneFlash: ((WorkspaceAttentionFlashReason) -> Void)? var onRequestAgentHibernationResume: ((Bool) -> Bool)? + var onRequestAgentHibernationTerminationRetry: (() -> Void)? private var cancellables = Set() @@ -160,10 +136,6 @@ final class TerminalPanel: Panel, ObservableObject { false } - var isAgentHibernated: Bool { - agentHibernationState != nil - } - /// The hosted NSView for embedding in SwiftUI var hostedView: GhosttySurfaceScrollView { surface.hostedView @@ -669,6 +641,11 @@ final class TerminalPanel: Panel, ObservableObject { func close() { isClosingPanel = true + AgentHibernationController.shared.discardTrackingStateForClosedPanel( + workspaceId: workspaceId, + panelId: id + ) + discardAgentHibernationPhaseForPermanentClose() discardTextBoxContentForClose() removeOwnedSessionScrollbackReplayArtifact() // Detach from the window portal on real close so stale hosted views @@ -697,37 +674,6 @@ final class TerminalPanel: Panel, ObservableObject { surface.teardownSurface() } - func enterAgentHibernation( - agent: SessionRestorableAgentSnapshot, - lastActivityAt: Date, - hibernatedAt: Date = Date() - ) { - agentHibernationState = AgentHibernationPanelState( - agent: agent, - hibernatedAt: hibernatedAt, - lastActivityAt: lastActivityAt - ) - unfocus() - searchState = nil - hostedView.setVisibleInUI(false) - TerminalWindowPortalRegistry.detach(hostedView: hostedView) - surface.suspendRuntimeSurfaceForAgentHibernation(reason: "agentHibernation") - requestViewReattach() - } - - @discardableResult - func prepareAgentHibernationResume() -> AgentHibernationResumePreparation { - guard let state = agentHibernationState else { - return .unavailable - } - let resumeStartupInput = state.agent.resumeStartupInput() - agentHibernationState = nil - surface.prepareAgentHibernationResume(initialInput: resumeStartupInput) - requestViewReattach() - surface.requestBackgroundSurfaceStartIfNeeded() - return .resumed(queuedStartupInput: resumeStartupInput != nil) - } - func requestViewReattach() { viewReattachToken &+= 1 } diff --git a/Sources/Panels/TerminalPanelView.swift b/Sources/Panels/TerminalPanelView.swift index dfe74810c324..9287b146e4df 100644 --- a/Sources/Panels/TerminalPanelView.swift +++ b/Sources/Panels/TerminalPanelView.swift @@ -35,10 +35,33 @@ struct TerminalPanelView: View { let onTriggerFlash: () -> Void var body: some View { - if let hibernationState = panel.agentHibernationState { - hibernationBody(hibernationState) - } else { + switch panel.agentHibernationPhase { + case .live: terminalBody + case .terminating: + Color(nsColor: appearance.contentBackgroundColor) + .frame(maxWidth: .infinity, maxHeight: .infinity) + .id("hibernation-terminating-\(panel.id.uuidString)") + case .recovering(let hibernationState): + AgentHibernationPlaceholderView( + state: hibernationState, + appearance: appearance, + mode: AgentHibernationPlaceholderMode.recovering, + onAction: nil + ) + .id("hibernation-termination-recovery-\(panel.id.uuidString)") + case .terminationFailed(let hibernationState): + AgentHibernationPlaceholderView( + state: hibernationState, + appearance: appearance, + mode: AgentHibernationPlaceholderMode.failed, + onAction: { + panel.retryAgentHibernationTermination() + } + ) + .id("hibernation-termination-failed-\(panel.id.uuidString)") + case .hibernated(let hibernationState): + hibernationBody(hibernationState) } } @@ -55,7 +78,8 @@ struct TerminalPanelView: View { AgentHibernationPlaceholderView( state: hibernationState, appearance: appearance, - onResume: onResumeAgentHibernation + mode: AgentHibernationPlaceholderMode.hibernated, + onAction: onResumeAgentHibernation ) .id("hibernated-\(panel.id.uuidString)") .onChange(of: isVisibleInUI) { _, visible in @@ -229,7 +253,42 @@ struct TerminalPanelView: View { private struct AgentHibernationPlaceholderView: View { let state: AgentHibernationPanelState let appearance: PanelAppearance - let onResume: () -> Void + let mode: AgentHibernationPlaceholderMode + let onAction: (() -> Void)? + + private var title: String { + switch mode { + case .hibernated: + String( + localized: "terminal.agentHibernation.title", + defaultValue: "Agent hibernated" + ) + case .recovering: + String( + localized: "terminal.agentHibernation.finishing", + defaultValue: "Finishing agent shutdown" + ) + case .failed: + String( + localized: "terminal.agentHibernation.failed", + defaultValue: "Agent shutdown needs attention" + ) + } + } + + private var actionTitle: String? { + switch mode { + case .hibernated: + String(localized: "terminal.agentHibernation.resume", defaultValue: "Resume") + case .recovering: + nil + case .failed: + String( + localized: "terminal.agentHibernation.retry", + defaultValue: "Retry shutdown" + ) + } + } private var lastActivityText: String { let formatter = RelativeDateTimeFormatter() @@ -239,10 +298,24 @@ private struct AgentHibernationPlaceholderView: View { var body: some View { VStack(spacing: 14) { - CmuxSystemSymbolImage(magnified: "pause.circle", pointSize: 34, weight: .regular) + switch mode { + case .recovering: + ProgressView() + .controlSize(.small) + .accessibilityIdentifier("AgentHibernationTerminationRecoveryProgress") + case .hibernated: + CmuxSystemSymbolImage(magnified: "pause.circle", pointSize: 34, weight: .regular) + .foregroundStyle(.secondary) + case .failed: + CmuxSystemSymbolImage( + magnified: "exclamationmark.triangle", + pointSize: 34, + weight: .regular + ) .foregroundStyle(.secondary) + } VStack(spacing: 4) { - Text(String(localized: "terminal.agentHibernation.title", defaultValue: "Agent hibernated")) + Text(title) .cmuxFont(.headline) Text(state.agentDisplayName) .cmuxFont(.subheadline) @@ -256,12 +329,18 @@ private struct AgentHibernationPlaceholderView: View { .cmuxFont(.caption) .foregroundStyle(.tertiary) } - Button(String(localized: "terminal.agentHibernation.resume", defaultValue: "Resume")) { - onResume() + if let actionTitle, let onAction { + Button(actionTitle) { + onAction() + } + .buttonStyle(.borderedProminent) + .controlSize(.small) + .accessibilityIdentifier( + mode == .failed + ? "AgentHibernationTerminationRetryButton" + : "AgentHibernationResumeButton" + ) } - .buttonStyle(.borderedProminent) - .controlSize(.small) - .accessibilityIdentifier("AgentHibernationResumeButton") } .frame(maxWidth: .infinity, maxHeight: .infinity) .background(Color(nsColor: appearance.contentBackgroundColor)) diff --git a/Sources/RestorableAgentSession.swift b/Sources/RestorableAgentSession.swift index 78bf60848306..bc0987686ee4 100644 --- a/Sources/RestorableAgentSession.swift +++ b/Sources/RestorableAgentSession.swift @@ -910,8 +910,13 @@ struct RestorableAgentSessionIndex: Sendable { /// Unlike an empty process ID set, this distinguishes an exited recorded process from no PID evidence. let processLiveness: RestorableAgentProcessLiveness let processIDs: Set + let processIdentities: [Int: AgentPIDProcessIdentity] let agentProcessIDs: Set let agentProcessIdentities: [Int: AgentPIDProcessIdentity] + let hibernationPanelProcessIDs: Set + let terminationProcessIDs: Set + let terminationProcessIdentities: [Int: AgentPIDProcessIdentity] + let containsUnrelatedProcess: Bool } enum ProcessDetectedSessionIDSource: Equatable, Sendable { @@ -929,6 +934,12 @@ struct RestorableAgentSessionIndex: Sendable { sessionIDSource: ProcessDetectedSessionIDSource ) + typealias HibernationProcessScope = ( + panelProcessIDs: Set, + terminationProcessIDs: Set, + containsUnrelatedProcess: Bool + ) + private struct SessionKey: Hashable { let kind: RestorableAgentKind let sessionId: String @@ -973,6 +984,10 @@ struct RestorableAgentSessionIndex: Sendable { entry(workspaceId: workspaceId, panelId: panelId)?.processIDs ?? [] } + func processIdentities(workspaceId: UUID, panelId: UUID) -> [Int: AgentPIDProcessIdentity] { + entry(workspaceId: workspaceId, panelId: panelId)?.processIdentities ?? [:] + } + func agentProcessIDs(workspaceId: UUID, panelId: UUID) -> Set { entry(workspaceId: workspaceId, panelId: panelId)?.agentProcessIDs ?? [] } @@ -1057,15 +1072,25 @@ struct RestorableAgentSessionIndex: Sendable { fileManager: FileManager = .default ) -> RestorableAgentSessionIndex { let registry = CmuxVaultAgentRegistry.load(homeDirectory: homeDirectory, fileManager: fileManager) + let processSnapshot = CmuxTopProcessSnapshot.capture(includeProcessDetails: true) let detectedSnapshots = processDetectedSnapshots( registry: registry, - fileManager: fileManager + fileManager: fileManager, + processSnapshot: processSnapshot, + capturedAt: processSnapshot.sampledAt.timeIntervalSince1970 ) + let hibernationProcessScopes = detectedSnapshots.mapValues { detected in + processSnapshot.agentHibernationProcessScope( + panelProcessIDs: detected.processIDs, + agentProcessIDs: detected.agentProcessIDs + ) + } return load( homeDirectory: homeDirectory, fileManager: fileManager, registry: registry, - detectedSnapshots: detectedSnapshots + detectedSnapshots: detectedSnapshots, + hibernationProcessScopes: hibernationProcessScopes ) } @@ -1074,6 +1099,7 @@ struct RestorableAgentSessionIndex: Sendable { fileManager: FileManager, registry: CmuxVaultAgentRegistry, detectedSnapshots: [PanelKey: ProcessDetectedSnapshotEntry], + hibernationProcessScopes: [PanelKey: HibernationProcessScope] = [:], processArgumentsProvider: (Int) -> CmuxTopProcessArguments? = { CmuxTopProcessSnapshot.processArgumentsAndEnvironment(for: $0) }, @@ -1209,8 +1235,16 @@ struct RestorableAgentSessionIndex: Sendable { updatedAt: effectiveRecord.updatedAt, processLiveness: processObservation.liveness, processIDs: liveProcessID.map { [$0] } ?? [], + processIdentities: liveProcessIdentities, agentProcessIDs: liveProcessID.map { [$0] } ?? [], - agentProcessIdentities: liveProcessIdentities + agentProcessIdentities: liveProcessIdentities, + hibernationPanelProcessIDs: liveProcessID.map { [$0] } ?? [], + terminationProcessIDs: liveProcessID.map { [$0] } ?? [], + terminationProcessIdentities: liveProcessIdentities, + // A saved hook PID proves liveness but cannot prove the + // surrounding pane is exclusive. Critical-pressure + // termination requires a fresh process-tree detection. + containsUnrelatedProcess: liveProcessID != nil ) if shouldReplaceHookEntry( existing: hookCandidatesByPanelAndKind[panelKindKey], @@ -1252,15 +1286,39 @@ struct RestorableAgentSessionIndex: Sendable { } } - func processDetectedEntry(snapshot: SessionRestorableAgentSnapshot, lifecycle: AgentHibernationLifecycleState?, updatedAt: TimeInterval, detected: ProcessDetectedSnapshotEntry) -> Entry { - Entry( + func processDetectedEntry( + key: PanelKey, + snapshot: SessionRestorableAgentSnapshot, + lifecycle: AgentHibernationLifecycleState?, + updatedAt: TimeInterval, + detected: ProcessDetectedSnapshotEntry + ) -> Entry { + let processIdentities = Self.processIdentities( + for: detected.processIDs, + processIdentityProvider: processIdentityProvider + ) + let hibernationScope = hibernationProcessScopes[key] ?? ( + detected.processIDs, + detected.agentProcessIDs, + !detected.processIDs.isSubset(of: detected.agentProcessIDs) + ) + let terminationProcessIdentities = Self.processIdentities( + for: hibernationScope.terminationProcessIDs, + processIdentityProvider: processIdentityProvider + ) + return Entry( snapshot: snapshot, lifecycle: lifecycle, updatedAt: updatedAt, processLiveness: .running, - processIDs: detected.processIDs, agentProcessIDs: detected.agentProcessIDs, - agentProcessIdentities: agentProcessIdentities( - for: detected.agentProcessIDs, - processIdentityProvider: processIdentityProvider - ) + processIDs: detected.processIDs, + processIdentities: processIdentities, + agentProcessIDs: detected.agentProcessIDs, + agentProcessIdentities: processIdentities.filter { + detected.agentProcessIDs.contains($0.key) + }, + hibernationPanelProcessIDs: hibernationScope.panelProcessIDs, + terminationProcessIDs: hibernationScope.terminationProcessIDs, + terminationProcessIdentities: terminationProcessIdentities, + containsUnrelatedProcess: hibernationScope.containsUnrelatedProcess ) } @@ -1286,7 +1344,7 @@ struct RestorableAgentSessionIndex: Sendable { detected: detected, processIdentityProvider: processIdentityProvider ) { - resolved[key] = processDetectedEntry(snapshot: panelCandidate.snapshot, lifecycle: panelCandidate.lifecycle, updatedAt: panelCandidate.updatedAt, detected: detected) + resolved[key] = processDetectedEntry(key: key, snapshot: panelCandidate.snapshot, lifecycle: panelCandidate.lifecycle, updatedAt: panelCandidate.updatedAt, detected: detected) } else if detected.sessionIDSource == .forkParentFallback, Self.forkParentFallbackMustYield(kind: detected.snapshot.kind, toExisting: resolved[key]) { // A nested fork process inside another agent's pane must not displace @@ -1298,7 +1356,7 @@ struct RestorableAgentSessionIndex: Sendable { // cwd. Prefer the hook-store identity for this stable panel/surface while still carrying // live process evidence for the restored panel. The workspace UUID can rotate during // session restore, but the surface id is intentionally reused on the normal restore path. - resolved[key] = processDetectedEntry(snapshot: panelCandidate.snapshot, lifecycle: panelCandidate.lifecycle, updatedAt: panelCandidate.updatedAt, detected: detected) + resolved[key] = processDetectedEntry(key: key, snapshot: panelCandidate.snapshot, lifecycle: panelCandidate.lifecycle, updatedAt: panelCandidate.updatedAt, detected: detected) } else if let existing = Self.matchingHookEntry( for: detected.snapshot, resolved: resolved[key], @@ -1307,9 +1365,9 @@ struct RestorableAgentSessionIndex: Sendable { SessionKey(kind: detected.snapshot.kind, sessionId: detected.snapshot.sessionId) ] ) { - resolved[key] = processDetectedEntry(snapshot: detected.snapshot, lifecycle: existing.lifecycle, updatedAt: existing.updatedAt, detected: detected) + resolved[key] = processDetectedEntry(key: key, snapshot: detected.snapshot, lifecycle: existing.lifecycle, updatedAt: existing.updatedAt, detected: detected) } else { - resolved[key] = processDetectedEntry(snapshot: detected.snapshot, lifecycle: nil, updatedAt: 0, detected: detected) + resolved[key] = processDetectedEntry(key: key, snapshot: detected.snapshot, lifecycle: nil, updatedAt: 0, detected: detected) } } @@ -1350,7 +1408,7 @@ struct RestorableAgentSessionIndex: Sendable { .max { $0.updatedAt < $1.updatedAt } } - private static func agentProcessIdentities( + private static func processIdentities( for processIDs: Set, processIdentityProvider: (Int) -> AgentPIDProcessIdentity? ) -> [Int: AgentPIDProcessIdentity] { diff --git a/Sources/TerminalController.swift b/Sources/TerminalController.swift index 8b1b190ee116..cc81aa6a724c 100644 --- a/Sources/TerminalController.swift +++ b/Sources/TerminalController.swift @@ -11013,7 +11013,7 @@ class TerminalController { simulate_app_active - Trigger app active handler set_status [--icon=X] [--color=#hex] [--url=X] [--priority=N] [--format=plain|markdown] [--tab=X] - Set a status entry set_agent_lifecycle [--tab=X] [--panel=ID] - Report coding-agent lifecycle for hibernation - agent_hibernation - Enable or disable Agent Hibernation + agent_hibernation - Enable or disable routine Agent Hibernation report_meta [--icon=X] [--color=#hex] [--url=X] [--priority=N] [--format=plain|markdown] [--tab=X] - Set sidebar metadata entry report_meta_block [--priority=N] [--tab=X] -- - Set freeform sidebar markdown block clear_status [--tab=X] - Remove a status entry diff --git a/Sources/TerminalFontConfigurationReloadReconciler.swift b/Sources/TerminalFontConfigurationReloadReconciler.swift index 1421ca41db85..e3b6d84692b3 100644 --- a/Sources/TerminalFontConfigurationReloadReconciler.swift +++ b/Sources/TerminalFontConfigurationReloadReconciler.swift @@ -8,7 +8,7 @@ final class TerminalFontConfigurationReloadReconciler { typealias CaptureNextWork = @MainActor () -> ReconciliationWork? typealias Scheduler = - @MainActor (@escaping @MainActor () -> Void) -> Void + @MainActor @Sendable (@escaping @MainActor @Sendable () -> Void) -> Void nonisolated static let defaultMaximumSurfaceVisitsPerDrain = 8 nonisolated static let defaultMaximumAttemptsPerWork = 3 diff --git a/Sources/TerminalSurfaceRuntimeWiring.swift b/Sources/TerminalSurfaceRuntimeWiring.swift index 4553b6e2f04e..338ac54d504a 100644 --- a/Sources/TerminalSurfaceRuntimeWiring.swift +++ b/Sources/TerminalSurfaceRuntimeWiring.swift @@ -125,19 +125,15 @@ extension RendererRealizationController: TerminalRendererRealizationScheduling { // MARK: Agent hibernation -/// The legacy `recordAgentHibernationTerminalInput` free helper as an -/// injected recorder: same gate, same timestamp capture, same main-actor hop. +/// The app-owned safety tracker injected into the terminal input path. +@MainActor final class TerminalAgentHibernationRecorder: AgentHibernationRecording { func recordTerminalInput(workspaceId: UUID, panelId: UUID) { guard AgentHibernationTrackingGate.isEnabled() else { return } - let recordedAt = Date() - Task { @MainActor in - AgentHibernationController.shared.recordTerminalInput( - workspaceId: workspaceId, - panelId: panelId, - recordedAt: recordedAt - ) - } + AgentHibernationController.shared.recordTerminalInput( + workspaceId: workspaceId, + panelId: panelId + ) } } diff --git a/Sources/Workspace+PanelLifecycle.swift b/Sources/Workspace+PanelLifecycle.swift index fd95fd36a32d..0f66c468416f 100644 --- a/Sources/Workspace+PanelLifecycle.swift +++ b/Sources/Workspace+PanelLifecycle.swift @@ -405,6 +405,7 @@ extension Workspace { publishSurfaceClosedEvent: Bool, clearSurfaceNotifications: Bool, requestTransferredRemoteCleanup: Bool, + discardAgentHibernationTracking: Bool = true, cleanupControllerSurfaceState: Bool = false, preservesTerminalForTransfer: Bool = false ) -> WorkspaceRemoteConfiguration? { @@ -447,6 +448,12 @@ extension Workspace { } let removedPanel = panels.removeValue(forKey: panelId) + if discardAgentHibernationTracking { + AgentHibernationController.shared.discardTrackingStateForClosedPanel( + workspaceId: id, + panelId: panelId + ) + } if let terminalPanel = (removedPanel ?? panel) as? TerminalPanel { terminalFontSizeChangeCoordinator? diff --git a/Sources/Workspace.swift b/Sources/Workspace.swift index fb09f84cf719..4845eb79d316 100644 --- a/Sources/Workspace.swift +++ b/Sources/Workspace.swift @@ -4837,16 +4837,24 @@ final class Workspace: Identifiable, ObservableObject { return snapshot } + @discardableResult func enterAgentHibernation( panelId: UUID, agent: SessionRestorableAgentSnapshot, lastActivityAt: Date - ) { + ) -> Bool { guard let terminalPanel = panels[panelId] as? TerminalPanel, - !terminalPanel.isAgentHibernated else { - return + !terminalPanel.isAgentHibernated || + terminalPanel.isAgentHibernationCommitPending else { + return false + } + guard agent.resumeCommand != nil, + terminalPanel.enterAgentHibernation( + agent: agent, + lastActivityAt: lastActivityAt + ) else { + return false } - guard agent.resumeCommand != nil else { return } restoredAgentSnapshotsByPanelId[panelId] = agent restoredAgentResumeStatesByPanelId[panelId] = .manualResumeAvailable invalidatedRestoredAgentFingerprintsByPanelId.removeValue(forKey: panelId) @@ -4857,7 +4865,7 @@ final class Workspace: Identifiable, ObservableObject { if !keys.isEmpty { refreshTrackedAgentPorts() } - terminalPanel.enterAgentHibernation(agent: agent, lastActivityAt: lastActivityAt) + return true } @discardableResult @@ -7904,6 +7912,7 @@ final class Workspace: Identifiable, ObservableObject { publishSurfaceClosedEvent: false, clearSurfaceNotifications: false, requestTransferredRemoteCleanup: true, + discardAgentHibernationTracking: false, cleanupControllerSurfaceState: false ) GhosttyApp.terminalSurfaceRegistry.unregister(oldPanel.surface) @@ -8818,6 +8827,7 @@ final class Workspace: Identifiable, ObservableObject { publishSurfaceClosedEvent: true, clearSurfaceNotifications: true, requestTransferredRemoteCleanup: true, + discardAgentHibernationTracking: true, cleanupControllerSurfaceState: true ) } @@ -9170,6 +9180,12 @@ final class Workspace: Identifiable, ObservableObject { } var detached = splitLayout.takeDetachedTransfer(tabId) + if detached == nil { + AgentHibernationController.shared.discardTrackingStateForClosedPanel( + workspaceId: id, + panelId: panelId + ) + } if shouldSkipControlMasterCleanupAfterDetach, let detachedTransfer = detached, detachedTransfer.isRemoteTerminal { skipControlMasterCleanupAfterDetachedRemoteTransfer = true if detachedTransfer.remoteCleanupConfiguration == nil { @@ -9333,6 +9349,11 @@ final class Workspace: Identifiable, ObservableObject { } else if let customSidebarPanel = detached.panel as? CustomSidebarPanel { customSidebarPanel.reattach(to: self) } + AgentHibernationController.shared.transferTrackingStateForMovedPanel( + panelId: detached.panelId, + from: detached.sourceWorkspaceId, + to: id + ) AppDelegate.shared?.notificationStore?.rebindSurfaceNotifications( fromTabId: detached.sourceWorkspaceId, toTabId: id, @@ -12131,6 +12152,7 @@ extension Workspace: BonsplitDelegate { publishSurfaceClosedEvent: !isDetaching, clearSurfaceNotifications: !preservesSurfaceForDetach, requestTransferredRemoteCleanup: false, + discardAgentHibernationTracking: !isDetaching, cleanupControllerSurfaceState: !isDetaching, preservesTerminalForTransfer: isDetaching ) @@ -12322,6 +12344,7 @@ extension Workspace: BonsplitDelegate { publishSurfaceClosedEvent: true, clearSurfaceNotifications: true, requestTransferredRemoteCleanup: true, + discardAgentHibernationTracking: !isDetachingCloseTransaction, cleanupControllerSurfaceState: !isDetachingCloseTransaction ) if !isDetachingCloseTransaction { diff --git a/cmux.xcodeproj/project.pbxproj b/cmux.xcodeproj/project.pbxproj index 5ebb88e74f30..ab80d59b16a1 100644 --- a/cmux.xcodeproj/project.pbxproj +++ b/cmux.xcodeproj/project.pbxproj @@ -50,25 +50,49 @@ C0DEF0D30000000000000001 /* AgentForkExecutableIdentityResolver.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0DEF0D30000000000000002 /* AgentForkExecutableIdentityResolver.swift */; }; C0DEF0C10000000000000001 /* AgentForkSupport.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0DEF0C10000000000000002 /* AgentForkSupport.swift */; }; C0DEF0D40000000000000001 /* AgentForkTimeoutResumeGate.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0DEF0D40000000000000002 /* AgentForkTimeoutResumeGate.swift */; }; + 8997C00E8997C00E8997C00E /* AgentHibernationController+CommittedTerminationCleanup.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D00E8997D00E8997D00E /* AgentHibernationController+CommittedTerminationCleanup.swift */; }; + 8997C00F8997C00F8997C00F /* AgentHibernationController+CommittedTerminationObservation.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D00F8997D00F8997D00F /* AgentHibernationController+CommittedTerminationObservation.swift */; }; F65760100000000000000001 /* AgentHibernationController+Confirmation.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760100000000000000002 /* AgentHibernationController+Confirmation.swift */; }; F65760110000000000000001 /* AgentHibernationController+InFlightTeardown.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760110000000000000002 /* AgentHibernationController+InFlightTeardown.swift */; }; + 8997C0018997C0018997C001 /* AgentHibernationController+MemoryPressure.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D0018997D0018997D001 /* AgentHibernationController+MemoryPressure.swift */; }; + 8997C0068997C0068997C006 /* AgentHibernationController+PanelLifecycle.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D0068997D0068997D006 /* AgentHibernationController+PanelLifecycle.swift */; }; F65760120000000000000001 /* AgentHibernationController+PostSnapshotValidationIndexTask.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760120000000000000002 /* AgentHibernationController+PostSnapshotValidationIndexTask.swift */; }; F65760260000000000000001 /* AgentHibernationController+PostTeardownRestoreCancellationState.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760260000000000000002 /* AgentHibernationController+PostTeardownRestoreCancellationState.swift */; }; F65760130000000000000001 /* AgentHibernationController+PostTeardownRestoreTask.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760130000000000000002 /* AgentHibernationController+PostTeardownRestoreTask.swift */; }; + 8997C0088997C0088997C008 /* AgentHibernationController+ProcessExitObservation.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D0088997D0088997D008 /* AgentHibernationController+ProcessExitObservation.swift */; }; + 8997C00A8997C00A8997C00A /* AgentHibernationController+ProcessExitWaiting.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D00A8997D00A8997D00A /* AgentHibernationController+ProcessExitWaiting.swift */; }; + 8997C00D8997C00D8997C00D /* AgentHibernationController+ProcessSignaling.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D00D8997D00D8997D00D /* AgentHibernationController+ProcessSignaling.swift */; }; + 8997C0038997C0038997C003 /* AgentHibernationController+ProcessTermination.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D0038997D0038997D003 /* AgentHibernationController+ProcessTermination.swift */; }; F65760030000000000000001 /* AgentHibernationController+Records.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760030000000000000002 /* AgentHibernationController+Records.swift */; }; + 8997C0108997C0108997C010 /* AgentHibernationController+ScopedProcessTerminationResult.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D0108997D0108997D010 /* AgentHibernationController+ScopedProcessTerminationResult.swift */; }; F65760140000000000000001 /* AgentHibernationController+TailFingerprintSample.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760140000000000000002 /* AgentHibernationController+TailFingerprintSample.swift */; }; F65760060000000000000001 /* AgentHibernationController+Teardown.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760060000000000000002 /* AgentHibernationController+Teardown.swift */; }; + 8997E0018997E0018997E001 /* AgentHibernationController+TeardownValidation.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997F0018997F0018997F001 /* AgentHibernationController+TeardownValidation.swift */; }; F65760150000000000000001 /* AgentHibernationController+UnableToProtectMarker.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760150000000000000002 /* AgentHibernationController+UnableToProtectMarker.swift */; }; D36A00010000000000000001 /* AgentHibernationController.swift in Sources */ = {isa = PBXBuildFile; fileRef = D36A00010000000000000002 /* AgentHibernationController.swift */; }; D36A00030000000000000001 /* AgentHibernationLifecycleState.swift in Sources */ = {isa = PBXBuildFile; fileRef = D36A00030000000000000002 /* AgentHibernationLifecycleState.swift */; }; D36A00030000000000000003 /* AgentHibernationLifecycleState.swift in Sources */ = {isa = PBXBuildFile; fileRef = D36A00030000000000000002 /* AgentHibernationLifecycleState.swift */; }; + 8997C0028997C0028997C002 /* AgentHibernationMemoryPressureResponder.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D0028997D0028997D002 /* AgentHibernationMemoryPressureResponder.swift */; }; + 9090E0039090E0039090E003 /* AgentHibernationPanelCloseCleanupTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9090F0039090F0039090F003 /* AgentHibernationPanelCloseCleanupTests.swift */; }; + 8997E0028997E0028997E002 /* AgentHibernationPanelPhase.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997F0028997F0028997F002 /* AgentHibernationPanelPhase.swift */; }; + 9090E0019090E0019090E001 /* AgentHibernationPanelState.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9090F0019090F0019090F001 /* AgentHibernationPanelState.swift */; }; + 8997C00C8997C00C8997C00C /* AgentHibernationPlaceholderMode.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D00C8997D00C8997D00C /* AgentHibernationPlaceholderMode.swift */; }; F65760160000000000000001 /* AgentHibernationPlanner.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760160000000000000002 /* AgentHibernationPlanner.swift */; }; F65760170000000000000001 /* AgentHibernationPlannerInput.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760170000000000000002 /* AgentHibernationPlannerInput.swift */; }; F65760010000000000000001 /* AgentHibernationPlannerSwiftTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760010000000000000002 /* AgentHibernationPlannerSwiftTests.swift */; }; D72260010000000000000001 /* AgentHibernationPortalVisibilityTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D72260010000000000000002 /* AgentHibernationPortalVisibilityTests.swift */; }; + 8997C0098997C0098997C009 /* AgentHibernationProcessExitCompletion.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D0098997D0098997D009 /* AgentHibernationProcessExitCompletion.swift */; }; + 8997C0118997C0118997C011 /* AgentHibernationProcessExitEpoch.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D0118997D0118997D011 /* AgentHibernationProcessExitEpoch.swift */; }; + 8997C0138997C0138997C013 /* AgentHibernationProcessSignalBoundaryTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D0138997D0138997D013 /* AgentHibernationProcessSignalBoundaryTests.swift */; }; + 8997C00B8997C00B8997C00B /* AgentHibernationProcessSnapshotCoordinator.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D00B8997D00B8997D00B /* AgentHibernationProcessSnapshotCoordinator.swift */; }; + 8997C0128997C0128997C012 /* AgentHibernationProcessSnapshotCoordinatorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D0128997D0128997D012 /* AgentHibernationProcessSnapshotCoordinatorTests.swift */; }; + 8997C0048997C0048997C004 /* AgentHibernationProcessTerminationTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D0048997D0048997D004 /* AgentHibernationProcessTerminationTests.swift */; }; F65760240000000000000001 /* AgentHibernationRestoreMonitorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760240000000000000002 /* AgentHibernationRestoreMonitorTests.swift */; }; + 9090E0029090E0029090E002 /* AgentHibernationResumePreparation.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9090F0029090F0029090F002 /* AgentHibernationResumePreparation.swift */; }; + 8997C0078997C0078997C007 /* AgentHibernationTerminationFailureTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D0078997D0078997D007 /* AgentHibernationTerminationFailureTests.swift */; }; F65760020000000000000001 /* AgentHibernationTestHelpers.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760020000000000000002 /* AgentHibernationTestHelpers.swift */; }; D36A00020000000000000001 /* AgentHibernationTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D36A00020000000000000002 /* AgentHibernationTests.swift */; }; + 8997C0058997C0058997C005 /* AgentHibernationTrackingLifecycleTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D0058997D0058997D005 /* AgentHibernationTrackingLifecycleTests.swift */; }; F65760090000000000000001 /* AgentHibernationTranscriptGuard+ClaudeWorkflow.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760090000000000000002 /* AgentHibernationTranscriptGuard+ClaudeWorkflow.swift */; }; F65760220000000000000001 /* AgentHibernationTranscriptGuard+PostTeardownRestore.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760220000000000000002 /* AgentHibernationTranscriptGuard+PostTeardownRestore.swift */; }; F65760230000000000000001 /* AgentHibernationTranscriptGuard+StableFileComparison.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760230000000000000002 /* AgentHibernationTranscriptGuard+StableFileComparison.swift */; }; @@ -874,6 +898,7 @@ C0DE71B10000000000000001 /* AppDelegate+AgentChatNotifications.swift in Sources DCDC1000000000000000B021 /* DockSplitStore+CloseConfirmation.swift in Sources */ = {isa = PBXBuildFile; fileRef = DCDC1000000000000000B022 /* DockSplitStore+CloseConfirmation.swift */; }; DCDC0000000000000000A003 /* DockSplitStore+Config.swift in Sources */ = {isa = PBXBuildFile; fileRef = DCDC0000000000000000A004 /* DockSplitStore+Config.swift */; }; DCDC1000000000000000B019 /* DockSplitStore+PaneFocus.swift in Sources */ = {isa = PBXBuildFile; fileRef = DCDC1000000000000000B020 /* DockSplitStore+PaneFocus.swift */; }; + 8997E0038997E0038997E003 /* DockSplitStore+PanelDestruction.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997F0038997F0038997F003 /* DockSplitStore+PanelDestruction.swift */; }; DCDC1000000000000000C040 /* DockSplitStore+PortalDrop.swift in Sources */ = {isa = PBXBuildFile; fileRef = DCDC1000000000000000C041 /* DockSplitStore+PortalDrop.swift */; }; D77800020000000000000001 /* DockSplitStore+PortalReconcile.swift in Sources */ = {isa = PBXBuildFile; fileRef = D77800020000000000000002 /* DockSplitStore+PortalReconcile.swift */; }; D86840000000000000000011 /* DockSplitStore+Reset.swift in Sources */ = {isa = PBXBuildFile; fileRef = D86840000000000000000012 /* DockSplitStore+Reset.swift */; }; @@ -2147,6 +2172,7 @@ C0DE71B10000000000000001 /* AppDelegate+AgentChatNotifications.swift in Sources 0A110711000000000000001E /* TerminalOutputTeeContext.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0A110711000000000000001F /* TerminalOutputTeeContext.swift */; }; C71500060000000000000001 /* TerminalPaneBackgroundView.swift in Sources */ = {isa = PBXBuildFile; fileRef = C71500050000000000000001 /* TerminalPaneBackgroundView.swift */; }; D0B10000A1B2C3D4E5F60001 /* TerminalPaneDropTargetView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0B10001A1B2C3D4E5F60001 /* TerminalPaneDropTargetView.swift */; }; + 8997E0048997E0048997E004 /* TerminalPanel+AgentHibernation.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997F0048997F0048997F004 /* TerminalPanel+AgentHibernation.swift */; }; A5F100000000000000000007 /* TerminalPanel+NotificationScroll.swift in Sources */ = {isa = PBXBuildFile; fileRef = A5F100000000000000000008 /* TerminalPanel+NotificationScroll.swift */; }; 791101010000000000000001 /* TerminalPanel+SessionScrollbackReplay.swift in Sources */ = {isa = PBXBuildFile; fileRef = 791101020000000000000001 /* TerminalPanel+SessionScrollbackReplay.swift */; }; A5001401 /* TerminalPanel.swift in Sources */ = {isa = PBXBuildFile; fileRef = A5001411 /* TerminalPanel.swift */; }; @@ -2592,24 +2618,48 @@ C0DE71B10000000000000001 /* AppDelegate+AgentChatNotifications.swift in Sources C0DEF0D30000000000000002 /* AgentForkExecutableIdentityResolver.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentForkExecutableIdentityResolver.swift; sourceTree = ""; }; C0DEF0C10000000000000002 /* AgentForkSupport.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentForkSupport.swift; sourceTree = ""; }; C0DEF0D40000000000000002 /* AgentForkTimeoutResumeGate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentForkTimeoutResumeGate.swift; sourceTree = ""; }; + 8997D00E8997D00E8997D00E /* AgentHibernationController+CommittedTerminationCleanup.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+CommittedTerminationCleanup.swift"; sourceTree = ""; }; + 8997D00F8997D00F8997D00F /* AgentHibernationController+CommittedTerminationObservation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+CommittedTerminationObservation.swift"; sourceTree = ""; }; F65760100000000000000002 /* AgentHibernationController+Confirmation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+Confirmation.swift"; sourceTree = ""; }; F65760110000000000000002 /* AgentHibernationController+InFlightTeardown.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+InFlightTeardown.swift"; sourceTree = ""; }; + 8997D0018997D0018997D001 /* AgentHibernationController+MemoryPressure.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+MemoryPressure.swift"; sourceTree = ""; }; + 8997D0068997D0068997D006 /* AgentHibernationController+PanelLifecycle.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+PanelLifecycle.swift"; sourceTree = ""; }; F65760120000000000000002 /* AgentHibernationController+PostSnapshotValidationIndexTask.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+PostSnapshotValidationIndexTask.swift"; sourceTree = ""; }; F65760260000000000000002 /* AgentHibernationController+PostTeardownRestoreCancellationState.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+PostTeardownRestoreCancellationState.swift"; sourceTree = ""; }; F65760130000000000000002 /* AgentHibernationController+PostTeardownRestoreTask.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+PostTeardownRestoreTask.swift"; sourceTree = ""; }; + 8997D0088997D0088997D008 /* AgentHibernationController+ProcessExitObservation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+ProcessExitObservation.swift"; sourceTree = ""; }; + 8997D00A8997D00A8997D00A /* AgentHibernationController+ProcessExitWaiting.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+ProcessExitWaiting.swift"; sourceTree = ""; }; + 8997D00D8997D00D8997D00D /* AgentHibernationController+ProcessSignaling.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+ProcessSignaling.swift"; sourceTree = ""; }; + 8997D0038997D0038997D003 /* AgentHibernationController+ProcessTermination.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+ProcessTermination.swift"; sourceTree = ""; }; F65760030000000000000002 /* AgentHibernationController+Records.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+Records.swift"; sourceTree = ""; }; + 8997D0108997D0108997D010 /* AgentHibernationController+ScopedProcessTerminationResult.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+ScopedProcessTerminationResult.swift"; sourceTree = ""; }; F65760140000000000000002 /* AgentHibernationController+TailFingerprintSample.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+TailFingerprintSample.swift"; sourceTree = ""; }; F65760060000000000000002 /* AgentHibernationController+Teardown.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+Teardown.swift"; sourceTree = ""; }; + 8997F0018997F0018997F001 /* AgentHibernationController+TeardownValidation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+TeardownValidation.swift"; sourceTree = ""; }; F65760150000000000000002 /* AgentHibernationController+UnableToProtectMarker.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+UnableToProtectMarker.swift"; sourceTree = ""; }; D36A00010000000000000002 /* AgentHibernationController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = App/AgentHibernationController.swift; sourceTree = ""; }; D36A00030000000000000002 /* AgentHibernationLifecycleState.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernation/AgentHibernationLifecycleState.swift; sourceTree = ""; }; + 8997D0028997D0028997D002 /* AgentHibernationMemoryPressureResponder.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = App/AgentHibernationMemoryPressureResponder.swift; sourceTree = ""; }; + 9090F0039090F0039090F003 /* AgentHibernationPanelCloseCleanupTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernationPanelCloseCleanupTests.swift; sourceTree = ""; }; + 8997F0028997F0028997F002 /* AgentHibernationPanelPhase.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernation/AgentHibernationPanelPhase.swift; sourceTree = ""; }; + 9090F0019090F0019090F001 /* AgentHibernationPanelState.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernation/AgentHibernationPanelState.swift; sourceTree = ""; }; + 8997D00C8997D00C8997D00C /* AgentHibernationPlaceholderMode.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Panels/AgentHibernationPlaceholderMode.swift; sourceTree = ""; }; F65760160000000000000002 /* AgentHibernationPlanner.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationPlanner.swift"; sourceTree = ""; }; F65760170000000000000002 /* AgentHibernationPlannerInput.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationPlannerInput.swift"; sourceTree = ""; }; F65760010000000000000002 /* AgentHibernationPlannerSwiftTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernationPlannerSwiftTests.swift; sourceTree = ""; }; D72260010000000000000002 /* AgentHibernationPortalVisibilityTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernationPortalVisibilityTests.swift; sourceTree = ""; }; + 8997D0098997D0098997D009 /* AgentHibernationProcessExitCompletion.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationProcessExitCompletion.swift"; sourceTree = ""; }; + 8997D0118997D0118997D011 /* AgentHibernationProcessExitEpoch.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationProcessExitEpoch.swift"; sourceTree = ""; }; + 8997D0138997D0138997D013 /* AgentHibernationProcessSignalBoundaryTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernationProcessSignalBoundaryTests.swift; sourceTree = ""; }; + 8997D00B8997D00B8997D00B /* AgentHibernationProcessSnapshotCoordinator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationProcessSnapshotCoordinator.swift"; sourceTree = ""; }; + 8997D0128997D0128997D012 /* AgentHibernationProcessSnapshotCoordinatorTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernationProcessSnapshotCoordinatorTests.swift; sourceTree = ""; }; + 8997D0048997D0048997D004 /* AgentHibernationProcessTerminationTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernationProcessTerminationTests.swift; sourceTree = ""; }; F65760240000000000000002 /* AgentHibernationRestoreMonitorTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernationRestoreMonitorTests.swift; sourceTree = ""; }; + 9090F0029090F0029090F002 /* AgentHibernationResumePreparation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernation/AgentHibernationResumePreparation.swift; sourceTree = ""; }; + 8997D0078997D0078997D007 /* AgentHibernationTerminationFailureTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernationTerminationFailureTests.swift; sourceTree = ""; }; F65760020000000000000002 /* AgentHibernationTestHelpers.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernationTestHelpers.swift; sourceTree = ""; }; D36A00020000000000000002 /* AgentHibernationTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernationTests.swift; sourceTree = ""; }; + 8997D0058997D0058997D005 /* AgentHibernationTrackingLifecycleTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernationTrackingLifecycleTests.swift; sourceTree = ""; }; F65760090000000000000002 /* AgentHibernationTranscriptGuard+ClaudeWorkflow.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationTranscriptGuard+ClaudeWorkflow.swift"; sourceTree = ""; }; F65760220000000000000002 /* AgentHibernationTranscriptGuard+PostTeardownRestore.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationTranscriptGuard+PostTeardownRestore.swift"; sourceTree = ""; }; F65760230000000000000002 /* AgentHibernationTranscriptGuard+StableFileComparison.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationTranscriptGuard+StableFileComparison.swift"; sourceTree = ""; }; @@ -3344,6 +3394,7 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = DCDC1000000000000000B022 /* DockSplitStore+CloseConfirmation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "DockSplitStore+CloseConfirmation.swift"; sourceTree = ""; }; DCDC0000000000000000A004 /* DockSplitStore+Config.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "DockSplitStore+Config.swift"; sourceTree = ""; }; DCDC1000000000000000B020 /* DockSplitStore+PaneFocus.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "DockSplitStore+PaneFocus.swift"; sourceTree = ""; }; + 8997F0038997F0038997F003 /* DockSplitStore+PanelDestruction.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "DockSplitStore+PanelDestruction.swift"; sourceTree = ""; }; DCDC1000000000000000C041 /* DockSplitStore+PortalDrop.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "DockSplitStore+PortalDrop.swift"; sourceTree = ""; }; D77800020000000000000002 /* DockSplitStore+PortalReconcile.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "DockSplitStore+PortalReconcile.swift"; sourceTree = ""; }; D86840000000000000000012 /* DockSplitStore+Reset.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "DockSplitStore+Reset.swift"; sourceTree = ""; }; @@ -4598,6 +4649,7 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = 0A110711000000000000001F /* TerminalOutputTeeContext.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TerminalOutputTeeContext.swift; sourceTree = ""; }; C71500050000000000000001 /* TerminalPaneBackgroundView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TerminalPaneBackgroundView.swift; sourceTree = ""; }; D0B10001A1B2C3D4E5F60001 /* TerminalPaneDropTargetView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TerminalPaneDropTargetView.swift; sourceTree = ""; }; + 8997F0048997F0048997F004 /* TerminalPanel+AgentHibernation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Panels/TerminalPanel+AgentHibernation.swift"; sourceTree = ""; }; A5F100000000000000000008 /* TerminalPanel+NotificationScroll.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Panels/TerminalPanel+NotificationScroll.swift"; sourceTree = ""; }; 791101020000000000000001 /* TerminalPanel+SessionScrollbackReplay.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Panels/TerminalPanel+SessionScrollbackReplay.swift"; sourceTree = ""; }; A5001411 /* TerminalPanel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Panels/TerminalPanel.swift; sourceTree = ""; }; @@ -5405,14 +5457,28 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = E9014000000000000000000C /* AgentSessionAutoRetrySettings.swift */, D5671002D5671002D5671002 /* TerminalScrollSpeedSettings.swift */, F65760100000000000000002 /* AgentHibernationController+Confirmation.swift */, + 8997D00E8997D00E8997D00E /* AgentHibernationController+CommittedTerminationCleanup.swift */, + 8997D00F8997D00F8997D00F /* AgentHibernationController+CommittedTerminationObservation.swift */, F65760110000000000000002 /* AgentHibernationController+InFlightTeardown.swift */, F65760120000000000000002 /* AgentHibernationController+PostSnapshotValidationIndexTask.swift */, F65760130000000000000002 /* AgentHibernationController+PostTeardownRestoreTask.swift */, F65760260000000000000002 /* AgentHibernationController+PostTeardownRestoreCancellationState.swift */, + 8997D0038997D0038997D003 /* AgentHibernationController+ProcessTermination.swift */, + 8997D0088997D0088997D008 /* AgentHibernationController+ProcessExitObservation.swift */, + 8997D00A8997D00A8997D00A /* AgentHibernationController+ProcessExitWaiting.swift */, + 8997D00D8997D00D8997D00D /* AgentHibernationController+ProcessSignaling.swift */, + 8997D0108997D0108997D010 /* AgentHibernationController+ScopedProcessTerminationResult.swift */, + 8997D0098997D0098997D009 /* AgentHibernationProcessExitCompletion.swift */, + 8997D0118997D0118997D011 /* AgentHibernationProcessExitEpoch.swift */, + 8997D00B8997D00B8997D00B /* AgentHibernationProcessSnapshotCoordinator.swift */, + 8997D0068997D0068997D006 /* AgentHibernationController+PanelLifecycle.swift */, + 8997F0018997F0018997F001 /* AgentHibernationController+TeardownValidation.swift */, F65760030000000000000002 /* AgentHibernationController+Records.swift */, F65760140000000000000002 /* AgentHibernationController+TailFingerprintSample.swift */, F65760060000000000000002 /* AgentHibernationController+Teardown.swift */, F65760150000000000000002 /* AgentHibernationController+UnableToProtectMarker.swift */, + 8997D0018997D0018997D001 /* AgentHibernationController+MemoryPressure.swift */, + 8997D0028997D0028997D002 /* AgentHibernationMemoryPressureResponder.swift */, D36A00010000000000000002 /* AgentHibernationController.swift */, F65760160000000000000002 /* AgentHibernationPlanner.swift */, F65760170000000000000002 /* AgentHibernationPlannerInput.swift */, @@ -5878,6 +5944,9 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = 0A1107110000000000000021 /* VaultAgentProcessScanner+Ollama.swift */, 0A1107110000000000000023 /* SessionRestorableAgentSnapshot+Commands.swift */, D36A00030000000000000002 /* AgentHibernationLifecycleState.swift */, + 8997F0028997F0028997F002 /* AgentHibernationPanelPhase.swift */, + 9090F0019090F0019090F001 /* AgentHibernationPanelState.swift */, + 9090F0029090F0029090F002 /* AgentHibernationResumePreparation.swift */, F0ACC0E0000000000000002 /* CachedAgentProcessIdentityValidator.swift */, B3575000000000000000000B /* SessionIndexModels.swift */, F0ACC0DE0000000000000008 /* SharedLiveAgentIndexLoader.swift */, @@ -6222,6 +6291,7 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = C54860060000000000000002 /* PanelStableSurfaceIdentity.swift */, A5F0C0010000000000000001 /* WorkspaceAttentionFlashRingView.swift */, A5001411 /* TerminalPanel.swift */, + 8997F0048997F0048997F004 /* TerminalPanel+AgentHibernation.swift */, 791101020000000000000001 /* TerminalPanel+SessionScrollbackReplay.swift */, A5F100000000000000000008 /* TerminalPanel+NotificationScroll.swift */, C0DE5A410000000000000002 /* TerminalPanelShellActivityModel.swift */, @@ -6299,6 +6369,7 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = D7632B14A1B2C3D4E5F60718 /* WKWebView+CmuxPrintOperation.swift */, A500RG00 /* ReactGrab.swift */, A5001413 /* TerminalPanelView.swift */, + 8997D00C8997D00C8997D00C /* AgentHibernationPlaceholderMode.swift */, C71500010000000000000001 /* SessionContentWidthModifier.swift */, C71500030000000000000001 /* SessionContentWidthPresentation.swift */, C71500050000000000000001 /* TerminalPaneBackgroundView.swift */, @@ -6723,6 +6794,7 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = D8684000000000000000000E /* DockSplitStore+SurfaceResume.swift */, D86840000000000000000010 /* ControlSurfaceResumeTarget.swift */, D86840000000000000000012 /* DockSplitStore+Reset.swift */, + 8997F0038997F0038997F003 /* DockSplitStore+PanelDestruction.swift */, DCDC1000000000000000B00E /* DockSurfaceKind.swift */, DCDC1000000000000000B010 /* DockTrustRequest.swift */, D0C0D0C0D0C0D0C0D0C0D004 /* DockEmptyView.swift */, @@ -6963,7 +7035,13 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = A9E050000000000000000001 /* AgentSessionWebRendererTests.swift */, A9E040000000000000000001 /* CodexAppServerSessionTests.swift */, F65760010000000000000002 /* AgentHibernationPlannerSwiftTests.swift */, + 9090F0039090F0039090F003 /* AgentHibernationPanelCloseCleanupTests.swift */, + 8997D0048997D0048997D004 /* AgentHibernationProcessTerminationTests.swift */, + 8997D0138997D0138997D013 /* AgentHibernationProcessSignalBoundaryTests.swift */, + 8997D0128997D0128997D012 /* AgentHibernationProcessSnapshotCoordinatorTests.swift */, + 8997D0058997D0058997D005 /* AgentHibernationTrackingLifecycleTests.swift */, F65760240000000000000002 /* AgentHibernationRestoreMonitorTests.swift */, + 8997D0078997D0078997D007 /* AgentHibernationTerminationFailureTests.swift */, F65760040000000000000002 /* AgentHibernationTranscriptGuardTests.swift */, F65760080000000000000002 /* AgentHibernationTranscriptGuardScanTests.swift */, F65760250000000000000002 /* AgentHibernationTranscriptSnapshotRaceTests.swift */, @@ -8034,19 +8112,37 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = C0DEF0D30000000000000001 /* AgentForkExecutableIdentityResolver.swift in Sources */, C0DEF0C10000000000000001 /* AgentForkSupport.swift in Sources */, C0DEF0D40000000000000001 /* AgentForkTimeoutResumeGate.swift in Sources */, + 8997C00E8997C00E8997C00E /* AgentHibernationController+CommittedTerminationCleanup.swift in Sources */, + 8997C00F8997C00F8997C00F /* AgentHibernationController+CommittedTerminationObservation.swift in Sources */, F65760100000000000000001 /* AgentHibernationController+Confirmation.swift in Sources */, F65760110000000000000001 /* AgentHibernationController+InFlightTeardown.swift in Sources */, + 8997C0018997C0018997C001 /* AgentHibernationController+MemoryPressure.swift in Sources */, + 8997C0068997C0068997C006 /* AgentHibernationController+PanelLifecycle.swift in Sources */, F65760120000000000000001 /* AgentHibernationController+PostSnapshotValidationIndexTask.swift in Sources */, F65760260000000000000001 /* AgentHibernationController+PostTeardownRestoreCancellationState.swift in Sources */, F65760130000000000000001 /* AgentHibernationController+PostTeardownRestoreTask.swift in Sources */, + 8997C0088997C0088997C008 /* AgentHibernationController+ProcessExitObservation.swift in Sources */, + 8997C00A8997C00A8997C00A /* AgentHibernationController+ProcessExitWaiting.swift in Sources */, + 8997C00D8997C00D8997C00D /* AgentHibernationController+ProcessSignaling.swift in Sources */, + 8997C0038997C0038997C003 /* AgentHibernationController+ProcessTermination.swift in Sources */, F65760030000000000000001 /* AgentHibernationController+Records.swift in Sources */, + 8997C0108997C0108997C010 /* AgentHibernationController+ScopedProcessTerminationResult.swift in Sources */, F65760140000000000000001 /* AgentHibernationController+TailFingerprintSample.swift in Sources */, F65760060000000000000001 /* AgentHibernationController+Teardown.swift in Sources */, + 8997E0018997E0018997E001 /* AgentHibernationController+TeardownValidation.swift in Sources */, F65760150000000000000001 /* AgentHibernationController+UnableToProtectMarker.swift in Sources */, D36A00010000000000000001 /* AgentHibernationController.swift in Sources */, D36A00030000000000000001 /* AgentHibernationLifecycleState.swift in Sources */, + 8997C0028997C0028997C002 /* AgentHibernationMemoryPressureResponder.swift in Sources */, + 8997E0028997E0028997E002 /* AgentHibernationPanelPhase.swift in Sources */, + 9090E0019090E0019090E001 /* AgentHibernationPanelState.swift in Sources */, + 8997C00C8997C00C8997C00C /* AgentHibernationPlaceholderMode.swift in Sources */, F65760160000000000000001 /* AgentHibernationPlanner.swift in Sources */, F65760170000000000000001 /* AgentHibernationPlannerInput.swift in Sources */, + 8997C0098997C0098997C009 /* AgentHibernationProcessExitCompletion.swift in Sources */, + 8997C0118997C0118997C011 /* AgentHibernationProcessExitEpoch.swift in Sources */, + 8997C00B8997C00B8997C00B /* AgentHibernationProcessSnapshotCoordinator.swift in Sources */, + 9090E0029090E0029090E002 /* AgentHibernationResumePreparation.swift in Sources */, F65760090000000000000001 /* AgentHibernationTranscriptGuard+ClaudeWorkflow.swift in Sources */, F65760220000000000000001 /* AgentHibernationTranscriptGuard+PostTeardownRestore.swift in Sources */, F65760230000000000000001 /* AgentHibernationTranscriptGuard+StableFileComparison.swift in Sources */, @@ -8461,6 +8557,7 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = DCDC1000000000000000B021 /* DockSplitStore+CloseConfirmation.swift in Sources */, DCDC0000000000000000A003 /* DockSplitStore+Config.swift in Sources */, DCDC1000000000000000B019 /* DockSplitStore+PaneFocus.swift in Sources */, + 8997E0038997E0038997E003 /* DockSplitStore+PanelDestruction.swift in Sources */, DCDC1000000000000000C040 /* DockSplitStore+PortalDrop.swift in Sources */, D77800020000000000000001 /* DockSplitStore+PortalReconcile.swift in Sources */, D86840000000000000000011 /* DockSplitStore+Reset.swift in Sources */, @@ -9341,6 +9438,7 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = 0A110711000000000000001E /* TerminalOutputTeeContext.swift in Sources */, C71500060000000000000001 /* TerminalPaneBackgroundView.swift in Sources */, D0B10000A1B2C3D4E5F60001 /* TerminalPaneDropTargetView.swift in Sources */, + 8997E0048997E0048997E004 /* TerminalPanel+AgentHibernation.swift in Sources */, A5F100000000000000000007 /* TerminalPanel+NotificationScroll.swift in Sources */, 791101010000000000000001 /* TerminalPanel+SessionScrollbackReplay.swift in Sources */, A5001401 /* TerminalPanel.swift in Sources */, @@ -9772,11 +9870,17 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = C7A51D100000000000000002 /* AgentChatSessionRegistryObservationReviewRegressionTests.swift in Sources */, C7A51D000000000000000002 /* AgentChatSessionRegistryObservationTests.swift in Sources */, A9E020000000000000000005 /* AgentExecutableResolverTests.swift in Sources */, + 9090E0039090E0039090E003 /* AgentHibernationPanelCloseCleanupTests.swift in Sources */, F65760010000000000000001 /* AgentHibernationPlannerSwiftTests.swift in Sources */, D72260010000000000000001 /* AgentHibernationPortalVisibilityTests.swift in Sources */, + 8997C0138997C0138997C013 /* AgentHibernationProcessSignalBoundaryTests.swift in Sources */, + 8997C0128997C0128997C012 /* AgentHibernationProcessSnapshotCoordinatorTests.swift in Sources */, + 8997C0048997C0048997C004 /* AgentHibernationProcessTerminationTests.swift in Sources */, F65760240000000000000001 /* AgentHibernationRestoreMonitorTests.swift in Sources */, + 8997C0078997C0078997C007 /* AgentHibernationTerminationFailureTests.swift in Sources */, F65760020000000000000001 /* AgentHibernationTestHelpers.swift in Sources */, D36A00020000000000000001 /* AgentHibernationTests.swift in Sources */, + 8997C0058997C0058997C005 /* AgentHibernationTrackingLifecycleTests.swift in Sources */, F65760080000000000000001 /* AgentHibernationTranscriptGuardScanTests.swift in Sources */, F65760040000000000000001 /* AgentHibernationTranscriptGuardTests.swift in Sources */, F65760250000000000000001 /* AgentHibernationTranscriptSnapshotRaceTests.swift in Sources */, diff --git a/cmuxTests/AgentHibernationPanelCloseCleanupTests.swift b/cmuxTests/AgentHibernationPanelCloseCleanupTests.swift new file mode 100644 index 000000000000..3f3fa8cbf620 --- /dev/null +++ b/cmuxTests/AgentHibernationPanelCloseCleanupTests.swift @@ -0,0 +1,47 @@ +import Foundation +import Testing + +#if canImport(cmux_DEV) +@testable import cmux_DEV +#elseif canImport(cmux) +@testable import cmux +#endif + +@MainActor +@Suite(.serialized) +struct AgentHibernationPanelCloseCleanupTests { + @Test + func rejectedCleanupDeadlineReleasesCleanupOwnership() async throws { + let controller = AgentHibernationController.shared + let panelID = UUID() + let requestID = UUID() + let processExitCompletion = AgentHibernationProcessExitCompletion() + controller.registerCommittedTerminationObservation( + panelID: panelID, + requestID: requestID, + processExitCompletion: processExitCompletion + ) + defer { + controller.removeCommittedTerminationObservation( + panelID: panelID, + requestID: requestID + ) + } + + controller.limitCommittedTerminationObservationAfterPanelClose( + panelID: panelID, + cleanupDelay: .zero, + sleepUntilDeadline: { _ in false } + ) + let cleanupTask = try #require( + controller.committedTerminationCleanupByPanelID[panelID]?.task + ) + await cleanupTask.value + + #expect(controller.committedTerminationCleanupByPanelID[panelID] == nil) + #expect( + controller.committedTerminationObservationsByPanelID[panelID]? + .requestID == requestID + ) + } +} diff --git a/cmuxTests/AgentHibernationPlannerSwiftTests.swift b/cmuxTests/AgentHibernationPlannerSwiftTests.swift index cf44c48602f7..919ae3990b79 100644 --- a/cmuxTests/AgentHibernationPlannerSwiftTests.swift +++ b/cmuxTests/AgentHibernationPlannerSwiftTests.swift @@ -134,7 +134,10 @@ struct AgentHibernationPlannerSwiftTests { lastActivityAt: 0, isProtected: false, hasLiveProcess: false, - processIDs: [] + containsUnrelatedProcess: false, + panelProcessIDs: [], + processIDs: [], + processIdentities: [:] ) #expect(record.isStillOwnedByOriginalWorkspace) @@ -230,6 +233,100 @@ struct AgentHibernationPlannerSwiftTests { #expect(selected == Set([safeAgent])) } + @Test + func criticalPressureSelectsBoundedSafeIdleBatchWhenScheduledHibernationIsDisabled() { + let workspaceId = UUID() + let now: TimeInterval = 1_000 + let idle = AgentHibernationPanelKey(workspaceId: workspaceId, panelId: UUID()) + let secondIdle = AgentHibernationPanelKey(workspaceId: workspaceId, panelId: UUID()) + let visible = AgentHibernationPanelKey(workspaceId: workspaceId, panelId: UUID()) + let running = AgentHibernationPanelKey(workspaceId: workspaceId, panelId: UUID()) + let liveProcess = AgentHibernationPanelKey(workspaceId: workspaceId, panelId: UUID()) + let unconfirmedInput = AgentHibernationPanelKey(workspaceId: workspaceId, panelId: UUID()) + let unableToProtect = AgentHibernationPanelKey(workspaceId: workspaceId, panelId: UUID()) + let settings = AgentHibernationSettings.Values( + enabled: false, + idleSeconds: 60, + maxLiveTerminals: 256, + confirmationSeconds: 5 + ) + + let selected = AgentHibernationPlanner.selectedPanelKeys( + inputs: [ + .init( + key: idle, + hasRestorableAgent: true, + isLive: true, + isProtected: false, + lifecycle: .idle, + hasUnconfirmedTerminalInput: false, + lastActivityAt: now + ), + .init( + key: secondIdle, + hasRestorableAgent: true, + isLive: true, + isProtected: false, + lifecycle: .idle, + hasUnconfirmedTerminalInput: false, + lastActivityAt: 1 + ), + .init( + key: visible, + hasRestorableAgent: true, + isLive: true, + isProtected: true, + lifecycle: .idle, + hasUnconfirmedTerminalInput: false, + lastActivityAt: 0 + ), + .init( + key: running, + hasRestorableAgent: true, + isLive: true, + isProtected: false, + lifecycle: .running, + hasUnconfirmedTerminalInput: false, + lastActivityAt: 0 + ), + .init( + key: liveProcess, + hasRestorableAgent: true, + isLive: true, + hasLiveProcess: true, + isProtected: false, + lifecycle: .idle, + hasUnconfirmedTerminalInput: false, + lastActivityAt: 0 + ), + .init( + key: unconfirmedInput, + hasRestorableAgent: true, + isLive: true, + isProtected: false, + lifecycle: .idle, + hasUnconfirmedTerminalInput: true, + lastActivityAt: 0 + ), + .init( + key: unableToProtect, + hasRestorableAgent: true, + isLive: true, + isProtected: false, + lifecycle: .idle, + isTemporarilyUnableToProtect: true, + hasUnconfirmedTerminalInput: false, + lastActivityAt: 0 + ), + ], + settings: settings, + now: now, + trigger: .systemMemoryPressure + ) + + #expect(selected == Set([secondIdle, liveProcess])) + } + @MainActor @Test func unableToProtectMarkerExpiresSoTransientSnapshotFailuresRetry() { @@ -371,7 +468,10 @@ struct AgentHibernationPlannerSwiftTests { lastActivityAt: 100, isProtected: false, hasLiveProcess: false, - processIDs: [] + containsUnrelatedProcess: false, + panelProcessIDs: [], + processIDs: [], + processIdentities: [:] ) #expect(controller.postSnapshotLifecycle(for: record, index: index) == .running) diff --git a/cmuxTests/AgentHibernationProcessSignalBoundaryTests.swift b/cmuxTests/AgentHibernationProcessSignalBoundaryTests.swift new file mode 100644 index 000000000000..c873d4c270bb --- /dev/null +++ b/cmuxTests/AgentHibernationProcessSignalBoundaryTests.swift @@ -0,0 +1,343 @@ +import Darwin +import Foundation +import os +import Testing + +#if canImport(cmux_DEV) +@testable import cmux_DEV +#elseif canImport(cmux) +@testable import cmux +#endif + +struct AgentHibernationProcessSignalBoundaryTests { + private nonisolated static let signalScopeKey = AgentHibernationPanelKey( + workspaceId: UUID(), + panelId: UUID() + ) + private nonisolated static let signalScopeArguments = CmuxTopProcessArguments( + arguments: ["/usr/bin/test-agent"], + environment: [ + "CMUX_WORKSPACE_ID": signalScopeKey.workspaceId.uuidString, + "CMUX_SURFACE_ID": signalScopeKey.panelId.uuidString, + ] + ) + + @MainActor + @Test + func rejectsScopeOrTTYDriftWithoutSendingSignals() async { + let firstIdentity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let secondIdentity = AgentPIDProcessIdentity( + pid: 202, + startSeconds: 20, + startMicroseconds: 2 + ) + let signaledTargets = OSAllocatedUnfairLock(initialState: [pid_t]()) + let wrongScopeArguments = CmuxTopProcessArguments( + arguments: ["/usr/bin/test-agent"], + environment: [ + "CMUX_WORKSPACE_ID": UUID().uuidString, + "CMUX_SURFACE_ID": UUID().uuidString, + ] + ) + + let wrongScopeResult = await AgentHibernationController + .terminateScopedProcessesForHibernation( + [ + .init( + processID: 101, + processIdentity: firstIdentity, + processGroupID: 101, + ttyDevice: 123 + ), + ], + processScopeKey: Self.signalScopeKey, + currentProcessID: 999, + currentProcessGroupID: 999, + processIdentityProvider: { _ in firstIdentity }, + processGroupProvider: { _ in 101 }, + processArgumentsProvider: { _ in wrongScopeArguments }, + processTTYDeviceProvider: { _ in 123 }, + signalErrorProvider: { target, _ in + signaledTargets.withLock { $0.append(target) } + return nil + } + ) + let ttyDriftResult = await AgentHibernationController + .terminateScopedProcessesForHibernation( + [ + .init( + processID: 101, + processIdentity: firstIdentity, + processGroupID: 101, + ttyDevice: 123 + ), + .init( + processID: 202, + processIdentity: secondIdentity, + processGroupID: 202, + ttyDevice: 123 + ), + ], + processScopeKey: Self.signalScopeKey, + currentProcessID: 999, + currentProcessGroupID: 999, + processIdentityProvider: { pid in + pid == 101 ? firstIdentity : secondIdentity + }, + processGroupProvider: { $0 }, + processArgumentsProvider: { _ in Self.signalScopeArguments }, + processTTYDeviceProvider: { pid in + pid == 101 ? 123 : 456 + }, + signalErrorProvider: { target, _ in + signaledTargets.withLock { $0.append(target) } + return nil + } + ) + + #expect(wrongScopeResult == .rejected) + #expect(ttyDriftResult == .rejected) + #expect(signaledTargets.withLock { $0 }.isEmpty) + } + + @MainActor + @Test + func rejectsTTYChangeBeforeFinalCommit() async { + let identity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let ttyProbeCount = OSAllocatedUnfairLock(initialState: 0) + let signaledTargets = OSAllocatedUnfairLock(initialState: [pid_t]()) + + let result = await AgentHibernationController + .terminateScopedProcessesForHibernation( + [ + .init( + processID: 101, + processIdentity: identity, + processGroupID: 101, + ttyDevice: 123 + ), + ], + processScopeKey: Self.signalScopeKey, + currentProcessID: 999, + currentProcessGroupID: 999, + processIdentityProvider: { _ in identity }, + processGroupProvider: { _ in 101 }, + processArgumentsProvider: { _ in Self.signalScopeArguments }, + processTTYDeviceProvider: { _ in + let probe = ttyProbeCount.withLock { + $0 += 1 + return $0 + } + return probe == 1 ? 123 : 456 + }, + signalErrorProvider: { target, _ in + signaledTargets.withLock { $0.append(target) } + return nil + } + ) + + #expect(result == .rejected) + #expect(ttyProbeCount.withLock { $0 } == 2) + #expect(signaledTargets.withLock { $0 }.isEmpty) + } + + @MainActor + @Test + func rejectsMultipleRecordedTTYsBeforeSendingSignals() async { + let firstIdentity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let secondIdentity = AgentPIDProcessIdentity( + pid: 202, + startSeconds: 20, + startMicroseconds: 2 + ) + let didProbe = OSAllocatedUnfairLock(initialState: false) + let signaledTargets = OSAllocatedUnfairLock(initialState: [pid_t]()) + + let result = await AgentHibernationController + .terminateScopedProcessesForHibernation( + [ + .init( + processID: 101, + processIdentity: firstIdentity, + processGroupID: 101, + ttyDevice: 123 + ), + .init( + processID: 202, + processIdentity: secondIdentity, + processGroupID: 202, + ttyDevice: 456 + ), + ], + processScopeKey: Self.signalScopeKey, + currentProcessID: 999, + currentProcessGroupID: 999, + processIdentityProvider: { _ in + didProbe.withLock { $0 = true } + return firstIdentity + }, + signalErrorProvider: { target, _ in + signaledTargets.withLock { $0.append(target) } + return nil + } + ) + + #expect(result == .rejected) + #expect(didProbe.withLock { $0 } == false) + #expect(signaledTargets.withLock { $0 }.isEmpty) + } + + @MainActor + @Test + func rejectsUnboundedSignalAuthorityBeforeKernelProbes() async { + let processCount = + AgentHibernationController.maximumScopedProcessTerminationCount + 1 + let terminations = (1...processCount).map { offset in + let processID = 100 + offset + return AgentHibernationController.ScopedProcessTermination( + processID: processID, + processIdentity: .init( + pid: pid_t(processID), + startSeconds: Int64(processID), + startMicroseconds: 1 + ), + processGroupID: pid_t(processID), + ttyDevice: 123 + ) + } + let didProbe = OSAllocatedUnfairLock(initialState: false) + let signaledTargets = OSAllocatedUnfairLock(initialState: [pid_t]()) + + let result = await AgentHibernationController + .terminateScopedProcessesForHibernation( + terminations, + processScopeKey: Self.signalScopeKey, + processIdentityProvider: { _ in + didProbe.withLock { $0 = true } + return nil + }, + signalErrorProvider: { target, _ in + signaledTargets.withLock { $0.append(target) } + return nil + } + ) + + #expect(result == .rejected) + #expect(didProbe.withLock { $0 } == false) + #expect(signaledTargets.withLock { $0 }.isEmpty) + } + + @Test + func rejectsUnboundedTerminationScopeBeforeKernelProbes() { + let processCount = + AgentHibernationController.maximumScopedProcessTerminationCount + 1 + let processIDs = Set(1...processCount) + let identities = Dictionary( + uniqueKeysWithValues: processIDs.map { processID in + ( + processID, + AgentPIDProcessIdentity( + pid: pid_t(processID), + startSeconds: Int64(processID), + startMicroseconds: 1 + ) + ) + } + ) + let didProbe = OSAllocatedUnfairLock(initialState: false) + + let terminations = AgentHibernationController + .validatedScopedProcessTerminations( + for: .init( + key: Self.signalScopeKey, + processIDs: processIDs, + processIdentities: identities + ), + processIdentityProvider: { _ in + didProbe.withLock { $0 = true } + return nil + }, + processGroupProvider: { _ in + didProbe.withLock { $0 = true } + return 0 + } + ) + + #expect(terminations == nil) + #expect(didProbe.withLock { $0 } == false) + } + + @Test + func boundsLateGenerationRefreshChurn() async { + let rootIdentity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let refreshCount = OSAllocatedUnfairLock(initialState: 0) + let waitedEpochCount = OSAllocatedUnfairLock(initialState: 0) + + let didExit = await AgentHibernationController + .waitForScopedProcessGenerationsToExitWithoutTimeout( + [ + .init( + processID: 101, + processIdentity: rootIdentity, + processGroupID: 101, + ttyDevice: 123 + ), + ], + processScopeKey: Self.signalScopeKey, + waitForExactEpoch: { _ in + waitedEpochCount.withLock { $0 += 1 } + return true + }, + nextEpochProvider: { processGroupLeaders, scopeKey, ttyDevice, _ in + #expect(scopeKey == Self.signalScopeKey) + #expect(ttyDevice == 123) + let refresh = refreshCount.withLock { + $0 += 1 + return $0 + } + let processID = 200 + refresh + return AgentHibernationProcessExitEpoch( + terminations: [ + .init( + processID: processID, + processIdentity: .init( + pid: pid_t(processID), + startSeconds: Int64(processID), + startMicroseconds: 1 + ), + processGroupID: 101, + ttyDevice: 123 + ), + ], + processGroupLeaders: processGroupLeaders + ) + } + ) + + #expect(didExit == false) + #expect( + refreshCount.withLock { $0 } == + AgentHibernationController.maximumProcessExitEpochRefreshCount + ) + #expect( + waitedEpochCount.withLock { $0 } == + AgentHibernationController.maximumProcessExitEpochRefreshCount + 1 + ) + } +} diff --git a/cmuxTests/AgentHibernationProcessSnapshotCoordinatorTests.swift b/cmuxTests/AgentHibernationProcessSnapshotCoordinatorTests.swift new file mode 100644 index 000000000000..1f70902226f6 --- /dev/null +++ b/cmuxTests/AgentHibernationProcessSnapshotCoordinatorTests.swift @@ -0,0 +1,164 @@ +import Darwin +import Foundation +import os +import Testing + +#if canImport(cmux_DEV) +@testable import cmux_DEV +#elseif canImport(cmux) +@testable import cmux +#endif + +@Suite(.serialized) +struct AgentHibernationProcessSnapshotCoordinatorTests { + @Test + func coalescesOneQueuedRefreshEpoch() async throws { + let captureScheduled = AsyncStream.makeStream() + let allowCapture = AsyncStream.makeStream() + let captureCount = OSAllocatedUnfairLock(initialState: 0) + let snapshot = CmuxTopProcessSnapshot( + processes: [], + sampledAt: .now, + includesProcessDetails: false + ) + let coordinator = AgentHibernationProcessSnapshotCoordinator( + beforeCapture: { + captureScheduled.continuation.yield() + for await _ in allowCapture.stream { return } + }, + captureSnapshot: { + captureCount.withLock { $0 += 1 } + return snapshot + } + ) + let first = Task { await coordinator.nextSnapshot() } + var captureScheduledIterator = captureScheduled.stream.makeAsyncIterator() + _ = await captureScheduledIterator.next() + let second = Task { await coordinator.nextSnapshot() } + let clock = ContinuousClock() + let registrationDeadline = clock.now.advanced(by: .seconds(1)) + while clock.now < registrationDeadline { + if await coordinator.queuedSnapshotWaiterCount == 2 { + break + } + await Task.yield() + } + #expect(await coordinator.queuedSnapshotWaiterCount == 2) + + allowCapture.continuation.yield() + allowCapture.continuation.finish() + let firstSnapshot = try #require(await first.value) + let secondSnapshot = try #require(await second.value) + + #expect(firstSnapshot === snapshot) + #expect(secondSnapshot === snapshot) + #expect(captureCount.withLock { $0 } == 1) + captureScheduled.continuation.finish() + } + + @Test + func cancelledWaiterDoesNotAwaitOrDuplicateCapture() async throws { + let captureScheduled = AsyncStream.makeStream() + let allowCapture = AsyncStream.makeStream() + let captureCount = OSAllocatedUnfairLock(initialState: 0) + let snapshot = CmuxTopProcessSnapshot( + processes: [], + sampledAt: .now, + includesProcessDetails: false + ) + let coordinator = AgentHibernationProcessSnapshotCoordinator( + beforeCapture: { + captureScheduled.continuation.yield() + for await _ in allowCapture.stream { return } + }, + captureSnapshot: { + captureCount.withLock { $0 += 1 } + return snapshot + } + ) + let cancelledRequest = Task { await coordinator.nextSnapshot() } + var captureScheduledIterator = captureScheduled.stream.makeAsyncIterator() + _ = await captureScheduledIterator.next() + + cancelledRequest.cancel() + #expect(await cancelledRequest.value == nil) + #expect(captureCount.withLock { $0 } == 0) + + allowCapture.continuation.yield() + allowCapture.continuation.finish() + let nextSnapshot = try #require(await coordinator.nextSnapshot()) + + #expect(nextSnapshot === snapshot) + #expect(captureCount.withLock { $0 } == 1) + captureScheduled.continuation.finish() + } + + @Test + func rejectsLateFanoutBeforePerCandidateProbes() async { + let scopeKey = AgentHibernationPanelKey( + workspaceId: UUID(), + panelId: UUID() + ) + let ttyDevice = Int64(123) + let processGroupID = 101 + let processes = (101...133).map { processID in + CmuxTopProcessInfo( + pid: processID, + parentPID: 1, + name: "test", + path: nil, + ttyDevice: ttyDevice, + cmuxWorkspaceID: nil, + cmuxSurfaceID: nil, + cmuxAttributionReason: nil, + processGroupID: processGroupID, + terminalProcessGroupID: processGroupID, + cpuPercent: 0, + residentBytes: 0, + virtualBytes: 0, + threadCount: 1 + ) + } + let snapshot = CmuxTopProcessSnapshot( + processes: processes, + sampledAt: .now, + includesProcessDetails: false, + includesCMUXScope: false + ) + let leaderIdentity = AgentPIDProcessIdentity( + pid: pid_t(processGroupID), + startSeconds: 10, + startMicroseconds: 1 + ) + let identityProbeIDs = OSAllocatedUnfairLock(initialState: [pid_t]()) + let argumentProbeCount = OSAllocatedUnfairLock(initialState: 0) + let processGroupProbeCount = OSAllocatedUnfairLock(initialState: 0) + let coordinator = AgentHibernationProcessSnapshotCoordinator( + captureSnapshot: { snapshot }, + processArgumentsProvider: { _ in + argumentProbeCount.withLock { $0 += 1 } + return nil + }, + processIdentityProvider: { processID in + identityProbeIDs.withLock { $0.append(processID) } + return processID == pid_t(processGroupID) ? leaderIdentity : nil + }, + processGroupProvider: { _ in + processGroupProbeCount.withLock { $0 += 1 } + return pid_t(processGroupID) + } + ) + + let epoch = await coordinator.refreshedExitEpoch( + processGroupLeaders: [pid_t(processGroupID): leaderIdentity], + processScopeKey: scopeKey, + ttyDevice: ttyDevice, + excluding: [] + ) + + #expect(epoch == nil) + #expect(identityProbeIDs.withLock { $0 } == [pid_t(processGroupID)]) + #expect(argumentProbeCount.withLock { $0 } == 0) + #expect(processGroupProbeCount.withLock { $0 } == 0) + } +} diff --git a/cmuxTests/AgentHibernationProcessTerminationTests.swift b/cmuxTests/AgentHibernationProcessTerminationTests.swift new file mode 100644 index 000000000000..1c645a80e6d9 --- /dev/null +++ b/cmuxTests/AgentHibernationProcessTerminationTests.swift @@ -0,0 +1,957 @@ +import Darwin +import Foundation +import os +import Testing + +#if canImport(cmux_DEV) +@testable import cmux_DEV +#elseif canImport(cmux) +@testable import cmux +#endif + +@Suite(.serialized) +struct AgentHibernationProcessTerminationTests { + private nonisolated static let signalScopeKey = AgentHibernationPanelKey( + workspaceId: UUID(), + panelId: UUID() + ) + private nonisolated static let signalScopeArguments = CmuxTopProcessArguments( + arguments: ["/usr/bin/test-agent"], + environment: [ + "CMUX_WORKSPACE_ID": signalScopeKey.workspaceId.uuidString, + "CMUX_SURFACE_ID": signalScopeKey.panelId.uuidString, + ] + ) + + private actor CompletionRecorder { + private(set) var value: Bool? + + func record(_ value: Bool) { + self.value = value + } + } + + @Test + func processScopeRejectsPanelScopedSiblingOutsideAgentSubtree() { + let workspaceID = UUID() + let panelID = UUID() + let ttyDevice = Int64(0x123) + let process: (Int, Int, Bool) -> CmuxTopProcessInfo = { + processID, parentProcessID, isEnvironmentScoped in + CmuxTopProcessInfo( + pid: processID, parentPID: parentProcessID, name: "test", path: nil, + ttyDevice: ttyDevice, + cmuxWorkspaceID: isEnvironmentScoped ? workspaceID : nil, + cmuxSurfaceID: isEnvironmentScoped ? panelID : nil, + cmuxAttributionReason: isEnvironmentScoped ? "environment" : nil, + processGroupID: nil, + terminalProcessGroupID: nil, cpuPercent: 0, residentBytes: 0, + virtualBytes: 0, threadCount: 1 + ) + } + let snapshot = CmuxTopProcessSnapshot( + processes: [ + process(10, 1, true), + process(20, 10, true), + process(21, 20, false), + process(30, 10, false), + ], + sampledAt: .now, + includesProcessDetails: true + ) + + let scope = snapshot.agentHibernationProcessScope( + panelProcessIDs: [10, 20], + agentProcessIDs: [20] + ) + + #expect(scope.terminationProcessIDs == [20, 21]) + #expect(scope.containsUnrelatedProcess) + } + + @Test + func freshExitEpochProbesOnlyTTYAndAuthorizedGroupCandidates() async throws { + let ttyDevice = Int64(0x123) + let process: (Int, Int64, Int) -> CmuxTopProcessInfo = { + processID, processTTYDevice, processGroupID in + CmuxTopProcessInfo( + pid: processID, + parentPID: 1, + name: "test", + path: nil, + ttyDevice: processTTYDevice, + cmuxWorkspaceID: nil, + cmuxSurfaceID: nil, + cmuxAttributionReason: nil, + processGroupID: processGroupID, + terminalProcessGroupID: processGroupID, + cpuPercent: 0, + residentBytes: 0, + virtualBytes: 0, + threadCount: 1 + ) + } + let snapshot = CmuxTopProcessSnapshot( + processes: [ + process(101, ttyDevice, 101), + process(202, ttyDevice, 202), + process(303, 0x999, 303), + ], + sampledAt: .now, + includesProcessDetails: false, + includesCMUXScope: false + ) + let rootIdentity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let lateIdentity = AgentPIDProcessIdentity( + pid: 202, + startSeconds: 20, + startMicroseconds: 2 + ) + let unrelatedIdentity = AgentPIDProcessIdentity( + pid: 303, + startSeconds: 30, + startMicroseconds: 3 + ) + let identities = [ + pid_t(101): rootIdentity, + pid_t(202): lateIdentity, + pid_t(303): unrelatedIdentity, + ] + let probedProcessIDs = OSAllocatedUnfairLock(initialState: Set()) + let coordinator = AgentHibernationProcessSnapshotCoordinator( + captureSnapshot: { snapshot }, + processArgumentsProvider: { processID in + probedProcessIDs.withLock { $0.insert(processID) } + return Self.signalScopeArguments + }, + processIdentityProvider: { identities[$0] }, + processGroupProvider: { $0 } + ) + + let epoch = try #require( + await coordinator.refreshedExitEpoch( + processGroupLeaders: [101: rootIdentity], + processScopeKey: Self.signalScopeKey, + ttyDevice: ttyDevice, + excluding: [] + ) + ) + + #expect(Set(epoch.terminations.map(\.processID)) == [101, 202]) + #expect(epoch.signalableProcessIdentities == [rootIdentity]) + #expect(probedProcessIDs.withLock { $0 } == [101, 202]) + } + + @MainActor + @Test + func terminationSignalsValidatedProcessGroupWithoutRedundantPIDSignal() async { + let identity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let signaledTargets = OSAllocatedUnfairLock(initialState: [pid_t]()) + + let result = await AgentHibernationController + .terminateScopedProcessesForHibernation( + [ + .init( + processID: 101, + processIdentity: identity, + processGroupID: 101, + ttyDevice: 123 + ), + ], + processScopeKey: Self.signalScopeKey, + currentProcessID: 999, + currentProcessGroupID: 999, + processIdentityProvider: { _ in identity }, + processGroupProvider: { _ in 101 }, + processArgumentsProvider: { _ in Self.signalScopeArguments }, + processTTYDeviceProvider: { _ in 123 }, + signalErrorProvider: { target, _ in + signaledTargets.withLock { $0.append(target) } + return nil + } + ) + + #expect(result == .committedAwaitingExit) + #expect(signaledTargets.withLock { $0 } == [-101]) + + let allowExit = AsyncStream.makeStream() + let escalatedTargets = OSAllocatedUnfairLock(initialState: [pid_t]()) + let deadlineCallCount = OSAllocatedUnfairLock(initialState: 0) + let postKillDeadline = AsyncStream.makeStream() + let lateIdentity = AgentPIDProcessIdentity( + pid: 202, + startSeconds: 20, + startMicroseconds: 2 + ) + let didExit = await AgentHibernationController + .waitForScopedProcessGenerationsToExitAfterEscalation( + [.init(processID: 101, processIdentity: identity, processGroupID: 101)], + waitForExit: { _ in + for await _ in allowExit.stream { return true } + return false + }, + sleepUntilDeadline: { _ in + let call = deadlineCallCount.withLock { + $0 += 1 + return $0 + } + guard call > 1 else { return true } + for await _ in postKillDeadline.stream { return true } + return false + }, + processIdentityProvider: { processID in + switch processID { + case 101: identity + case 202: lateIdentity + default: nil + } + }, + processGroupProvider: { $0 }, + signalErrorProvider: { target, signal in + escalatedTargets.withLock { $0.append(target) } + #expect(signal == SIGKILL) + allowExit.continuation.yield() + allowExit.continuation.finish() + return nil + }, + nextEpochProvider: { processGroupLeaders, _, _, _ in + AgentHibernationProcessExitEpoch( + terminations: [ + .init( + processID: 101, + processIdentity: identity, + processGroupID: 101 + ), + .init( + processID: 202, + processIdentity: lateIdentity, + processGroupID: 202 + ), + ], + processGroupLeaders: processGroupLeaders, + signalableProcessIdentities: [identity] + ) + } + ) + #expect(didExit) + #expect(escalatedTargets.withLock { $0 } == [-101]) + postKillDeadline.continuation.finish() + } + + @MainActor + @Test + func mutableCommitGateRunsAfterBlockingProcessProbes() async { + let identity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let didProbe = OSAllocatedUnfairLock(initialState: false) + let signaledTargets = OSAllocatedUnfairLock(initialState: [pid_t]()) + + let result = await AgentHibernationController + .terminateScopedProcessesForHibernation( + [ + .init( + processID: 101, + processIdentity: identity, + processGroupID: 101, + ttyDevice: 123 + ), + ], + processScopeKey: Self.signalScopeKey, + shouldCommit: { + #expect(didProbe.withLock { $0 }) + return false + }, + currentProcessID: 999, + currentProcessGroupID: 999, + processIdentityProvider: { _ in identity }, + processGroupProvider: { _ in 101 }, + processArgumentsProvider: { _ in + didProbe.withLock { $0 = true } + return Self.signalScopeArguments + }, + processTTYDeviceProvider: { _ in 123 }, + signalErrorProvider: { target, _ in + signaledTargets.withLock { $0.append(target) } + return nil + } + ) + + #expect(result == .rejected) + #expect(signaledTargets.withLock { $0 }.isEmpty) + } + + @Test + func sigkillEscalationRejectsFreshScopeDrift() async { + let identity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let termination = AgentHibernationController.ScopedProcessTermination( + processID: 101, + processIdentity: identity, + processGroupID: 101, + ttyDevice: 123 + ) + let exitWait = AsyncStream.makeStream() + let signaledTargets = OSAllocatedUnfairLock(initialState: [pid_t]()) + let wrongScopeArguments = CmuxTopProcessArguments( + arguments: ["/usr/bin/test-agent"], + environment: [ + "CMUX_WORKSPACE_ID": UUID().uuidString, + "CMUX_SURFACE_ID": UUID().uuidString, + ] + ) + + let didExit = await AgentHibernationController + .waitForScopedProcessGenerationsToExitAfterEscalation( + [termination], + processScopeKey: Self.signalScopeKey, + gracePeriod: .zero, + postKillExitPeriod: .zero, + waitForExit: { _ in + for await _ in exitWait.stream {} + return false + }, + sleepUntilDeadline: { _ in true }, + processIdentityProvider: { _ in identity }, + processGroupProvider: { _ in 101 }, + processArgumentsProvider: { _ in wrongScopeArguments }, + processTTYDeviceProvider: { _ in 123 }, + signalErrorProvider: { target, _ in + signaledTargets.withLock { $0.append(target) } + return nil + }, + nextEpochProvider: { processGroupLeaders, _, _, _ in + AgentHibernationProcessExitEpoch( + terminations: [termination], + processGroupLeaders: processGroupLeaders, + signalableProcessIdentities: [identity] + ) + } + ) + + #expect(didExit == false) + #expect(signaledTargets.withLock { $0 }.isEmpty) + exitWait.continuation.finish() + } + + @MainActor + @Test + func signalBatchFinishesBeforeCommittedUITransition() async { + let firstIdentity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let secondIdentity = AgentPIDProcessIdentity( + pid: 202, + startSeconds: 20, + startMicroseconds: 2 + ) + let events = OSAllocatedUnfairLock(initialState: [String]()) + + let result = await AgentHibernationController + .terminateScopedProcessesForHibernation( + [ + .init( + processID: 101, + processIdentity: firstIdentity, + processGroupID: 101, + ttyDevice: 123 + ), + .init( + processID: 202, + processIdentity: secondIdentity, + processGroupID: 202, + ttyDevice: 123 + ), + ], + processScopeKey: Self.signalScopeKey, + onTeardownCommit: { + events.withLock { $0.append("commit") } + }, + currentProcessID: 999, + currentProcessGroupID: 999, + processIdentityProvider: { pid in + pid == 101 ? firstIdentity : secondIdentity + }, + processGroupProvider: { $0 }, + processArgumentsProvider: { _ in Self.signalScopeArguments }, + processTTYDeviceProvider: { _ in 123 }, + signalErrorProvider: { target, _ in + events.withLock { $0.append("signal:\(target)") } + return nil + } + ) + + #expect(result == .committedAwaitingExit) + #expect(events.withLock { $0 } == ["signal:-101", "signal:-202", "commit"]) + } + + @Test + func processScopeIncludesRelatedWrapperGroupLeader() { + let workspaceID = UUID() + let panelID = UUID() + let ttyDevice = Int64(0x123) + let process: (Int, Int, Int) -> CmuxTopProcessInfo = { + processID, parentProcessID, processGroupID in + CmuxTopProcessInfo( + pid: processID, parentPID: parentProcessID, name: "test", path: nil, + ttyDevice: ttyDevice, + cmuxWorkspaceID: workspaceID, + cmuxSurfaceID: panelID, + cmuxAttributionReason: "environment", + processGroupID: processGroupID, + terminalProcessGroupID: processGroupID, cpuPercent: 0, residentBytes: 0, + virtualBytes: 0, threadCount: 1 + ) + } + let snapshot = CmuxTopProcessSnapshot( + processes: [ + process(100, 1, 100), + process(101, 100, 100), + process(102, 101, 100), + ], + sampledAt: .now, + includesProcessDetails: true + ) + + let scope = snapshot.agentHibernationProcessScope( + panelProcessIDs: [100, 101, 102], + agentProcessIDs: [101] + ) + + #expect(scope.terminationProcessIDs == [100, 101, 102]) + #expect(scope.containsUnrelatedProcess == false) + } + + @Test + func validatesExactProcessGeneration() throws { + let workspaceID = UUID() + let panelID = UUID() + let firstIdentity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let secondIdentity = AgentPIDProcessIdentity( + pid: 202, + startSeconds: 20, + startMicroseconds: 2 + ) + let identities = [101: firstIdentity, 202: secondIdentity] + let scope = AgentHibernationController.ProcessTerminationScope( + key: AgentHibernationPanelKey(workspaceId: workspaceID, panelId: panelID), + processIDs: Set(identities.keys), + processIdentities: identities + ) + + let terminations = try #require( + AgentHibernationController.validatedScopedProcessTerminations( + for: scope, + processIdentityProvider: { identities[$0] }, + processGroupProvider: { pid_t($0 + 1_000) } + ) + ) + + #expect( + terminations == [ + .init( + processID: 202, + processIdentity: secondIdentity, + processGroupID: 1_202 + ), + .init( + processID: 101, + processIdentity: firstIdentity, + processGroupID: 1_101 + ), + ] + ) + } + + @Test + func rejectsReusedProcessIdentity() { + let workspaceID = UUID() + let panelID = UUID() + let capturedIdentity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let scope = AgentHibernationController.ProcessTerminationScope( + key: AgentHibernationPanelKey(workspaceId: workspaceID, panelId: panelID), + processIDs: [101], + processIdentities: [101: capturedIdentity] + ) + + let terminations = AgentHibernationController.validatedScopedProcessTerminations( + for: scope, + processIdentityProvider: { _ in + AgentPIDProcessIdentity( + pid: 101, + startSeconds: 11, + startMicroseconds: 0 + ) + }, + processGroupProvider: { _ in 1_101 } + ) + + #expect(terminations == nil) + } + + @Test + func rejectsScopeWithUnrecordedProcessIdentity() { + let workspaceID = UUID() + let panelID = UUID() + let identity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let scope = AgentHibernationController.ProcessTerminationScope( + key: AgentHibernationPanelKey(workspaceId: workspaceID, panelId: panelID), + processIDs: [101, 202], + processIdentities: [101: identity] + ) + + let terminations = AgentHibernationController.validatedScopedProcessTerminations( + for: scope, + processIdentityProvider: { _ in identity }, + processGroupProvider: { _ in 1_101 } + ) + + #expect(terminations == nil) + } + + @MainActor + @Test + func committedObservationDoesNotCompleteUntilTheExactProcessGenerationExits() async { + let identity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let termination = AgentHibernationController.ScopedProcessTermination( + processID: 101, + processIdentity: identity, + processGroupID: 101, + ttyDevice: 123 + ) + let waitStarted = AsyncStream.makeStream() + let allowExit = AsyncStream.makeStream() + let controller = AgentHibernationController.shared + let panelID = UUID() + var didComplete = false + controller.observeCommittedTermination( + panelID: panelID, + terminations: [termination], + waitForExit: { _ in + waitStarted.continuation.yield() + for await _ in allowExit.stream { + break + } + return true + }, + onExit: { + didComplete = true + return true + }, + onRecovery: {} + ) + let task = controller.committedTerminationObservationsByPanelID[panelID]?.task + var waitStartedIterator = waitStarted.stream.makeAsyncIterator() + _ = await waitStartedIterator.next() + + #expect(!didComplete) + + allowExit.continuation.yield() + allowExit.continuation.finish() + await task?.value + #expect(didComplete) + waitStarted.continuation.finish() + } + + @MainActor + @Test + func signalFailureOtherThanMissingProcessAbortsBeforeCommit() async { + let identity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let termination = AgentHibernationController.ScopedProcessTermination( + processID: 101, + processIdentity: identity, + processGroupID: 101 + ) + + let result = await AgentHibernationController + .terminateScopedProcessesForHibernation( + [termination], + processScopeKey: Self.signalScopeKey, + currentProcessID: 999, + currentProcessGroupID: 999, + processIdentityProvider: { _ in identity }, + processGroupProvider: { _ in 101 }, + processArgumentsProvider: { _ in Self.signalScopeArguments }, + processTTYDeviceProvider: { _ in 123 }, + signalErrorProvider: { _, _ in EPERM } + ) + + #expect(result == .rejected) + } + + @MainActor + @Test + func signalFailureAfterTeardownCommitStillWaitsForOriginalProcesses() async { + let firstIdentity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let secondIdentity = AgentPIDProcessIdentity( + pid: 202, + startSeconds: 20, + startMicroseconds: 2 + ) + let terminations = [ + AgentHibernationController.ScopedProcessTermination( + processID: 101, + processIdentity: firstIdentity, + processGroupID: 101, + ttyDevice: 123 + ), + AgentHibernationController.ScopedProcessTermination( + processID: 202, + processIdentity: secondIdentity, + processGroupID: 202, + ttyDevice: 123 + ), + ] + let waitRecorder = CompletionRecorder() + + let controller = AgentHibernationController.shared + let result = await AgentHibernationController + .terminateScopedProcessesForHibernation( + terminations, + processScopeKey: Self.signalScopeKey, + currentProcessID: 999, + currentProcessGroupID: 999, + processIdentityProvider: { pid in + pid == 101 ? firstIdentity : secondIdentity + }, + processGroupProvider: { pid in + pid + }, + processArgumentsProvider: { _ in Self.signalScopeArguments }, + processTTYDeviceProvider: { _ in 123 }, + signalErrorProvider: { target, _ in + target == -101 || target == 101 ? nil : EPERM + } + ) + let panelID = UUID() + controller.observeCommittedTermination( + panelID: panelID, + terminations: terminations, + waitForExit: { observedTerminations in + await waitRecorder.record(observedTerminations == terminations) + return observedTerminations == terminations + }, + onExit: { true }, + onRecovery: {} + ) + let observationTask = controller + .committedTerminationObservationsByPanelID[panelID]? + .task + await observationTask?.value + #expect(await waitRecorder.value == true) + #expect(result == .committedAwaitingExit) + } + + @MainActor + @Test + func resumeStaysUnavailableUntilCommittedTerminationCompletes() { + let panel = TerminalPanel(workspaceId: UUID()) + defer { panel.close() } + let agent = SessionRestorableAgentSnapshot( + kind: .custom("test-agent"), + sessionId: "committed-hibernation", + workingDirectory: "/tmp", + launchCommand: nil + ) + + panel.beginAgentHibernationTermination( + agent: agent, + lastActivityAt: Date(timeIntervalSince1970: 1) + ) + + #expect(panel.isAgentHibernated) + #expect(panel.isAgentHibernationTerminating) + #expect(panel.prepareAgentHibernationResume() == .unavailable) + + panel.completeAgentHibernationTermination() + + #expect(!panel.isAgentHibernationTerminating) + #expect(panel.prepareAgentHibernationResume() == .resumed(queuedStartupInput: false)) + #expect(!panel.isAgentHibernated) + } + + @MainActor + @Test + func exitDeadlineDoesNotBlockLaterWorkWhileCommittedPanelAwaitsExit() async { + let identity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let termination = AgentHibernationController.ScopedProcessTermination( + processID: 101, + processIdentity: identity, + processGroupID: 101, + ttyDevice: 123 + ) + let panel = TerminalPanel(workspaceId: UUID()) + defer { panel.close() } + let agent = SessionRestorableAgentSnapshot( + kind: .custom("test-agent"), + sessionId: "eventual-exit-hibernation", + workingDirectory: "/tmp", + launchCommand: nil + ) + let eventualWaitStarted = AsyncStream.makeStream() + let allowEventualExit = AsyncStream.makeStream() + let controller = AgentHibernationController.shared + let result = await AgentHibernationController.terminateScopedProcessesForHibernation( + [termination], + processScopeKey: Self.signalScopeKey, + onTeardownCommit: { [weak panel] in + panel?.beginAgentHibernationTermination( + agent: agent, + lastActivityAt: Date(timeIntervalSince1970: 1) + ) + }, + currentProcessID: 999, + currentProcessGroupID: 999, + processIdentityProvider: { _ in identity }, + processGroupProvider: { _ in 101 }, + processArgumentsProvider: { _ in Self.signalScopeArguments }, + processTTYDeviceProvider: { _ in 123 }, + signalErrorProvider: { _, _ in nil } + ) + #expect(result == .committedAwaitingExit) + controller.observeCommittedTermination( + panelID: panel.id, + terminations: [termination], + waitForExit: { _ in + eventualWaitStarted.continuation.yield() + for await _ in allowEventualExit.stream { + break + } + return true + }, + onExit: { + panel.completeAgentHibernationTermination() + return true + }, + onRecovery: { + panel.completeAgentHibernationTermination() + } + ) + let observationTask = controller + .committedTerminationObservationsByPanelID[panel.id]? + .task + var eventualWaitStartedIterator = eventualWaitStarted.stream.makeAsyncIterator() + _ = await eventualWaitStartedIterator.next() + + #expect(panel.isAgentHibernationTerminating) + #expect(panel.prepareAgentHibernationResume() == .unavailable) + + allowEventualExit.continuation.yield() + allowEventualExit.continuation.finish() + await observationTask?.value + + #expect(!panel.isAgentHibernationTerminating) + #expect(panel.isAgentHibernated) + eventualWaitStarted.continuation.finish() + } + + @Test + func aReusedPIDMeansTheOriginalProcessGenerationExited() async { + let originalIdentity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let reusedIdentity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 11, + startMicroseconds: 0 + ) + + let didExit = await AgentHibernationController + .waitForExactProcessGenerationsToExitWithoutTimeout( + [ + .init( + processID: 101, + processIdentity: originalIdentity, + processGroupID: 1 + ), + ], + processIdentityProvider: { _ in reusedIdentity } + ) + + #expect(didExit) + } + + @Test + func scopedExitWaitIncludesLateProcessInAuthorizedGroup() async { + let processScopeKey = AgentHibernationPanelKey( + workspaceId: UUID(), + panelId: UUID() + ) + let rootIdentity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let lateIdentity = AgentPIDProcessIdentity( + pid: 202, + startSeconds: 20, + startMicroseconds: 2 + ) + let nextEpochCount = OSAllocatedUnfairLock(initialState: 0) + let waitCompleted = OSAllocatedUnfairLock(initialState: false) + let lateEpochStarted = AsyncStream.makeStream() + let allowLateExit = AsyncStream.makeStream() + let wait = Task { + let didExit = await AgentHibernationController + .waitForScopedProcessGenerationsToExitWithoutTimeout( + [ + .init( + processID: 101, + processIdentity: rootIdentity, + processGroupID: 101, + ttyDevice: 123 + ), + ], + processScopeKey: processScopeKey, + waitForExactEpoch: { epoch in + guard epoch.contains(where: { + $0.processIdentity == lateIdentity + }) else { + return true + } + lateEpochStarted.continuation.yield() + for await _ in allowLateExit.stream { return true } + return false + }, + nextEpochProvider: { + processGroupLeaders, + receivedScopeKey, + receivedTTYDevice, + exitedIdentities in + #expect(receivedScopeKey == processScopeKey) + #expect(receivedTTYDevice == 123) + let call = nextEpochCount.withLock { + $0 += 1 + return $0 + } + guard call == 1 else { + #expect(processGroupLeaders[101] == rootIdentity) + #expect(exitedIdentities == [rootIdentity, lateIdentity]) + return AgentHibernationProcessExitEpoch( + terminations: [], + processGroupLeaders: [:] + ) + } + #expect(exitedIdentities == [rootIdentity]) + return AgentHibernationProcessExitEpoch( + terminations: [ + .init( + processID: 202, + processIdentity: lateIdentity, + processGroupID: 101 + ), + ], + processGroupLeaders: [101: rootIdentity] + ) + } + ) + waitCompleted.withLock { $0 = true } + return didExit + } + var lateEpochIterator = lateEpochStarted.stream.makeAsyncIterator() + _ = await lateEpochIterator.next() + + #expect(waitCompleted.withLock { $0 } == false) + + allowLateExit.continuation.yield() + allowLateExit.continuation.finish() + #expect(await wait.value) + lateEpochStarted.continuation.finish() + } + + @Test + func scopedExitWaitObservesLateProcessWithoutAuthorizingItsGroup() async { + let rootIdentity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let lateIdentity = AgentPIDProcessIdentity( + pid: 202, + startSeconds: 20, + startMicroseconds: 2 + ) + + let refreshCount = OSAllocatedUnfairLock(initialState: 0) + let didExit = await AgentHibernationController + .waitForScopedProcessGenerationsToExitWithoutTimeout( + [ + .init( + processID: 101, + processIdentity: rootIdentity, + processGroupID: 101 + ), + ], + waitForExactEpoch: { _ in true }, + nextEpochProvider: { processGroupLeaders, _, _, _ in + let refresh = refreshCount.withLock { + $0 += 1 + return $0 + } + guard refresh == 1 else { + return AgentHibernationProcessExitEpoch( + terminations: [], + processGroupLeaders: processGroupLeaders + ) + } + return AgentHibernationProcessExitEpoch( + terminations: [ + .init( + processID: 202, + processIdentity: lateIdentity, + processGroupID: 202 + ), + ], + processGroupLeaders: processGroupLeaders + ) + } + ) + + #expect(didExit) + } +} diff --git a/cmuxTests/AgentHibernationRestoreMonitorTests.swift b/cmuxTests/AgentHibernationRestoreMonitorTests.swift index 51c9298be560..8dc8a4186379 100644 --- a/cmuxTests/AgentHibernationRestoreMonitorTests.swift +++ b/cmuxTests/AgentHibernationRestoreMonitorTests.swift @@ -117,6 +117,51 @@ struct AgentHibernationRestoreMonitorTests { ) == false) } + @MainActor + @Test + func lateAbortReleasesArmedMonitorAndDisposesItsSnapshot() async throws { + let controller = AgentHibernationController.shared + defer { resetSharedHibernationState(controller) } + + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("cmux-hibernation-late-abort-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + + let live = directory.appendingPathComponent("live.jsonl") + let snapshotURL = directory.appendingPathComponent("snapshot.jsonl") + let content = #"{"type":"user","message":{"content":"protected"}}"# + "\n" + try content.write(to: live, atomically: true, encoding: .utf8) + try content.write(to: snapshotURL, atomically: true, encoding: .utf8) + let snapshot = try #require( + AgentHibernationTranscriptGuard.snapshotStillMatchesLive( + .init( + transcriptPath: live.path, + snapshotPath: snapshotURL.path + ) + ) + ) + var restoreOwnedSnapshotPaths: Set = [snapshot.snapshotPath] + #expect(controller.armPostTeardownRestoreMonitor( + snapshot: snapshot, + processIDs: [] + )) + + await controller.releaseArmedRestoreMonitorAfterAbortedTeardown( + snapshot, + sessionId: "late-abort", + restoreOwnedSnapshotPaths: &restoreOwnedSnapshotPaths + ) + + let monitorKey = AgentHibernationController.postTeardownRestoreTaskKey( + transcriptPath: live.path + ) + #expect(controller.postTeardownRestoreTasksByTranscriptPath[monitorKey] == nil) + #expect(restoreOwnedSnapshotPaths.isEmpty) + #expect(FileManager.default.fileExists(atPath: snapshotURL.path) == false) + #expect(try String(contentsOf: live, encoding: .utf8) == content) + } + @MainActor @Test func armedForfeitMonitorRestoresClobberedTranscriptAndRefusesDuplicates() async throws { @@ -226,6 +271,83 @@ struct AgentHibernationRestoreMonitorTests { #expect(try String(contentsOf: retained, encoding: .utf8) == snapshotContent) } + @Test + func committedMonitorDefersRestoreChecksUntilExactProcessExit() async throws { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("cmux-hibernation-exact-exit-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + + let live = directory.appendingPathComponent("live.jsonl") + let snapshot = directory.appendingPathComponent("snapshot.jsonl") + let metadataStub = #"{"type":"last-prompt","prompt":"continue"}"# + "\n" + let snapshotContent = #"{"type":"user","message":{"content":"protected"}}"# + "\n" + try metadataStub.write(to: live, atomically: true, encoding: .utf8) + try snapshotContent.write(to: snapshot, atomically: true, encoding: .utf8) + let waitStarted = AsyncStream.makeStream() + let allowExit = AsyncStream.makeStream() + + let task = Task { + await AgentHibernationTranscriptGuard.runPostTeardownRestoreChecks( + snapshot: .init(transcriptPath: live.path, snapshotPath: snapshot.path), + processIDs: [101], + initialRetryDelaysNanoseconds: [0], + backstopDelaysSeconds: [], + awaitProcessExit: { + waitStarted.continuation.yield() + for await _ in allowExit.stream { + break + } + return true + } + ) + } + var waitStartedIterator = waitStarted.stream.makeAsyncIterator() + _ = await waitStartedIterator.next() + + #expect(try String(contentsOf: live, encoding: .utf8) == metadataStub) + #expect(FileManager.default.fileExists(atPath: snapshot.path)) + + allowExit.continuation.yield() + allowExit.continuation.finish() + await task.value + #expect(try String(contentsOf: live, encoding: .utf8).hasPrefix(snapshotContent)) + #expect(FileManager.default.fileExists(atPath: snapshot.path) == false) + waitStarted.continuation.finish() + } + + @Test + func cappedExitObservationStillRunsTranscriptRecoveryChecks() async throws { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent( + "cmux-hibernation-capped-exit-\(UUID().uuidString)", + isDirectory: true + ) + try FileManager.default.createDirectory( + at: directory, + withIntermediateDirectories: true + ) + defer { try? FileManager.default.removeItem(at: directory) } + + let live = directory.appendingPathComponent("live.jsonl") + let snapshot = directory.appendingPathComponent("snapshot.jsonl") + let metadataStub = #"{"type":"last-prompt","prompt":"continue"}"# + "\n" + let snapshotContent = #"{"type":"user","message":{"content":"protected"}}"# + "\n" + try metadataStub.write(to: live, atomically: true, encoding: .utf8) + try snapshotContent.write(to: snapshot, atomically: true, encoding: .utf8) + + await AgentHibernationTranscriptGuard.runPostTeardownRestoreChecks( + snapshot: .init(transcriptPath: live.path, snapshotPath: snapshot.path), + processIDs: [101], + initialRetryDelaysNanoseconds: [0], + backstopDelaysSeconds: [], + awaitProcessExit: { false } + ) + + #expect(try String(contentsOf: live, encoding: .utf8).hasPrefix(snapshotContent)) + #expect(FileManager.default.fileExists(atPath: snapshot.path) == false) + } + @MainActor @Test func monitorKeyResolvesSymlinkedTranscriptPathAliases() throws { @@ -272,6 +394,54 @@ struct AgentHibernationRestoreMonitorTests { #expect(FileManager.default.fileExists(atPath: snapshot.path)) } + @MainActor + @Test + func committedTerminationRecoveryKeepsRestoreMonitorArmedUntilExactExit() async throws { + let controller = AgentHibernationController.shared + defer { resetSharedHibernationState(controller) } + + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("cmux-hibernation-termination-failure-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + + let live = directory.appendingPathComponent("live.jsonl") + let snapshotURL = directory.appendingPathComponent("snapshot.jsonl") + let protectedContent = #"{"type":"user","message":{"content":"protected"}}"# + "\n" + let clobberedContent = #"{"type":"last-prompt","prompt":"continue"}"# + "\n" + try protectedContent.write(to: live, atomically: true, encoding: .utf8) + try protectedContent.write(to: snapshotURL, atomically: true, encoding: .utf8) + let snapshot = AgentHibernationTranscriptGuard.TeardownTranscriptSnapshot( + transcriptPath: live.path, + snapshotPath: snapshotURL.path + ) + let processExit = AsyncStream.makeStream() + #expect(controller.armPostTeardownRestoreMonitor( + snapshot: snapshot, + processIDs: [101], + awaitProcessExit: { + for await _ in processExit.stream { return true } + return false + } + )) + let monitor = try #require( + controller.postTeardownRestoreTasksByTranscriptPath.values.first?.task + ) + + try clobberedContent.write(to: live, atomically: true, encoding: .utf8) + + #expect(controller.postTeardownRestoreTasksByTranscriptPath.isEmpty == false) + #expect(FileManager.default.fileExists(atPath: snapshotURL.path)) + + processExit.continuation.yield() + processExit.continuation.finish() + await monitor.value + + #expect(controller.postTeardownRestoreTasksByTranscriptPath.isEmpty) + #expect(try String(contentsOf: live, encoding: .utf8).hasPrefix(protectedContent)) + #expect(FileManager.default.fileExists(atPath: snapshotURL.path) == false) + } + @MainActor private func restoreTask( live: URL, diff --git a/cmuxTests/AgentHibernationTerminationFailureTests.swift b/cmuxTests/AgentHibernationTerminationFailureTests.swift new file mode 100644 index 000000000000..6c75d99bfdbd --- /dev/null +++ b/cmuxTests/AgentHibernationTerminationFailureTests.swift @@ -0,0 +1,355 @@ +import Darwin +import Foundation +import os +import Testing + +#if canImport(cmux_DEV) +@testable import cmux_DEV +#elseif canImport(cmux) +@testable import cmux +#endif + +@Suite(.serialized) +struct AgentHibernationTerminationFailureTests { + @Test(arguments: [Int32?.none, Int32(EPERM)]) + func escalationFailureOrPostKillDeadlineFailsClosed(signalError: Int32?) async { + let identity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let processExit = AsyncStream.makeStream() + + let didExit = await AgentHibernationController + .waitForScopedProcessGenerationsToExitAfterEscalation( + [.init(processID: 101, processIdentity: identity, processGroupID: 1)], + gracePeriod: .zero, + postKillExitPeriod: .zero, + waitForExit: { _ in + for await _ in processExit.stream { return true } + return false + }, + sleepUntilDeadline: { _ in true }, + processIdentityProvider: { _ in identity }, + signalErrorProvider: { _, _ in signalError } + ) + + #expect(didExit == false) + processExit.continuation.finish() + } + + @Test + func emptyRefreshDoesNotProveOriginalGenerationExited() async { + let identity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let processExit = AsyncStream.makeStream() + + let didExit = await AgentHibernationController + .waitForScopedProcessGenerationsToExitAfterEscalation( + [ + .init( + processID: 101, + processIdentity: identity, + processGroupID: 101, + ttyDevice: 123 + ), + ], + gracePeriod: .zero, + waitForExit: { _ in + for await _ in processExit.stream { return true } + return false + }, + sleepUntilDeadline: { _ in true }, + processIdentityProvider: { _ in identity }, + processGroupProvider: { _ in 101 }, + nextEpochProvider: { processGroupLeaders, _, _, _ in + AgentHibernationProcessExitEpoch( + terminations: [], + processGroupLeaders: processGroupLeaders + ) + } + ) + + #expect(didExit == false) + processExit.continuation.finish() + } + + @MainActor + @Test + func failedCommittedTerminationRecoversAfterExactExit() async { + let panel = TerminalPanel(workspaceId: UUID()) + defer { panel.close() } + let agent = SessionRestorableAgentSnapshot( + kind: .custom("test-agent"), + sessionId: "failed-termination", + workingDirectory: "/tmp", + launchCommand: nil + ) + panel.beginAgentHibernationTermination( + agent: agent, + lastActivityAt: Date(timeIntervalSince1970: 1) + ) + let controller = AgentHibernationController.shared + let recoveryExit = AsyncStream.makeStream() + controller.observeCommittedTermination( + panelID: panel.id, + terminations: [], + waitForExit: { _ in false }, + waitForRecoveryExit: { _ in + for await _ in recoveryExit.stream { return true } + return false + }, + onExit: { true }, + onFailure: { + panel.beginAgentHibernationTerminationRecovery() + }, + onRecovery: { + panel.completeAgentHibernationTermination() + } + ) + let observation = controller.committedTerminationObservationsByPanelID[panel.id]?.task + + await observation?.value + + #expect(panel.agentHibernationTerminationFailed == false) + #expect(panel.isAgentHibernationCommitPending) + #expect(panel.prepareAgentHibernationResume() == .unavailable) + let recovery = controller.committedTerminationObservationsByPanelID[panel.id]?.task + #expect(recovery != nil) + + recoveryExit.continuation.yield() + recoveryExit.continuation.finish() + await recovery?.value + + #expect(panel.agentHibernationTerminationFailed == false) + #expect(controller.committedTerminationObservationsByPanelID[panel.id] == nil) + } + + @MainActor + @Test + func failedTerminationKeepsRecoveryUntilNativeTeardownCompletes() async { + let panel = TerminalPanel(workspaceId: UUID()) + defer { panel.close() } + panel.beginAgentHibernationTermination( + agent: .init( + kind: .custom("test-agent"), + sessionId: "delayed-native-teardown", + workingDirectory: "/tmp", + launchCommand: nil + ), + lastActivityAt: Date(timeIntervalSince1970: 1) + ) + let controller = AgentHibernationController.shared + let nativeTeardownCompleted = AsyncStream.makeStream() + controller.observeCommittedTermination( + panelID: panel.id, + terminations: [], + waitForExit: { _ in false }, + waitForRecoveryExit: { _ in true }, + waitForRecoveryReadiness: { + for await _ in nativeTeardownCompleted.stream { return true } + return false + }, + onExit: { true }, + onFailure: { + panel.beginAgentHibernationTerminationRecovery() + }, + onRecovery: { + panel.completeAgentHibernationTermination() + } + ) + let initialObservation = + controller.committedTerminationObservationsByPanelID[panel.id]?.task + await initialObservation?.value + + #expect(panel.agentHibernationTerminationFailed == false) + #expect(panel.isAgentHibernationCommitPending) + let recovery = controller.committedTerminationObservationsByPanelID[panel.id]?.task + #expect(recovery != nil) + + nativeTeardownCompleted.continuation.yield() + nativeTeardownCompleted.continuation.finish() + await recovery?.value + + #expect(panel.isAgentHibernated) + #expect(controller.committedTerminationObservationsByPanelID[panel.id] == nil) + } + + @MainActor + @Test + func failedNoProcessRecoveryRetrySkipsExitWaitAndSucceeds() async { + let panel = TerminalPanel(workspaceId: UUID()) + defer { panel.close() } + panel.beginAgentHibernationTermination( + agent: .init( + kind: .custom("test-agent"), + sessionId: "retryable-recovery", + workingDirectory: "/tmp", + launchCommand: nil + ), + lastActivityAt: Date(timeIntervalSince1970: 1) + ) + let controller = AgentHibernationController.shared + let readinessAttempts = OSAllocatedUnfairLock(initialState: 0) + let terminationRetryCount = OSAllocatedUnfairLock(initialState: 0) + controller.observeCommittedTermination( + panelID: panel.id, + terminations: [], + waitForExit: { _ in false }, + waitForRecoveryExit: { _ in true }, + retryTermination: { + terminationRetryCount.withLock { $0 += 1 } + return .exited + }, + waitForRecoveryReadiness: { + readinessAttempts.withLock { + $0 += 1 + return $0 > 1 + } + }, + onExit: { true }, + onFailure: { + panel.beginAgentHibernationTerminationRecovery() + }, + onRecovery: { + panel.completeAgentHibernationTermination() + }, + onRecoveryFailure: { + panel.failAgentHibernationTermination() + }, + onRecoveryRetry: { + panel.beginAgentHibernationTerminationRecovery() + } + ) + let initial = controller.committedTerminationObservationsByPanelID[panel.id]?.task + await initial?.value + let failedRecovery = + controller.committedTerminationObservationsByPanelID[panel.id]?.task + await failedRecovery?.value + + #expect(panel.agentHibernationTerminationFailed) + #expect(controller.committedTerminationObservationsByPanelID[panel.id] != nil) + + controller.retryCommittedTerminationRecovery(panelID: panel.id) + let retry = controller.committedTerminationObservationsByPanelID[panel.id]?.task + await retry?.value + + #expect(panel.isAgentHibernated) + #expect(controller.committedTerminationObservationsByPanelID[panel.id] == nil) + #expect(terminationRetryCount.withLock { $0 } == 1) + } + + @MainActor + @Test + func recoveryDeadlineExposesRetryableFailure() async { + let panel = TerminalPanel(workspaceId: UUID()) + defer { panel.close() } + panel.beginAgentHibernationTermination( + agent: .init( + kind: .custom("test-agent"), + sessionId: "recovery-deadline", + workingDirectory: "/tmp", + launchCommand: nil + ), + lastActivityAt: Date(timeIntervalSince1970: 1) + ) + let controller = AgentHibernationController.shared + let readiness = AsyncStream.makeStream() + controller.observeCommittedTermination( + panelID: panel.id, + terminations: [], + waitForExit: { _ in false }, + waitForRecoveryReadiness: { + for await _ in readiness.stream { return true } + return false + }, + recoveryDeadline: .zero, + sleepUntilRecoveryDeadline: { _ in true }, + onExit: { true }, + onFailure: { + panel.beginAgentHibernationTerminationRecovery() + }, + onRecovery: { + panel.completeAgentHibernationTermination() + }, + onRecoveryFailure: { + panel.failAgentHibernationTermination() + } + ) + let initial = controller.committedTerminationObservationsByPanelID[panel.id]?.task + await initial?.value + let recovery = controller.committedTerminationObservationsByPanelID[panel.id]?.task + await recovery?.value + + #expect(panel.agentHibernationTerminationFailed) + #expect(controller.committedTerminationObservationsByPanelID[panel.id] != nil) + readiness.continuation.finish() + } + + @MainActor + @Test + func recoveredFailureRunsWorkspaceHibernationCommit() throws { + let workspace = Workspace() + let panelID = try #require(workspace.focusedPanelId) + let panel = try #require(workspace.panels[panelID] as? TerminalPanel) + defer { panel.close() } + let agent = SessionRestorableAgentSnapshot( + kind: .codex, + sessionId: "recovered-workspace-commit", + workingDirectory: "/tmp", + launchCommand: nil + ) + try #require(agent.resumeCommand != nil) + panel.beginAgentHibernationTermination( + agent: agent, + lastActivityAt: Date(timeIntervalSince1970: 1) + ) + panel.beginAgentHibernationTerminationRecovery() + panel.failAgentHibernationTermination() + workspace.restoredAgentResumeStatesByPanelId[panelID] = .completedAgentExit + + workspace.enterAgentHibernation( + panelId: panelID, + agent: agent, + lastActivityAt: Date(timeIntervalSince1970: 1) + ) + + #expect(panel.agentHibernationTerminationFailed == false) + #expect(panel.isAgentHibernated) + #expect(workspace.restoredAgentSnapshotsByPanelId[panelID]?.sessionId == agent.sessionId) + #expect(workspace.restoredAgentResumeStatesByPanelId[panelID] == .manualResumeAvailable) + } + + @MainActor + @Test + func unknownProcessAbsenceIsUnsafeForPressureTeardown() throws { + let workspace = Workspace() + let panelID = try #require(workspace.focusedPanelId) + let panel = try #require(workspace.panels[panelID] as? TerminalPanel) + let record = AgentHibernationRecord( + key: .init(workspaceId: workspace.id, panelId: panelID), + workspace: workspace, + terminalPanel: panel, + agent: .init( + kind: .custom("test-agent"), + sessionId: "unknown-process-evidence", + workingDirectory: "/tmp", + launchCommand: nil + ), + lifecycle: .idle, + hasUnconfirmedTerminalInput: false, + lastActivityAt: 0, + isProtected: false, + hasLiveProcess: false, + containsUnrelatedProcess: false, + panelProcessIDs: [], + processIDs: [], + processIdentities: [:] + ) + + #expect(record.hasPressureSafeProcessEvidence == false) + } +} diff --git a/cmuxTests/AgentHibernationTrackingLifecycleTests.swift b/cmuxTests/AgentHibernationTrackingLifecycleTests.swift new file mode 100644 index 000000000000..7e6924ba6c7b --- /dev/null +++ b/cmuxTests/AgentHibernationTrackingLifecycleTests.swift @@ -0,0 +1,402 @@ +import Foundation +import Testing + +#if canImport(cmux_DEV) +@testable import cmux_DEV +#elseif canImport(cmux) +@testable import cmux +#endif + +@MainActor +@Suite(.serialized) +struct AgentHibernationTrackingLifecycleTests { + @Test + func permanentPanelTeardownPrunesOnlyTheClosedPanelTracking() throws { + let controller = AgentHibernationController.shared + let workspace = Workspace() + let panelID = try #require(workspace.focusedPanelId) + let closedKey = AgentHibernationPanelKey(workspaceId: workspace.id, panelId: panelID) + let retainedKey = AgentHibernationPanelKey(workspaceId: workspace.id, panelId: UUID()) + defer { + for key in [closedKey, retainedKey] { + controller.discardTrackingStateForClosedPanel( + workspaceId: key.workspaceId, + panelId: key.panelId + ) + } + } + + for key in [closedKey, retainedKey] { + controller.activityByPanel[key] = 1 + controller.terminalInputByPanel[key] = 2 + controller.lifecycleChangeByPanel[key] = 3 + controller.teardownValidationEpochByPanel[key] = 4 + } + + workspace.teardownAllPanels() + + #expect(workspace.panels[panelID] == nil) + #expect(controller.activityByPanel[closedKey] == nil) + #expect(controller.terminalInputByPanel[closedKey] == nil) + #expect(controller.lifecycleChangeByPanel[closedKey] == nil) + #expect(controller.teardownValidationEpochByPanel[closedKey] == nil) + #expect(controller.activityByPanel[retainedKey] == 1) + #expect(controller.terminalInputByPanel[retainedKey] == 2) + #expect(controller.lifecycleChangeByPanel[retainedKey] == 3) + #expect(controller.teardownValidationEpochByPanel[retainedKey] == 4) + } + + @Test + func detachedPanelPreservesTrackingUntilItsNewOwnerAdoptsIt() throws { + let controller = AgentHibernationController.shared + let workspace = Workspace() + let panelID = try #require(workspace.focusedPanelId) + let panel = try #require(workspace.panels[panelID]) + let key = AgentHibernationPanelKey(workspaceId: workspace.id, panelId: panelID) + defer { + panel.close() + controller.discardTrackingStateForClosedPanel( + workspaceId: key.workspaceId, + panelId: key.panelId + ) + } + controller.terminalInputByPanel[key] = 2 + controller.lifecycleChangeByPanel[key] = 1 + + workspace.discardClosedPanelLifecycleState( + panelId: panelID, + tabId: workspace.surfaceIdFromPanelId(panelID), + paneId: workspace.paneId(forPanelId: panelID), + panel: panel, + origin: "test_detach", + closePanel: false, + publishSurfaceClosedEvent: false, + clearSurfaceNotifications: false, + requestTransferredRemoteCleanup: false, + discardAgentHibernationTracking: false + ) + + #expect(controller.terminalInputByPanel[key] == 2) + #expect(controller.lifecycleChangeByPanel[key] == 1) + } + + @Test + func movedPanelTransfersUnconfirmedInputSafetyToItsNewOwner() { + let controller = AgentHibernationController.shared + let panelID = UUID() + let sourceKey = AgentHibernationPanelKey(workspaceId: UUID(), panelId: panelID) + let destinationKey = AgentHibernationPanelKey(workspaceId: UUID(), panelId: panelID) + defer { + for key in [sourceKey, destinationKey] { + controller.discardTrackingStateForClosedPanel( + workspaceId: key.workspaceId, + panelId: key.panelId + ) + } + } + controller.activityByPanel[sourceKey] = 1 + controller.terminalInputByPanel[sourceKey] = 3 + controller.lifecycleChangeByPanel[sourceKey] = 2 + controller.teardownValidationEpochByPanel[sourceKey] = 4 + + controller.transferTrackingStateForMovedPanel( + panelId: panelID, + from: sourceKey.workspaceId, + to: destinationKey.workspaceId + ) + + #expect(controller.activityByPanel[sourceKey] == nil) + #expect(controller.terminalInputByPanel[sourceKey] == nil) + #expect(controller.lifecycleChangeByPanel[sourceKey] == nil) + #expect(controller.teardownValidationEpochByPanel[sourceKey] == nil) + #expect(controller.activityByPanel[destinationKey] == 1) + #expect(controller.terminalInputByPanel[destinationKey] == 3) + #expect(controller.lifecycleChangeByPanel[destinationKey] == 2) + #expect(controller.teardownValidationEpochByPanel[destinationKey] == 5) + } + + @Test + func dockReconcilePrunesTrackingForPermanentlyClosedPanel() throws { + let controller = AgentHibernationController.shared + let store = DockSplitStore(workspaceId: UUID(), baseDirectoryProvider: { nil }) + defer { store.closeAllPanels() } + let paneID = try #require(store.bonsplitController.allPaneIds.first) + let panelID = try #require( + store.newSurface(kind: .terminal, inPane: paneID, focus: false) + ) + let tabID = try #require(store.surfaceId(forPanelId: panelID)) + let key = AgentHibernationPanelKey(workspaceId: store.workspaceId, panelId: panelID) + controller.activityByPanel[key] = 1 + controller.terminalInputByPanel[key] = 2 + controller.lifecycleChangeByPanel[key] = 3 + controller.teardownValidationEpochByPanel[key] = 4 + + store.forceCloseDockTabIds.insert(tabID) + defer { store.forceCloseDockTabIds.remove(tabID) } + #expect(store.bonsplitController.closeTab(tabID)) + store.reconcilePanels() + + #expect(store.panels[panelID] == nil) + expectTrackingWasDiscarded(for: key, controller: controller) + } + + @Test + func dockResetPrunesTrackingForEveryPermanentlyClosedPanel() throws { + let controller = AgentHibernationController.shared + let store = DockSplitStore(workspaceId: UUID(), baseDirectoryProvider: { nil }) + let paneID = try #require(store.bonsplitController.allPaneIds.first) + let panelID = try #require( + store.newSurface(kind: .terminal, inPane: paneID, focus: false) + ) + let key = AgentHibernationPanelKey(workspaceId: store.workspaceId, panelId: panelID) + controller.activityByPanel[key] = 1 + controller.terminalInputByPanel[key] = 2 + controller.lifecycleChangeByPanel[key] = 3 + controller.teardownValidationEpochByPanel[key] = 4 + + store.removeAllPanels() + + #expect(store.panels.isEmpty) + expectTrackingWasDiscarded(for: key, controller: controller) + } + + @Test + func permanentPanelCloseKeepsTranscriptExitObservationUntilExit() async { + let controller = AgentHibernationController.shared + let panel = TerminalPanel(workspaceId: UUID()) + let identity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let exitEvents = AsyncStream.makeStream() + var didCompleteHibernation = false + controller.observeCommittedTermination( + panelID: panel.id, + terminations: [ + .init( + processID: 101, + processIdentity: identity, + processGroupID: 1 + ), + ], + waitForExit: { _ in + for await _ in exitEvents.stream {} + return true + }, + onExit: { + didCompleteHibernation = true + return true + }, + onRecovery: {} + ) + let observationTask = controller + .committedTerminationObservationsByPanelID[panel.id]? + .task + + panel.close() + exitEvents.continuation.finish() + await observationTask?.value + + #expect(controller.committedTerminationObservationsByPanelID[panel.id] == nil) + #expect(didCompleteHibernation) + } + + @Test + func permanentPanelCloseBoundsTranscriptExitObservation() async { + let controller = AgentHibernationController.shared + let panel = TerminalPanel(workspaceId: UUID()) + let identity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let exitEvents = AsyncStream.makeStream() + let processExitCompletion = AgentHibernationProcessExitCompletion() + var didCompleteHibernation = false + controller.observeCommittedTermination( + panelID: panel.id, + terminations: [ + .init( + processID: 101, + processIdentity: identity, + processGroupID: 1 + ), + ], + processExitCompletion: processExitCompletion, + waitForExit: { _ in + for await _ in exitEvents.stream {} + return true + }, + onExit: { + didCompleteHibernation = true + return true + }, + onRecovery: {} + ) + + panel.close() + controller.limitCommittedTerminationObservationAfterPanelClose( + panelID: panel.id, + cleanupDelay: .zero, + sleepUntilDeadline: { _ in true } + ) + let didExit = await processExitCompletion.wait() + + #expect(!didExit) + #expect(controller.committedTerminationObservationsByPanelID[panel.id] == nil) + #expect(!didCompleteHibernation) + exitEvents.continuation.finish() + } + + @Test + func panelCloseCleanupFollowsObservationIntoRecovery() async throws { + let controller = AgentHibernationController.shared + let panelID = UUID() + let identity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let initialWaitStarted = AsyncStream.makeStream() + let allowInitialFailure = AsyncStream.makeStream() + let nativeReadiness = AsyncStream.makeStream() + let allowCleanup = AsyncStream.makeStream() + let processExitCompletion = AgentHibernationProcessExitCompletion() + controller.observeCommittedTermination( + panelID: panelID, + terminations: [ + .init( + processID: 101, + processIdentity: identity, + processGroupID: 1 + ), + ], + processExitCompletion: processExitCompletion, + waitForExit: { _ in + initialWaitStarted.continuation.yield() + for await _ in allowInitialFailure.stream { return false } + return false + }, + waitForRecoveryReadiness: { + for await _ in nativeReadiness.stream { return true } + return false + }, + onExit: { true }, + onRecovery: {} + ) + let initialObservation = try #require( + controller.committedTerminationObservationsByPanelID[panelID] + ) + let initialObservationTask = try #require(initialObservation.task) + var initialWaitIterator = initialWaitStarted.stream.makeAsyncIterator() + _ = await initialWaitIterator.next() + + controller.limitCommittedTerminationObservationAfterPanelClose( + panelID: panelID, + cleanupDelay: .zero, + sleepUntilDeadline: { _ in + for await _ in allowCleanup.stream { return true } + return false + } + ) + let cleanupTask = try #require( + controller.committedTerminationCleanupByPanelID[panelID]?.task + ) + allowInitialFailure.continuation.yield() + allowInitialFailure.continuation.finish() + await initialObservationTask.value + + let recoveryObservation = try #require( + controller.committedTerminationObservationsByPanelID[panelID] + ) + #expect(recoveryObservation.requestID == initialObservation.requestID) + + allowCleanup.continuation.yield() + allowCleanup.continuation.finish() + await cleanupTask.value + + #expect(controller.committedTerminationObservationsByPanelID[panelID] == nil) + #expect(await processExitCompletion.wait() == false) + initialWaitStarted.continuation.finish() + nativeReadiness.continuation.finish() + } + + @Test + func panelCloseCleanupOwnsCommitBeforeNativeRecoveryCanSuspend() async throws { + let controller = AgentHibernationController.shared + let workspaceID = UUID() + let panelID = UUID() + let requestID = UUID() + let processExitCompletion = AgentHibernationProcessExitCompletion() + let allowCleanup = AsyncStream.makeStream() + let allowLateRecovery = AsyncStream.makeStream() + var didRecover = false + + controller.registerCommittedTerminationObservation( + panelID: panelID, + requestID: requestID, + processExitCompletion: processExitCompletion + ) + #expect( + controller.committedTerminationObservationsByPanelID[panelID]? + .requestID == requestID + ) + + controller.discardTrackingStateForClosedPanel( + workspaceId: workspaceID, + panelId: panelID + ) + #expect(controller.committedTerminationCleanupByPanelID[panelID] != nil) + controller.limitCommittedTerminationObservationAfterPanelClose( + panelID: panelID, + cleanupDelay: .zero, + sleepUntilDeadline: { _ in + for await _ in allowCleanup.stream { return true } + return false + } + ) + let cleanupTask = try #require( + controller.committedTerminationCleanupByPanelID[panelID]?.task + ) + + let lateRecoveryTask = Task { @MainActor in + for await _ in allowLateRecovery.stream { + controller.observeCommittedTerminationRecovery( + panelID: panelID, + requestID: requestID, + terminations: [], + processExitCompletion: processExitCompletion, + onRecovery: { + didRecover = true + } + ) + return + } + } + + allowCleanup.continuation.yield() + allowCleanup.continuation.finish() + await cleanupTask.value + + #expect(controller.committedTerminationObservationsByPanelID[panelID] == nil) + #expect(await processExitCompletion.wait() == false) + + allowLateRecovery.continuation.yield() + allowLateRecovery.continuation.finish() + await lateRecoveryTask.value + + #expect(controller.committedTerminationObservationsByPanelID[panelID] == nil) + #expect(!didRecover) + } + + private func expectTrackingWasDiscarded( + for key: AgentHibernationPanelKey, + controller: AgentHibernationController + ) { + #expect(controller.activityByPanel[key] == nil) + #expect(controller.terminalInputByPanel[key] == nil) + #expect(controller.lifecycleChangeByPanel[key] == nil) + #expect(controller.teardownValidationEpochByPanel[key] == nil) + } +} diff --git a/cmuxTests/AgentResumeLivenessTests.swift b/cmuxTests/AgentResumeLivenessTests.swift index c2081198b1d5..63018b0c7702 100644 --- a/cmuxTests/AgentResumeLivenessTests.swift +++ b/cmuxTests/AgentResumeLivenessTests.swift @@ -28,8 +28,13 @@ struct AgentResumeLivenessTests { // one value that would contradict the empty-PID case. processLiveness: processIDs.isEmpty ? .exited : .running, processIDs: processIDs, + processIdentities: [:], agentProcessIDs: processIDs, - agentProcessIdentities: [:] + agentProcessIdentities: [:], + hibernationPanelProcessIDs: processIDs, + terminationProcessIDs: processIDs, + terminationProcessIdentities: [:], + containsUnrelatedProcess: false ) } diff --git a/cmuxTests/AppDelegateEqualizeSplitsShortcutTests.swift b/cmuxTests/AppDelegateEqualizeSplitsShortcutTests.swift index 8e3fba2e1b0f..3eef18d97cac 100644 --- a/cmuxTests/AppDelegateEqualizeSplitsShortcutTests.swift +++ b/cmuxTests/AppDelegateEqualizeSplitsShortcutTests.swift @@ -804,7 +804,7 @@ final class AppDelegateEqualizeSplitsShortcutTests { } @Test - func testDefaultWorkspaceTerminalFontSizeDrainScheduleIsOneShotAndCancellable() + func testWorkspaceTerminalFontSizeDrainScheduleContractIsOneShotAndCancellable() async { var firedCount = 0 let fired = expectation( @@ -819,21 +819,20 @@ final class AppDelegateEqualizeSplitsShortcutTests { await waitWhileSuspended(for: [fired], timeout: 1) completedCancellation() - try? await Task.sleep( - nanoseconds: 50_000_000 - ) + completedCancellation() XCTAssertEqual(firedCount, 1) var cancelledFireCount = 0 - let pendingCancellation = - WorkspaceTerminalFontSizeCoordinator - .scheduleDefaultDrain(after: 0.05) { - cancelledFireCount += 1 - } + // Drive cancellation through the coordinator's injected scheduler + // instead of waiting for a real deadline. + let scheduler = ManualWorkspaceFontSizeDrainScheduler() + let pendingCancellation = scheduler.schedule( + delay: 0.05 + ) { + cancelledFireCount += 1 + } pendingCancellation() - try? await Task.sleep( - nanoseconds: 100_000_000 - ) + scheduler.fire(at: 0) XCTAssertEqual(cancelledFireCount, 0) } @@ -8045,10 +8044,10 @@ private final class ManualWorkspaceFontSizeDrainScheduler { @MainActor private final class ManualTerminalFontConfigurationReloadScheduler { - private var actions: [@MainActor () -> Void] = [] + private var actions: [@MainActor @Sendable () -> Void] = [] func schedule( - action: @escaping @MainActor () -> Void + action: @escaping @MainActor @Sendable () -> Void ) { actions.append(action) } diff --git a/cmuxTests/CompletedRestoredAgentGenerationTests.swift b/cmuxTests/CompletedRestoredAgentGenerationTests.swift index fe1256b74538..d144a278b21f 100644 --- a/cmuxTests/CompletedRestoredAgentGenerationTests.swift +++ b/cmuxTests/CompletedRestoredAgentGenerationTests.swift @@ -235,8 +235,13 @@ struct CompletedRestoredAgentGenerationTests { updatedAt: updatedAt, processLiveness: .running, processIDs: [Int(identity.pid)], + processIdentities: [Int(identity.pid): identity], agentProcessIDs: [Int(identity.pid)], - agentProcessIdentities: [Int(identity.pid): identity] + agentProcessIdentities: [Int(identity.pid): identity], + hibernationPanelProcessIDs: [Int(identity.pid)], + terminationProcessIDs: [Int(identity.pid)], + terminationProcessIdentities: [Int(identity.pid): identity], + containsUnrelatedProcess: false ) } diff --git a/cmuxTests/DockRuntimeParityTests.swift b/cmuxTests/DockRuntimeParityTests.swift index a8432c4327a4..ed10f8397347 100644 --- a/cmuxTests/DockRuntimeParityTests.swift +++ b/cmuxTests/DockRuntimeParityTests.swift @@ -22,12 +22,15 @@ private final class DockRuntimeParityPanel: Panel, ObservableObject { var isDirty = false private(set) var flashReasons: [WorkspaceAttentionFlashReason] = [] + private(set) var closeCount = 0 init(title: String) { displayTitle = title } - func close() {} + func close() { + closeCount += 1 + } func focus() {} func unfocus() {} @@ -61,6 +64,30 @@ private extension DockSplitStore { struct DockRuntimeParityTests { private static let socketWorker = DispatchQueue(label: "DockRuntimeParityTests.socketWorker") + @Test("Reconciling a stale tab alias preserves the live panel owner") + func reconcilingStaleTabAliasPreservesLivePanelOwner() throws { + let dock = DockSplitStore(workspaceId: UUID(), baseDirectoryProvider: { nil }) + let panel = DockRuntimeParityPanel(title: "Shared panel") + let paneID = try dock.seedRuntimeParityPanel(panel) + let liveTabID = try #require(dock.surfaceId(forPanelId: panel.id)) + let staleAliasID = try #require( + dock.bonsplitController.createTab( + title: "Stale alias", + icon: panel.displayIcon, + kind: panel.panelType.rawValue, + isDirty: false, + inPane: paneID + ) + ) + dock.surfaceIdToPanelId[staleAliasID] = panel.id + + #expect(dock.bonsplitController.closeTab(staleAliasID)) + + #expect(dock.panel(for: liveTabID) === panel) + #expect(dock.surfaceIdToPanelId[staleAliasID] == nil) + #expect(panel.closeCount == 0) + } + private func socketEnvelope( method: String, params: [String: Any] = [:] diff --git a/cmuxTests/SharedLiveAgentIndexAgentLivenessTests.swift b/cmuxTests/SharedLiveAgentIndexAgentLivenessTests.swift index 85dbc39c351b..f1630dd58c89 100644 --- a/cmuxTests/SharedLiveAgentIndexAgentLivenessTests.swift +++ b/cmuxTests/SharedLiveAgentIndexAgentLivenessTests.swift @@ -28,6 +28,7 @@ struct SharedLiveAgentIndexAgentLivenessTests { let agentPID = 7_286 let childPID = 7_287 let agentIdentity = AgentPIDProcessIdentity(pid: pid_t(agentPID), startSeconds: 42, startMicroseconds: 7) + let childIdentity = AgentPIDProcessIdentity(pid: pid_t(childPID), startSeconds: 43, startMicroseconds: 8) let executable = "/usr/local/bin/\(agentId)" let registry = CmuxVaultAgentRegistry(registrations: [ CmuxVaultAgentRegistration( @@ -98,7 +99,7 @@ struct SharedLiveAgentIndexAgentLivenessTests { return processArguments.withLock { $0 } }, processIdentityProvider: { pid in - pid == agentPID ? agentIdentity : nil + [agentPID: agentIdentity, childPID: childIdentity][pid] } ) .loadResultSynchronously() @@ -111,6 +112,10 @@ struct SharedLiveAgentIndexAgentLivenessTests { await sharedIndex.refreshForkAvailabilityNow(workspaceId: workspaceId, panelId: panelId) #expect(sharedIndex.index?.processIDs(workspaceId: workspaceId, panelId: panelId) == Set([agentPID, childPID])) + #expect(sharedIndex.index?.processIdentities( + workspaceId: workspaceId, + panelId: panelId + ) == [agentPID: agentIdentity, childPID: childIdentity]) #expect(sharedIndex.index?.agentProcessIDs(workspaceId: workspaceId, panelId: panelId) == Set([agentPID])) #expect(sharedIndex.index?.agentProcessIdentities(workspaceId: workspaceId, panelId: panelId) == [agentPID: agentIdentity]) #expect(sharedIndex.prepareForkAvailabilityProbe(workspaceId: workspaceId, panelId: panelId)) diff --git a/cmuxUITests/SettingsTerminalBehaviorUITests.swift b/cmuxUITests/SettingsTerminalBehaviorUITests.swift index 9f474430706b..21ec38ce67b1 100644 --- a/cmuxUITests/SettingsTerminalBehaviorUITests.swift +++ b/cmuxUITests/SettingsTerminalBehaviorUITests.swift @@ -161,7 +161,7 @@ final class SettingsTerminalBehaviorUITests: SettingsUITestCase { static let resumeOn = "automatically run their resume command" static let resumeOff = "stay idle until you resume them manually" static let hibernateOn = "Idle background agent terminals can be suspended" - static let hibernateOff = "Agent terminals stay live until you close them" + static let hibernateOff = "Scheduled hibernation is off" static let showTextBoxOn = "open with the TextBox visible" static let showTextBoxOff = "start with the TextBox hidden" static let focusTextBoxOn = "put keyboard focus in the TextBox" @@ -277,7 +277,7 @@ final class SettingsTerminalBehaviorUITests: SettingsUITestCase { control.click() XCTAssertTrue( waitForStaticText(window, Subtitle.hibernateOff), - "Disabling hibernation should restore the stay-live sentence" + "Disabling scheduled hibernation should restore the critical-pressure disclosure" ) closeSettings(app, window) diff --git a/docs/agent-hooks.md b/docs/agent-hooks.md index 00277246688a..ccd2c13c0282 100644 --- a/docs/agent-hooks.md +++ b/docs/agent-hooks.md @@ -59,11 +59,11 @@ When the opt-in `automation.workspaceAutoNaming` setting is enabled, turn-end ho ## Agent Hibernation -Agent Hibernation kills idle background agent processes to free their RAM and CPU, then resumes each one with its saved session when you return to its tab. It is opt-in and off by default. cmux knows which process belongs to which terminal because the agent hooks associate each session ID with its surface (see the session-restore section above), so it can terminate the right process and bring back the right session. +Agent Hibernation kills idle background agent processes to free their RAM and CPU, then resumes each one with its saved session when you return to its tab. Routine hibernation based on the live-terminal limit is opt-in and off by default. A separate bounded safety path remains active for critical system memory pressure. cmux knows which process belongs to which terminal because the agent hooks associate each session ID with its surface (see the session-restore section above), so it can terminate the right process and bring back the right session. ### When a terminal hibernates -A live terminal is only ever a candidate when all of these hold: +For routine hibernation, a live terminal is only a candidate when all of these hold: - it has a saved restorable agent session, and the saved launch data can build a resume command - the agent lifecycle is `idle` (not running, not waiting on input) @@ -75,7 +75,11 @@ The live-terminal limit is the first gate. Under the limit, nothing hibernates n Before killing, cmux watches the terminal tail. It samples the last lines of output and a fingerprint of the process, and waits a short confirmation window (`confirmationSeconds`, ~60s) during which the output and process must stay unchanged. Any new output, input, lifecycle change, or PID change cancels the pending hibernation. This is why a small `idleSeconds` is safe: a freshly idle agent that resumes work on its own is never killed mid-task. -So with the defaults, hibernation only affects power users running more than 12 agents at once, and even then only ~1 minute after an agent has gone quiet off-screen. +So with the defaults, routine hibernation only affects power users running more than 12 agents at once, and even then only ~1 minute after an agent has gone quiet off-screen. + +### Critical memory pressure + +When macOS reports critical memory pressure, cmux can run the same protected teardown path independently of the `enabled` setting and live-terminal limit. Each pass selects at most two of the oldest eligible background agents. The agent must still be restorable, off-screen, explicitly idle, free of unconfirmed input, stable through the confirmation window, and backed by a transcript cmux can protect. Visible, running, needs-input, recently changed, or unprotectable agents are never selected. Before signaling anything, cmux revalidates the exact process generation and workspace/surface scope. ### What gets killed and how it comes back @@ -83,7 +87,7 @@ cmux sends `SIGTERM` to the agent's process group (scoped to that workspace and ### Enable and configure -Enable from the command palette (`⌘⇧P` -> **Enable Agent Hibernation**), from **Settings > Terminal > Agent Hibernation**, or from the CLI: +Enable routine hibernation from the command palette (`⌘⇧P` -> **Enable Agent Hibernation**), from **Settings > Terminal > Agent Hibernation**, or from the CLI: ```bash cmux agent-hibernation on diff --git a/docs/cli-contract.md b/docs/cli-contract.md index 4b33a75021d7..af103eaad3c5 100644 --- a/docs/cli-contract.md +++ b/docs/cli-contract.md @@ -66,7 +66,7 @@ Environment: | `disable-browser` | Disable cmux browser creation and link interception until re-enabled. | | `enable-browser` | Re-enable cmux browser creation and link interception. | | `browser-status` | Print whether cmux browser creation and link interception are enabled. | -| `agent-hibernation` | Enable or disable Agent Hibernation. | +| `agent-hibernation` | Enable or disable routine Agent Hibernation. | | `restore-session` | Restore the previously saved cmux session. | | `open` | Open files, directories, or URLs in cmux. | | `feedback` | Open feedback UI or submit feedback with `--email`, `--body`, and repeated `--image`. | diff --git a/docs/configuration.md b/docs/configuration.md index ac419d2cea0c..d01d3ce61c5c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -103,7 +103,7 @@ Default: `cloudFirst`. ## `terminal.agentHibernation` -Opt-in Agent Hibernation. cmux kills idle background agent processes to free RAM and CPU, then resumes each one with its saved session when you visit its tab. See [agent-hooks.md](agent-hooks.md#agent-hibernation) for the full behavior, including the confirmation settle window and how resume works. +Routine Agent Hibernation is opt-in. cmux kills idle background agent processes to free RAM and CPU, then resumes each one with its saved session when you visit its tab. Independently, critical memory pressure can trigger a bounded safety pass over eligible idle background agents even when routine hibernation is disabled. See [agent-hooks.md](agent-hooks.md#agent-hibernation) for the full eligibility rules, confirmation settle window, and resume behavior. ```json { @@ -117,11 +117,11 @@ Opt-in Agent Hibernation. cmux kills idle background agent processes to free RAM } ``` -- `enabled`: turn Agent Hibernation on. Default: `false`. +- `enabled`: turn routine Agent Hibernation on. Default: `false`. Critical-pressure safety hibernation remains active when this is `false`. - `idleSeconds`: seconds a background idle agent terminal must be quiet before it can hibernate. A ~60s confirmation settle window still applies on top of this. Default: `5`. Range: `5`-`604800`. - `maxLiveTerminals`: how many live restorable agent terminals to keep before cmux hibernates the oldest idle background ones. Nothing hibernates while you are at or under this count. Default: `12`. Range: `1`-`256`. -Enable it from the command palette (`⌘⇧P` -> Enable Agent Hibernation), from **Settings > Terminal > Agent Hibernation**, or with `cmux agent-hibernation on`. +Enable routine hibernation from the command palette (`⌘⇧P` -> Enable Agent Hibernation), from **Settings > Terminal > Agent Hibernation**, or with `cmux agent-hibernation on`. ## `sidebar.showAgentActivity` diff --git a/web/data/cmux.schema.json b/web/data/cmux.schema.json index d38e7fc9e5db..ac75292b6860 100644 --- a/web/data/cmux.schema.json +++ b/web/data/cmux.schema.json @@ -611,12 +611,14 @@ "agentHibernation": { "type": "object", "additionalProperties": false, - "description": "Opt-in Agent Hibernation settings. cmux kills idle background agent processes to free RAM and CPU, then resumes them with their saved session when their tab is visited. A terminal is only suspended when it runs a restorable coding agent, the agent lifecycle reports idle, the terminal is off-screen, the live-terminal limit is exceeded, and its output has stayed unchanged for the idle window plus a short confirmation settle window. The placeholder Resume button is a manual fallback.", + "descriptionKey": "schemaDescriptions.terminal.agentHibernation", + "description": "Routine Agent Hibernation settings. cmux kills idle background agent processes to free RAM and CPU, then resumes them with their saved session when their tab is visited. Routine hibernation requires a restorable coding agent whose lifecycle reports idle, an off-screen terminal, a live-terminal count above the configured limit, and unchanged output through the idle and confirmation windows. Independently, during critical memory pressure cmux may hibernate a bounded batch of safe idle background agents even when enabled is false; visible, running, needs-input, recently changed, and unprotectable agents remain excluded. The placeholder Resume button is a manual fallback.", "properties": { "enabled": { "type": "boolean", "default": false, - "description": "Enable Agent Hibernation." + "descriptionKey": "schemaDescriptions.terminal.agentHibernationEnabled", + "description": "Enable routine Agent Hibernation based on the live-terminal limit. Critical-pressure safety hibernation remains active when false." }, "idleSeconds": { "type": "integer", diff --git a/web/messages/en.json b/web/messages/en.json index ccd92fefb134..e85aeee3a962 100644 --- a/web/messages/en.json +++ b/web/messages/en.json @@ -1812,6 +1812,8 @@ "globalFontMagnification": "Scales cmux-owned terminals, tab titles, sidebars, settings, overlays, and app chrome by this percentage. Rendered browser page content is excluded." }, "terminal": { + "agentHibernation": "Routine Agent Hibernation settings. cmux kills idle background agent processes to free RAM and CPU, then resumes them with their saved session when their tab is visited. Routine hibernation requires a restorable coding agent whose lifecycle reports idle, an off-screen terminal, a live-terminal count above the configured limit, and unchanged output through the idle and confirmation windows. Independently, during critical memory pressure cmux may hibernate a bounded batch of safe idle background agents even when enabled is false; visible, running, needs-input, recently changed, and unprotectable agents remain excluded. The placeholder Resume button is a manual fallback.", + "agentHibernationEnabled": "Enable routine Agent Hibernation based on the live-terminal limit. Critical-pressure safety hibernation remains active when false.", "autoRetryAgentSessions": "Automatically resume a managed agent session when its active command exits with a nonzero, non-signal error status. Retries are bounded and use exponential backoff; normal completion and explicit user termination are never retried.", "copyOnSelect": "When true, copy selected terminal text to the system clipboard when the selection is committed. When false, cmux does not emit a Ghostty copy-on-select override; Ghostty config and defaults control selection-clipboard behavior.", "scrollSpeed": "Multiplier applied to terminal scroll wheel and trackpad deltas. Higher values scroll faster; lower values scroll slower.", diff --git a/web/messages/ja.json b/web/messages/ja.json index 1457cb6b0eb7..056c4bf7827f 100644 --- a/web/messages/ja.json +++ b/web/messages/ja.json @@ -1735,6 +1735,8 @@ "globalFontMagnification": "cmux が所有するターミナル、タブタイトル、サイドバー、設定、オーバーレイ、アプリの chrome をこの割合で拡大縮小します。レンダリングされたブラウザページ内容は対象外です。" }, "terminal": { + "agentHibernation": "通常のエージェント休止の設定です。cmux はアイドル中のバックグラウンドエージェントプロセスを終了して RAM と CPU を解放し、そのタブを開いたときに保存済みセッションから再開します。通常の休止には、再開可能なコーディングエージェントであること、ライフサイクルがアイドルと報告していること、ターミナルが画面外にあること、ライブターミナル数が設定上限を超えていること、アイドル時間と確認時間を通して出力が変化していないことが必要です。これとは別に、メモリ負荷が危険な状態では enabled が false でも、安全なアイドル中のバックグラウンドエージェントを cmux が上限付きのバッチで休止する場合があります。表示中、実行中、入力待ち、最近変更された、またはトランスクリプトを保護できないエージェントは対象外です。プレースホルダーの「再開」ボタンは手動の代替手段です。", + "agentHibernationEnabled": "ライブターミナル数の上限に基づく通常のエージェント休止を有効にします。false の場合も、メモリ負荷が危険な状態での安全休止は有効です。", "autoRetryAgentSessions": "アクティブな管理対象エージェントのコマンドが、シグナル以外の 0 でないエラーステータスで終了した場合に、セッションを自動的に再開します。再試行は回数制限付きの指数バックオフで行い、正常終了やユーザーによる明示的な終了は再試行しません。", "copyOnSelect": "true の場合、選択が確定したときにターミナルで選択したテキストをシステムクリップボードへコピーします。false の場合、cmux は Ghostty の copy-on-select 上書きを出力せず、選択クリップボードの動作は Ghostty の設定と既定値に従います。", "scrollSpeed": "ターミナルのスクロールホイールとトラックパッドの移動量に適用される倍率です。値を大きくするとスクロールが速くなり、小さくすると遅くなります。", diff --git a/web/tests/client-config-env.test.ts b/web/tests/client-config-env.test.ts index d5644f936b87..71d89ff18b0d 100644 --- a/web/tests/client-config-env.test.ts +++ b/web/tests/client-config-env.test.ts @@ -10,6 +10,8 @@ const requiredEnv = { STACK_SECRET_SERVER_KEY: "stack-secret", NEXT_PUBLIC_STACK_PROJECT_ID: "stack-project", NEXT_PUBLIC_STACK_PUBLISHABLE_CLIENT_KEY: "stack-public", + SUBROUTER_ENFORCE_STACK_PERMISSIONS: "0", + SUBROUTER_ALLOWED_TEAM_IDS: "*", }; const requiredIrohProductionEnv = { diff --git a/web/tests/subrouter-accounts-route.test.ts b/web/tests/subrouter-accounts-route.test.ts index 68fb209be207..7b832fa0311b 100644 --- a/web/tests/subrouter-accounts-route.test.ts +++ b/web/tests/subrouter-accounts-route.test.ts @@ -85,12 +85,7 @@ describe("subrouter accounts route", () => { const operation = withSubrouterAuthorizationDeadline( async () => await new Promise(() => {}), ); - const result = await Promise.race([ - operation.catch((error: unknown) => error), - new Promise<"still-pending">((resolve) => - setTimeout(() => resolve("still-pending"), 100) - ), - ]); + const result = await operation.catch((error: unknown) => error); expect(result).toBeInstanceOf(SubrouterAuthorizationTimeoutError); }); @@ -998,9 +993,8 @@ describe("subrouter accounts route", () => { { length: 12 }, () => teamsRoute.GET(request("/api/subrouter/teams")), ); - await new Promise((resolve) => setTimeout(resolve, 50)); - releasePermissions(); const responses = await Promise.all(routes); + releasePermissions(); expect(responses.every((response) => response.status === 503)).toBe(true); expect(maxActive).toBeLessThanOrEqual(8);