From a9069343ababe010d7a136715671265bd70ce0b6 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 28 Jul 2026 15:44:36 -0700 Subject: [PATCH 01/20] test: cover critical-pressure agent reclamation --- Sources/App/AgentHibernationPlanner.swift | 8 +- .../AgentHibernationPlannerSwiftTests.swift | 84 +++++++++++++++++++ 2 files changed, 91 insertions(+), 1 deletion(-) diff --git a/Sources/App/AgentHibernationPlanner.swift b/Sources/App/AgentHibernationPlanner.swift index d1d14d8e2b3f..3f02a7dd9b79 100644 --- a/Sources/App/AgentHibernationPlanner.swift +++ b/Sources/App/AgentHibernationPlanner.swift @@ -1,10 +1,16 @@ import Foundation +enum AgentHibernationReclaimTrigger: Sendable { + case scheduled + case systemMemoryPressure +} + enum AgentHibernationPlanner { static func selectedPanelKeys( inputs: [AgentHibernationPlannerInput], settings: AgentHibernationSettings.Values, - now: TimeInterval + now: TimeInterval, + trigger: AgentHibernationReclaimTrigger = .scheduled ) -> Set { guard settings.enabled else { return [] } let liveRestorable = inputs.filter { $0.hasRestorableAgent && $0.isLive } diff --git a/cmuxTests/AgentHibernationPlannerSwiftTests.swift b/cmuxTests/AgentHibernationPlannerSwiftTests.swift index cf44c48602f7..1d2727f244f5 100644 --- a/cmuxTests/AgentHibernationPlannerSwiftTests.swift +++ b/cmuxTests/AgentHibernationPlannerSwiftTests.swift @@ -230,6 +230,90 @@ struct AgentHibernationPlannerSwiftTests { #expect(selected == Set([safeAgent])) } + @Test + func criticalPressureSelectsOnlySafeIdleAgentsWhenScheduledHibernationIsDisabled() { + let workspaceId = UUID() + let now: TimeInterval = 1_000 + let idle = AgentHibernationPanelKey(workspaceId: workspaceId, panelId: UUID()) + let visible = AgentHibernationPanelKey(workspaceId: workspaceId, panelId: UUID()) + let running = AgentHibernationPanelKey(workspaceId: workspaceId, panelId: UUID()) + let liveProcess = AgentHibernationPanelKey(workspaceId: workspaceId, panelId: UUID()) + let unconfirmedInput = AgentHibernationPanelKey(workspaceId: workspaceId, panelId: UUID()) + let unableToProtect = AgentHibernationPanelKey(workspaceId: workspaceId, panelId: UUID()) + let settings = AgentHibernationSettings.Values( + enabled: false, + idleSeconds: 60, + maxLiveTerminals: 256, + confirmationSeconds: 5 + ) + + let selected = AgentHibernationPlanner.selectedPanelKeys( + inputs: [ + .init( + key: idle, + hasRestorableAgent: true, + isLive: true, + isProtected: false, + lifecycle: .idle, + hasUnconfirmedTerminalInput: false, + lastActivityAt: now + ), + .init( + key: visible, + hasRestorableAgent: true, + isLive: true, + isProtected: true, + lifecycle: .idle, + hasUnconfirmedTerminalInput: false, + lastActivityAt: 0 + ), + .init( + key: running, + hasRestorableAgent: true, + isLive: true, + isProtected: false, + lifecycle: .running, + hasUnconfirmedTerminalInput: false, + lastActivityAt: 0 + ), + .init( + key: liveProcess, + hasRestorableAgent: true, + isLive: true, + hasLiveProcess: true, + isProtected: false, + lifecycle: .idle, + hasUnconfirmedTerminalInput: false, + lastActivityAt: 0 + ), + .init( + key: unconfirmedInput, + hasRestorableAgent: true, + isLive: true, + isProtected: false, + lifecycle: .idle, + hasUnconfirmedTerminalInput: true, + lastActivityAt: 0 + ), + .init( + key: unableToProtect, + hasRestorableAgent: true, + isLive: true, + isProtected: false, + lifecycle: .idle, + isTemporarilyUnableToProtect: true, + hasUnconfirmedTerminalInput: false, + lastActivityAt: 0 + ), + ], + settings: settings, + now: now, + trigger: .systemMemoryPressure + ) + + #expect(selected == Set([idle])) + } + @MainActor @Test func unableToProtectMarkerExpiresSoTransientSnapshotFailuresRetry() { From 131781c47bb28ed4969be88f0c0796d284fdf067 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 28 Jul 2026 16:01:24 -0700 Subject: [PATCH 02/20] fix: hibernate safe agents under critical pressure --- ...ntHibernationController+Confirmation.swift | 1 + ...HibernationController+MemoryPressure.swift | 71 ++++++++++++++++ .../AgentHibernationController+Teardown.swift | 27 ++++-- Sources/App/AgentHibernationController.swift | 85 +++++++++++++------ ...ntHibernationMemoryPressureResponder.swift | 40 +++++++++ Sources/App/AgentHibernationPlanner.swift | 28 ++++-- Sources/AppDelegate+PaneMemoryGuardrail.swift | 5 ++ cmux.xcodeproj/project.pbxproj | 8 ++ .../AgentHibernationPlannerSwiftTests.swift | 2 +- 9 files changed, 230 insertions(+), 37 deletions(-) create mode 100644 Sources/App/AgentHibernationController+MemoryPressure.swift create mode 100644 Sources/App/AgentHibernationMemoryPressureResponder.swift diff --git a/Sources/App/AgentHibernationController+Confirmation.swift b/Sources/App/AgentHibernationController+Confirmation.swift index 9892bf9221a2..aced833e985f 100644 --- a/Sources/App/AgentHibernationController+Confirmation.swift +++ b/Sources/App/AgentHibernationController+Confirmation.swift @@ -2,6 +2,7 @@ import Foundation extension AgentHibernationController { struct Confirmation: Sendable { + let trigger: AgentHibernationReclaimTrigger let fingerprint: String let sampledAt: TimeInterval let dueAt: TimeInterval diff --git a/Sources/App/AgentHibernationController+MemoryPressure.swift b/Sources/App/AgentHibernationController+MemoryPressure.swift new file mode 100644 index 000000000000..900d5aaf1ca3 --- /dev/null +++ b/Sources/App/AgentHibernationController+MemoryPressure.swift @@ -0,0 +1,71 @@ +import Foundation + +extension AgentHibernationController { + /// Starts one asynchronous critical-pressure evaluation. + /// + /// The existing hibernation lifecycle remains the sole teardown owner: + /// pressure only changes which safe idle agents it selects. Transcript + /// protection, confirmation, activity revalidation, and scoped process + /// termination are unchanged. + @discardableResult + func reclaimIdleAgentsForSystemMemoryPressure( + now: Date, + onHibernationCompleted: @escaping @MainActor (Int) -> Void + ) -> Bool { + guard memoryPressureEvaluation == nil else { return false } + + let requestID = UUID() + AgentHibernationTrackingGate.setEnabled(true) + let task = Task { @MainActor [weak self] in + guard let self else { return } + var awaitsTeardownCompletion = false + defer { + if !awaitsTeardownCompletion { + self.finishMemoryPressureEvaluation(requestID: requestID) + } + } + + let settings = AgentHibernationSettings.values() + let index = await RestorableAgentSessionIndex.loadIncludingProcessDetectedSnapshots() + guard !Task.isCancelled else { return } + let initialEvaluation = self.evaluate( + index: index, + settings: settings, + now: now, + trigger: .systemMemoryPressure + ) + guard initialEvaluation.hasCandidates else { return } + + do { + try await ContinuousClock().sleep(for: .seconds(settings.confirmationSeconds)) + } catch { + return + } + let confirmationIndex = await RestorableAgentSessionIndex + .loadIncludingProcessDetectedSnapshots() + guard !Task.isCancelled else { return } + let confirmationEvaluation = self.evaluate( + index: confirmationIndex, + settings: AgentHibernationSettings.values(), + now: .now, + trigger: .systemMemoryPressure, + onHibernationCompleted: { [weak self] hibernatedCount in + self?.finishMemoryPressureEvaluation(requestID: requestID) + onHibernationCompleted(hibernatedCount) + } + ) + awaitsTeardownCompletion = confirmationEvaluation.beganTeardowns + } + memoryPressureEvaluation = (requestID, task) + return true + } + + private func finishMemoryPressureEvaluation(requestID: UUID) { + guard memoryPressureEvaluation?.id == requestID else { return } + memoryPressureEvaluation = nil + clearMemoryPressureConfirmations() + AgentHibernationTrackingGate.setEnabled( + AgentHibernationSettings.isEnabled() + ) + } +} diff --git a/Sources/App/AgentHibernationController+Teardown.swift b/Sources/App/AgentHibernationController+Teardown.swift index 3022da25907f..bd899cbbda41 100644 --- a/Sources/App/AgentHibernationController+Teardown.swift +++ b/Sources/App/AgentHibernationController+Teardown.swift @@ -17,6 +17,7 @@ extension AgentHibernationController { let requestID: UUID let epoch: UInt64 let generation: UInt64 + let trigger: AgentHibernationReclaimTrigger } /// Runs the transcript snapshot off the main actor, then resumes teardown on the @@ -24,13 +25,18 @@ extension AgentHibernationController { /// SIGTERM / pty-close can trigger Claude's interrupted-exit transcript rewrite, /// so the teardown is sequenced after it rather than racing it; the re-validation /// below covers disable/stop and anything else that changed during the brief I/O hop. - func beginConfirmedTeardowns(_ requests: [ConfirmedTeardownRequest]) { + func beginConfirmedTeardowns( + _ requests: [ConfirmedTeardownRequest], + onCompletion: (@MainActor (Int) -> Void)? = nil + ) { guard !requests.isEmpty else { return } Task { @MainActor in + var hibernatedCount = 0 defer { for request in requests { self.clearInFlightTeardown(request.record.key, requestID: request.requestID) } + onCompletion?(hibernatedCount) } var snapshotOutcomes = await Self.snapshotOutcomes(for: requests) @@ -98,12 +104,22 @@ extension AgentHibernationController { index: postSnapshotIndex ) // Re-validate: the pane must still be exactly as confirmed. Any activity, - // scrollback change, visibility/protection change, hibernation disable, - // hibernation, or surface loss during the hop aborts; the regular 30s - // tick will re-arm if still idle. + // process-set or scrollback change, visibility/protection change, + // hibernation, or surface loss during the hop aborts; a later evaluation + // can re-arm it if it is still idle. guard AgentHibernationTrackingGate.isEnabled(), record.isStillOwnedByOriginalWorkspace, - !postSnapshotIndex.hasLiveProcess(workspaceId: record.key.workspaceId, panelId: record.key.panelId), + ( + request.trigger == .systemMemoryPressure || + !postSnapshotIndex.hasLiveProcess( + workspaceId: record.key.workspaceId, + panelId: record.key.panelId + ) + ), + postSnapshotIndex.processIDs( + workspaceId: record.key.workspaceId, + panelId: record.key.panelId + ) == record.processIDs, TabManager.restorableAgentSnapshotFingerprint(currentAgent) == TabManager.restorableAgentSnapshotFingerprint(record.agent), !record.terminalPanel.isAgentHibernated, @@ -178,6 +194,7 @@ extension AgentHibernationController { agent: record.agent, lastActivityAt: Date(timeIntervalSince1970: request.effectiveLastActivityAt) ) + hibernatedCount += 1 guard let snapshot else { continue } if self.armPostTeardownRestoreMonitor(snapshot: snapshot, processIDs: record.processIDs) { restoreOwnedSnapshotPaths.insert(snapshot.snapshotPath) diff --git a/Sources/App/AgentHibernationController.swift b/Sources/App/AgentHibernationController.swift index e9e66810d380..a196fadf09bc 100644 --- a/Sources/App/AgentHibernationController.swift +++ b/Sources/App/AgentHibernationController.swift @@ -43,6 +43,7 @@ final class AgentHibernationController { private var teardownInFlightByPanel: [AgentHibernationPanelKey: InFlightTeardown] = [:] private var confirmations: [AgentHibernationPanelKey: Confirmation] = [:] private var tailFingerprintSamples: [AgentHibernationPanelKey: TailFingerprintSample] = [:] + var memoryPressureEvaluation: (id: UUID, task: Task)? private init() {} @@ -66,6 +67,8 @@ final class AgentHibernationController { func stop() { timer?.cancel() timer = nil + memoryPressureEvaluation?.task.cancel() + memoryPressureEvaluation = nil AgentHibernationTrackingGate.setEnabled(false) clearTrackingState() if let settingsObserver { @@ -116,18 +119,22 @@ final class AgentHibernationController { private func recordSettingsChange() { teardownValidationGeneration = teardownValidationGeneration &+ 1 - confirmations.removeAll(keepingCapacity: false) + confirmations = confirmations.filter { $0.value.trigger == .systemMemoryPressure } unableToProtectByPanel.removeAll(keepingCapacity: false) updateTimerForCurrentSettings() } private func updateTimerForCurrentSettings() { let enabled = AgentHibernationSettings.isEnabled() - AgentHibernationTrackingGate.setEnabled(enabled) + AgentHibernationTrackingGate.setEnabled( + enabled || memoryPressureEvaluation != nil + ) guard enabled else { timer?.cancel() timer = nil - clearTrackingState() + if memoryPressureEvaluation == nil { + clearTrackingState() + } return } guard timer == nil else { return } @@ -148,17 +155,20 @@ final class AgentHibernationController { self.timer = timer } - private func evaluate( + @discardableResult + func evaluate( index: RestorableAgentSessionIndex, settings: AgentHibernationSettings.Values, - now: Date - ) { - guard settings.enabled else { - AgentHibernationTrackingGate.setEnabled(false) - clearTrackingState() - return + now: Date, + trigger: AgentHibernationReclaimTrigger = .scheduled, + onHibernationCompleted: (@MainActor (Int) -> Void)? = nil + ) -> (hasCandidates: Bool, beganTeardowns: Bool) { + guard trigger != .scheduled || settings.enabled else { + return (false, false) + } + guard let appDelegate = AppDelegate.shared else { + return (false, false) } - guard let appDelegate = AppDelegate.shared else { return } let records = appDelegate.agentHibernationRecords( index: index, @@ -183,8 +193,10 @@ final class AgentHibernationController { if record.hasLiveProcess { bumpTeardownValidationEpoch(record.key) tailFingerprintSamples.removeValue(forKey: record.key) - confirmations.removeValue(forKey: record.key) - unableToProtectByPanel.removeValue(forKey: record.key) + if trigger == .scheduled { + confirmations.removeValue(forKey: record.key) + unableToProtectByPanel.removeValue(forKey: record.key) + } } if shouldMaintainTailSamples, isLive, @@ -216,10 +228,15 @@ final class AgentHibernationController { let selectedKeys = AgentHibernationPlanner.selectedPanelKeys( inputs: plannerInputs, settings: settings, - now: nowTime + now: nowTime, + trigger: trigger ) let currentKeys = Set(records.map(\.key)) - pruneTrackingState(currentKeys: currentKeys, selectedKeys: selectedKeys) + pruneTrackingState( + currentKeys: currentKeys, + selectedKeys: selectedKeys, + trigger: trigger + ) let confirmedTeardowns = records.compactMap { record -> ConfirmedTeardownRequest? in guard selectedKeys.contains(record.key) else { return nil } @@ -227,22 +244,30 @@ final class AgentHibernationController { record: record, effectiveLastActivityAt: effectiveActivityByKey[record.key] ?? record.lastActivityAt, settings: settings, - now: nowTime + now: nowTime, + trigger: trigger ) } - if !confirmedTeardowns.isEmpty { beginConfirmedTeardowns(confirmedTeardowns) } + if !confirmedTeardowns.isEmpty { + beginConfirmedTeardowns( + confirmedTeardowns, + onCompletion: onHibernationCompleted + ) + } + return (!selectedKeys.isEmpty, !confirmedTeardowns.isEmpty) } private func evaluateConfirmation( record: AgentHibernationRecord, effectiveLastActivityAt: TimeInterval, settings: AgentHibernationSettings.Values, - now: TimeInterval + now: TimeInterval, + trigger: AgentHibernationReclaimTrigger ) -> ConfirmedTeardownRequest? { guard record.lifecycle.allowsHibernation, !record.hasUnconfirmedTerminalInput, !record.isProtected, - !record.hasLiveProcess, + (trigger == .systemMemoryPressure || !record.hasLiveProcess), record.terminalPanel.surface.hasLiveSurface, !record.terminalPanel.isAgentHibernated else { confirmations.removeValue(forKey: record.key) @@ -251,7 +276,8 @@ final class AgentHibernationController { } if teardownInFlightByPanel[record.key] != nil { confirmations.removeValue(forKey: record.key); return nil } - if let confirmation = confirmations[record.key] { + if let confirmation = confirmations[record.key], + confirmation.trigger == trigger { guard now >= confirmation.dueAt else { return nil } guard effectiveLastActivityAt <= confirmation.sampledAt else { confirmations.removeValue(forKey: record.key) @@ -271,9 +297,13 @@ final class AgentHibernationController { effectiveLastActivityAt: effectiveLastActivityAt, requestID: requestID, epoch: teardownValidationEpochByPanel[record.key] ?? 0, - generation: teardownValidationGeneration + generation: teardownValidationGeneration, + trigger: trigger ) } + if confirmations[record.key]?.trigger == .systemMemoryPressure { + return nil + } guard let fingerprint = hibernationFingerprint(for: record) else { return nil } if let marker = unableToProtectByPanel[record.key], @@ -287,6 +317,7 @@ final class AgentHibernationController { } unableToProtectByPanel.removeValue(forKey: record.key) confirmations[record.key] = Confirmation( + trigger: trigger, fingerprint: fingerprint, sampledAt: now, dueAt: now + settings.confirmationSeconds @@ -429,7 +460,8 @@ final class AgentHibernationController { private func pruneTrackingState( currentKeys: Set, - selectedKeys: Set + selectedKeys: Set, + trigger: AgentHibernationReclaimTrigger ) { activityByPanel = activityByPanel.filter { currentKeys.contains($0.key) } terminalInputByPanel = terminalInputByPanel.filter { currentKeys.contains($0.key) } @@ -437,12 +469,17 @@ final class AgentHibernationController { teardownValidationEpochByPanel = teardownValidationEpochByPanel.filter { currentKeys.contains($0.key) } unableToProtectByPanel = unableToProtectByPanel.filter { currentKeys.contains($0.key) } teardownInFlightByPanel = teardownInFlightByPanel.filter { currentKeys.contains($0.key) } - confirmations = confirmations.filter { key, _ in - currentKeys.contains(key) && selectedKeys.contains(key) + confirmations = confirmations.filter { key, confirmation in + currentKeys.contains(key) && + (confirmation.trigger != trigger || selectedKeys.contains(key)) } tailFingerprintSamples = tailFingerprintSamples.filter { currentKeys.contains($0.key) } } + func clearMemoryPressureConfirmations() { + confirmations = confirmations.filter { $0.value.trigger != .systemMemoryPressure } + } + func clearInFlightTeardown(_ key: AgentHibernationPanelKey, requestID: UUID) { guard teardownInFlightByPanel[key]?.requestID == requestID else { return } teardownInFlightByPanel.removeValue(forKey: key) diff --git a/Sources/App/AgentHibernationMemoryPressureResponder.swift b/Sources/App/AgentHibernationMemoryPressureResponder.swift new file mode 100644 index 000000000000..ff013350ecaa --- /dev/null +++ b/Sources/App/AgentHibernationMemoryPressureResponder.swift @@ -0,0 +1,40 @@ +import Foundation + +@MainActor +final class AgentHibernationMemoryPressureResponder: MemoryPressureResponder { + let memoryPressureResponderID = "idle-agent-hibernation" + let memoryPressureMinimumSeverity: MemoryPressureSeverity = .critical + let memoryPressurePriority = 80 + + private let controller: AgentHibernationController + + init(controller: AgentHibernationController) { + self.controller = controller + } + + func shedMemory(for snapshot: MemoryPressureSnapshot) -> MemoryPressureShedResult { + let responderID = memoryPressureResponderID + let severity = snapshot.severity + let didSchedule = controller.reclaimIdleAgentsForSystemMemoryPressure( + now: snapshot.sampledAt + ) { hibernatedCount in + guard hibernatedCount > 0 else { return } + MemoryPressureResponderRegistry.logShedAction( + MemoryPressureShedAction( + responderID: responderID, + severity: severity, + reclaimedItemCount: hibernatedCount, + estimatedBytes: nil, + detail: "hidden-idle-agents", + performedAt: .now + ) + ) + } + return MemoryPressureShedResult( + reclaimedItemCount: 0, + detail: didSchedule + ? "hidden-idle-agent-evaluation" + : "hidden-idle-agent-evaluation-in-flight" + ) + } +} diff --git a/Sources/App/AgentHibernationPlanner.swift b/Sources/App/AgentHibernationPlanner.swift index 3f02a7dd9b79..b46420554c00 100644 --- a/Sources/App/AgentHibernationPlanner.swift +++ b/Sources/App/AgentHibernationPlanner.swift @@ -1,6 +1,6 @@ import Foundation -enum AgentHibernationReclaimTrigger: Sendable { +enum AgentHibernationReclaimTrigger: Equatable, Sendable { case scheduled case systemMemoryPressure } @@ -12,21 +12,35 @@ enum AgentHibernationPlanner { now: TimeInterval, trigger: AgentHibernationReclaimTrigger = .scheduled ) -> Set { - guard settings.enabled else { return [] } let liveRestorable = inputs.filter { $0.hasRestorableAgent && $0.isLive } - let excess = liveRestorable.count - settings.maxLiveTerminals + let excess: Int + switch trigger { + case .scheduled: + guard settings.enabled else { return [] } + excess = liveRestorable.count - settings.maxLiveTerminals + case .systemMemoryPressure: + excess = liveRestorable.count + } guard excess > 0 else { return [] } - // Live scoped processes still create cap pressure, but they are not - // eligible for teardown; reclaim safe idle panes first instead. + // Scheduled hibernation never reaps a live scoped process. Critical + // pressure may select one only while its lifecycle is explicitly idle; + // the controller still confirms stability, protects the transcript, + // and revalidates the exact process set before teardown. let eligible = liveRestorable .filter { input in !input.isProtected && - !input.hasLiveProcess && + ( + trigger == .systemMemoryPressure || + !input.hasLiveProcess + ) && input.lifecycle.allowsHibernation && !input.isTemporarilyUnableToProtect && !input.hasUnconfirmedTerminalInput && - now - input.lastActivityAt >= settings.idleSeconds + ( + trigger == .systemMemoryPressure || + now - input.lastActivityAt >= settings.idleSeconds + ) } .sorted { lhs, rhs in if lhs.lastActivityAt == rhs.lastActivityAt { diff --git a/Sources/AppDelegate+PaneMemoryGuardrail.swift b/Sources/AppDelegate+PaneMemoryGuardrail.swift index cd24f1f9bd56..5f396f68387b 100644 --- a/Sources/AppDelegate+PaneMemoryGuardrail.swift +++ b/Sources/AppDelegate+PaneMemoryGuardrail.swift @@ -34,6 +34,11 @@ extension AppDelegate { self?.paneMemoryGuardrailTabManagers() ?? [] } ) + monitor.registry.register( + AgentHibernationMemoryPressureResponder( + controller: AgentHibernationController.shared + ) + ) if let notificationStore { monitor.registry.register( NotificationCacheMemoryPressureResponder(store: notificationStore) diff --git a/cmux.xcodeproj/project.pbxproj b/cmux.xcodeproj/project.pbxproj index 07346a32966d..63696cc97ce3 100644 --- a/cmux.xcodeproj/project.pbxproj +++ b/cmux.xcodeproj/project.pbxproj @@ -52,6 +52,7 @@ C0DEF0D40000000000000001 /* AgentForkTimeoutResumeGate.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0DEF0D40000000000000002 /* AgentForkTimeoutResumeGate.swift */; }; F65760100000000000000001 /* AgentHibernationController+Confirmation.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760100000000000000002 /* AgentHibernationController+Confirmation.swift */; }; F65760110000000000000001 /* AgentHibernationController+InFlightTeardown.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760110000000000000002 /* AgentHibernationController+InFlightTeardown.swift */; }; + 8997C0018997C0018997C001 /* AgentHibernationController+MemoryPressure.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D0018997D0018997D001 /* AgentHibernationController+MemoryPressure.swift */; }; F65760120000000000000001 /* AgentHibernationController+PostSnapshotValidationIndexTask.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760120000000000000002 /* AgentHibernationController+PostSnapshotValidationIndexTask.swift */; }; F65760260000000000000001 /* AgentHibernationController+PostTeardownRestoreCancellationState.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760260000000000000002 /* AgentHibernationController+PostTeardownRestoreCancellationState.swift */; }; F65760130000000000000001 /* AgentHibernationController+PostTeardownRestoreTask.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760130000000000000002 /* AgentHibernationController+PostTeardownRestoreTask.swift */; }; @@ -62,6 +63,7 @@ D36A00010000000000000001 /* AgentHibernationController.swift in Sources */ = {isa = PBXBuildFile; fileRef = D36A00010000000000000002 /* AgentHibernationController.swift */; }; D36A00030000000000000001 /* AgentHibernationLifecycleState.swift in Sources */ = {isa = PBXBuildFile; fileRef = D36A00030000000000000002 /* AgentHibernationLifecycleState.swift */; }; D36A00030000000000000003 /* AgentHibernationLifecycleState.swift in Sources */ = {isa = PBXBuildFile; fileRef = D36A00030000000000000002 /* AgentHibernationLifecycleState.swift */; }; + 8997C0028997C0028997C002 /* AgentHibernationMemoryPressureResponder.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D0028997D0028997D002 /* AgentHibernationMemoryPressureResponder.swift */; }; F65760160000000000000001 /* AgentHibernationPlanner.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760160000000000000002 /* AgentHibernationPlanner.swift */; }; F65760170000000000000001 /* AgentHibernationPlannerInput.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760170000000000000002 /* AgentHibernationPlannerInput.swift */; }; F65760010000000000000001 /* AgentHibernationPlannerSwiftTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760010000000000000002 /* AgentHibernationPlannerSwiftTests.swift */; }; @@ -2511,6 +2513,7 @@ C0DE71B10000000000000001 /* AppDelegate+AgentChatNotifications.swift in Sources C0DEF0D40000000000000002 /* AgentForkTimeoutResumeGate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentForkTimeoutResumeGate.swift; sourceTree = ""; }; F65760100000000000000002 /* AgentHibernationController+Confirmation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+Confirmation.swift"; sourceTree = ""; }; F65760110000000000000002 /* AgentHibernationController+InFlightTeardown.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+InFlightTeardown.swift"; sourceTree = ""; }; + 8997D0018997D0018997D001 /* AgentHibernationController+MemoryPressure.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+MemoryPressure.swift"; sourceTree = ""; }; F65760120000000000000002 /* AgentHibernationController+PostSnapshotValidationIndexTask.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+PostSnapshotValidationIndexTask.swift"; sourceTree = ""; }; F65760260000000000000002 /* AgentHibernationController+PostTeardownRestoreCancellationState.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+PostTeardownRestoreCancellationState.swift"; sourceTree = ""; }; F65760130000000000000002 /* AgentHibernationController+PostTeardownRestoreTask.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+PostTeardownRestoreTask.swift"; sourceTree = ""; }; @@ -2520,6 +2523,7 @@ C0DE71B10000000000000001 /* AppDelegate+AgentChatNotifications.swift in Sources F65760150000000000000002 /* AgentHibernationController+UnableToProtectMarker.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+UnableToProtectMarker.swift"; sourceTree = ""; }; D36A00010000000000000002 /* AgentHibernationController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = App/AgentHibernationController.swift; sourceTree = ""; }; D36A00030000000000000002 /* AgentHibernationLifecycleState.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernation/AgentHibernationLifecycleState.swift; sourceTree = ""; }; + 8997D0028997D0028997D002 /* AgentHibernationMemoryPressureResponder.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = App/AgentHibernationMemoryPressureResponder.swift; sourceTree = ""; }; F65760160000000000000002 /* AgentHibernationPlanner.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationPlanner.swift"; sourceTree = ""; }; F65760170000000000000002 /* AgentHibernationPlannerInput.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationPlannerInput.swift"; sourceTree = ""; }; F65760010000000000000002 /* AgentHibernationPlannerSwiftTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernationPlannerSwiftTests.swift; sourceTree = ""; }; @@ -5243,6 +5247,8 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = F65760140000000000000002 /* AgentHibernationController+TailFingerprintSample.swift */, F65760060000000000000002 /* AgentHibernationController+Teardown.swift */, F65760150000000000000002 /* AgentHibernationController+UnableToProtectMarker.swift */, + 8997D0018997D0018997D001 /* AgentHibernationController+MemoryPressure.swift */, + 8997D0028997D0028997D002 /* AgentHibernationMemoryPressureResponder.swift */, D36A00010000000000000002 /* AgentHibernationController.swift */, F65760160000000000000002 /* AgentHibernationPlanner.swift */, F65760170000000000000002 /* AgentHibernationPlannerInput.swift */, @@ -7785,6 +7791,7 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = C0DEF0D40000000000000001 /* AgentForkTimeoutResumeGate.swift in Sources */, F65760100000000000000001 /* AgentHibernationController+Confirmation.swift in Sources */, F65760110000000000000001 /* AgentHibernationController+InFlightTeardown.swift in Sources */, + 8997C0018997C0018997C001 /* AgentHibernationController+MemoryPressure.swift in Sources */, F65760120000000000000001 /* AgentHibernationController+PostSnapshotValidationIndexTask.swift in Sources */, F65760260000000000000001 /* AgentHibernationController+PostTeardownRestoreCancellationState.swift in Sources */, F65760130000000000000001 /* AgentHibernationController+PostTeardownRestoreTask.swift in Sources */, @@ -7794,6 +7801,7 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = F65760150000000000000001 /* AgentHibernationController+UnableToProtectMarker.swift in Sources */, D36A00010000000000000001 /* AgentHibernationController.swift in Sources */, D36A00030000000000000001 /* AgentHibernationLifecycleState.swift in Sources */, + 8997C0028997C0028997C002 /* AgentHibernationMemoryPressureResponder.swift in Sources */, F65760160000000000000001 /* AgentHibernationPlanner.swift in Sources */, F65760170000000000000001 /* AgentHibernationPlannerInput.swift in Sources */, F65760090000000000000001 /* AgentHibernationTranscriptGuard+ClaudeWorkflow.swift in Sources */, diff --git a/cmuxTests/AgentHibernationPlannerSwiftTests.swift b/cmuxTests/AgentHibernationPlannerSwiftTests.swift index 1d2727f244f5..d7c2eda2f969 100644 --- a/cmuxTests/AgentHibernationPlannerSwiftTests.swift +++ b/cmuxTests/AgentHibernationPlannerSwiftTests.swift @@ -311,7 +311,7 @@ struct AgentHibernationPlannerSwiftTests { trigger: .systemMemoryPressure ) - #expect(selected == Set([idle])) + #expect(selected == Set([idle, liveProcess])) } @MainActor From 4cfff09d8dec2fa2d97f5ed6927aa6522ec8f1e9 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 28 Jul 2026 16:16:03 -0700 Subject: [PATCH 03/20] fix: harden critical-pressure agent teardown --- ...bernationController+InFlightTeardown.swift | 1 + ...HibernationController+MemoryPressure.swift | 18 +++++- ...rnationController+ProcessTermination.swift | 44 ++++++++++++++ .../AgentHibernationController+Records.swift | 8 ++- .../AgentHibernationController+Teardown.swift | 59 +++++++++++++++---- Sources/App/AgentHibernationController.swift | 45 ++++++-------- ...ntHibernationMemoryPressureResponder.swift | 10 +++- Sources/App/AgentHibernationPlanner.swift | 4 +- Sources/AppDelegate+PaneMemoryGuardrail.swift | 5 +- cmux.xcodeproj/project.pbxproj | 4 ++ .../AgentHibernationPlannerSwiftTests.swift | 18 +++++- 11 files changed, 163 insertions(+), 53 deletions(-) create mode 100644 Sources/App/AgentHibernationController+ProcessTermination.swift diff --git a/Sources/App/AgentHibernationController+InFlightTeardown.swift b/Sources/App/AgentHibernationController+InFlightTeardown.swift index aec7c0429bf3..ae5ea092be01 100644 --- a/Sources/App/AgentHibernationController+InFlightTeardown.swift +++ b/Sources/App/AgentHibernationController+InFlightTeardown.swift @@ -3,5 +3,6 @@ import Foundation extension AgentHibernationController { struct InFlightTeardown: Sendable { let requestID: UUID + let trigger: AgentHibernationReclaimTrigger } } diff --git a/Sources/App/AgentHibernationController+MemoryPressure.swift b/Sources/App/AgentHibernationController+MemoryPressure.swift index 900d5aaf1ca3..ff06f6a38e70 100644 --- a/Sources/App/AgentHibernationController+MemoryPressure.swift +++ b/Sources/App/AgentHibernationController+MemoryPressure.swift @@ -10,9 +10,13 @@ extension AgentHibernationController { @discardableResult func reclaimIdleAgentsForSystemMemoryPressure( now: Date, + isPressureStillCritical: @escaping @MainActor () -> Bool, onHibernationCompleted: @escaping @MainActor (Int) -> Void ) -> Bool { - guard memoryPressureEvaluation == nil else { return false } + guard memoryPressureEvaluation == nil, + isPressureStillCritical() else { + return false + } let requestID = UUID() AgentHibernationTrackingGate.setEnabled(true) @@ -27,7 +31,10 @@ extension AgentHibernationController { let settings = AgentHibernationSettings.values() let index = await RestorableAgentSessionIndex.loadIncludingProcessDetectedSnapshots() - guard !Task.isCancelled else { return } + guard !Task.isCancelled, + isPressureStillCritical() else { + return + } let initialEvaluation = self.evaluate( index: index, settings: settings, @@ -41,14 +48,19 @@ extension AgentHibernationController { } catch { return } + guard isPressureStillCritical() else { return } let confirmationIndex = await RestorableAgentSessionIndex .loadIncludingProcessDetectedSnapshots() - guard !Task.isCancelled else { return } + guard !Task.isCancelled, + isPressureStillCritical() else { + return + } let confirmationEvaluation = self.evaluate( index: confirmationIndex, settings: AgentHibernationSettings.values(), now: .now, trigger: .systemMemoryPressure, + teardownShouldProceed: isPressureStillCritical, onHibernationCompleted: { [weak self] hibernatedCount in self?.finishMemoryPressureEvaluation(requestID: requestID) onHibernationCompleted(hibernatedCount) diff --git a/Sources/App/AgentHibernationController+ProcessTermination.swift b/Sources/App/AgentHibernationController+ProcessTermination.swift new file mode 100644 index 000000000000..d0dc7d121748 --- /dev/null +++ b/Sources/App/AgentHibernationController+ProcessTermination.swift @@ -0,0 +1,44 @@ +import Darwin +import Foundation + +extension AgentHibernationController { + nonisolated static func processIdentities( + for processIDs: Set + ) -> [Int: AgentPIDProcessIdentity] { + Dictionary(uniqueKeysWithValues: processIDs.compactMap { processID in + guard processID > 0, + processID <= Int(Int32.max), + let identity = AgentPIDProcessIdentity(pid: pid_t(processID)) else { + return nil + } + return (processID, identity) + }) + } + + func terminateScopedProcessesForHibernation(record: AgentHibernationRecord) { + guard !record.processIDs.isEmpty else { return } + let currentProcessID = getpid() + let currentProcessGroupID = getpgrp() + var signaledProcessGroups: Set = [] + for rawPID in record.processIDs.sorted(by: >) { + guard rawPID > 0, rawPID <= Int(Int32.max) else { continue } + let pid = pid_t(rawPID) + guard pid != currentProcessID, + let expectedIdentity = record.processIdentities[rawPID], + AgentPIDProcessIdentity(pid: pid) == expectedIdentity else { + continue + } + guard let process = CmuxTopProcessSnapshot.processArgumentsAndEnvironment(for: rawPID), + process.matchesCMUXScope(workspaceId: record.key.workspaceId, surfaceId: record.key.panelId) else { + continue + } + let processGroupID = getpgid(pid) + if processGroupID > 1, + processGroupID != currentProcessGroupID, + signaledProcessGroups.insert(processGroupID).inserted { + _ = kill(-processGroupID, SIGTERM) + } + _ = kill(pid, SIGTERM) + } + } +} diff --git a/Sources/App/AgentHibernationController+Records.swift b/Sources/App/AgentHibernationController+Records.swift index aea21c9b7e3d..51291d1b69ef 100644 --- a/Sources/App/AgentHibernationController+Records.swift +++ b/Sources/App/AgentHibernationController+Records.swift @@ -49,6 +49,7 @@ extension AppDelegate { panelId: panelId, fallback: index.lifecycle(workspaceId: workspace.id, panelId: panelId) ) + let processIDs = index.processIDs(workspaceId: workspace.id, panelId: panelId) records.append( AgentHibernationRecord( key: key, @@ -59,8 +60,11 @@ extension AppDelegate { hasUnconfirmedTerminalInput: terminalInputAt > lifecycleChangeAt, lastActivityAt: max(indexActivity, localActivity, createdAt), isProtected: workspaceIsVisible && visiblePanelIds.contains(panelId), - hasLiveProcess: index.hasLiveProcess(workspaceId: workspace.id, panelId: panelId), - processIDs: index.processIDs(workspaceId: workspace.id, panelId: panelId) + hasLiveProcess: !processIDs.isEmpty, + processIDs: processIDs, + processIdentities: AgentHibernationController.processIdentities( + for: processIDs + ) ) ) } diff --git a/Sources/App/AgentHibernationController+Teardown.swift b/Sources/App/AgentHibernationController+Teardown.swift index bd899cbbda41..6bf9fae35424 100644 --- a/Sources/App/AgentHibernationController+Teardown.swift +++ b/Sources/App/AgentHibernationController+Teardown.swift @@ -27,6 +27,7 @@ extension AgentHibernationController { /// below covers disable/stop and anything else that changed during the brief I/O hop. func beginConfirmedTeardowns( _ requests: [ConfirmedTeardownRequest], + shouldProceed: (@MainActor () -> Bool)? = nil, onCompletion: (@MainActor (Int) -> Void)? = nil ) { guard !requests.isEmpty else { return } @@ -107,7 +108,8 @@ extension AgentHibernationController { // process-set or scrollback change, visibility/protection change, // hibernation, or surface loss during the hop aborts; a later evaluation // can re-arm it if it is still idle. - guard AgentHibernationTrackingGate.isEnabled(), + guard (shouldProceed?() ?? true), + AgentHibernationTrackingGate.isEnabled(), record.isStillOwnedByOriginalWorkspace, ( request.trigger == .systemMemoryPressure || @@ -120,6 +122,7 @@ extension AgentHibernationController { workspaceId: record.key.workspaceId, panelId: record.key.panelId ) == record.processIDs, + Self.processIdentities(for: record.processIDs) == record.processIdentities, TabManager.restorableAgentSnapshotFingerprint(currentAgent) == TabManager.restorableAgentSnapshotFingerprint(record.agent), !record.terminalPanel.isAgentHibernated, @@ -164,6 +167,14 @@ extension AgentHibernationController { await cancelPostTeardownRestoreTaskForReplacement( transcriptPath: snapshot.transcriptPath ) + guard shouldProceed?() ?? true else { + preserveSnapshotAfterAbortedTeardown( + snapshot, + record: record, + restoreOwnedSnapshotPaths: &restoreOwnedSnapshotPaths + ) + continue + } guard AgentHibernationTranscriptGuard.liveFileVersionStillMatches(snapshot) else { self.unableToProtectByPanel[record.key] = UnableToProtectMarker( fingerprint: request.confirmationFingerprint, @@ -173,21 +184,24 @@ extension AgentHibernationController { // The quiesce above disarmed the path's previous monitor, so // forfeiting must re-arm protection with the fresh snapshot; // its restore checks fail closed if the live file has turns. - if self.armPostTeardownRestoreMonitor( - snapshot: snapshot, - processIDs: record.processIDs, - snapshotDisposal: .retainForRecovery(sessionId: record.agent.sessionId) - ) { - restoreOwnedSnapshotPaths.insert(snapshot.snapshotPath) - } else { - AgentHibernationTranscriptGuard.retainSnapshotForRecovery( - snapshot, - sessionId: record.agent.sessionId - ) - } + preserveSnapshotAfterAbortedTeardown( + snapshot, + record: record, + restoreOwnedSnapshotPaths: &restoreOwnedSnapshotPaths + ) continue } } + if let shouldProceed, !shouldProceed() { + if let snapshot { + preserveSnapshotAfterAbortedTeardown( + snapshot, + record: record, + restoreOwnedSnapshotPaths: &restoreOwnedSnapshotPaths + ) + } + continue + } self.terminateScopedProcessesForHibernation(record: record) record.workspace.enterAgentHibernation( panelId: record.key.panelId, @@ -203,6 +217,25 @@ extension AgentHibernationController { } } + private func preserveSnapshotAfterAbortedTeardown( + _ snapshot: AgentHibernationTranscriptGuard.TeardownTranscriptSnapshot, + record: AgentHibernationRecord, + restoreOwnedSnapshotPaths: inout Set + ) { + if armPostTeardownRestoreMonitor( + snapshot: snapshot, + processIDs: record.processIDs, + snapshotDisposal: .retainForRecovery(sessionId: record.agent.sessionId) + ) { + restoreOwnedSnapshotPaths.insert(snapshot.snapshotPath) + } else { + AgentHibernationTranscriptGuard.retainSnapshotForRecovery( + snapshot, + sessionId: record.agent.sessionId + ) + } + } + private static func snapshotOutcomes( for requests: [ConfirmedTeardownRequest] ) async -> [AgentHibernationPanelKey: AgentHibernationTranscriptGuard.TeardownSnapshotOutcome] { diff --git a/Sources/App/AgentHibernationController.swift b/Sources/App/AgentHibernationController.swift index a196fadf09bc..683932afb345 100644 --- a/Sources/App/AgentHibernationController.swift +++ b/Sources/App/AgentHibernationController.swift @@ -1,5 +1,3 @@ -import AppKit -import Darwin import Foundation struct AgentHibernationPanelKey: Hashable, Sendable { @@ -19,6 +17,7 @@ struct AgentHibernationRecord { let isProtected: Bool let hasLiveProcess: Bool let processIDs: Set + let processIdentities: [Int: AgentPIDProcessIdentity] } @MainActor @@ -161,6 +160,7 @@ final class AgentHibernationController { settings: AgentHibernationSettings.Values, now: Date, trigger: AgentHibernationReclaimTrigger = .scheduled, + teardownShouldProceed: (@MainActor () -> Bool)? = nil, onHibernationCompleted: (@MainActor (Int) -> Void)? = nil ) -> (hasCandidates: Bool, beganTeardowns: Bool) { guard trigger != .scheduled || settings.enabled else { @@ -191,10 +191,14 @@ final class AgentHibernationController { let isLive = isLiveByKey[record.key] ?? false var effectiveLastActivityAt = record.lastActivityAt if record.hasLiveProcess { - bumpTeardownValidationEpoch(record.key) tailFingerprintSamples.removeValue(forKey: record.key) - if trigger == .scheduled { + if confirmations[record.key]?.trigger == .scheduled { confirmations.removeValue(forKey: record.key) + } + if teardownInFlightByPanel[record.key]?.trigger == .scheduled { + bumpTeardownValidationEpoch(record.key) + } + if trigger == .scheduled { unableToProtectByPanel.removeValue(forKey: record.key) } } @@ -251,6 +255,7 @@ final class AgentHibernationController { if !confirmedTeardowns.isEmpty { beginConfirmedTeardowns( confirmedTeardowns, + shouldProceed: teardownShouldProceed, onCompletion: onHibernationCompleted ) } @@ -268,6 +273,10 @@ final class AgentHibernationController { !record.hasUnconfirmedTerminalInput, !record.isProtected, (trigger == .systemMemoryPressure || !record.hasLiveProcess), + ( + trigger != .systemMemoryPressure || + record.processIdentities.count == record.processIDs.count + ), record.terminalPanel.surface.hasLiveSurface, !record.terminalPanel.isAgentHibernated else { confirmations.removeValue(forKey: record.key) @@ -289,7 +298,10 @@ final class AgentHibernationController { return nil } let requestID = UUID() - teardownInFlightByPanel[record.key] = InFlightTeardown(requestID: requestID) + teardownInFlightByPanel[record.key] = InFlightTeardown( + requestID: requestID, + trigger: trigger + ) confirmations.removeValue(forKey: record.key) return ConfirmedTeardownRequest( record: record, @@ -422,29 +434,6 @@ final class AgentHibernationController { ) } - func terminateScopedProcessesForHibernation(record: AgentHibernationRecord) { - guard !record.processIDs.isEmpty else { return } - let currentProcessID = getpid() - let currentProcessGroupID = getpgrp() - var signaledProcessGroups: Set = [] - for rawPID in record.processIDs.sorted(by: >) { - guard rawPID > 0, rawPID <= Int(Int32.max) else { continue } - let pid = pid_t(rawPID) - guard pid != currentProcessID else { continue } - guard let process = CmuxTopProcessSnapshot.processArgumentsAndEnvironment(for: rawPID), - process.matchesCMUXScope(workspaceId: record.key.workspaceId, surfaceId: record.key.panelId) else { - continue - } - let processGroupID = getpgid(pid) - if processGroupID > 1, - processGroupID != currentProcessGroupID, - signaledProcessGroups.insert(processGroupID).inserted { - _ = kill(-processGroupID, SIGTERM) - } - _ = kill(pid, SIGTERM) - } - } - private func clearTrackingState() { cancelPostTeardownRestoreTasks() teardownValidationGeneration = teardownValidationGeneration &+ 1 diff --git a/Sources/App/AgentHibernationMemoryPressureResponder.swift b/Sources/App/AgentHibernationMemoryPressureResponder.swift index ff013350ecaa..fa145d8612e0 100644 --- a/Sources/App/AgentHibernationMemoryPressureResponder.swift +++ b/Sources/App/AgentHibernationMemoryPressureResponder.swift @@ -7,16 +7,22 @@ final class AgentHibernationMemoryPressureResponder: MemoryPressureResponder { let memoryPressurePriority = 80 private let controller: AgentHibernationController + private let isPressureCritical: @MainActor () -> Bool - init(controller: AgentHibernationController) { + init( + controller: AgentHibernationController, + isPressureCritical: @escaping @MainActor () -> Bool + ) { self.controller = controller + self.isPressureCritical = isPressureCritical } func shedMemory(for snapshot: MemoryPressureSnapshot) -> MemoryPressureShedResult { let responderID = memoryPressureResponderID let severity = snapshot.severity let didSchedule = controller.reclaimIdleAgentsForSystemMemoryPressure( - now: snapshot.sampledAt + now: snapshot.sampledAt, + isPressureStillCritical: isPressureCritical ) { hibernatedCount in guard hibernatedCount > 0 else { return } MemoryPressureResponderRegistry.logShedAction( diff --git a/Sources/App/AgentHibernationPlanner.swift b/Sources/App/AgentHibernationPlanner.swift index b46420554c00..0f85fb9154b1 100644 --- a/Sources/App/AgentHibernationPlanner.swift +++ b/Sources/App/AgentHibernationPlanner.swift @@ -6,6 +6,8 @@ enum AgentHibernationReclaimTrigger: Equatable, Sendable { } enum AgentHibernationPlanner { + private static let systemMemoryPressureBatchLimit = 2 + static func selectedPanelKeys( inputs: [AgentHibernationPlannerInput], settings: AgentHibernationSettings.Values, @@ -19,7 +21,7 @@ enum AgentHibernationPlanner { guard settings.enabled else { return [] } excess = liveRestorable.count - settings.maxLiveTerminals case .systemMemoryPressure: - excess = liveRestorable.count + excess = min(liveRestorable.count, systemMemoryPressureBatchLimit) } guard excess > 0 else { return [] } diff --git a/Sources/AppDelegate+PaneMemoryGuardrail.swift b/Sources/AppDelegate+PaneMemoryGuardrail.swift index 5f396f68387b..f2fc17729773 100644 --- a/Sources/AppDelegate+PaneMemoryGuardrail.swift +++ b/Sources/AppDelegate+PaneMemoryGuardrail.swift @@ -36,7 +36,10 @@ extension AppDelegate { ) monitor.registry.register( AgentHibernationMemoryPressureResponder( - controller: AgentHibernationController.shared + controller: AgentHibernationController.shared, + isPressureCritical: { [weak monitor] in + monitor?.currentSeverity == .critical + } ) ) if let notificationStore { diff --git a/cmux.xcodeproj/project.pbxproj b/cmux.xcodeproj/project.pbxproj index 63696cc97ce3..3853d6273841 100644 --- a/cmux.xcodeproj/project.pbxproj +++ b/cmux.xcodeproj/project.pbxproj @@ -56,6 +56,7 @@ F65760120000000000000001 /* AgentHibernationController+PostSnapshotValidationIndexTask.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760120000000000000002 /* AgentHibernationController+PostSnapshotValidationIndexTask.swift */; }; F65760260000000000000001 /* AgentHibernationController+PostTeardownRestoreCancellationState.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760260000000000000002 /* AgentHibernationController+PostTeardownRestoreCancellationState.swift */; }; F65760130000000000000001 /* AgentHibernationController+PostTeardownRestoreTask.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760130000000000000002 /* AgentHibernationController+PostTeardownRestoreTask.swift */; }; + 8997C0038997C0038997C003 /* AgentHibernationController+ProcessTermination.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D0038997D0038997D003 /* AgentHibernationController+ProcessTermination.swift */; }; F65760030000000000000001 /* AgentHibernationController+Records.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760030000000000000002 /* AgentHibernationController+Records.swift */; }; F65760140000000000000001 /* AgentHibernationController+TailFingerprintSample.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760140000000000000002 /* AgentHibernationController+TailFingerprintSample.swift */; }; F65760060000000000000001 /* AgentHibernationController+Teardown.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760060000000000000002 /* AgentHibernationController+Teardown.swift */; }; @@ -2517,6 +2518,7 @@ C0DE71B10000000000000001 /* AppDelegate+AgentChatNotifications.swift in Sources F65760120000000000000002 /* AgentHibernationController+PostSnapshotValidationIndexTask.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+PostSnapshotValidationIndexTask.swift"; sourceTree = ""; }; F65760260000000000000002 /* AgentHibernationController+PostTeardownRestoreCancellationState.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+PostTeardownRestoreCancellationState.swift"; sourceTree = ""; }; F65760130000000000000002 /* AgentHibernationController+PostTeardownRestoreTask.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+PostTeardownRestoreTask.swift"; sourceTree = ""; }; + 8997D0038997D0038997D003 /* AgentHibernationController+ProcessTermination.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+ProcessTermination.swift"; sourceTree = ""; }; F65760030000000000000002 /* AgentHibernationController+Records.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+Records.swift"; sourceTree = ""; }; F65760140000000000000002 /* AgentHibernationController+TailFingerprintSample.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+TailFingerprintSample.swift"; sourceTree = ""; }; F65760060000000000000002 /* AgentHibernationController+Teardown.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+Teardown.swift"; sourceTree = ""; }; @@ -5243,6 +5245,7 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = F65760120000000000000002 /* AgentHibernationController+PostSnapshotValidationIndexTask.swift */, F65760130000000000000002 /* AgentHibernationController+PostTeardownRestoreTask.swift */, F65760260000000000000002 /* AgentHibernationController+PostTeardownRestoreCancellationState.swift */, + 8997D0038997D0038997D003 /* AgentHibernationController+ProcessTermination.swift */, F65760030000000000000002 /* AgentHibernationController+Records.swift */, F65760140000000000000002 /* AgentHibernationController+TailFingerprintSample.swift */, F65760060000000000000002 /* AgentHibernationController+Teardown.swift */, @@ -7795,6 +7798,7 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = F65760120000000000000001 /* AgentHibernationController+PostSnapshotValidationIndexTask.swift in Sources */, F65760260000000000000001 /* AgentHibernationController+PostTeardownRestoreCancellationState.swift in Sources */, F65760130000000000000001 /* AgentHibernationController+PostTeardownRestoreTask.swift in Sources */, + 8997C0038997C0038997C003 /* AgentHibernationController+ProcessTermination.swift in Sources */, F65760030000000000000001 /* AgentHibernationController+Records.swift in Sources */, F65760140000000000000001 /* AgentHibernationController+TailFingerprintSample.swift in Sources */, F65760060000000000000001 /* AgentHibernationController+Teardown.swift in Sources */, diff --git a/cmuxTests/AgentHibernationPlannerSwiftTests.swift b/cmuxTests/AgentHibernationPlannerSwiftTests.swift index d7c2eda2f969..20052859c6ce 100644 --- a/cmuxTests/AgentHibernationPlannerSwiftTests.swift +++ b/cmuxTests/AgentHibernationPlannerSwiftTests.swift @@ -134,7 +134,8 @@ struct AgentHibernationPlannerSwiftTests { lastActivityAt: 0, isProtected: false, hasLiveProcess: false, - processIDs: [] + processIDs: [], + processIdentities: [:] ) #expect(record.isStillOwnedByOriginalWorkspace) @@ -235,6 +236,7 @@ struct AgentHibernationPlannerSwiftTests { let workspaceId = UUID() let now: TimeInterval = 1_000 let idle = AgentHibernationPanelKey(workspaceId: workspaceId, panelId: UUID()) + let secondIdle = AgentHibernationPanelKey(workspaceId: workspaceId, panelId: UUID()) let visible = AgentHibernationPanelKey(workspaceId: workspaceId, panelId: UUID()) let running = AgentHibernationPanelKey(workspaceId: workspaceId, panelId: UUID()) let liveProcess = AgentHibernationPanelKey(workspaceId: workspaceId, panelId: UUID()) @@ -258,6 +260,15 @@ struct AgentHibernationPlannerSwiftTests { hasUnconfirmedTerminalInput: false, lastActivityAt: now ), + .init( + key: secondIdle, + hasRestorableAgent: true, + isLive: true, + isProtected: false, + lifecycle: .idle, + hasUnconfirmedTerminalInput: false, + lastActivityAt: 1 + ), .init( key: visible, hasRestorableAgent: true, @@ -311,7 +322,7 @@ struct AgentHibernationPlannerSwiftTests { trigger: .systemMemoryPressure ) - #expect(selected == Set([idle, liveProcess])) + #expect(selected == Set([secondIdle, liveProcess])) } @MainActor @@ -455,7 +466,8 @@ struct AgentHibernationPlannerSwiftTests { lastActivityAt: 100, isProtected: false, hasLiveProcess: false, - processIDs: [] + processIDs: [], + processIdentities: [:] ) #expect(controller.postSnapshotLifecycle(for: record, index: index) == .running) From 102e73302661d093ab1fedbca5de96338059fdd0 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 28 Jul 2026 16:18:14 -0700 Subject: [PATCH 04/20] fix: retain confirmed process generations --- Sources/App/AgentHibernationController+Confirmation.swift | 1 + Sources/App/AgentHibernationController.swift | 4 +++- Sources/App/AgentHibernationPlanner.swift | 1 + cmuxTests/AgentHibernationPlannerSwiftTests.swift | 2 +- 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Sources/App/AgentHibernationController+Confirmation.swift b/Sources/App/AgentHibernationController+Confirmation.swift index aced833e985f..5453efc7189c 100644 --- a/Sources/App/AgentHibernationController+Confirmation.swift +++ b/Sources/App/AgentHibernationController+Confirmation.swift @@ -4,6 +4,7 @@ extension AgentHibernationController { struct Confirmation: Sendable { let trigger: AgentHibernationReclaimTrigger let fingerprint: String + let processIdentities: [Int: AgentPIDProcessIdentity] let sampledAt: TimeInterval let dueAt: TimeInterval } diff --git a/Sources/App/AgentHibernationController.swift b/Sources/App/AgentHibernationController.swift index 683932afb345..1ab12635f09c 100644 --- a/Sources/App/AgentHibernationController.swift +++ b/Sources/App/AgentHibernationController.swift @@ -293,7 +293,8 @@ final class AgentHibernationController { return nil } guard let fingerprint = hibernationFingerprint(for: record), - fingerprint == confirmation.fingerprint else { + fingerprint == confirmation.fingerprint, + record.processIdentities == confirmation.processIdentities else { confirmations.removeValue(forKey: record.key) return nil } @@ -331,6 +332,7 @@ final class AgentHibernationController { confirmations[record.key] = Confirmation( trigger: trigger, fingerprint: fingerprint, + processIdentities: record.processIdentities, sampledAt: now, dueAt: now + settings.confirmationSeconds ) diff --git a/Sources/App/AgentHibernationPlanner.swift b/Sources/App/AgentHibernationPlanner.swift index 0f85fb9154b1..b0627fbab875 100644 --- a/Sources/App/AgentHibernationPlanner.swift +++ b/Sources/App/AgentHibernationPlanner.swift @@ -21,6 +21,7 @@ enum AgentHibernationPlanner { guard settings.enabled else { return [] } excess = liveRestorable.count - settings.maxLiveTerminals case .systemMemoryPressure: + // Snapshot and reclaim a small batch before the next pressure pass. excess = min(liveRestorable.count, systemMemoryPressureBatchLimit) } guard excess > 0 else { return [] } diff --git a/cmuxTests/AgentHibernationPlannerSwiftTests.swift b/cmuxTests/AgentHibernationPlannerSwiftTests.swift index 20052859c6ce..51e482463eed 100644 --- a/cmuxTests/AgentHibernationPlannerSwiftTests.swift +++ b/cmuxTests/AgentHibernationPlannerSwiftTests.swift @@ -232,7 +232,7 @@ struct AgentHibernationPlannerSwiftTests { } @Test - func criticalPressureSelectsOnlySafeIdleAgentsWhenScheduledHibernationIsDisabled() { + func criticalPressureSelectsBoundedSafeIdleBatchWhenScheduledHibernationIsDisabled() { let workspaceId = UUID() let now: TimeInterval = 1_000 let idle = AgentHibernationPanelKey(workspaceId: workspaceId, panelId: UUID()) From 4d5f1428a23b41cb9428c1e62ab49ceff19bf5be Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 28 Jul 2026 16:43:48 -0700 Subject: [PATCH 05/20] fix: harden emergency agent hibernation --- CLI/cmux.swift | 2 +- .../Sections/TerminalSection.swift | 2 +- .../Runtime/AgentHibernationRecording.swift | 6 +- .../FakeHibernationRecorder.swift | 1 + Resources/Localizable.xcstrings | 4 +- ...HibernationController+MemoryPressure.swift | 4 - ...rnationController+ProcessTermination.swift | 122 ++++++++++++++-- .../AgentHibernationController+Teardown.swift | 19 ++- Sources/App/AgentHibernationController.swift | 14 +- Sources/App/WorkspaceRuntimeSettings.swift | 18 +-- Sources/GhosttyTerminalView.swift | 13 +- Sources/TerminalController.swift | 2 +- Sources/TerminalSurfaceRuntimeWiring.swift | 16 +-- cmux.xcodeproj/project.pbxproj | 4 + ...ntHibernationProcessTerminationTests.swift | 132 ++++++++++++++++++ .../SettingsTerminalBehaviorUITests.swift | 4 +- docs/agent-hooks.md | 12 +- docs/cli-contract.md | 2 +- docs/configuration.md | 6 +- web/data/cmux.schema.json | 6 +- web/messages/en.json | 2 + web/messages/ja.json | 2 + 22 files changed, 316 insertions(+), 77 deletions(-) create mode 100644 cmuxTests/AgentHibernationProcessTerminationTests.swift diff --git a/CLI/cmux.swift b/CLI/cmux.swift index 91a534545f82..56fd1423c1a5 100644 --- a/CLI/cmux.swift +++ b/CLI/cmux.swift @@ -15379,7 +15379,7 @@ struct CMUXCLI { return """ Usage: cmux agent-hibernation [--json] - Enable or disable Agent Hibernation. + Enable or disable routine Agent Hibernation. Configure idle and live-terminal limits from Settings or cmux settings JSON. """ case "restore-session": diff --git a/Packages/macOS/CmuxSettingsUI/Sources/CmuxSettingsUI/Sections/TerminalSection.swift b/Packages/macOS/CmuxSettingsUI/Sources/CmuxSettingsUI/Sections/TerminalSection.swift index 410497794eb8..ee00342b6986 100644 --- a/Packages/macOS/CmuxSettingsUI/Sources/CmuxSettingsUI/Sections/TerminalSection.swift +++ b/Packages/macOS/CmuxSettingsUI/Sources/CmuxSettingsUI/Sections/TerminalSection.swift @@ -386,7 +386,7 @@ public struct TerminalSection: View { String(localized: "settings.terminal.agentHibernation", defaultValue: "Agent Hibernation"), subtitle: hibernation.current ? String(localized: "settings.terminal.agentHibernation.subtitleOn", defaultValue: "Idle background agent terminals can be suspended when the live-terminal limit is exceeded.") - : String(localized: "settings.terminal.agentHibernation.subtitleOff", defaultValue: "Agent terminals stay live until you close them or quit cmux.") + : String(localized: "settings.terminal.agentHibernation.subtitleOff", defaultValue: "Scheduled hibernation is off. During critical memory pressure, cmux may still hibernate safe idle background agents.") ) { Toggle("", isOn: Binding(get: { hibernation.current }, set: { hibernation.set($0) })) .labelsHidden() diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Runtime/AgentHibernationRecording.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Runtime/AgentHibernationRecording.swift index bfef80ce3b09..bc0d00e2f39f 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Runtime/AgentHibernationRecording.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Runtime/AgentHibernationRecording.swift @@ -3,14 +3,14 @@ public import Foundation /// Records terminal input liveness for agent-hibernation tracking. /// /// Implemented in the app over `AgentHibernationController`; the recorder is -/// called synchronously on the input hot path and is responsible for its own -/// enablement gate and main-actor hop, exactly like the legacy -/// `recordAgentHibernationTerminalInput` helper it replaces. +/// called synchronously on the main-actor input hot path and is responsible for +/// its own lock-free enablement gate. public protocol AgentHibernationRecording: AnyObject, Sendable { /// Records that a terminal surface received input. /// /// - Parameters: /// - workspaceId: The owning workspace id. /// - panelId: The surface/panel id that received input. + @MainActor func recordTerminalInput(workspaceId: UUID, panelId: UUID) } diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/FakeHibernationRecorder.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/FakeHibernationRecorder.swift index 11616275d93f..c6e64610eb63 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/FakeHibernationRecorder.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/FakeHibernationRecorder.swift @@ -2,5 +2,6 @@ import Foundation @testable import CmuxTerminal final class FakeHibernationRecorder: AgentHibernationRecording { + @MainActor func recordTerminalInput(workspaceId: UUID, panelId: UUID) {} } diff --git a/Resources/Localizable.xcstrings b/Resources/Localizable.xcstrings index 719cc4018166..0557b21f3372 100644 --- a/Resources/Localizable.xcstrings +++ b/Resources/Localizable.xcstrings @@ -194112,13 +194112,13 @@ "en": { "stringUnit": { "state": "translated", - "value": "Agent terminals stay live until you close them or quit cmux." + "value": "Scheduled hibernation is off. During critical memory pressure, cmux may still hibernate safe idle background agents." } }, "ja": { "stringUnit": { "state": "translated", - "value": "エージェント端末は、閉じるか cmux を終了するまでライブのままです。" + "value": "通常の休止はオフです。メモリ負荷が危険な状態では、安全なアイドル中のバックグラウンドエージェントを cmux が休止する場合があります。" } } } diff --git a/Sources/App/AgentHibernationController+MemoryPressure.swift b/Sources/App/AgentHibernationController+MemoryPressure.swift index ff06f6a38e70..a0404c10da71 100644 --- a/Sources/App/AgentHibernationController+MemoryPressure.swift +++ b/Sources/App/AgentHibernationController+MemoryPressure.swift @@ -19,7 +19,6 @@ extension AgentHibernationController { } let requestID = UUID() - AgentHibernationTrackingGate.setEnabled(true) let task = Task { @MainActor [weak self] in guard let self else { return } var awaitsTeardownCompletion = false @@ -76,8 +75,5 @@ extension AgentHibernationController { guard memoryPressureEvaluation?.id == requestID else { return } memoryPressureEvaluation = nil clearMemoryPressureConfirmations() - AgentHibernationTrackingGate.setEnabled( - AgentHibernationSettings.isEnabled() - ) } } diff --git a/Sources/App/AgentHibernationController+ProcessTermination.swift b/Sources/App/AgentHibernationController+ProcessTermination.swift index d0dc7d121748..961c657ea76c 100644 --- a/Sources/App/AgentHibernationController+ProcessTermination.swift +++ b/Sources/App/AgentHibernationController+ProcessTermination.swift @@ -1,7 +1,29 @@ import Darwin import Foundation +extension AgentHibernationRecord { + var processTerminationScope: AgentHibernationController.ProcessTerminationScope { + AgentHibernationController.ProcessTerminationScope( + key: key, + processIDs: processIDs, + processIdentities: processIdentities + ) + } +} + extension AgentHibernationController { + struct ProcessTerminationScope: Sendable { + let key: AgentHibernationPanelKey + let processIDs: Set + let processIdentities: [Int: AgentPIDProcessIdentity] + } + + struct ScopedProcessTermination: Equatable, Sendable { + let processID: Int + let processIdentity: AgentPIDProcessIdentity + let processGroupID: pid_t + } + nonisolated static func processIdentities( for processIDs: Set ) -> [Int: AgentPIDProcessIdentity] { @@ -15,24 +37,93 @@ extension AgentHibernationController { }) } - func terminateScopedProcessesForHibernation(record: AgentHibernationRecord) { - guard !record.processIDs.isEmpty else { return } + nonisolated static func scopedProcessTerminations( + for scopes: [ProcessTerminationScope] + ) async -> [AgentHibernationPanelKey: [ScopedProcessTermination]] { + await withTaskGroup( + of: (AgentHibernationPanelKey, [ScopedProcessTermination]?).self, + returning: [AgentHibernationPanelKey: [ScopedProcessTermination]].self + ) { group in + var terminationsByPanel: [AgentHibernationPanelKey: [ScopedProcessTermination]] = Dictionary( + uniqueKeysWithValues: scopes.compactMap { scope in + scope.processIDs.isEmpty ? (scope.key, []) : nil + } + ) + for scope in scopes where !scope.processIDs.isEmpty { + group.addTask(priority: .utility) { + ( + scope.key, + validatedScopedProcessTerminations( + for: scope, + processIdentityProvider: { + AgentPIDProcessIdentity(pid: pid_t($0)) + }, + processArgumentsProvider: + CmuxTopProcessSnapshot.processArgumentsAndEnvironment(for:), + processGroupProvider: { getpgid(pid_t($0)) } + ) + ) + } + } + for await (key, terminations) in group { + if let terminations { + terminationsByPanel[key] = terminations + } + } + return terminationsByPanel + } + } + + nonisolated static func validatedScopedProcessTerminations( + for scope: ProcessTerminationScope, + processIdentityProvider: (Int) -> AgentPIDProcessIdentity?, + processArgumentsProvider: (Int) -> CmuxTopProcessArguments?, + processGroupProvider: (Int) -> pid_t + ) -> [ScopedProcessTermination]? { + guard Set(scope.processIdentities.keys) == scope.processIDs else { return nil } + var terminations: [ScopedProcessTermination] = [] + for processID in scope.processIDs.sorted(by: >) { + guard processID > 0, + processID <= Int(Int32.max), + let expectedIdentity = scope.processIdentities[processID], + processIdentityProvider(processID) == expectedIdentity, + let process = processArgumentsProvider(processID), + process.matchesCMUXScope( + workspaceId: scope.key.workspaceId, + surfaceId: scope.key.panelId + ) else { + return nil + } + terminations.append( + ScopedProcessTermination( + processID: processID, + processIdentity: expectedIdentity, + processGroupID: processGroupProvider(processID) + ) + ) + } + return terminations + } + + @discardableResult + func terminateScopedProcessesForHibernation( + _ terminations: [ScopedProcessTermination] + ) -> Bool { + guard !terminations.isEmpty else { return true } let currentProcessID = getpid() let currentProcessGroupID = getpgrp() + guard terminations.allSatisfy({ termination in + let pid = pid_t(termination.processID) + return pid != currentProcessID && + AgentPIDProcessIdentity(pid: pid) == termination.processIdentity && + getpgid(pid) == termination.processGroupID + }) else { + return false + } var signaledProcessGroups: Set = [] - for rawPID in record.processIDs.sorted(by: >) { - guard rawPID > 0, rawPID <= Int(Int32.max) else { continue } - let pid = pid_t(rawPID) - guard pid != currentProcessID, - let expectedIdentity = record.processIdentities[rawPID], - AgentPIDProcessIdentity(pid: pid) == expectedIdentity else { - continue - } - guard let process = CmuxTopProcessSnapshot.processArgumentsAndEnvironment(for: rawPID), - process.matchesCMUXScope(workspaceId: record.key.workspaceId, surfaceId: record.key.panelId) else { - continue - } - let processGroupID = getpgid(pid) + for termination in terminations { + let pid = pid_t(termination.processID) + let processGroupID = termination.processGroupID if processGroupID > 1, processGroupID != currentProcessGroupID, signaledProcessGroups.insert(processGroupID).inserted { @@ -40,5 +131,6 @@ extension AgentHibernationController { } _ = kill(pid, SIGTERM) } + return true } } diff --git a/Sources/App/AgentHibernationController+Teardown.swift b/Sources/App/AgentHibernationController+Teardown.swift index 6bf9fae35424..aff0e169b5ab 100644 --- a/Sources/App/AgentHibernationController+Teardown.swift +++ b/Sources/App/AgentHibernationController+Teardown.swift @@ -87,6 +87,9 @@ extension AgentHibernationController { } } + let scopedProcessTerminationsByPanel = await Self.scopedProcessTerminations( + for: requests.map { $0.record.processTerminationScope } + ) let postSnapshotSequence = markPostSnapshotValidationPoint() let postSnapshotIndex = await sharedPostSnapshotValidationIndexTask( minimumStartSequence: postSnapshotSequence @@ -94,7 +97,10 @@ extension AgentHibernationController { for request in requests { let record = request.record - guard let snapshotOutcome = snapshotOutcomes[record.key] else { continue } + guard let snapshotOutcome = snapshotOutcomes[record.key], + let scopedProcessTerminations = scopedProcessTerminationsByPanel[record.key] else { + continue + } let currentAgent = record.workspace.restorableAgentForHibernation( panelId: record.key.panelId, index: postSnapshotIndex @@ -202,7 +208,16 @@ extension AgentHibernationController { } continue } - self.terminateScopedProcessesForHibernation(record: record) + guard self.terminateScopedProcessesForHibernation(scopedProcessTerminations) else { + if let snapshot { + preserveSnapshotAfterAbortedTeardown( + snapshot, + record: record, + restoreOwnedSnapshotPaths: &restoreOwnedSnapshotPaths + ) + } + continue + } record.workspace.enterAgentHibernation( panelId: record.key.panelId, agent: record.agent, diff --git a/Sources/App/AgentHibernationController.swift b/Sources/App/AgentHibernationController.swift index 1ab12635f09c..7ae86493ce74 100644 --- a/Sources/App/AgentHibernationController.swift +++ b/Sources/App/AgentHibernationController.swift @@ -47,6 +47,7 @@ final class AgentHibernationController { private init() {} func start() { + AgentHibernationTrackingGate.setEnabled(true) guard settingsObserver == nil else { updateTimerForCurrentSettings() return @@ -78,8 +79,13 @@ final class AgentHibernationController { func recordTerminalInput(workspaceId: UUID, panelId: UUID, recordedAt: Date? = nil) { guard AgentHibernationTrackingGate.isEnabled() else { return } + let key = AgentHibernationPanelKey(workspaceId: workspaceId, panelId: panelId) + if let terminalInputAt = terminalInputByPanel[key], + terminalInputAt > (lifecycleChangeByPanel[key] ?? 0) { + return + } let recordedAt = recordedAt ?? Date() - let key = recordActivity(workspaceId: workspaceId, panelId: panelId, recordedAt: recordedAt) + recordActivity(workspaceId: workspaceId, panelId: panelId, recordedAt: recordedAt) terminalInputByPanel[key] = recordedAt.timeIntervalSince1970 } @@ -125,15 +131,9 @@ final class AgentHibernationController { private func updateTimerForCurrentSettings() { let enabled = AgentHibernationSettings.isEnabled() - AgentHibernationTrackingGate.setEnabled( - enabled || memoryPressureEvaluation != nil - ) guard enabled else { timer?.cancel() timer = nil - if memoryPressureEvaluation == nil { - clearTrackingState() - } return } guard timer == nil else { return } diff --git a/Sources/App/WorkspaceRuntimeSettings.swift b/Sources/App/WorkspaceRuntimeSettings.swift index d984e3f68a20..784696e6c73c 100644 --- a/Sources/App/WorkspaceRuntimeSettings.swift +++ b/Sources/App/WorkspaceRuntimeSettings.swift @@ -1,4 +1,5 @@ import Darwin +import CmuxFoundation import Foundation enum WorkspaceTitlebarSettings { static let showTitlebarKey = "workspaceTitlebarVisible" @@ -269,7 +270,7 @@ enum AgentHibernationSettings { static let confirmationSecondsKey = "terminal.agentHibernation.confirmationSeconds" static let defaultEnabled = false - // Hibernation is opt-in. Once enabled, reclaim idle background agents quickly: + // Routine hibernation is opt-in. Once enabled, reclaim idle background agents quickly: // the maxLiveTerminals cap and the confirmationSeconds settle window keep this safe. static let defaultIdleSeconds: TimeInterval = 5 static let defaultMaxLiveTerminals = 12 @@ -364,8 +365,8 @@ enum AgentHibernationSettings { } /// Settings for non-destructive offscreen renderer reclamation. Unlike -/// `AgentHibernationSettings` (which kills a resumable agent's PTY and is opt-in), -/// this only releases an offscreen terminal's GPU renderer (Metal swap chain / +/// routine `AgentHibernationSettings` (which kills a resumable agent's PTY and is +/// opt-in), this only releases an offscreen terminal's GPU renderer (Metal swap chain / /// IOSurface) while keeping its PTY and terminal state alive, rebuilding it on /// re-show. It is therefore safe to default ON. The cap keeps recently-used tabs /// warm so switching stays instant; the idle window avoids reclaiming a tab the @@ -462,19 +463,14 @@ enum RendererRealizationSettings { } enum AgentHibernationTrackingGate { - private static let lock = NSLock() - private static var enabled = AgentHibernationSettings.isEnabled() + private static let gate = AtomicBooleanGate(false) static func isEnabled() -> Bool { - lock.lock() - defer { lock.unlock() } - return enabled + gate.loadRelaxed() } static func setEnabled(_ nextEnabled: Bool) { - lock.lock() - enabled = nextEnabled - lock.unlock() + gate.storeRelease(nextEnabled) } } diff --git a/Sources/GhosttyTerminalView.swift b/Sources/GhosttyTerminalView.swift index ddc21b99ca1b..d49548a1e605 100644 --- a/Sources/GhosttyTerminalView.swift +++ b/Sources/GhosttyTerminalView.swift @@ -3102,16 +3102,13 @@ private final class TerminalSharedBackdropCutoutFilter: CIFilter { // TerminalSurfaceFocusPlacement moved to CmuxTerminalCore (SurfaceRegistry/). +@MainActor private func recordAgentHibernationTerminalInput(workspaceId: UUID, panelId: UUID) { guard AgentHibernationTrackingGate.isEnabled() else { return } - let recordedAt = Date() - Task { @MainActor in - AgentHibernationController.shared.recordTerminalInput( - workspaceId: workspaceId, - panelId: panelId, - recordedAt: recordedAt - ) - } + AgentHibernationController.shared.recordTerminalInput( + workspaceId: workspaceId, + panelId: panelId + ) } // TerminalSurface and its SearchState moved to the CmuxTerminal package diff --git a/Sources/TerminalController.swift b/Sources/TerminalController.swift index f7c4b518f6d8..da7af208274c 100644 --- a/Sources/TerminalController.swift +++ b/Sources/TerminalController.swift @@ -10963,7 +10963,7 @@ class TerminalController { simulate_app_active - Trigger app active handler set_status [--icon=X] [--color=#hex] [--url=X] [--priority=N] [--format=plain|markdown] [--tab=X] - Set a status entry set_agent_lifecycle [--tab=X] [--panel=ID] - Report coding-agent lifecycle for hibernation - agent_hibernation - Enable or disable Agent Hibernation + agent_hibernation - Enable or disable routine Agent Hibernation report_meta [--icon=X] [--color=#hex] [--url=X] [--priority=N] [--format=plain|markdown] [--tab=X] - Set sidebar metadata entry report_meta_block [--priority=N] [--tab=X] -- - Set freeform sidebar markdown block clear_status [--tab=X] - Remove a status entry diff --git a/Sources/TerminalSurfaceRuntimeWiring.swift b/Sources/TerminalSurfaceRuntimeWiring.swift index e21c60f7d856..78f3f7dd81a2 100644 --- a/Sources/TerminalSurfaceRuntimeWiring.swift +++ b/Sources/TerminalSurfaceRuntimeWiring.swift @@ -121,19 +121,15 @@ extension RendererRealizationController: TerminalRendererRealizationScheduling { // MARK: Agent hibernation -/// The legacy `recordAgentHibernationTerminalInput` free helper as an -/// injected recorder: same gate, same timestamp capture, same main-actor hop. +/// The app-owned safety tracker injected into the terminal input path. +@MainActor final class TerminalAgentHibernationRecorder: AgentHibernationRecording { func recordTerminalInput(workspaceId: UUID, panelId: UUID) { guard AgentHibernationTrackingGate.isEnabled() else { return } - let recordedAt = Date() - Task { @MainActor in - AgentHibernationController.shared.recordTerminalInput( - workspaceId: workspaceId, - panelId: panelId, - recordedAt: recordedAt - ) - } + AgentHibernationController.shared.recordTerminalInput( + workspaceId: workspaceId, + panelId: panelId + ) } } diff --git a/cmux.xcodeproj/project.pbxproj b/cmux.xcodeproj/project.pbxproj index 3853d6273841..0de87e1cb952 100644 --- a/cmux.xcodeproj/project.pbxproj +++ b/cmux.xcodeproj/project.pbxproj @@ -69,6 +69,7 @@ F65760170000000000000001 /* AgentHibernationPlannerInput.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760170000000000000002 /* AgentHibernationPlannerInput.swift */; }; F65760010000000000000001 /* AgentHibernationPlannerSwiftTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760010000000000000002 /* AgentHibernationPlannerSwiftTests.swift */; }; D72260010000000000000001 /* AgentHibernationPortalVisibilityTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D72260010000000000000002 /* AgentHibernationPortalVisibilityTests.swift */; }; + 8997C0048997C0048997C004 /* AgentHibernationProcessTerminationTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D0048997D0048997D004 /* AgentHibernationProcessTerminationTests.swift */; }; F65760240000000000000001 /* AgentHibernationRestoreMonitorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760240000000000000002 /* AgentHibernationRestoreMonitorTests.swift */; }; F65760020000000000000001 /* AgentHibernationTestHelpers.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760020000000000000002 /* AgentHibernationTestHelpers.swift */; }; D36A00020000000000000001 /* AgentHibernationTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D36A00020000000000000002 /* AgentHibernationTests.swift */; }; @@ -2530,6 +2531,7 @@ C0DE71B10000000000000001 /* AppDelegate+AgentChatNotifications.swift in Sources F65760170000000000000002 /* AgentHibernationPlannerInput.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationPlannerInput.swift"; sourceTree = ""; }; F65760010000000000000002 /* AgentHibernationPlannerSwiftTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernationPlannerSwiftTests.swift; sourceTree = ""; }; D72260010000000000000002 /* AgentHibernationPortalVisibilityTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernationPortalVisibilityTests.swift; sourceTree = ""; }; + 8997D0048997D0048997D004 /* AgentHibernationProcessTerminationTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernationProcessTerminationTests.swift; sourceTree = ""; }; F65760240000000000000002 /* AgentHibernationRestoreMonitorTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernationRestoreMonitorTests.swift; sourceTree = ""; }; F65760020000000000000002 /* AgentHibernationTestHelpers.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernationTestHelpers.swift; sourceTree = ""; }; D36A00020000000000000002 /* AgentHibernationTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernationTests.swift; sourceTree = ""; }; @@ -6728,6 +6730,7 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = A9E050000000000000000001 /* AgentSessionWebRendererTests.swift */, A9E040000000000000000001 /* CodexAppServerSessionTests.swift */, F65760010000000000000002 /* AgentHibernationPlannerSwiftTests.swift */, + 8997D0048997D0048997D004 /* AgentHibernationProcessTerminationTests.swift */, F65760240000000000000002 /* AgentHibernationRestoreMonitorTests.swift */, F65760040000000000000002 /* AgentHibernationTranscriptGuardTests.swift */, F65760080000000000000002 /* AgentHibernationTranscriptGuardScanTests.swift */, @@ -9461,6 +9464,7 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = A9E020000000000000000005 /* AgentExecutableResolverTests.swift in Sources */, F65760010000000000000001 /* AgentHibernationPlannerSwiftTests.swift in Sources */, D72260010000000000000001 /* AgentHibernationPortalVisibilityTests.swift in Sources */, + 8997C0048997C0048997C004 /* AgentHibernationProcessTerminationTests.swift in Sources */, F65760240000000000000001 /* AgentHibernationRestoreMonitorTests.swift in Sources */, F65760020000000000000001 /* AgentHibernationTestHelpers.swift in Sources */, D36A00020000000000000001 /* AgentHibernationTests.swift in Sources */, diff --git a/cmuxTests/AgentHibernationProcessTerminationTests.swift b/cmuxTests/AgentHibernationProcessTerminationTests.swift new file mode 100644 index 000000000000..0e1eb126059f --- /dev/null +++ b/cmuxTests/AgentHibernationProcessTerminationTests.swift @@ -0,0 +1,132 @@ +import Darwin +import Foundation +import Testing + +#if canImport(cmux_DEV) +@testable import cmux_DEV +#elseif canImport(cmux) +@testable import cmux +#endif + +struct AgentHibernationProcessTerminationTests { + @Test + func validatesExactProcessGenerationAndCmuxScope() throws { + let workspaceID = UUID() + let panelID = UUID() + let firstIdentity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let secondIdentity = AgentPIDProcessIdentity( + pid: 202, + startSeconds: 20, + startMicroseconds: 2 + ) + let identities = [101: firstIdentity, 202: secondIdentity] + let scope = AgentHibernationController.ProcessTerminationScope( + key: AgentHibernationPanelKey(workspaceId: workspaceID, panelId: panelID), + processIDs: Set(identities.keys), + processIdentities: identities + ) + + let terminations = try #require( + AgentHibernationController.validatedScopedProcessTerminations( + for: scope, + processIdentityProvider: { identities[$0] }, + processArgumentsProvider: { _ in + Self.processArguments(workspaceID: workspaceID, panelID: panelID) + }, + processGroupProvider: { pid_t($0 + 1_000) } + ) + ) + + #expect( + terminations == [ + .init( + processID: 202, + processIdentity: secondIdentity, + processGroupID: 1_202 + ), + .init( + processID: 101, + processIdentity: firstIdentity, + processGroupID: 1_101 + ), + ] + ) + } + + @Test + func rejectsReusedProcessIdentity() { + let workspaceID = UUID() + let panelID = UUID() + let capturedIdentity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let scope = AgentHibernationController.ProcessTerminationScope( + key: AgentHibernationPanelKey(workspaceId: workspaceID, panelId: panelID), + processIDs: [101], + processIdentities: [101: capturedIdentity] + ) + + let terminations = AgentHibernationController.validatedScopedProcessTerminations( + for: scope, + processIdentityProvider: { _ in + AgentPIDProcessIdentity( + pid: 101, + startSeconds: 11, + startMicroseconds: 0 + ) + }, + processArgumentsProvider: { _ in + Self.processArguments(workspaceID: workspaceID, panelID: panelID) + }, + processGroupProvider: { _ in 1_101 } + ) + + #expect(terminations == nil) + } + + @Test + func rejectsProcessOutsidePaneScope() { + let workspaceID = UUID() + let panelID = UUID() + let identity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let scope = AgentHibernationController.ProcessTerminationScope( + key: AgentHibernationPanelKey(workspaceId: workspaceID, panelId: panelID), + processIDs: [101], + processIdentities: [101: identity] + ) + + let terminations = AgentHibernationController.validatedScopedProcessTerminations( + for: scope, + processIdentityProvider: { _ in identity }, + processArgumentsProvider: { _ in + Self.processArguments(workspaceID: workspaceID, panelID: UUID()) + }, + processGroupProvider: { _ in 1_101 } + ) + + #expect(terminations == nil) + } + + private static func processArguments( + workspaceID: UUID, + panelID: UUID + ) -> CmuxTopProcessArguments { + CmuxTopProcessArguments( + arguments: ["/usr/bin/agent"], + environment: [ + "CMUX_WORKSPACE_ID": workspaceID.uuidString, + "CMUX_SURFACE_ID": panelID.uuidString, + ] + ) + } +} diff --git a/cmuxUITests/SettingsTerminalBehaviorUITests.swift b/cmuxUITests/SettingsTerminalBehaviorUITests.swift index 9f474430706b..21ec38ce67b1 100644 --- a/cmuxUITests/SettingsTerminalBehaviorUITests.swift +++ b/cmuxUITests/SettingsTerminalBehaviorUITests.swift @@ -161,7 +161,7 @@ final class SettingsTerminalBehaviorUITests: SettingsUITestCase { static let resumeOn = "automatically run their resume command" static let resumeOff = "stay idle until you resume them manually" static let hibernateOn = "Idle background agent terminals can be suspended" - static let hibernateOff = "Agent terminals stay live until you close them" + static let hibernateOff = "Scheduled hibernation is off" static let showTextBoxOn = "open with the TextBox visible" static let showTextBoxOff = "start with the TextBox hidden" static let focusTextBoxOn = "put keyboard focus in the TextBox" @@ -277,7 +277,7 @@ final class SettingsTerminalBehaviorUITests: SettingsUITestCase { control.click() XCTAssertTrue( waitForStaticText(window, Subtitle.hibernateOff), - "Disabling hibernation should restore the stay-live sentence" + "Disabling scheduled hibernation should restore the critical-pressure disclosure" ) closeSettings(app, window) diff --git a/docs/agent-hooks.md b/docs/agent-hooks.md index 00277246688a..ccd2c13c0282 100644 --- a/docs/agent-hooks.md +++ b/docs/agent-hooks.md @@ -59,11 +59,11 @@ When the opt-in `automation.workspaceAutoNaming` setting is enabled, turn-end ho ## Agent Hibernation -Agent Hibernation kills idle background agent processes to free their RAM and CPU, then resumes each one with its saved session when you return to its tab. It is opt-in and off by default. cmux knows which process belongs to which terminal because the agent hooks associate each session ID with its surface (see the session-restore section above), so it can terminate the right process and bring back the right session. +Agent Hibernation kills idle background agent processes to free their RAM and CPU, then resumes each one with its saved session when you return to its tab. Routine hibernation based on the live-terminal limit is opt-in and off by default. A separate bounded safety path remains active for critical system memory pressure. cmux knows which process belongs to which terminal because the agent hooks associate each session ID with its surface (see the session-restore section above), so it can terminate the right process and bring back the right session. ### When a terminal hibernates -A live terminal is only ever a candidate when all of these hold: +For routine hibernation, a live terminal is only a candidate when all of these hold: - it has a saved restorable agent session, and the saved launch data can build a resume command - the agent lifecycle is `idle` (not running, not waiting on input) @@ -75,7 +75,11 @@ The live-terminal limit is the first gate. Under the limit, nothing hibernates n Before killing, cmux watches the terminal tail. It samples the last lines of output and a fingerprint of the process, and waits a short confirmation window (`confirmationSeconds`, ~60s) during which the output and process must stay unchanged. Any new output, input, lifecycle change, or PID change cancels the pending hibernation. This is why a small `idleSeconds` is safe: a freshly idle agent that resumes work on its own is never killed mid-task. -So with the defaults, hibernation only affects power users running more than 12 agents at once, and even then only ~1 minute after an agent has gone quiet off-screen. +So with the defaults, routine hibernation only affects power users running more than 12 agents at once, and even then only ~1 minute after an agent has gone quiet off-screen. + +### Critical memory pressure + +When macOS reports critical memory pressure, cmux can run the same protected teardown path independently of the `enabled` setting and live-terminal limit. Each pass selects at most two of the oldest eligible background agents. The agent must still be restorable, off-screen, explicitly idle, free of unconfirmed input, stable through the confirmation window, and backed by a transcript cmux can protect. Visible, running, needs-input, recently changed, or unprotectable agents are never selected. Before signaling anything, cmux revalidates the exact process generation and workspace/surface scope. ### What gets killed and how it comes back @@ -83,7 +87,7 @@ cmux sends `SIGTERM` to the agent's process group (scoped to that workspace and ### Enable and configure -Enable from the command palette (`⌘⇧P` -> **Enable Agent Hibernation**), from **Settings > Terminal > Agent Hibernation**, or from the CLI: +Enable routine hibernation from the command palette (`⌘⇧P` -> **Enable Agent Hibernation**), from **Settings > Terminal > Agent Hibernation**, or from the CLI: ```bash cmux agent-hibernation on diff --git a/docs/cli-contract.md b/docs/cli-contract.md index 4b33a75021d7..af103eaad3c5 100644 --- a/docs/cli-contract.md +++ b/docs/cli-contract.md @@ -66,7 +66,7 @@ Environment: | `disable-browser` | Disable cmux browser creation and link interception until re-enabled. | | `enable-browser` | Re-enable cmux browser creation and link interception. | | `browser-status` | Print whether cmux browser creation and link interception are enabled. | -| `agent-hibernation` | Enable or disable Agent Hibernation. | +| `agent-hibernation` | Enable or disable routine Agent Hibernation. | | `restore-session` | Restore the previously saved cmux session. | | `open` | Open files, directories, or URLs in cmux. | | `feedback` | Open feedback UI or submit feedback with `--email`, `--body`, and repeated `--image`. | diff --git a/docs/configuration.md b/docs/configuration.md index 921256b446a4..e7966054428b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -103,7 +103,7 @@ Default: `cloudFirst`. ## `terminal.agentHibernation` -Opt-in Agent Hibernation. cmux kills idle background agent processes to free RAM and CPU, then resumes each one with its saved session when you visit its tab. See [agent-hooks.md](agent-hooks.md#agent-hibernation) for the full behavior, including the confirmation settle window and how resume works. +Routine Agent Hibernation is opt-in. cmux kills idle background agent processes to free RAM and CPU, then resumes each one with its saved session when you visit its tab. Independently, critical memory pressure can trigger a bounded safety pass over eligible idle background agents even when routine hibernation is disabled. See [agent-hooks.md](agent-hooks.md#agent-hibernation) for the full eligibility rules, confirmation settle window, and resume behavior. ```json { @@ -117,11 +117,11 @@ Opt-in Agent Hibernation. cmux kills idle background agent processes to free RAM } ``` -- `enabled`: turn Agent Hibernation on. Default: `false`. +- `enabled`: turn routine Agent Hibernation on. Default: `false`. Critical-pressure safety hibernation remains active when this is `false`. - `idleSeconds`: seconds a background idle agent terminal must be quiet before it can hibernate. A ~60s confirmation settle window still applies on top of this. Default: `5`. Range: `5`-`604800`. - `maxLiveTerminals`: how many live restorable agent terminals to keep before cmux hibernates the oldest idle background ones. Nothing hibernates while you are at or under this count. Default: `12`. Range: `1`-`256`. -Enable it from the command palette (`⌘⇧P` -> Enable Agent Hibernation), from **Settings > Terminal > Agent Hibernation**, or with `cmux agent-hibernation on`. +Enable routine hibernation from the command palette (`⌘⇧P` -> Enable Agent Hibernation), from **Settings > Terminal > Agent Hibernation**, or with `cmux agent-hibernation on`. ## `sidebar.showAgentActivity` diff --git a/web/data/cmux.schema.json b/web/data/cmux.schema.json index b507d47ce05b..479119d60245 100644 --- a/web/data/cmux.schema.json +++ b/web/data/cmux.schema.json @@ -605,12 +605,14 @@ "agentHibernation": { "type": "object", "additionalProperties": false, - "description": "Opt-in Agent Hibernation settings. cmux kills idle background agent processes to free RAM and CPU, then resumes them with their saved session when their tab is visited. A terminal is only suspended when it runs a restorable coding agent, the agent lifecycle reports idle, the terminal is off-screen, the live-terminal limit is exceeded, and its output has stayed unchanged for the idle window plus a short confirmation settle window. The placeholder Resume button is a manual fallback.", + "descriptionKey": "schemaDescriptions.terminal.agentHibernation", + "description": "Routine Agent Hibernation settings. cmux kills idle background agent processes to free RAM and CPU, then resumes them with their saved session when their tab is visited. Routine hibernation requires a restorable coding agent whose lifecycle reports idle, an off-screen terminal, a live-terminal count above the configured limit, and unchanged output through the idle and confirmation windows. Independently, during critical memory pressure cmux may hibernate a bounded batch of safe idle background agents even when enabled is false; visible, running, needs-input, recently changed, and unprotectable agents remain excluded. The placeholder Resume button is a manual fallback.", "properties": { "enabled": { "type": "boolean", "default": false, - "description": "Enable Agent Hibernation." + "descriptionKey": "schemaDescriptions.terminal.agentHibernationEnabled", + "description": "Enable routine Agent Hibernation based on the live-terminal limit. Critical-pressure safety hibernation remains active when false." }, "idleSeconds": { "type": "integer", diff --git a/web/messages/en.json b/web/messages/en.json index 96bcf82c3338..2628bdfd384c 100644 --- a/web/messages/en.json +++ b/web/messages/en.json @@ -1722,6 +1722,8 @@ "globalFontMagnification": "Scales cmux-owned terminals, tab titles, sidebars, settings, overlays, and app chrome by this percentage. Rendered browser page content is excluded." }, "terminal": { + "agentHibernation": "Routine Agent Hibernation settings. cmux kills idle background agent processes to free RAM and CPU, then resumes them with their saved session when their tab is visited. Routine hibernation requires a restorable coding agent whose lifecycle reports idle, an off-screen terminal, a live-terminal count above the configured limit, and unchanged output through the idle and confirmation windows. Independently, during critical memory pressure cmux may hibernate a bounded batch of safe idle background agents even when enabled is false; visible, running, needs-input, recently changed, and unprotectable agents remain excluded. The placeholder Resume button is a manual fallback.", + "agentHibernationEnabled": "Enable routine Agent Hibernation based on the live-terminal limit. Critical-pressure safety hibernation remains active when false.", "copyOnSelect": "When true, copy selected terminal text to the system clipboard when the selection is committed. When false, cmux does not emit a Ghostty copy-on-select override; Ghostty config and defaults control selection-clipboard behavior.", "scrollSpeed": "Multiplier applied to terminal scroll wheel and trackpad deltas. Higher values scroll faster; lower values scroll slower.", "sessionContentMaxWidth": "Optional maximum width, in points, for terminal and built-in agent chat content. Set false to use the full pane width.", diff --git a/web/messages/ja.json b/web/messages/ja.json index 9c8716fbf4a9..6e3e5c141ad8 100644 --- a/web/messages/ja.json +++ b/web/messages/ja.json @@ -1645,6 +1645,8 @@ "globalFontMagnification": "cmux が所有するターミナル、タブタイトル、サイドバー、設定、オーバーレイ、アプリの chrome をこの割合で拡大縮小します。レンダリングされたブラウザページ内容は対象外です。" }, "terminal": { + "agentHibernation": "通常のエージェント休止の設定です。cmux はアイドル中のバックグラウンドエージェントプロセスを終了して RAM と CPU を解放し、そのタブを開いたときに保存済みセッションから再開します。通常の休止には、再開可能なコーディングエージェントであること、ライフサイクルがアイドルと報告していること、ターミナルが画面外にあること、ライブターミナル数が設定上限を超えていること、アイドル時間と確認時間を通して出力が変化していないことが必要です。これとは別に、メモリ負荷が危険な状態では enabled が false でも、安全なアイドル中のバックグラウンドエージェントを cmux が上限付きのバッチで休止する場合があります。表示中、実行中、入力待ち、最近変更された、またはトランスクリプトを保護できないエージェントは対象外です。プレースホルダーの「再開」ボタンは手動の代替手段です。", + "agentHibernationEnabled": "ライブターミナル数の上限に基づく通常のエージェント休止を有効にします。false の場合も、メモリ負荷が危険な状態での安全休止は有効です。", "copyOnSelect": "true の場合、選択が確定したときにターミナルで選択したテキストをシステムクリップボードへコピーします。false の場合、cmux は Ghostty の copy-on-select 上書きを出力せず、選択クリップボードの動作は Ghostty の設定と既定値に従います。", "scrollSpeed": "ターミナルのスクロールホイールとトラックパッドの移動量に適用される倍率です。値を大きくするとスクロールが速くなり、小さくすると遅くなります。", "sessionContentMaxWidth": "ターミナルと組み込みエージェントチャットのコンテンツ幅の上限をポイント単位で指定します。ペインの全幅を使うには false を設定します。", From ca7ca8fff28d82c24e38a8c61baa9891286cba54 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 28 Jul 2026 17:19:42 -0700 Subject: [PATCH 06/20] fix: await emergency agent process exit --- ...HibernationController+PanelLifecycle.swift | 67 +++++ ...rnationController+ProcessTermination.swift | 232 +++++++++++++++++- .../AgentHibernationController+Records.swift | 5 +- .../AgentHibernationController+Teardown.swift | 94 +------ Sources/App/AgentHibernationController.swift | 6 +- Sources/DockSplitStore+SurfaceTransfer.swift | 10 + Sources/RestorableAgentSession.swift | 25 +- Sources/Workspace+PanelLifecycle.swift | 7 + Sources/Workspace.swift | 15 ++ cmux.xcodeproj/project.pbxproj | 8 + ...ntHibernationProcessTerminationTests.swift | 114 +++++++++ ...entHibernationTrackingLifecycleTests.swift | 117 +++++++++ cmuxTests/AgentResumeLivenessTests.swift | 1 + ...ompletedRestoredAgentGenerationTests.swift | 1 + ...aredLiveAgentIndexAgentLivenessTests.swift | 7 +- 15 files changed, 606 insertions(+), 103 deletions(-) create mode 100644 Sources/App/AgentHibernationController+PanelLifecycle.swift create mode 100644 cmuxTests/AgentHibernationTrackingLifecycleTests.swift diff --git a/Sources/App/AgentHibernationController+PanelLifecycle.swift b/Sources/App/AgentHibernationController+PanelLifecycle.swift new file mode 100644 index 000000000000..f8d3b1da4f72 --- /dev/null +++ b/Sources/App/AgentHibernationController+PanelLifecycle.swift @@ -0,0 +1,67 @@ +import Foundation + +extension AgentHibernationController { + func discardTrackingStateForClosedPanel(workspaceId: UUID, panelId: UUID) { + let key = AgentHibernationPanelKey(workspaceId: workspaceId, panelId: panelId) + activityByPanel.removeValue(forKey: key) + terminalInputByPanel.removeValue(forKey: key) + lifecycleChangeByPanel.removeValue(forKey: key) + teardownValidationEpochByPanel.removeValue(forKey: key) + discardTransientTrackingState(for: key) + } + + func transferTrackingStateForMovedPanel( + panelId: UUID, + from sourceWorkspaceId: UUID, + to destinationWorkspaceId: UUID + ) { + guard sourceWorkspaceId != destinationWorkspaceId else { return } + let sourceKey = AgentHibernationPanelKey( + workspaceId: sourceWorkspaceId, + panelId: panelId + ) + let destinationKey = AgentHibernationPanelKey( + workspaceId: destinationWorkspaceId, + panelId: panelId + ) + let hadTrackingState = + activityByPanel[sourceKey] != nil || + terminalInputByPanel[sourceKey] != nil || + lifecycleChangeByPanel[sourceKey] != nil || + teardownValidationEpochByPanel[sourceKey] != nil || + activityByPanel[destinationKey] != nil || + terminalInputByPanel[destinationKey] != nil || + lifecycleChangeByPanel[destinationKey] != nil || + teardownValidationEpochByPanel[destinationKey] != nil + + transferTimestamp(&activityByPanel, from: sourceKey, to: destinationKey) + transferTimestamp(&terminalInputByPanel, from: sourceKey, to: destinationKey) + transferTimestamp(&lifecycleChangeByPanel, from: sourceKey, to: destinationKey) + let sourceEpoch = teardownValidationEpochByPanel.removeValue(forKey: sourceKey) + let destinationEpoch = teardownValidationEpochByPanel.removeValue(forKey: destinationKey) + if hadTrackingState { + teardownValidationEpochByPanel[destinationKey] = + max(sourceEpoch ?? 0, destinationEpoch ?? 0) &+ 1 + } + // Confirmation, fingerprints, and retries are ownership-specific. A move + // preserves raw input/lifecycle safety but requires a fresh qualification. + discardTransientTrackingState(for: sourceKey) + discardTransientTrackingState(for: destinationKey) + } + + private func transferTimestamp( + _ timestamps: inout [AgentHibernationPanelKey: TimeInterval], + from sourceKey: AgentHibernationPanelKey, + to destinationKey: AgentHibernationPanelKey + ) { + guard let sourceTimestamp = timestamps.removeValue(forKey: sourceKey) else { return } + timestamps[destinationKey] = max(sourceTimestamp, timestamps[destinationKey] ?? 0) + } + + private func discardTransientTrackingState(for key: AgentHibernationPanelKey) { + unableToProtectByPanel.removeValue(forKey: key) + teardownInFlightByPanel.removeValue(forKey: key) + confirmations.removeValue(forKey: key) + tailFingerprintSamples.removeValue(forKey: key) + } +} diff --git a/Sources/App/AgentHibernationController+ProcessTermination.swift b/Sources/App/AgentHibernationController+ProcessTermination.swift index 961c657ea76c..73c6bfd0bc6f 100644 --- a/Sources/App/AgentHibernationController+ProcessTermination.swift +++ b/Sources/App/AgentHibernationController+ProcessTermination.swift @@ -12,6 +12,8 @@ extension AgentHibernationRecord { } extension AgentHibernationController { + private nonisolated static let processExitTimeout: Duration = .seconds(30) + struct ProcessTerminationScope: Sendable { let key: AgentHibernationPanelKey let processIDs: Set @@ -105,18 +107,148 @@ extension AgentHibernationController { return terminations } + func commitConfirmedTeardown( + _ request: ConfirmedTeardownRequest, + snapshotOutcome: AgentHibernationTranscriptGuard.TeardownSnapshotOutcome, + scopedProcessTerminations: [ScopedProcessTermination], + shouldProceed: (@MainActor () -> Bool)?, + restoreOwnedSnapshotPaths: inout Set + ) async -> Bool { + let record = request.record + let snapshot: AgentHibernationTranscriptGuard.TeardownTranscriptSnapshot? + switch snapshotOutcome { + case .snapshot(let value): + snapshot = value + case .nothingToProtect: + snapshot = nil + case .unableToProtect: + // Forfeit hibernation rather than risk issue #6565 transcript loss. + unableToProtectByPanel[record.key] = UnableToProtectMarker( + fingerprint: request.confirmationFingerprint, + lastActivityAt: request.effectiveLastActivityAt, + retryAfter: Date.now.timeIntervalSince1970 + Self.unableToProtectRetrySeconds + ) + return false + } + + if let snapshot { + // Hand off any older monitor only after every other precondition passed. + await cancelPostTeardownRestoreTaskForReplacement( + transcriptPath: snapshot.transcriptPath + ) + guard shouldProceed?() ?? true else { + preserveSnapshotAfterAbortedTeardown( + snapshot, + record: record, + restoreOwnedSnapshotPaths: &restoreOwnedSnapshotPaths + ) + return false + } + guard AgentHibernationTranscriptGuard.liveFileVersionStillMatches(snapshot) else { + unableToProtectByPanel[record.key] = UnableToProtectMarker( + fingerprint: request.confirmationFingerprint, + lastActivityAt: request.effectiveLastActivityAt, + retryAfter: Date.now.timeIntervalSince1970 + Self.unableToProtectRetrySeconds + ) + preserveSnapshotAfterAbortedTeardown( + snapshot, + record: record, + restoreOwnedSnapshotPaths: &restoreOwnedSnapshotPaths + ) + return false + } + // Protection must already be active when SIGTERM or PTY closure can + // trigger an interrupted-exit transcript rewrite. + guard armPostTeardownRestoreMonitor( + snapshot: snapshot, + processIDs: record.processIDs + ) else { + preserveSnapshotAfterAbortedTeardown( + snapshot, + record: record, + restoreOwnedSnapshotPaths: &restoreOwnedSnapshotPaths + ) + return false + } + restoreOwnedSnapshotPaths.insert(snapshot.snapshotPath) + } + + let terminalInputAtCommit = terminalInputByPanel[record.key] ?? 0 + guard shouldProceed?() ?? true, + await terminateScopedProcessesForHibernation(scopedProcessTerminations) else { + return false + } + let postTerminationIndex: RestorableAgentSessionIndex? + if scopedProcessTerminations.isEmpty { + postTerminationIndex = nil + } else { + postTerminationIndex = await RestorableAgentSessionIndex + .loadIncludingProcessDetectedSnapshots() + } + let postTerminationStateIsValid: Bool + if let postTerminationIndex { + postTerminationStateIsValid = + postTerminationIndex.processIDs( + workspaceId: record.key.workspaceId, + panelId: record.key.panelId + ).isEmpty && + (terminalInputByPanel[record.key] ?? 0) == terminalInputAtCommit + } else { + postTerminationStateIsValid = + record.workspace.agentHibernationLifecycleState( + panelId: record.key.panelId, + fallback: record.lifecycle + ).allowsHibernation && + (terminalInputByPanel[record.key] ?? 0) <= + (lifecycleChangeByPanel[record.key] ?? 0) && + (teardownValidationEpochByPanel[record.key] ?? 0) == request.epoch && + hibernationFingerprint(for: record) == request.confirmationFingerprint && + (activityByPanel[record.key] ?? 0) <= request.effectiveLastActivityAt + } + guard postTerminationStateIsValid, + shouldProceed?() ?? true, + AgentHibernationTrackingGate.isEnabled(), + record.isStillOwnedByOriginalWorkspace, + !record.terminalPanel.isAgentHibernated, + record.terminalPanel.surface.hasLiveSurface, + AppDelegate.shared?.agentHibernationPanelIsProtected( + workspace: record.workspace, + panelId: record.key.panelId + ) == false, + teardownValidationGeneration == request.generation else { + return false + } + + record.workspace.enterAgentHibernation( + panelId: record.key.panelId, + agent: record.agent, + lastActivityAt: Date(timeIntervalSince1970: request.effectiveLastActivityAt) + ) + return true + } + @discardableResult func terminateScopedProcessesForHibernation( - _ terminations: [ScopedProcessTermination] - ) -> Bool { + _ terminations: [ScopedProcessTermination], + currentProcessID: pid_t = getpid(), + currentProcessGroupID: pid_t = getpgrp(), + processIdentityProvider: (pid_t) -> AgentPIDProcessIdentity? = { + AgentPIDProcessIdentity(pid: $0) + }, + processGroupProvider: (pid_t) -> pid_t = { getpgid($0) }, + signalErrorProvider: (pid_t, Int32) -> Int32? = { target, signal in + kill(target, signal) == 0 ? nil : errno + }, + waitForExit: @escaping @Sendable ([ScopedProcessTermination]) async -> Bool = { + await AgentHibernationController.waitForExactProcessGenerationsToExit($0) + } + ) async -> Bool { guard !terminations.isEmpty else { return true } - let currentProcessID = getpid() - let currentProcessGroupID = getpgrp() guard terminations.allSatisfy({ termination in let pid = pid_t(termination.processID) return pid != currentProcessID && - AgentPIDProcessIdentity(pid: pid) == termination.processIdentity && - getpgid(pid) == termination.processGroupID + processIdentityProvider(pid) == termination.processIdentity && + processGroupProvider(pid) == termination.processGroupID }) else { return false } @@ -127,10 +259,92 @@ extension AgentHibernationController { if processGroupID > 1, processGroupID != currentProcessGroupID, signaledProcessGroups.insert(processGroupID).inserted { - _ = kill(-processGroupID, SIGTERM) + if let error = signalErrorProvider(-processGroupID, SIGTERM), + error != ESRCH { + return false + } + } + if let error = signalErrorProvider(pid, SIGTERM), + error != ESRCH { + return false } - _ = kill(pid, SIGTERM) } - return true + return await waitForExit(terminations) + } + + nonisolated static func waitForExactProcessGenerationsToExit( + _ terminations: [ScopedProcessTermination], + timeout: Duration = processExitTimeout, + processIdentityProvider: @escaping @Sendable (pid_t) -> AgentPIDProcessIdentity? = { + AgentPIDProcessIdentity(pid: $0) + } + ) async -> Bool { + await withTaskGroup(of: Bool.self) { group in + for termination in terminations { + group.addTask(priority: .utility) { + await waitForExactProcessGenerationToExit( + termination, + timeout: timeout, + processIdentityProvider: processIdentityProvider + ) + } + } + for await didExit in group where !didExit { + group.cancelAll() + return false + } + return true + } + } + + private nonisolated static func waitForExactProcessGenerationToExit( + _ termination: ScopedProcessTermination, + timeout: Duration, + processIdentityProvider: @escaping @Sendable (pid_t) -> AgentPIDProcessIdentity? + ) async -> Bool { + let processID = pid_t(termination.processID) + guard processIdentityProvider(processID) == termination.processIdentity else { + return true + } + + let exitEvents = AsyncStream(bufferingPolicy: .bufferingNewest(1)) { continuation in + let source = DispatchSource.makeProcessSource( + identifier: processID, + eventMask: .exit, + queue: .global(qos: .utility) + ) + source.setEventHandler { + continuation.yield() + continuation.finish() + } + continuation.onTermination = { @Sendable _ in + source.cancel() + } + source.resume() + } + // Close the registration race without ever accepting a reused PID. + guard processIdentityProvider(processID) == termination.processIdentity else { + return true + } + + return await withTaskGroup(of: Bool.self) { group in + group.addTask { + for await _ in exitEvents { + return true + } + return false + } + group.addTask { + do { + try await ContinuousClock().sleep(for: timeout) + } catch { + return false + } + return processIdentityProvider(processID) != termination.processIdentity + } + let didExit = await group.next() ?? false + group.cancelAll() + return didExit + } } } diff --git a/Sources/App/AgentHibernationController+Records.swift b/Sources/App/AgentHibernationController+Records.swift index 51291d1b69ef..36c06b0b64f1 100644 --- a/Sources/App/AgentHibernationController+Records.swift +++ b/Sources/App/AgentHibernationController+Records.swift @@ -62,8 +62,9 @@ extension AppDelegate { isProtected: workspaceIsVisible && visiblePanelIds.contains(panelId), hasLiveProcess: !processIDs.isEmpty, processIDs: processIDs, - processIdentities: AgentHibernationController.processIdentities( - for: processIDs + processIdentities: index.processIdentities( + workspaceId: workspace.id, + panelId: panelId ) ) ) diff --git a/Sources/App/AgentHibernationController+Teardown.swift b/Sources/App/AgentHibernationController+Teardown.swift index aff0e169b5ab..366db67d4937 100644 --- a/Sources/App/AgentHibernationController+Teardown.swift +++ b/Sources/App/AgentHibernationController+Teardown.swift @@ -128,7 +128,10 @@ extension AgentHibernationController { workspaceId: record.key.workspaceId, panelId: record.key.panelId ) == record.processIDs, - Self.processIdentities(for: record.processIDs) == record.processIdentities, + postSnapshotIndex.processIdentities( + workspaceId: record.key.workspaceId, + panelId: record.key.panelId + ) == record.processIdentities, TabManager.restorableAgentSnapshotFingerprint(currentAgent) == TabManager.restorableAgentSnapshotFingerprint(record.agent), !record.terminalPanel.isAgentHibernated, @@ -148,91 +151,20 @@ extension AgentHibernationController { continue } - let snapshot: AgentHibernationTranscriptGuard.TeardownTranscriptSnapshot? - switch snapshotOutcome { - case .snapshot(let value): - snapshot = value - case .nothingToProtect: - snapshot = nil - case .unableToProtect: - // Forfeit hibernation rather than risk issue #6565 transcript loss. - self.unableToProtectByPanel[record.key] = UnableToProtectMarker( - fingerprint: request.confirmationFingerprint, - lastActivityAt: request.effectiveLastActivityAt, - retryAfter: Date().timeIntervalSince1970 + Self.unableToProtectRetrySeconds - ) - continue - } - - if let snapshot { - // An armed monitor for this transcript (a prior hibernation's, - // or one stored earlier in this batch) hands off here: quiesce - // it only now that this request is otherwise committed, and - // re-check the path version. Nothing may suspend between the - // check and SIGTERM, or a rewrite can invalidate the snapshot. - await cancelPostTeardownRestoreTaskForReplacement( - transcriptPath: snapshot.transcriptPath - ) - guard shouldProceed?() ?? true else { - preserveSnapshotAfterAbortedTeardown( - snapshot, - record: record, - restoreOwnedSnapshotPaths: &restoreOwnedSnapshotPaths - ) - continue - } - guard AgentHibernationTranscriptGuard.liveFileVersionStillMatches(snapshot) else { - self.unableToProtectByPanel[record.key] = UnableToProtectMarker( - fingerprint: request.confirmationFingerprint, - lastActivityAt: request.effectiveLastActivityAt, - retryAfter: Date().timeIntervalSince1970 + Self.unableToProtectRetrySeconds - ) - // The quiesce above disarmed the path's previous monitor, so - // forfeiting must re-arm protection with the fresh snapshot; - // its restore checks fail closed if the live file has turns. - preserveSnapshotAfterAbortedTeardown( - snapshot, - record: record, - restoreOwnedSnapshotPaths: &restoreOwnedSnapshotPaths - ) - continue - } - } - if let shouldProceed, !shouldProceed() { - if let snapshot { - preserveSnapshotAfterAbortedTeardown( - snapshot, - record: record, - restoreOwnedSnapshotPaths: &restoreOwnedSnapshotPaths - ) - } - continue - } - guard self.terminateScopedProcessesForHibernation(scopedProcessTerminations) else { - if let snapshot { - preserveSnapshotAfterAbortedTeardown( - snapshot, - record: record, - restoreOwnedSnapshotPaths: &restoreOwnedSnapshotPaths - ) - } - continue - } - record.workspace.enterAgentHibernation( - panelId: record.key.panelId, - agent: record.agent, - lastActivityAt: Date(timeIntervalSince1970: request.effectiveLastActivityAt) - ) - hibernatedCount += 1 - guard let snapshot else { continue } - if self.armPostTeardownRestoreMonitor(snapshot: snapshot, processIDs: record.processIDs) { - restoreOwnedSnapshotPaths.insert(snapshot.snapshotPath) + if await commitConfirmedTeardown( + request, + snapshotOutcome: snapshotOutcome, + scopedProcessTerminations: scopedProcessTerminations, + shouldProceed: shouldProceed, + restoreOwnedSnapshotPaths: &restoreOwnedSnapshotPaths + ) { + hibernatedCount += 1 } } } } - private func preserveSnapshotAfterAbortedTeardown( + func preserveSnapshotAfterAbortedTeardown( _ snapshot: AgentHibernationTranscriptGuard.TeardownTranscriptSnapshot, record: AgentHibernationRecord, restoreOwnedSnapshotPaths: inout Set diff --git a/Sources/App/AgentHibernationController.swift b/Sources/App/AgentHibernationController.swift index 7ae86493ce74..bda7714cd7bb 100644 --- a/Sources/App/AgentHibernationController.swift +++ b/Sources/App/AgentHibernationController.swift @@ -39,9 +39,9 @@ final class AgentHibernationController { var postTeardownRestoreDrainTask: Task? var postSnapshotValidationIndexSequence: UInt64 = 0 var postSnapshotValidationIndexTask: PostSnapshotValidationIndexTask? - private var teardownInFlightByPanel: [AgentHibernationPanelKey: InFlightTeardown] = [:] - private var confirmations: [AgentHibernationPanelKey: Confirmation] = [:] - private var tailFingerprintSamples: [AgentHibernationPanelKey: TailFingerprintSample] = [:] + var teardownInFlightByPanel: [AgentHibernationPanelKey: InFlightTeardown] = [:] + var confirmations: [AgentHibernationPanelKey: Confirmation] = [:] + var tailFingerprintSamples: [AgentHibernationPanelKey: TailFingerprintSample] = [:] var memoryPressureEvaluation: (id: UUID, task: Task)? private init() {} diff --git a/Sources/DockSplitStore+SurfaceTransfer.swift b/Sources/DockSplitStore+SurfaceTransfer.swift index 05e6272c1445..60595378006b 100644 --- a/Sources/DockSplitStore+SurfaceTransfer.swift +++ b/Sources/DockSplitStore+SurfaceTransfer.swift @@ -269,6 +269,11 @@ extension DockSplitStore { return nil } surfaceIdToPanelId[newTabId] = detached.panelId + AgentHibernationController.shared.transferTrackingStateForMovedPanel( + panelId: detached.panelId, + from: detached.sourceWorkspaceId, + to: workspaceId + ) if let index { _ = bonsplitController.reorderTab(newTabId, toIndex: index) } @@ -340,6 +345,11 @@ extension DockSplitStore { clearSessionRestoreState(panelId: detached.panelId) return nil } + AgentHibernationController.shared.transferTrackingStateForMovedPanel( + panelId: detached.panelId, + from: detached.sourceWorkspaceId, + to: workspaceId + ) repairPlaceholderOnlyDockPane(paneId) finishAttachingDetachedSurface( diff --git a/Sources/RestorableAgentSession.swift b/Sources/RestorableAgentSession.swift index 78bf60848306..fb9507b1f6aa 100644 --- a/Sources/RestorableAgentSession.swift +++ b/Sources/RestorableAgentSession.swift @@ -910,6 +910,7 @@ struct RestorableAgentSessionIndex: Sendable { /// Unlike an empty process ID set, this distinguishes an exited recorded process from no PID evidence. let processLiveness: RestorableAgentProcessLiveness let processIDs: Set + let processIdentities: [Int: AgentPIDProcessIdentity] let agentProcessIDs: Set let agentProcessIdentities: [Int: AgentPIDProcessIdentity] } @@ -973,6 +974,10 @@ struct RestorableAgentSessionIndex: Sendable { entry(workspaceId: workspaceId, panelId: panelId)?.processIDs ?? [] } + func processIdentities(workspaceId: UUID, panelId: UUID) -> [Int: AgentPIDProcessIdentity] { + entry(workspaceId: workspaceId, panelId: panelId)?.processIdentities ?? [:] + } + func agentProcessIDs(workspaceId: UUID, panelId: UUID) -> Set { entry(workspaceId: workspaceId, panelId: panelId)?.agentProcessIDs ?? [] } @@ -1209,6 +1214,7 @@ struct RestorableAgentSessionIndex: Sendable { updatedAt: effectiveRecord.updatedAt, processLiveness: processObservation.liveness, processIDs: liveProcessID.map { [$0] } ?? [], + processIdentities: liveProcessIdentities, agentProcessIDs: liveProcessID.map { [$0] } ?? [], agentProcessIdentities: liveProcessIdentities ) @@ -1253,14 +1259,19 @@ struct RestorableAgentSessionIndex: Sendable { } func processDetectedEntry(snapshot: SessionRestorableAgentSnapshot, lifecycle: AgentHibernationLifecycleState?, updatedAt: TimeInterval, detected: ProcessDetectedSnapshotEntry) -> Entry { - Entry( + let processIdentities = processIdentities( + for: detected.processIDs, + processIdentityProvider: processIdentityProvider + ) + return Entry( snapshot: snapshot, lifecycle: lifecycle, updatedAt: updatedAt, processLiveness: .running, - processIDs: detected.processIDs, agentProcessIDs: detected.agentProcessIDs, - agentProcessIdentities: agentProcessIdentities( - for: detected.agentProcessIDs, - processIdentityProvider: processIdentityProvider - ) + processIDs: detected.processIDs, + processIdentities: processIdentities, + agentProcessIDs: detected.agentProcessIDs, + agentProcessIdentities: processIdentities.filter { + detected.agentProcessIDs.contains($0.key) + } ) } @@ -1350,7 +1361,7 @@ struct RestorableAgentSessionIndex: Sendable { .max { $0.updatedAt < $1.updatedAt } } - private static func agentProcessIdentities( + private static func processIdentities( for processIDs: Set, processIdentityProvider: (Int) -> AgentPIDProcessIdentity? ) -> [Int: AgentPIDProcessIdentity] { diff --git a/Sources/Workspace+PanelLifecycle.swift b/Sources/Workspace+PanelLifecycle.swift index 5726a83c3559..1f2d5e4b08b6 100644 --- a/Sources/Workspace+PanelLifecycle.swift +++ b/Sources/Workspace+PanelLifecycle.swift @@ -405,6 +405,7 @@ extension Workspace { publishSurfaceClosedEvent: Bool, clearSurfaceNotifications: Bool, requestTransferredRemoteCleanup: Bool, + discardAgentHibernationTracking: Bool, cleanupControllerSurfaceState: Bool = false ) -> WorkspaceRemoteConfiguration? { if publishSurfaceClosedEvent { @@ -446,6 +447,12 @@ extension Workspace { } panels.removeValue(forKey: panelId) + if discardAgentHibernationTracking { + AgentHibernationController.shared.discardTrackingStateForClosedPanel( + workspaceId: id, + panelId: panelId + ) + } untrackRemoteTerminalSurface(panelId) discardRemoteDirectoryTrustState(panelId: panelId) pendingRemoteTerminalChildExitSurfaceIds.remove(panelId) diff --git a/Sources/Workspace.swift b/Sources/Workspace.swift index 9589a68f0ec5..d33e41555dd3 100644 --- a/Sources/Workspace.swift +++ b/Sources/Workspace.swift @@ -7605,6 +7605,7 @@ final class Workspace: Identifiable, ObservableObject { publishSurfaceClosedEvent: false, clearSurfaceNotifications: false, requestTransferredRemoteCleanup: true, + discardAgentHibernationTracking: false, cleanupControllerSurfaceState: false ) GhosttyApp.terminalSurfaceRegistry.unregister(oldPanel.surface) @@ -8519,6 +8520,7 @@ final class Workspace: Identifiable, ObservableObject { publishSurfaceClosedEvent: true, clearSurfaceNotifications: true, requestTransferredRemoteCleanup: true, + discardAgentHibernationTracking: true, cleanupControllerSurfaceState: true ) } @@ -8864,6 +8866,12 @@ final class Workspace: Identifiable, ObservableObject { } var detached = splitLayout.takeDetachedTransfer(tabId) + if detached == nil { + AgentHibernationController.shared.discardTrackingStateForClosedPanel( + workspaceId: id, + panelId: panelId + ) + } if shouldSkipControlMasterCleanupAfterDetach, let detachedTransfer = detached, detachedTransfer.isRemoteTerminal { skipControlMasterCleanupAfterDetachedRemoteTransfer = true if detachedTransfer.remoteCleanupConfiguration == nil { @@ -9019,6 +9027,11 @@ final class Workspace: Identifiable, ObservableObject { } else if let customSidebarPanel = detached.panel as? CustomSidebarPanel { customSidebarPanel.reattach(to: self) } + AgentHibernationController.shared.transferTrackingStateForMovedPanel( + panelId: detached.panelId, + from: detached.sourceWorkspaceId, + to: id + ) AppDelegate.shared?.notificationStore?.rebindSurfaceNotifications( fromTabId: detached.sourceWorkspaceId, toTabId: id, @@ -11825,6 +11838,7 @@ extension Workspace: BonsplitDelegate { publishSurfaceClosedEvent: !isDetaching, clearSurfaceNotifications: !preservesSurfaceForDetach, requestTransferredRemoteCleanup: false, + discardAgentHibernationTracking: !isDetaching, cleanupControllerSurfaceState: !isDetaching ) if !isDetaching { @@ -12015,6 +12029,7 @@ extension Workspace: BonsplitDelegate { publishSurfaceClosedEvent: true, clearSurfaceNotifications: true, requestTransferredRemoteCleanup: true, + discardAgentHibernationTracking: !isDetachingCloseTransaction, cleanupControllerSurfaceState: !isDetachingCloseTransaction ) if !isDetachingCloseTransaction { diff --git a/cmux.xcodeproj/project.pbxproj b/cmux.xcodeproj/project.pbxproj index 2baf816eeab8..92d0f716b13b 100644 --- a/cmux.xcodeproj/project.pbxproj +++ b/cmux.xcodeproj/project.pbxproj @@ -53,6 +53,7 @@ F65760100000000000000001 /* AgentHibernationController+Confirmation.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760100000000000000002 /* AgentHibernationController+Confirmation.swift */; }; F65760110000000000000001 /* AgentHibernationController+InFlightTeardown.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760110000000000000002 /* AgentHibernationController+InFlightTeardown.swift */; }; 8997C0018997C0018997C001 /* AgentHibernationController+MemoryPressure.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D0018997D0018997D001 /* AgentHibernationController+MemoryPressure.swift */; }; + 8997C0068997C0068997C006 /* AgentHibernationController+PanelLifecycle.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D0068997D0068997D006 /* AgentHibernationController+PanelLifecycle.swift */; }; F65760120000000000000001 /* AgentHibernationController+PostSnapshotValidationIndexTask.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760120000000000000002 /* AgentHibernationController+PostSnapshotValidationIndexTask.swift */; }; F65760260000000000000001 /* AgentHibernationController+PostTeardownRestoreCancellationState.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760260000000000000002 /* AgentHibernationController+PostTeardownRestoreCancellationState.swift */; }; F65760130000000000000001 /* AgentHibernationController+PostTeardownRestoreTask.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760130000000000000002 /* AgentHibernationController+PostTeardownRestoreTask.swift */; }; @@ -73,6 +74,7 @@ F65760240000000000000001 /* AgentHibernationRestoreMonitorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760240000000000000002 /* AgentHibernationRestoreMonitorTests.swift */; }; F65760020000000000000001 /* AgentHibernationTestHelpers.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760020000000000000002 /* AgentHibernationTestHelpers.swift */; }; D36A00020000000000000001 /* AgentHibernationTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D36A00020000000000000002 /* AgentHibernationTests.swift */; }; + 8997C0058997C0058997C005 /* AgentHibernationTrackingLifecycleTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D0058997D0058997D005 /* AgentHibernationTrackingLifecycleTests.swift */; }; F65760090000000000000001 /* AgentHibernationTranscriptGuard+ClaudeWorkflow.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760090000000000000002 /* AgentHibernationTranscriptGuard+ClaudeWorkflow.swift */; }; F65760220000000000000001 /* AgentHibernationTranscriptGuard+PostTeardownRestore.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760220000000000000002 /* AgentHibernationTranscriptGuard+PostTeardownRestore.swift */; }; F65760230000000000000001 /* AgentHibernationTranscriptGuard+StableFileComparison.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760230000000000000002 /* AgentHibernationTranscriptGuard+StableFileComparison.swift */; }; @@ -2518,6 +2520,7 @@ C0DE71B10000000000000001 /* AppDelegate+AgentChatNotifications.swift in Sources F65760100000000000000002 /* AgentHibernationController+Confirmation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+Confirmation.swift"; sourceTree = ""; }; F65760110000000000000002 /* AgentHibernationController+InFlightTeardown.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+InFlightTeardown.swift"; sourceTree = ""; }; 8997D0018997D0018997D001 /* AgentHibernationController+MemoryPressure.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+MemoryPressure.swift"; sourceTree = ""; }; + 8997D0068997D0068997D006 /* AgentHibernationController+PanelLifecycle.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+PanelLifecycle.swift"; sourceTree = ""; }; F65760120000000000000002 /* AgentHibernationController+PostSnapshotValidationIndexTask.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+PostSnapshotValidationIndexTask.swift"; sourceTree = ""; }; F65760260000000000000002 /* AgentHibernationController+PostTeardownRestoreCancellationState.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+PostTeardownRestoreCancellationState.swift"; sourceTree = ""; }; F65760130000000000000002 /* AgentHibernationController+PostTeardownRestoreTask.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+PostTeardownRestoreTask.swift"; sourceTree = ""; }; @@ -2537,6 +2540,7 @@ C0DE71B10000000000000001 /* AppDelegate+AgentChatNotifications.swift in Sources F65760240000000000000002 /* AgentHibernationRestoreMonitorTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernationRestoreMonitorTests.swift; sourceTree = ""; }; F65760020000000000000002 /* AgentHibernationTestHelpers.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernationTestHelpers.swift; sourceTree = ""; }; D36A00020000000000000002 /* AgentHibernationTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernationTests.swift; sourceTree = ""; }; + 8997D0058997D0058997D005 /* AgentHibernationTrackingLifecycleTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernationTrackingLifecycleTests.swift; sourceTree = ""; }; F65760090000000000000002 /* AgentHibernationTranscriptGuard+ClaudeWorkflow.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationTranscriptGuard+ClaudeWorkflow.swift"; sourceTree = ""; }; F65760220000000000000002 /* AgentHibernationTranscriptGuard+PostTeardownRestore.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationTranscriptGuard+PostTeardownRestore.swift"; sourceTree = ""; }; F65760230000000000000002 /* AgentHibernationTranscriptGuard+StableFileComparison.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationTranscriptGuard+StableFileComparison.swift"; sourceTree = ""; }; @@ -5253,6 +5257,7 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = F65760130000000000000002 /* AgentHibernationController+PostTeardownRestoreTask.swift */, F65760260000000000000002 /* AgentHibernationController+PostTeardownRestoreCancellationState.swift */, 8997D0038997D0038997D003 /* AgentHibernationController+ProcessTermination.swift */, + 8997D0068997D0068997D006 /* AgentHibernationController+PanelLifecycle.swift */, F65760030000000000000002 /* AgentHibernationController+Records.swift */, F65760140000000000000002 /* AgentHibernationController+TailFingerprintSample.swift */, F65760060000000000000002 /* AgentHibernationController+Teardown.swift */, @@ -6736,6 +6741,7 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = A9E040000000000000000001 /* CodexAppServerSessionTests.swift */, F65760010000000000000002 /* AgentHibernationPlannerSwiftTests.swift */, 8997D0048997D0048997D004 /* AgentHibernationProcessTerminationTests.swift */, + 8997D0058997D0058997D005 /* AgentHibernationTrackingLifecycleTests.swift */, F65760240000000000000002 /* AgentHibernationRestoreMonitorTests.swift */, F65760040000000000000002 /* AgentHibernationTranscriptGuardTests.swift */, F65760080000000000000002 /* AgentHibernationTranscriptGuardScanTests.swift */, @@ -7804,6 +7810,7 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = F65760100000000000000001 /* AgentHibernationController+Confirmation.swift in Sources */, F65760110000000000000001 /* AgentHibernationController+InFlightTeardown.swift in Sources */, 8997C0018997C0018997C001 /* AgentHibernationController+MemoryPressure.swift in Sources */, + 8997C0068997C0068997C006 /* AgentHibernationController+PanelLifecycle.swift in Sources */, F65760120000000000000001 /* AgentHibernationController+PostSnapshotValidationIndexTask.swift in Sources */, F65760260000000000000001 /* AgentHibernationController+PostTeardownRestoreCancellationState.swift in Sources */, F65760130000000000000001 /* AgentHibernationController+PostTeardownRestoreTask.swift in Sources */, @@ -9475,6 +9482,7 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = F65760240000000000000001 /* AgentHibernationRestoreMonitorTests.swift in Sources */, F65760020000000000000001 /* AgentHibernationTestHelpers.swift in Sources */, D36A00020000000000000001 /* AgentHibernationTests.swift in Sources */, + 8997C0058997C0058997C005 /* AgentHibernationTrackingLifecycleTests.swift in Sources */, F65760080000000000000001 /* AgentHibernationTranscriptGuardScanTests.swift in Sources */, F65760040000000000000001 /* AgentHibernationTranscriptGuardTests.swift in Sources */, F65760250000000000000001 /* AgentHibernationTranscriptSnapshotRaceTests.swift in Sources */, diff --git a/cmuxTests/AgentHibernationProcessTerminationTests.swift b/cmuxTests/AgentHibernationProcessTerminationTests.swift index 0e1eb126059f..6f0c414dd3b9 100644 --- a/cmuxTests/AgentHibernationProcessTerminationTests.swift +++ b/cmuxTests/AgentHibernationProcessTerminationTests.swift @@ -9,6 +9,14 @@ import Testing #endif struct AgentHibernationProcessTerminationTests { + private actor CompletionRecorder { + private(set) var value: Bool? + + func record(_ value: Bool) { + self.value = value + } + } + @Test func validatesExactProcessGenerationAndCmuxScope() throws { let workspaceID = UUID() @@ -117,6 +125,112 @@ struct AgentHibernationProcessTerminationTests { #expect(terminations == nil) } + @MainActor + @Test + func terminationDoesNotCompleteUntilTheExactProcessGenerationExits() async { + let identity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let termination = AgentHibernationController.ScopedProcessTermination( + processID: 101, + processIdentity: identity, + processGroupID: 1 + ) + let waitStarted = AsyncStream.makeStream() + let allowExit = AsyncStream.makeStream() + let completion = CompletionRecorder() + let task = Task { @MainActor in + let result = await AgentHibernationController.shared + .terminateScopedProcessesForHibernation( + [termination], + currentProcessID: 999, + currentProcessGroupID: 999, + processIdentityProvider: { _ in identity }, + processGroupProvider: { _ in 1 }, + signalErrorProvider: { _, _ in nil }, + waitForExit: { _ in + waitStarted.continuation.yield() + for await _ in allowExit.stream { + break + } + return true + } + ) + await completion.record(result) + } + var waitStartedIterator = waitStarted.stream.makeAsyncIterator() + _ = await waitStartedIterator.next() + + #expect(await completion.value == nil) + + allowExit.continuation.yield() + allowExit.continuation.finish() + await task.value + #expect(await completion.value == true) + waitStarted.continuation.finish() + } + + @MainActor + @Test + func signalFailureOtherThanMissingProcessAbortsBeforeExitWait() async { + let identity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let termination = AgentHibernationController.ScopedProcessTermination( + processID: 101, + processIdentity: identity, + processGroupID: 1 + ) + + let result = await AgentHibernationController.shared + .terminateScopedProcessesForHibernation( + [termination], + currentProcessID: 999, + currentProcessGroupID: 999, + processIdentityProvider: { _ in identity }, + processGroupProvider: { _ in 1 }, + signalErrorProvider: { _, _ in EPERM }, + waitForExit: { _ in + Issue.record("Exit wait must not begin after a failed SIGTERM") + return true + } + ) + + #expect(result == false) + } + + @Test + func aReusedPIDMeansTheOriginalProcessGenerationExited() async { + let originalIdentity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let reusedIdentity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 11, + startMicroseconds: 0 + ) + + let didExit = await AgentHibernationController.waitForExactProcessGenerationsToExit( + [ + .init( + processID: 101, + processIdentity: originalIdentity, + processGroupID: 1 + ), + ], + timeout: .zero, + processIdentityProvider: { _ in reusedIdentity } + ) + + #expect(didExit) + } + private static func processArguments( workspaceID: UUID, panelID: UUID diff --git a/cmuxTests/AgentHibernationTrackingLifecycleTests.swift b/cmuxTests/AgentHibernationTrackingLifecycleTests.swift new file mode 100644 index 000000000000..587ff11bd77b --- /dev/null +++ b/cmuxTests/AgentHibernationTrackingLifecycleTests.swift @@ -0,0 +1,117 @@ +import Foundation +import Testing + +#if canImport(cmux_DEV) +@testable import cmux_DEV +#elseif canImport(cmux) +@testable import cmux +#endif + +@MainActor +@Suite(.serialized) +struct AgentHibernationTrackingLifecycleTests { + @Test + func permanentPanelTeardownPrunesOnlyTheClosedPanelTracking() throws { + let controller = AgentHibernationController.shared + let workspace = Workspace() + let panelID = try #require(workspace.focusedPanelId) + let closedKey = AgentHibernationPanelKey(workspaceId: workspace.id, panelId: panelID) + let retainedKey = AgentHibernationPanelKey(workspaceId: workspace.id, panelId: UUID()) + defer { + for key in [closedKey, retainedKey] { + controller.discardTrackingStateForClosedPanel( + workspaceId: key.workspaceId, + panelId: key.panelId + ) + } + } + + for key in [closedKey, retainedKey] { + controller.activityByPanel[key] = 1 + controller.terminalInputByPanel[key] = 2 + controller.lifecycleChangeByPanel[key] = 3 + controller.teardownValidationEpochByPanel[key] = 4 + } + + workspace.teardownAllPanels() + + #expect(workspace.panels[panelID] == nil) + #expect(controller.activityByPanel[closedKey] == nil) + #expect(controller.terminalInputByPanel[closedKey] == nil) + #expect(controller.lifecycleChangeByPanel[closedKey] == nil) + #expect(controller.teardownValidationEpochByPanel[closedKey] == nil) + #expect(controller.activityByPanel[retainedKey] == 1) + #expect(controller.terminalInputByPanel[retainedKey] == 2) + #expect(controller.lifecycleChangeByPanel[retainedKey] == 3) + #expect(controller.teardownValidationEpochByPanel[retainedKey] == 4) + } + + @Test + func detachedPanelPreservesTrackingUntilItsNewOwnerAdoptsIt() throws { + let controller = AgentHibernationController.shared + let workspace = Workspace() + let panelID = try #require(workspace.focusedPanelId) + let panel = try #require(workspace.panels[panelID]) + let key = AgentHibernationPanelKey(workspaceId: workspace.id, panelId: panelID) + defer { + panel.close() + controller.discardTrackingStateForClosedPanel( + workspaceId: key.workspaceId, + panelId: key.panelId + ) + } + controller.terminalInputByPanel[key] = 2 + controller.lifecycleChangeByPanel[key] = 1 + + workspace.discardClosedPanelLifecycleState( + panelId: panelID, + tabId: workspace.surfaceIdFromPanelId(panelID), + paneId: workspace.paneId(forPanelId: panelID), + panel: panel, + origin: "test_detach", + closePanel: false, + publishSurfaceClosedEvent: false, + clearSurfaceNotifications: false, + requestTransferredRemoteCleanup: false, + discardAgentHibernationTracking: false + ) + + #expect(controller.terminalInputByPanel[key] == 2) + #expect(controller.lifecycleChangeByPanel[key] == 1) + } + + @Test + func movedPanelTransfersUnconfirmedInputSafetyToItsNewOwner() { + let controller = AgentHibernationController.shared + let panelID = UUID() + let sourceKey = AgentHibernationPanelKey(workspaceId: UUID(), panelId: panelID) + let destinationKey = AgentHibernationPanelKey(workspaceId: UUID(), panelId: panelID) + defer { + for key in [sourceKey, destinationKey] { + controller.discardTrackingStateForClosedPanel( + workspaceId: key.workspaceId, + panelId: key.panelId + ) + } + } + controller.activityByPanel[sourceKey] = 1 + controller.terminalInputByPanel[sourceKey] = 3 + controller.lifecycleChangeByPanel[sourceKey] = 2 + controller.teardownValidationEpochByPanel[sourceKey] = 4 + + controller.transferTrackingStateForMovedPanel( + panelId: panelID, + from: sourceKey.workspaceId, + to: destinationKey.workspaceId + ) + + #expect(controller.activityByPanel[sourceKey] == nil) + #expect(controller.terminalInputByPanel[sourceKey] == nil) + #expect(controller.lifecycleChangeByPanel[sourceKey] == nil) + #expect(controller.teardownValidationEpochByPanel[sourceKey] == nil) + #expect(controller.activityByPanel[destinationKey] == 1) + #expect(controller.terminalInputByPanel[destinationKey] == 3) + #expect(controller.lifecycleChangeByPanel[destinationKey] == 2) + #expect(controller.teardownValidationEpochByPanel[destinationKey] == 5) + } +} diff --git a/cmuxTests/AgentResumeLivenessTests.swift b/cmuxTests/AgentResumeLivenessTests.swift index c2081198b1d5..5aa4ea3bd769 100644 --- a/cmuxTests/AgentResumeLivenessTests.swift +++ b/cmuxTests/AgentResumeLivenessTests.swift @@ -28,6 +28,7 @@ struct AgentResumeLivenessTests { // one value that would contradict the empty-PID case. processLiveness: processIDs.isEmpty ? .exited : .running, processIDs: processIDs, + processIdentities: [:], agentProcessIDs: processIDs, agentProcessIdentities: [:] ) diff --git a/cmuxTests/CompletedRestoredAgentGenerationTests.swift b/cmuxTests/CompletedRestoredAgentGenerationTests.swift index fe1256b74538..e8a00adc2aec 100644 --- a/cmuxTests/CompletedRestoredAgentGenerationTests.swift +++ b/cmuxTests/CompletedRestoredAgentGenerationTests.swift @@ -235,6 +235,7 @@ struct CompletedRestoredAgentGenerationTests { updatedAt: updatedAt, processLiveness: .running, processIDs: [Int(identity.pid)], + processIdentities: [Int(identity.pid): identity], agentProcessIDs: [Int(identity.pid)], agentProcessIdentities: [Int(identity.pid): identity] ) diff --git a/cmuxTests/SharedLiveAgentIndexAgentLivenessTests.swift b/cmuxTests/SharedLiveAgentIndexAgentLivenessTests.swift index 85dbc39c351b..f1630dd58c89 100644 --- a/cmuxTests/SharedLiveAgentIndexAgentLivenessTests.swift +++ b/cmuxTests/SharedLiveAgentIndexAgentLivenessTests.swift @@ -28,6 +28,7 @@ struct SharedLiveAgentIndexAgentLivenessTests { let agentPID = 7_286 let childPID = 7_287 let agentIdentity = AgentPIDProcessIdentity(pid: pid_t(agentPID), startSeconds: 42, startMicroseconds: 7) + let childIdentity = AgentPIDProcessIdentity(pid: pid_t(childPID), startSeconds: 43, startMicroseconds: 8) let executable = "/usr/local/bin/\(agentId)" let registry = CmuxVaultAgentRegistry(registrations: [ CmuxVaultAgentRegistration( @@ -98,7 +99,7 @@ struct SharedLiveAgentIndexAgentLivenessTests { return processArguments.withLock { $0 } }, processIdentityProvider: { pid in - pid == agentPID ? agentIdentity : nil + [agentPID: agentIdentity, childPID: childIdentity][pid] } ) .loadResultSynchronously() @@ -111,6 +112,10 @@ struct SharedLiveAgentIndexAgentLivenessTests { await sharedIndex.refreshForkAvailabilityNow(workspaceId: workspaceId, panelId: panelId) #expect(sharedIndex.index?.processIDs(workspaceId: workspaceId, panelId: panelId) == Set([agentPID, childPID])) + #expect(sharedIndex.index?.processIdentities( + workspaceId: workspaceId, + panelId: panelId + ) == [agentPID: agentIdentity, childPID: childIdentity]) #expect(sharedIndex.index?.agentProcessIDs(workspaceId: workspaceId, panelId: panelId) == Set([agentPID])) #expect(sharedIndex.index?.agentProcessIdentities(workspaceId: workspaceId, panelId: panelId) == [agentPID: agentIdentity]) #expect(sharedIndex.prepareForkAvailabilityProbe(workspaceId: workspaceId, panelId: panelId)) From 76c6ebd2ea8ad2f68e4c75a525b9207ce970b818 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 28 Jul 2026 17:49:07 -0700 Subject: [PATCH 07/20] test: cover committed hibernation and Dock cleanup --- ...ntHibernationProcessTerminationTests.swift | 49 +++++++++++++++++ ...entHibernationTrackingLifecycleTests.swift | 55 +++++++++++++++++++ 2 files changed, 104 insertions(+) diff --git a/cmuxTests/AgentHibernationProcessTerminationTests.swift b/cmuxTests/AgentHibernationProcessTerminationTests.swift index 6f0c414dd3b9..ce099b7f56e3 100644 --- a/cmuxTests/AgentHibernationProcessTerminationTests.swift +++ b/cmuxTests/AgentHibernationProcessTerminationTests.swift @@ -203,6 +203,55 @@ struct AgentHibernationProcessTerminationTests { #expect(result == false) } + @MainActor + @Test + func signalFailureAfterTeardownCommitStillWaitsForOriginalProcesses() async { + let firstIdentity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let secondIdentity = AgentPIDProcessIdentity( + pid: 202, + startSeconds: 20, + startMicroseconds: 2 + ) + let terminations = [ + AgentHibernationController.ScopedProcessTermination( + processID: 101, + processIdentity: firstIdentity, + processGroupID: 1 + ), + AgentHibernationController.ScopedProcessTermination( + processID: 202, + processIdentity: secondIdentity, + processGroupID: 1 + ), + ] + let waitRecorder = CompletionRecorder() + + let result = await AgentHibernationController.shared + .terminateScopedProcessesForHibernation( + terminations, + currentProcessID: 999, + currentProcessGroupID: 999, + processIdentityProvider: { pid in + pid == 101 ? firstIdentity : secondIdentity + }, + processGroupProvider: { _ in 1 }, + signalErrorProvider: { target, _ in + target == 101 ? nil : EPERM + }, + waitForExit: { observedTerminations in + await waitRecorder.record(observedTerminations == terminations) + return observedTerminations == terminations + } + ) + + #expect(await waitRecorder.value == true) + #expect(result) + } + @Test func aReusedPIDMeansTheOriginalProcessGenerationExited() async { let originalIdentity = AgentPIDProcessIdentity( diff --git a/cmuxTests/AgentHibernationTrackingLifecycleTests.swift b/cmuxTests/AgentHibernationTrackingLifecycleTests.swift index 587ff11bd77b..8524fc838088 100644 --- a/cmuxTests/AgentHibernationTrackingLifecycleTests.swift +++ b/cmuxTests/AgentHibernationTrackingLifecycleTests.swift @@ -114,4 +114,59 @@ struct AgentHibernationTrackingLifecycleTests { #expect(controller.lifecycleChangeByPanel[destinationKey] == 2) #expect(controller.teardownValidationEpochByPanel[destinationKey] == 5) } + + @Test + func dockReconcilePrunesTrackingForPermanentlyClosedPanel() throws { + let controller = AgentHibernationController.shared + let store = DockSplitStore(workspaceId: UUID(), baseDirectoryProvider: { nil }) + defer { store.closeAllPanels() } + let paneID = try #require(store.bonsplitController.allPaneIds.first) + let panelID = try #require( + store.newSurface(kind: .terminal, inPane: paneID, focus: false) + ) + let tabID = try #require(store.surfaceId(forPanelId: panelID)) + let key = AgentHibernationPanelKey(workspaceId: store.workspaceId, panelId: panelID) + controller.activityByPanel[key] = 1 + controller.terminalInputByPanel[key] = 2 + controller.lifecycleChangeByPanel[key] = 3 + controller.teardownValidationEpochByPanel[key] = 4 + + store.forceCloseDockTabIds.insert(tabID) + defer { store.forceCloseDockTabIds.remove(tabID) } + #expect(store.bonsplitController.closeTab(tabID)) + store.reconcilePanels() + + #expect(store.panels[panelID] == nil) + expectTrackingWasDiscarded(for: key, controller: controller) + } + + @Test + func dockResetPrunesTrackingForEveryPermanentlyClosedPanel() throws { + let controller = AgentHibernationController.shared + let store = DockSplitStore(workspaceId: UUID(), baseDirectoryProvider: { nil }) + let paneID = try #require(store.bonsplitController.allPaneIds.first) + let panelID = try #require( + store.newSurface(kind: .terminal, inPane: paneID, focus: false) + ) + let key = AgentHibernationPanelKey(workspaceId: store.workspaceId, panelId: panelID) + controller.activityByPanel[key] = 1 + controller.terminalInputByPanel[key] = 2 + controller.lifecycleChangeByPanel[key] = 3 + controller.teardownValidationEpochByPanel[key] = 4 + + store.removeAllPanels() + + #expect(store.panels.isEmpty) + expectTrackingWasDiscarded(for: key, controller: controller) + } + + private func expectTrackingWasDiscarded( + for key: AgentHibernationPanelKey, + controller: AgentHibernationController + ) { + #expect(controller.activityByPanel[key] == nil) + #expect(controller.terminalInputByPanel[key] == nil) + #expect(controller.lifecycleChangeByPanel[key] == nil) + #expect(controller.teardownValidationEpochByPanel[key] == nil) + } } From 53914adf1ff58b334e13f5218eef919acd764763 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 28 Jul 2026 20:27:13 -0700 Subject: [PATCH 08/20] fix: make agent hibernation teardown irreversible --- .../AgentHibernationPanelPhase.swift | 51 ++++ ...bernationController+InFlightTeardown.swift | 11 + ...HibernationController+PanelLifecycle.swift | 1 + ...rnationController+ProcessTermination.swift | 218 +++++++++++------- .../AgentHibernationController+Teardown.swift | 48 +--- ...rnationController+TeardownValidation.swift | 53 +++++ Sources/App/AgentHibernationController.swift | 1 + Sources/DockSplitStore+PanelDestruction.swift | 26 +++ Sources/DockSplitStore+Reset.swift | 5 +- Sources/DockSplitStore+SessionRestore.swift | 13 +- Sources/DockSplitStore.swift | 18 +- .../TerminalPanel+AgentHibernation.swift | 77 +++++++ Sources/Panels/TerminalPanel.swift | 67 +----- Sources/Panels/TerminalPanelView.swift | 11 +- Sources/Workspace.swift | 2 +- cmux.xcodeproj/project.pbxproj | 16 ++ ...ntHibernationProcessTerminationTests.swift | 185 +++++++++++---- ...entHibernationTrackingLifecycleTests.swift | 40 ++++ 18 files changed, 591 insertions(+), 252 deletions(-) create mode 100644 Sources/AgentHibernation/AgentHibernationPanelPhase.swift create mode 100644 Sources/App/AgentHibernationController+TeardownValidation.swift create mode 100644 Sources/DockSplitStore+PanelDestruction.swift create mode 100644 Sources/Panels/TerminalPanel+AgentHibernation.swift diff --git a/Sources/AgentHibernation/AgentHibernationPanelPhase.swift b/Sources/AgentHibernation/AgentHibernationPanelPhase.swift new file mode 100644 index 000000000000..90a185f80164 --- /dev/null +++ b/Sources/AgentHibernation/AgentHibernationPanelPhase.swift @@ -0,0 +1,51 @@ +import Foundation + +struct AgentHibernationPanelState { + let agent: SessionRestorableAgentSnapshot + let hibernatedAt: Date + let lastActivityAt: Date + + var agentDisplayName: String { + agent.agentDisplayName + } +} + +enum AgentHibernationPanelPhase { + case live + case terminating(AgentHibernationPanelState) + case hibernated(AgentHibernationPanelState) + + var state: AgentHibernationPanelState? { + switch self { + case .live: + nil + case .terminating(let state), .hibernated(let state): + state + } + } + + var isCommitted: Bool { + if case .live = self { return false } + return true + } + + var isTerminating: Bool { + if case .terminating = self { return true } + return false + } +} + +enum AgentHibernationResumePreparation: Equatable { + case unavailable + case resumed(queuedStartupInput: Bool) + + var didResume: Bool { + if case .resumed = self { return true } + return false + } + + var queuedStartupInput: Bool { + if case .resumed(let queuedStartupInput) = self { return queuedStartupInput } + return false + } +} diff --git a/Sources/App/AgentHibernationController+InFlightTeardown.swift b/Sources/App/AgentHibernationController+InFlightTeardown.swift index ae5ea092be01..0abae83af6f9 100644 --- a/Sources/App/AgentHibernationController+InFlightTeardown.swift +++ b/Sources/App/AgentHibernationController+InFlightTeardown.swift @@ -5,4 +5,15 @@ extension AgentHibernationController { let requestID: UUID let trigger: AgentHibernationReclaimTrigger } + + enum ScopedProcessTerminationResult: Equatable, Sendable { + case rejected + case exited + case committedAwaitingExit + } + + struct CommittedTerminationObservation { + let requestID: UUID + let task: Task + } } diff --git a/Sources/App/AgentHibernationController+PanelLifecycle.swift b/Sources/App/AgentHibernationController+PanelLifecycle.swift index f8d3b1da4f72..5516ffa43350 100644 --- a/Sources/App/AgentHibernationController+PanelLifecycle.swift +++ b/Sources/App/AgentHibernationController+PanelLifecycle.swift @@ -2,6 +2,7 @@ import Foundation extension AgentHibernationController { func discardTrackingStateForClosedPanel(workspaceId: UUID, panelId: UUID) { + committedTerminationObservationsByPanelID.removeValue(forKey: panelId)?.task.cancel() let key = AgentHibernationPanelKey(workspaceId: workspaceId, panelId: panelId) activityByPanel.removeValue(forKey: key) terminalInputByPanel.removeValue(forKey: key) diff --git a/Sources/App/AgentHibernationController+ProcessTermination.swift b/Sources/App/AgentHibernationController+ProcessTermination.swift index 73c6bfd0bc6f..d1b535e9b38e 100644 --- a/Sources/App/AgentHibernationController+ProcessTermination.swift +++ b/Sources/App/AgentHibernationController+ProcessTermination.swift @@ -12,8 +12,6 @@ extension AgentHibernationRecord { } extension AgentHibernationController { - private nonisolated static let processExitTimeout: Duration = .seconds(30) - struct ProcessTerminationScope: Sendable { let key: AgentHibernationPanelKey let processIDs: Set @@ -39,6 +37,11 @@ extension AgentHibernationController { }) } + #if compiler(>=6.2) + @concurrent + #else + @Sendable + #endif nonisolated static func scopedProcessTerminations( for scopes: [ProcessTerminationScope] ) async -> [AgentHibernationPanelKey: [ScopedProcessTermination]] { @@ -173,63 +176,76 @@ extension AgentHibernationController { restoreOwnedSnapshotPaths.insert(snapshot.snapshotPath) } - let terminalInputAtCommit = terminalInputByPanel[record.key] ?? 0 - guard shouldProceed?() ?? true, - await terminateScopedProcessesForHibernation(scopedProcessTerminations) else { + // The monitor handoff above awaits an older task and makes the main actor + // reentrant. Refresh the process generation snapshot, then re-check every + // mutable safety condition at the last synchronous boundary before SIGTERM. + let preSignalIndex = await RestorableAgentSessionIndex + .loadIncludingProcessDetectedSnapshots() + guard teardownIsStillSafe( + request, + index: preSignalIndex, + shouldProceed: shouldProceed + ), + snapshot.map(AgentHibernationTranscriptGuard.liveFileVersionStillMatches) ?? true else { return false } - let postTerminationIndex: RestorableAgentSessionIndex? - if scopedProcessTerminations.isEmpty { - postTerminationIndex = nil - } else { - postTerminationIndex = await RestorableAgentSessionIndex - .loadIncludingProcessDetectedSnapshots() - } - let postTerminationStateIsValid: Bool - if let postTerminationIndex { - postTerminationStateIsValid = - postTerminationIndex.processIDs( - workspaceId: record.key.workspaceId, - panelId: record.key.panelId - ).isEmpty && - (terminalInputByPanel[record.key] ?? 0) == terminalInputAtCommit - } else { - postTerminationStateIsValid = - record.workspace.agentHibernationLifecycleState( - panelId: record.key.panelId, - fallback: record.lifecycle - ).allowsHibernation && - (terminalInputByPanel[record.key] ?? 0) <= - (lifecycleChangeByPanel[record.key] ?? 0) && - (teardownValidationEpochByPanel[record.key] ?? 0) == request.epoch && - hibernationFingerprint(for: record) == request.confirmationFingerprint && - (activityByPanel[record.key] ?? 0) <= request.effectiveLastActivityAt + + let lastActivityAt = Date( + timeIntervalSince1970: request.effectiveLastActivityAt + ) + let panelID = record.key.panelId + let workspaceID = record.key.workspaceId + let agent = record.agent + let finishTeardown: @MainActor () -> Void = { + [weak workspace = record.workspace, weak terminalPanel = record.terminalPanel] in + guard let terminalPanel else { return } + if let workspace, + let currentPanel = workspace.panels[panelID] as? TerminalPanel, + currentPanel === terminalPanel, + terminalPanel.workspaceId == workspaceID, + terminalPanel.isAgentHibernationTerminating { + workspace.enterAgentHibernation( + panelId: panelID, + agent: agent, + lastActivityAt: lastActivityAt + ) + } else { + // A live move carries the phase on TerminalPanel. Its new owner + // gets the same resumable state without consulting the source. + terminalPanel.completeAgentHibernationTermination() + } } - guard postTerminationStateIsValid, - shouldProceed?() ?? true, - AgentHibernationTrackingGate.isEnabled(), - record.isStillOwnedByOriginalWorkspace, - !record.terminalPanel.isAgentHibernated, - record.terminalPanel.surface.hasLiveSurface, - AppDelegate.shared?.agentHibernationPanelIsProtected( - workspace: record.workspace, - panelId: record.key.panelId - ) == false, - teardownValidationGeneration == request.generation else { + let terminationResult = terminateScopedProcessesForHibernation( + scopedProcessTerminations, + onTeardownCommit: { + record.terminalPanel.beginAgentHibernationTermination( + agent: record.agent, + lastActivityAt: lastActivityAt + ) + } + ) + switch terminationResult { + case .rejected: + return false + case .exited: + finishTeardown() + case .committedAwaitingExit: + observeCommittedTermination( + panelID: panelID, + terminations: scopedProcessTerminations, + onExit: finishTeardown + ) return false } - record.workspace.enterAgentHibernation( - panelId: record.key.panelId, - agent: record.agent, - lastActivityAt: Date(timeIntervalSince1970: request.effectiveLastActivityAt) - ) - return true + return record.terminalPanel.isAgentHibernated && + !record.terminalPanel.isAgentHibernationTerminating } @discardableResult func terminateScopedProcessesForHibernation( _ terminations: [ScopedProcessTermination], + onTeardownCommit: @MainActor () -> Void = {}, currentProcessID: pid_t = getpid(), currentProcessGroupID: pid_t = getpgrp(), processIdentityProvider: (pid_t) -> AgentPIDProcessIdentity? = { @@ -238,19 +254,25 @@ extension AgentHibernationController { processGroupProvider: (pid_t) -> pid_t = { getpgid($0) }, signalErrorProvider: (pid_t, Int32) -> Int32? = { target, signal in kill(target, signal) == 0 ? nil : errno - }, - waitForExit: @escaping @Sendable ([ScopedProcessTermination]) async -> Bool = { - await AgentHibernationController.waitForExactProcessGenerationsToExit($0) } - ) async -> Bool { - guard !terminations.isEmpty else { return true } + ) -> ScopedProcessTerminationResult { + guard !terminations.isEmpty else { + onTeardownCommit() + return .exited + } guard terminations.allSatisfy({ termination in let pid = pid_t(termination.processID) return pid != currentProcessID && processIdentityProvider(pid) == termination.processIdentity && processGroupProvider(pid) == termination.processGroupID }) else { - return false + return .rejected + } + var teardownIsCommitted = false + let commitTeardownIfNeeded = { + guard !teardownIsCommitted else { return } + teardownIsCommitted = true + onTeardownCommit() } var signaledProcessGroups: Set = [] for termination in terminations { @@ -259,22 +281,69 @@ extension AgentHibernationController { if processGroupID > 1, processGroupID != currentProcessGroupID, signaledProcessGroups.insert(processGroupID).inserted { - if let error = signalErrorProvider(-processGroupID, SIGTERM), - error != ESRCH { - return false + if let error = signalErrorProvider(-processGroupID, SIGTERM) { + if error != ESRCH, !teardownIsCommitted { + return .rejected + } + } else { + commitTeardownIfNeeded() } } - if let error = signalErrorProvider(pid, SIGTERM), - error != ESRCH { - return false + if let error = signalErrorProvider(pid, SIGTERM) { + if error != ESRCH, !teardownIsCommitted { + return .rejected + } + } else { + commitTeardownIfNeeded() + } + } + // Every target may have exited between validation and signaling. That is + // still a committed teardown: there is no live generation left to kill. + commitTeardownIfNeeded() + // A later signal error cannot roll back a teardown after the first signal. + // Exact-generation exit is completed by a stored per-panel observation so + // one slow process cannot block later critical-pressure reclamation batches. + return .committedAwaitingExit + } + + func observeCommittedTermination( + panelID: UUID, + terminations: [ScopedProcessTermination], + waitForExit: @escaping @Sendable ([ScopedProcessTermination]) async -> Bool = { + await AgentHibernationController + .waitForExactProcessGenerationsToExitWithoutTimeout($0) + }, + onExit: @escaping @MainActor () -> Void + ) { + committedTerminationObservationsByPanelID + .removeValue(forKey: panelID)? + .task + .cancel() + let requestID = UUID() + let task = Task { @MainActor [weak self] in + let didExit = await waitForExit(terminations) + guard let self, + self.committedTerminationObservationsByPanelID[panelID]?.requestID == + requestID else { + return } + self.committedTerminationObservationsByPanelID.removeValue(forKey: panelID) + guard didExit, !Task.isCancelled else { return } + onExit() } - return await waitForExit(terminations) + committedTerminationObservationsByPanelID[panelID] = CommittedTerminationObservation( + requestID: requestID, + task: task + ) } - nonisolated static func waitForExactProcessGenerationsToExit( + #if compiler(>=6.2) + @concurrent + #else + @Sendable + #endif + nonisolated static func waitForExactProcessGenerationsToExitWithoutTimeout( _ terminations: [ScopedProcessTermination], - timeout: Duration = processExitTimeout, processIdentityProvider: @escaping @Sendable (pid_t) -> AgentPIDProcessIdentity? = { AgentPIDProcessIdentity(pid: $0) } @@ -284,7 +353,6 @@ extension AgentHibernationController { group.addTask(priority: .utility) { await waitForExactProcessGenerationToExit( termination, - timeout: timeout, processIdentityProvider: processIdentityProvider ) } @@ -299,7 +367,6 @@ extension AgentHibernationController { private nonisolated static func waitForExactProcessGenerationToExit( _ termination: ScopedProcessTermination, - timeout: Duration, processIdentityProvider: @escaping @Sendable (pid_t) -> AgentPIDProcessIdentity? ) async -> Bool { let processID = pid_t(termination.processID) @@ -327,24 +394,9 @@ extension AgentHibernationController { return true } - return await withTaskGroup(of: Bool.self) { group in - group.addTask { - for await _ in exitEvents { - return true - } - return false - } - group.addTask { - do { - try await ContinuousClock().sleep(for: timeout) - } catch { - return false - } - return processIdentityProvider(processID) != termination.processIdentity - } - let didExit = await group.next() ?? false - group.cancelAll() - return didExit + for await _ in exitEvents { + return true } + return processIdentityProvider(processID) != termination.processIdentity } } diff --git a/Sources/App/AgentHibernationController+Teardown.swift b/Sources/App/AgentHibernationController+Teardown.swift index 366db67d4937..d327c0102812 100644 --- a/Sources/App/AgentHibernationController+Teardown.swift +++ b/Sources/App/AgentHibernationController+Teardown.swift @@ -101,53 +101,15 @@ extension AgentHibernationController { let scopedProcessTerminations = scopedProcessTerminationsByPanel[record.key] else { continue } - let currentAgent = record.workspace.restorableAgentForHibernation( - panelId: record.key.panelId, - index: postSnapshotIndex - ) - let currentLifecycle = postSnapshotLifecycle(for: record, index: postSnapshotIndex) - let currentEffectiveLastActivityAt = postSnapshotEffectiveLastActivityAt( - for: record, - index: postSnapshotIndex - ) // Re-validate: the pane must still be exactly as confirmed. Any activity, // process-set or scrollback change, visibility/protection change, // hibernation, or surface loss during the hop aborts; a later evaluation // can re-arm it if it is still idle. - guard (shouldProceed?() ?? true), - AgentHibernationTrackingGate.isEnabled(), - record.isStillOwnedByOriginalWorkspace, - ( - request.trigger == .systemMemoryPressure || - !postSnapshotIndex.hasLiveProcess( - workspaceId: record.key.workspaceId, - panelId: record.key.panelId - ) - ), - postSnapshotIndex.processIDs( - workspaceId: record.key.workspaceId, - panelId: record.key.panelId - ) == record.processIDs, - postSnapshotIndex.processIdentities( - workspaceId: record.key.workspaceId, - panelId: record.key.panelId - ) == record.processIdentities, - TabManager.restorableAgentSnapshotFingerprint(currentAgent) == - TabManager.restorableAgentSnapshotFingerprint(record.agent), - !record.terminalPanel.isAgentHibernated, - record.terminalPanel.surface.hasLiveSurface, - AppDelegate.shared?.agentHibernationPanelIsProtected( - workspace: record.workspace, - panelId: record.key.panelId - ) == false, - currentLifecycle.allowsHibernation, - (self.terminalInputByPanel[record.key] ?? 0) <= - (self.lifecycleChangeByPanel[record.key] ?? 0), - self.teardownValidationGeneration == request.generation, - (self.teardownValidationEpochByPanel[record.key] ?? 0) == request.epoch, - let currentFingerprint = self.hibernationFingerprint(for: record), - currentFingerprint == request.confirmationFingerprint, - currentEffectiveLastActivityAt <= request.effectiveLastActivityAt else { + guard teardownIsStillSafe( + request, + index: postSnapshotIndex, + shouldProceed: shouldProceed + ) else { continue } diff --git a/Sources/App/AgentHibernationController+TeardownValidation.swift b/Sources/App/AgentHibernationController+TeardownValidation.swift new file mode 100644 index 000000000000..9e1fcab0c4a6 --- /dev/null +++ b/Sources/App/AgentHibernationController+TeardownValidation.swift @@ -0,0 +1,53 @@ +import Foundation + +extension AgentHibernationController { + func teardownIsStillSafe( + _ request: ConfirmedTeardownRequest, + index: RestorableAgentSessionIndex, + shouldProceed: (@MainActor () -> Bool)? + ) -> Bool { + let record = request.record + let currentAgent = record.workspace.restorableAgentForHibernation( + panelId: record.key.panelId, + index: index + ) + let currentLifecycle = postSnapshotLifecycle(for: record, index: index) + let currentEffectiveLastActivityAt = postSnapshotEffectiveLastActivityAt( + for: record, + index: index + ) + return (shouldProceed?() ?? true) && + AgentHibernationTrackingGate.isEnabled() && + record.isStillOwnedByOriginalWorkspace && + ( + request.trigger == .systemMemoryPressure || + !index.hasLiveProcess( + workspaceId: record.key.workspaceId, + panelId: record.key.panelId + ) + ) && + index.processIDs( + workspaceId: record.key.workspaceId, + panelId: record.key.panelId + ) == record.processIDs && + index.processIdentities( + workspaceId: record.key.workspaceId, + panelId: record.key.panelId + ) == record.processIdentities && + TabManager.restorableAgentSnapshotFingerprint(currentAgent) == + TabManager.restorableAgentSnapshotFingerprint(record.agent) && + !record.terminalPanel.isAgentHibernated && + record.terminalPanel.surface.hasLiveSurface && + AppDelegate.shared?.agentHibernationPanelIsProtected( + workspace: record.workspace, + panelId: record.key.panelId + ) == false && + currentLifecycle.allowsHibernation && + (terminalInputByPanel[record.key] ?? 0) <= + (lifecycleChangeByPanel[record.key] ?? 0) && + teardownValidationGeneration == request.generation && + (teardownValidationEpochByPanel[record.key] ?? 0) == request.epoch && + hibernationFingerprint(for: record) == request.confirmationFingerprint && + currentEffectiveLastActivityAt <= request.effectiveLastActivityAt + } +} diff --git a/Sources/App/AgentHibernationController.swift b/Sources/App/AgentHibernationController.swift index bda7714cd7bb..50e81a9346a0 100644 --- a/Sources/App/AgentHibernationController.swift +++ b/Sources/App/AgentHibernationController.swift @@ -40,6 +40,7 @@ final class AgentHibernationController { var postSnapshotValidationIndexSequence: UInt64 = 0 var postSnapshotValidationIndexTask: PostSnapshotValidationIndexTask? var teardownInFlightByPanel: [AgentHibernationPanelKey: InFlightTeardown] = [:] + var committedTerminationObservationsByPanelID: [UUID: CommittedTerminationObservation] = [:] var confirmations: [AgentHibernationPanelKey: Confirmation] = [:] var tailFingerprintSamples: [AgentHibernationPanelKey: TailFingerprintSample] = [:] var memoryPressureEvaluation: (id: UUID, task: Task)? diff --git a/Sources/DockSplitStore+PanelDestruction.swift b/Sources/DockSplitStore+PanelDestruction.swift new file mode 100644 index 000000000000..b1d9d8c0ae37 --- /dev/null +++ b/Sources/DockSplitStore+PanelDestruction.swift @@ -0,0 +1,26 @@ +import Bonsplit +import Foundation + +extension DockSplitStore { + @discardableResult + func discardPanelOwnershipAndClose(panelId: UUID) -> (any Panel)? { + let tabIDs = surfaceIdToPanelId.compactMap { tabID, ownedPanelID in + ownedPanelID == panelId ? tabID : nil + } + for tabID in tabIDs { + surfaceIdToPanelId.removeValue(forKey: tabID) + } + panelCancellables[panelId]?.cancel() + panelCancellables.removeValue(forKey: panelId) + AppDelegate.shared?.notificationStore?.clearNotifications( + forTabId: workspaceId, + surfaceId: panelId + ) + detachedSurfaceTransfersByPanelId.removeValue(forKey: panelId) + clearSessionRestoreState(panelId: panelId) + + guard let panel = panels.removeValue(forKey: panelId) else { return nil } + panel.close() + return panel + } +} diff --git a/Sources/DockSplitStore+Reset.swift b/Sources/DockSplitStore+Reset.swift index caaa79b16071..78edb3c10ad3 100644 --- a/Sources/DockSplitStore+Reset.swift +++ b/Sources/DockSplitStore+Reset.swift @@ -10,8 +10,9 @@ extension DockSplitStore { for tabId in tabIds { _ = bonsplitController.closeTab(tabId) } collapseToSingleEmptyPane() reconcilePanels() - for panel in panels.values { panel.close() } - panels.removeAll() + for panelId in Array(panels.keys) { + discardPanelOwnershipAndClose(panelId: panelId) + } surfaceIdToPanelId.removeAll() detachedSurfaceTransfersByPanelId.removeAll() restoredTerminalScrollbackByPanelId.removeAll() diff --git a/Sources/DockSplitStore+SessionRestore.swift b/Sources/DockSplitStore+SessionRestore.swift index af23b1f09b90..cd553ca5561c 100644 --- a/Sources/DockSplitStore+SessionRestore.swift +++ b/Sources/DockSplitStore+SessionRestore.swift @@ -91,9 +91,15 @@ extension DockSplitStore { snapshot, snapshotWorkspaceId: sourceWorkspaceId, excludingStableIdentities: excludingStableIdentities - ) { + ) { let restoredPanelId = attachDetachedSurface(detached, inPane: paneId, focus: false) - if restoredPanelId == nil { detached.panel.close() } + if restoredPanelId == nil { + AgentHibernationController.shared.discardTrackingStateForClosedPanel( + workspaceId: detached.sourceWorkspaceId, + panelId: detached.panelId + ) + detached.panel.close() + } return restoredPanelId } if sourceWorkspaceId != nil, snapshot.terminal?.isRemoteTerminal == true { @@ -331,8 +337,7 @@ extension DockSplitStore { isPinned: false, inPane: paneId ) else { - panels.removeValue(forKey: panel.id) - panel.close() + discardPanelOwnershipAndClose(panelId: panel.id) return nil } surfaceIdToPanelId[tabId] = panel.id diff --git a/Sources/DockSplitStore.swift b/Sources/DockSplitStore.swift index ac66007041ed..15a1805bfd64 100644 --- a/Sources/DockSplitStore.swift +++ b/Sources/DockSplitStore.swift @@ -371,9 +371,7 @@ final class DockSplitStore: BonsplitDelegate { ) } guard splitResult != nil else { - surfaceIdToPanelId.removeValue(forKey: newTab.id) - panels.removeValue(forKey: panel.id) - panel.close() + discardPanelOwnershipAndClose(panelId: panel.id) return nil } installSubscription(for: panel, tracksTerminalTitle: true) @@ -530,8 +528,7 @@ final class DockSplitStore: BonsplitDelegate { isPinned: false, inPane: paneId ) else { - panels.removeValue(forKey: panel.id) - panel.close() + discardPanelOwnershipAndClose(panelId: panel.id) return nil } surfaceIdToPanelId[tabId] = panel.id @@ -607,12 +604,7 @@ final class DockSplitStore: BonsplitDelegate { let staleTabIds = surfaceIdToPanelId.keys.filter { !live.contains($0) } for tabId in staleTabIds { guard let panelId = surfaceIdToPanelId.removeValue(forKey: tabId) else { continue } - panelCancellables[panelId]?.cancel() - panelCancellables.removeValue(forKey: panelId) - AppDelegate.shared?.notificationStore?.clearNotifications(forTabId: workspaceId, surfaceId: panelId) - detachedSurfaceTransfersByPanelId.removeValue(forKey: panelId) - clearSessionRestoreState(panelId: panelId) - if let panel = panels.removeValue(forKey: panelId) { panel.close() } + discardPanelOwnershipAndClose(panelId: panelId) } } @@ -799,9 +791,7 @@ final class DockSplitStore: BonsplitDelegate { ) } guard seedSplitResult != nil else { - surfaceIdToPanelId.removeValue(forKey: newTab.id) - panels.removeValue(forKey: panel.id) - panel.close() + discardPanelOwnershipAndClose(panelId: panel.id) continue } installSubscription(for: panel, tracksTerminalTitle: tracksTitle) diff --git a/Sources/Panels/TerminalPanel+AgentHibernation.swift b/Sources/Panels/TerminalPanel+AgentHibernation.swift new file mode 100644 index 000000000000..838ff252b8b6 --- /dev/null +++ b/Sources/Panels/TerminalPanel+AgentHibernation.swift @@ -0,0 +1,77 @@ +import Foundation + +extension TerminalPanel { + var isAgentHibernated: Bool { + agentHibernationPhase.isCommitted + } + + var isAgentHibernationTerminating: Bool { + agentHibernationPhase.isTerminating + } + + var agentHibernationState: AgentHibernationPanelState? { + agentHibernationPhase.state + } + + func enterAgentHibernation( + agent: SessionRestorableAgentSnapshot, + lastActivityAt: Date, + hibernatedAt: Date = .now + ) { + if isAgentHibernationTerminating { + completeAgentHibernationTermination() + return + } + agentHibernationPhase = .hibernated(AgentHibernationPanelState( + agent: agent, + hibernatedAt: hibernatedAt, + lastActivityAt: lastActivityAt + )) + suspendRuntimeForAgentHibernation(reason: "agentHibernation") + } + + func beginAgentHibernationTermination( + agent: SessionRestorableAgentSnapshot, + lastActivityAt: Date, + committedAt: Date = .now + ) { + guard case .live = agentHibernationPhase else { return } + agentHibernationPhase = .terminating(AgentHibernationPanelState( + agent: agent, + hibernatedAt: committedAt, + lastActivityAt: lastActivityAt + )) + suspendRuntimeForAgentHibernation(reason: "agentHibernation.terminating") + } + + func completeAgentHibernationTermination() { + guard case .terminating(let state) = agentHibernationPhase else { return } + agentHibernationPhase = .hibernated(state) + } + + func discardAgentHibernationPhaseForPermanentClose() { + agentHibernationPhase = .live + } + + private func suspendRuntimeForAgentHibernation(reason: String) { + unfocus() + searchState = nil + hostedView.setVisibleInUI(false) + TerminalWindowPortalRegistry.detach(hostedView: hostedView) + surface.suspendRuntimeSurfaceForAgentHibernation(reason: reason) + requestViewReattach() + } + + @discardableResult + func prepareAgentHibernationResume() -> AgentHibernationResumePreparation { + guard case .hibernated(let state) = agentHibernationPhase else { + return .unavailable + } + let resumeStartupInput = state.agent.resumeStartupInput() + agentHibernationPhase = .live + surface.prepareAgentHibernationResume(initialInput: resumeStartupInput) + requestViewReattach() + surface.requestBackgroundSurfaceStartIfNeeded() + return .resumed(queuedStartupInput: resumeStartupInput != nil) + } +} diff --git a/Sources/Panels/TerminalPanel.swift b/Sources/Panels/TerminalPanel.swift index e741b6eb8c0e..201aa3a84bee 100644 --- a/Sources/Panels/TerminalPanel.swift +++ b/Sources/Panels/TerminalPanel.swift @@ -6,31 +6,6 @@ import Bonsplit import CmuxTerminal import CmuxWorkspaces -struct AgentHibernationPanelState { - let agent: SessionRestorableAgentSnapshot - let hibernatedAt: Date - let lastActivityAt: Date - - var agentDisplayName: String { - agent.agentDisplayName - } -} - -enum AgentHibernationResumePreparation: Equatable { - case unavailable - case resumed(queuedStartupInput: Bool) - - var didResume: Bool { - if case .resumed = self { return true } - return false - } - - var queuedStartupInput: Bool { - if case .resumed(let queuedStartupInput) = self { return queuedStartupInput } - return false - } -} - /// TerminalPanel wraps an existing TerminalSurface and conforms to the Panel protocol. /// This allows TerminalSurface to be used within the bonsplit-based layout system. @MainActor @@ -115,7 +90,7 @@ final class TerminalPanel: Panel, ObservableObject { /// (hostedView.window == nil) until the user switches workspaces. @Published var viewReattachToken: UInt64 = 0 - @Published private(set) var agentHibernationState: AgentHibernationPanelState? + @Published var agentHibernationPhase: AgentHibernationPanelPhase = .live var onRequestWorkspacePaneFlash: ((WorkspaceAttentionFlashReason) -> Void)? var onRequestAgentHibernationResume: ((Bool) -> Bool)? @@ -158,10 +133,6 @@ final class TerminalPanel: Panel, ObservableObject { false } - var isAgentHibernated: Bool { - agentHibernationState != nil - } - /// The hosted NSView for embedding in SwiftUI var hostedView: GhosttySurfaceScrollView { surface.hostedView @@ -667,6 +638,11 @@ final class TerminalPanel: Panel, ObservableObject { func close() { isClosingPanel = true + AgentHibernationController.shared.discardTrackingStateForClosedPanel( + workspaceId: workspaceId, + panelId: id + ) + discardAgentHibernationPhaseForPermanentClose() discardTextBoxContentForClose() removeOwnedSessionScrollbackReplayArtifact() // Detach from the window portal on real close so stale hosted views @@ -695,37 +671,6 @@ final class TerminalPanel: Panel, ObservableObject { surface.teardownSurface() } - func enterAgentHibernation( - agent: SessionRestorableAgentSnapshot, - lastActivityAt: Date, - hibernatedAt: Date = Date() - ) { - agentHibernationState = AgentHibernationPanelState( - agent: agent, - hibernatedAt: hibernatedAt, - lastActivityAt: lastActivityAt - ) - unfocus() - searchState = nil - hostedView.setVisibleInUI(false) - TerminalWindowPortalRegistry.detach(hostedView: hostedView) - surface.suspendRuntimeSurfaceForAgentHibernation(reason: "agentHibernation") - requestViewReattach() - } - - @discardableResult - func prepareAgentHibernationResume() -> AgentHibernationResumePreparation { - guard let state = agentHibernationState else { - return .unavailable - } - let resumeStartupInput = state.agent.resumeStartupInput() - agentHibernationState = nil - surface.prepareAgentHibernationResume(initialInput: resumeStartupInput) - requestViewReattach() - surface.requestBackgroundSurfaceStartIfNeeded() - return .resumed(queuedStartupInput: resumeStartupInput != nil) - } - func requestViewReattach() { viewReattachToken &+= 1 } diff --git a/Sources/Panels/TerminalPanelView.swift b/Sources/Panels/TerminalPanelView.swift index dfe74810c324..b66669b8cacf 100644 --- a/Sources/Panels/TerminalPanelView.swift +++ b/Sources/Panels/TerminalPanelView.swift @@ -35,10 +35,15 @@ struct TerminalPanelView: View { let onTriggerFlash: () -> Void var body: some View { - if let hibernationState = panel.agentHibernationState { - hibernationBody(hibernationState) - } else { + switch panel.agentHibernationPhase { + case .live: terminalBody + case .terminating: + Color(nsColor: appearance.contentBackgroundColor) + .frame(maxWidth: .infinity, maxHeight: .infinity) + .id("hibernation-terminating-\(panel.id.uuidString)") + case .hibernated(let hibernationState): + hibernationBody(hibernationState) } } diff --git a/Sources/Workspace.swift b/Sources/Workspace.swift index 411777086746..afc870b72c2d 100644 --- a/Sources/Workspace.swift +++ b/Sources/Workspace.swift @@ -4756,7 +4756,7 @@ final class Workspace: Identifiable, ObservableObject { lastActivityAt: Date ) { guard let terminalPanel = panels[panelId] as? TerminalPanel, - !terminalPanel.isAgentHibernated else { + !terminalPanel.isAgentHibernated || terminalPanel.isAgentHibernationTerminating else { return } guard agent.resumeCommand != nil else { return } diff --git a/cmux.xcodeproj/project.pbxproj b/cmux.xcodeproj/project.pbxproj index 38949fadf9c5..fc325207a3e7 100644 --- a/cmux.xcodeproj/project.pbxproj +++ b/cmux.xcodeproj/project.pbxproj @@ -61,11 +61,13 @@ F65760030000000000000001 /* AgentHibernationController+Records.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760030000000000000002 /* AgentHibernationController+Records.swift */; }; F65760140000000000000001 /* AgentHibernationController+TailFingerprintSample.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760140000000000000002 /* AgentHibernationController+TailFingerprintSample.swift */; }; F65760060000000000000001 /* AgentHibernationController+Teardown.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760060000000000000002 /* AgentHibernationController+Teardown.swift */; }; + 8997E0018997E0018997E001 /* AgentHibernationController+TeardownValidation.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997F0018997F0018997F001 /* AgentHibernationController+TeardownValidation.swift */; }; F65760150000000000000001 /* AgentHibernationController+UnableToProtectMarker.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760150000000000000002 /* AgentHibernationController+UnableToProtectMarker.swift */; }; D36A00010000000000000001 /* AgentHibernationController.swift in Sources */ = {isa = PBXBuildFile; fileRef = D36A00010000000000000002 /* AgentHibernationController.swift */; }; D36A00030000000000000001 /* AgentHibernationLifecycleState.swift in Sources */ = {isa = PBXBuildFile; fileRef = D36A00030000000000000002 /* AgentHibernationLifecycleState.swift */; }; D36A00030000000000000003 /* AgentHibernationLifecycleState.swift in Sources */ = {isa = PBXBuildFile; fileRef = D36A00030000000000000002 /* AgentHibernationLifecycleState.swift */; }; 8997C0028997C0028997C002 /* AgentHibernationMemoryPressureResponder.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D0028997D0028997D002 /* AgentHibernationMemoryPressureResponder.swift */; }; + 8997E0028997E0028997E002 /* AgentHibernationPanelPhase.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997F0028997F0028997F002 /* AgentHibernationPanelPhase.swift */; }; F65760160000000000000001 /* AgentHibernationPlanner.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760160000000000000002 /* AgentHibernationPlanner.swift */; }; F65760170000000000000001 /* AgentHibernationPlannerInput.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760170000000000000002 /* AgentHibernationPlannerInput.swift */; }; F65760010000000000000001 /* AgentHibernationPlannerSwiftTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760010000000000000002 /* AgentHibernationPlannerSwiftTests.swift */; }; @@ -878,6 +880,7 @@ C0DE71B10000000000000001 /* AppDelegate+AgentChatNotifications.swift in Sources DCDC1000000000000000B021 /* DockSplitStore+CloseConfirmation.swift in Sources */ = {isa = PBXBuildFile; fileRef = DCDC1000000000000000B022 /* DockSplitStore+CloseConfirmation.swift */; }; DCDC0000000000000000A003 /* DockSplitStore+Config.swift in Sources */ = {isa = PBXBuildFile; fileRef = DCDC0000000000000000A004 /* DockSplitStore+Config.swift */; }; DCDC1000000000000000B019 /* DockSplitStore+PaneFocus.swift in Sources */ = {isa = PBXBuildFile; fileRef = DCDC1000000000000000B020 /* DockSplitStore+PaneFocus.swift */; }; + 8997E0038997E0038997E003 /* DockSplitStore+PanelDestruction.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997F0038997F0038997F003 /* DockSplitStore+PanelDestruction.swift */; }; DCDC1000000000000000C040 /* DockSplitStore+PortalDrop.swift in Sources */ = {isa = PBXBuildFile; fileRef = DCDC1000000000000000C041 /* DockSplitStore+PortalDrop.swift */; }; D77800020000000000000001 /* DockSplitStore+PortalReconcile.swift in Sources */ = {isa = PBXBuildFile; fileRef = D77800020000000000000002 /* DockSplitStore+PortalReconcile.swift */; }; D86840000000000000000011 /* DockSplitStore+Reset.swift in Sources */ = {isa = PBXBuildFile; fileRef = D86840000000000000000012 /* DockSplitStore+Reset.swift */; }; @@ -2137,6 +2140,7 @@ C0DE71B10000000000000001 /* AppDelegate+AgentChatNotifications.swift in Sources 0A110711000000000000001E /* TerminalOutputTeeContext.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0A110711000000000000001F /* TerminalOutputTeeContext.swift */; }; C71500060000000000000001 /* TerminalPaneBackgroundView.swift in Sources */ = {isa = PBXBuildFile; fileRef = C71500050000000000000001 /* TerminalPaneBackgroundView.swift */; }; D0B10000A1B2C3D4E5F60001 /* TerminalPaneDropTargetView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0B10001A1B2C3D4E5F60001 /* TerminalPaneDropTargetView.swift */; }; + 8997E0048997E0048997E004 /* TerminalPanel+AgentHibernation.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997F0048997F0048997F004 /* TerminalPanel+AgentHibernation.swift */; }; A5F100000000000000000007 /* TerminalPanel+NotificationScroll.swift in Sources */ = {isa = PBXBuildFile; fileRef = A5F100000000000000000008 /* TerminalPanel+NotificationScroll.swift */; }; 791101010000000000000001 /* TerminalPanel+SessionScrollbackReplay.swift in Sources */ = {isa = PBXBuildFile; fileRef = 791101020000000000000001 /* TerminalPanel+SessionScrollbackReplay.swift */; }; A5001401 /* TerminalPanel.swift in Sources */ = {isa = PBXBuildFile; fileRef = A5001411 /* TerminalPanel.swift */; }; @@ -2554,10 +2558,12 @@ C0DE71B10000000000000001 /* AppDelegate+AgentChatNotifications.swift in Sources F65760030000000000000002 /* AgentHibernationController+Records.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+Records.swift"; sourceTree = ""; }; F65760140000000000000002 /* AgentHibernationController+TailFingerprintSample.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+TailFingerprintSample.swift"; sourceTree = ""; }; F65760060000000000000002 /* AgentHibernationController+Teardown.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+Teardown.swift"; sourceTree = ""; }; + 8997F0018997F0018997F001 /* AgentHibernationController+TeardownValidation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+TeardownValidation.swift"; sourceTree = ""; }; F65760150000000000000002 /* AgentHibernationController+UnableToProtectMarker.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+UnableToProtectMarker.swift"; sourceTree = ""; }; D36A00010000000000000002 /* AgentHibernationController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = App/AgentHibernationController.swift; sourceTree = ""; }; D36A00030000000000000002 /* AgentHibernationLifecycleState.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernation/AgentHibernationLifecycleState.swift; sourceTree = ""; }; 8997D0028997D0028997D002 /* AgentHibernationMemoryPressureResponder.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = App/AgentHibernationMemoryPressureResponder.swift; sourceTree = ""; }; + 8997F0028997F0028997F002 /* AgentHibernationPanelPhase.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernation/AgentHibernationPanelPhase.swift; sourceTree = ""; }; F65760160000000000000002 /* AgentHibernationPlanner.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationPlanner.swift"; sourceTree = ""; }; F65760170000000000000002 /* AgentHibernationPlannerInput.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationPlannerInput.swift"; sourceTree = ""; }; F65760010000000000000002 /* AgentHibernationPlannerSwiftTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernationPlannerSwiftTests.swift; sourceTree = ""; }; @@ -3299,6 +3305,7 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = DCDC1000000000000000B022 /* DockSplitStore+CloseConfirmation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "DockSplitStore+CloseConfirmation.swift"; sourceTree = ""; }; DCDC0000000000000000A004 /* DockSplitStore+Config.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "DockSplitStore+Config.swift"; sourceTree = ""; }; DCDC1000000000000000B020 /* DockSplitStore+PaneFocus.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "DockSplitStore+PaneFocus.swift"; sourceTree = ""; }; + 8997F0038997F0038997F003 /* DockSplitStore+PanelDestruction.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "DockSplitStore+PanelDestruction.swift"; sourceTree = ""; }; DCDC1000000000000000C041 /* DockSplitStore+PortalDrop.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "DockSplitStore+PortalDrop.swift"; sourceTree = ""; }; D77800020000000000000002 /* DockSplitStore+PortalReconcile.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "DockSplitStore+PortalReconcile.swift"; sourceTree = ""; }; D86840000000000000000012 /* DockSplitStore+Reset.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "DockSplitStore+Reset.swift"; sourceTree = ""; }; @@ -4538,6 +4545,7 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = 0A110711000000000000001F /* TerminalOutputTeeContext.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TerminalOutputTeeContext.swift; sourceTree = ""; }; C71500050000000000000001 /* TerminalPaneBackgroundView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TerminalPaneBackgroundView.swift; sourceTree = ""; }; D0B10001A1B2C3D4E5F60001 /* TerminalPaneDropTargetView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TerminalPaneDropTargetView.swift; sourceTree = ""; }; + 8997F0048997F0048997F004 /* TerminalPanel+AgentHibernation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Panels/TerminalPanel+AgentHibernation.swift"; sourceTree = ""; }; A5F100000000000000000008 /* TerminalPanel+NotificationScroll.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Panels/TerminalPanel+NotificationScroll.swift"; sourceTree = ""; }; 791101020000000000000001 /* TerminalPanel+SessionScrollbackReplay.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Panels/TerminalPanel+SessionScrollbackReplay.swift"; sourceTree = ""; }; A5001411 /* TerminalPanel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Panels/TerminalPanel.swift; sourceTree = ""; }; @@ -5311,6 +5319,7 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = F65760260000000000000002 /* AgentHibernationController+PostTeardownRestoreCancellationState.swift */, 8997D0038997D0038997D003 /* AgentHibernationController+ProcessTermination.swift */, 8997D0068997D0068997D006 /* AgentHibernationController+PanelLifecycle.swift */, + 8997F0018997F0018997F001 /* AgentHibernationController+TeardownValidation.swift */, F65760030000000000000002 /* AgentHibernationController+Records.swift */, F65760140000000000000002 /* AgentHibernationController+TailFingerprintSample.swift */, F65760060000000000000002 /* AgentHibernationController+Teardown.swift */, @@ -5728,6 +5737,7 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = 0A1107110000000000000021 /* VaultAgentProcessScanner+Ollama.swift */, 0A1107110000000000000023 /* SessionRestorableAgentSnapshot+Commands.swift */, D36A00030000000000000002 /* AgentHibernationLifecycleState.swift */, + 8997F0028997F0028997F002 /* AgentHibernationPanelPhase.swift */, F0ACC0E0000000000000002 /* CachedAgentProcessIdentityValidator.swift */, B3575000000000000000000B /* SessionIndexModels.swift */, F0ACC0DE0000000000000008 /* SharedLiveAgentIndexLoader.swift */, @@ -6071,6 +6081,7 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = C54860060000000000000002 /* PanelStableSurfaceIdentity.swift */, A5F0C0010000000000000001 /* WorkspaceAttentionFlashRingView.swift */, A5001411 /* TerminalPanel.swift */, + 8997F0048997F0048997F004 /* TerminalPanel+AgentHibernation.swift */, 791101020000000000000001 /* TerminalPanel+SessionScrollbackReplay.swift */, A5F100000000000000000008 /* TerminalPanel+NotificationScroll.swift */, C0DE5A410000000000000002 /* TerminalPanelShellActivityModel.swift */, @@ -6573,6 +6584,7 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = D8684000000000000000000E /* DockSplitStore+SurfaceResume.swift */, D86840000000000000000010 /* ControlSurfaceResumeTarget.swift */, D86840000000000000000012 /* DockSplitStore+Reset.swift */, + 8997F0038997F0038997F003 /* DockSplitStore+PanelDestruction.swift */, DCDC1000000000000000B00E /* DockSurfaceKind.swift */, DCDC1000000000000000B010 /* DockTrustRequest.swift */, D0C0D0C0D0C0D0C0D0C0D004 /* DockEmptyView.swift */, @@ -7896,10 +7908,12 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = F65760030000000000000001 /* AgentHibernationController+Records.swift in Sources */, F65760140000000000000001 /* AgentHibernationController+TailFingerprintSample.swift in Sources */, F65760060000000000000001 /* AgentHibernationController+Teardown.swift in Sources */, + 8997E0018997E0018997E001 /* AgentHibernationController+TeardownValidation.swift in Sources */, F65760150000000000000001 /* AgentHibernationController+UnableToProtectMarker.swift in Sources */, D36A00010000000000000001 /* AgentHibernationController.swift in Sources */, D36A00030000000000000001 /* AgentHibernationLifecycleState.swift in Sources */, 8997C0028997C0028997C002 /* AgentHibernationMemoryPressureResponder.swift in Sources */, + 8997E0028997E0028997E002 /* AgentHibernationPanelPhase.swift in Sources */, F65760160000000000000001 /* AgentHibernationPlanner.swift in Sources */, F65760170000000000000001 /* AgentHibernationPlannerInput.swift in Sources */, F65760090000000000000001 /* AgentHibernationTranscriptGuard+ClaudeWorkflow.swift in Sources */, @@ -8314,6 +8328,7 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = DCDC1000000000000000B021 /* DockSplitStore+CloseConfirmation.swift in Sources */, DCDC0000000000000000A003 /* DockSplitStore+Config.swift in Sources */, DCDC1000000000000000B019 /* DockSplitStore+PaneFocus.swift in Sources */, + 8997E0038997E0038997E003 /* DockSplitStore+PanelDestruction.swift in Sources */, DCDC1000000000000000C040 /* DockSplitStore+PortalDrop.swift in Sources */, D77800020000000000000001 /* DockSplitStore+PortalReconcile.swift in Sources */, D86840000000000000000011 /* DockSplitStore+Reset.swift in Sources */, @@ -9179,6 +9194,7 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = 0A110711000000000000001E /* TerminalOutputTeeContext.swift in Sources */, C71500060000000000000001 /* TerminalPaneBackgroundView.swift in Sources */, D0B10000A1B2C3D4E5F60001 /* TerminalPaneDropTargetView.swift in Sources */, + 8997E0048997E0048997E004 /* TerminalPanel+AgentHibernation.swift in Sources */, A5F100000000000000000007 /* TerminalPanel+NotificationScroll.swift in Sources */, 791101010000000000000001 /* TerminalPanel+SessionScrollbackReplay.swift in Sources */, A5001401 /* TerminalPanel.swift in Sources */, diff --git a/cmuxTests/AgentHibernationProcessTerminationTests.swift b/cmuxTests/AgentHibernationProcessTerminationTests.swift index ce099b7f56e3..435e2b508958 100644 --- a/cmuxTests/AgentHibernationProcessTerminationTests.swift +++ b/cmuxTests/AgentHibernationProcessTerminationTests.swift @@ -127,7 +127,7 @@ struct AgentHibernationProcessTerminationTests { @MainActor @Test - func terminationDoesNotCompleteUntilTheExactProcessGenerationExits() async { + func committedObservationDoesNotCompleteUntilTheExactProcessGenerationExits() async { let identity = AgentPIDProcessIdentity( pid: 101, startSeconds: 10, @@ -140,41 +140,39 @@ struct AgentHibernationProcessTerminationTests { ) let waitStarted = AsyncStream.makeStream() let allowExit = AsyncStream.makeStream() - let completion = CompletionRecorder() - let task = Task { @MainActor in - let result = await AgentHibernationController.shared - .terminateScopedProcessesForHibernation( - [termination], - currentProcessID: 999, - currentProcessGroupID: 999, - processIdentityProvider: { _ in identity }, - processGroupProvider: { _ in 1 }, - signalErrorProvider: { _, _ in nil }, - waitForExit: { _ in - waitStarted.continuation.yield() - for await _ in allowExit.stream { - break - } - return true - } - ) - await completion.record(result) - } + let controller = AgentHibernationController.shared + let panelID = UUID() + var didComplete = false + controller.observeCommittedTermination( + panelID: panelID, + terminations: [termination], + waitForExit: { _ in + waitStarted.continuation.yield() + for await _ in allowExit.stream { + break + } + return true + }, + onExit: { + didComplete = true + } + ) + let task = controller.committedTerminationObservationsByPanelID[panelID]?.task var waitStartedIterator = waitStarted.stream.makeAsyncIterator() _ = await waitStartedIterator.next() - #expect(await completion.value == nil) + #expect(!didComplete) allowExit.continuation.yield() allowExit.continuation.finish() - await task.value - #expect(await completion.value == true) + await task?.value + #expect(didComplete) waitStarted.continuation.finish() } @MainActor @Test - func signalFailureOtherThanMissingProcessAbortsBeforeExitWait() async { + func signalFailureOtherThanMissingProcessAbortsBeforeCommit() { let identity = AgentPIDProcessIdentity( pid: 101, startSeconds: 10, @@ -186,21 +184,17 @@ struct AgentHibernationProcessTerminationTests { processGroupID: 1 ) - let result = await AgentHibernationController.shared + let result = AgentHibernationController.shared .terminateScopedProcessesForHibernation( [termination], currentProcessID: 999, currentProcessGroupID: 999, processIdentityProvider: { _ in identity }, processGroupProvider: { _ in 1 }, - signalErrorProvider: { _, _ in EPERM }, - waitForExit: { _ in - Issue.record("Exit wait must not begin after a failed SIGTERM") - return true - } + signalErrorProvider: { _, _ in EPERM } ) - #expect(result == false) + #expect(result == .rejected) } @MainActor @@ -230,7 +224,8 @@ struct AgentHibernationProcessTerminationTests { ] let waitRecorder = CompletionRecorder() - let result = await AgentHibernationController.shared + let controller = AgentHibernationController.shared + let result = controller .terminateScopedProcessesForHibernation( terminations, currentProcessID: 999, @@ -241,15 +236,123 @@ struct AgentHibernationProcessTerminationTests { processGroupProvider: { _ in 1 }, signalErrorProvider: { target, _ in target == 101 ? nil : EPERM - }, - waitForExit: { observedTerminations in - await waitRecorder.record(observedTerminations == terminations) - return observedTerminations == terminations } ) - + let panelID = UUID() + controller.observeCommittedTermination( + panelID: panelID, + terminations: terminations, + waitForExit: { observedTerminations in + await waitRecorder.record(observedTerminations == terminations) + return observedTerminations == terminations + }, + onExit: {} + ) + let observationTask = controller + .committedTerminationObservationsByPanelID[panelID]? + .task + await observationTask?.value #expect(await waitRecorder.value == true) - #expect(result) + #expect(result == .committedAwaitingExit) + } + + @MainActor + @Test + func resumeStaysUnavailableUntilCommittedTerminationCompletes() { + let panel = TerminalPanel(workspaceId: UUID()) + defer { panel.close() } + let agent = SessionRestorableAgentSnapshot( + kind: .custom("test-agent"), + sessionId: "committed-hibernation", + workingDirectory: "/tmp", + launchCommand: nil + ) + + panel.beginAgentHibernationTermination( + agent: agent, + lastActivityAt: Date(timeIntervalSince1970: 1) + ) + + #expect(panel.isAgentHibernated) + #expect(panel.isAgentHibernationTerminating) + #expect(panel.prepareAgentHibernationResume() == .unavailable) + + panel.completeAgentHibernationTermination() + + #expect(!panel.isAgentHibernationTerminating) + #expect(panel.prepareAgentHibernationResume() == .resumed(queuedStartupInput: false)) + #expect(!panel.isAgentHibernated) + } + + @MainActor + @Test + func exitDeadlineDoesNotBlockLaterWorkWhileCommittedPanelAwaitsExit() async { + let identity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let termination = AgentHibernationController.ScopedProcessTermination( + processID: 101, + processIdentity: identity, + processGroupID: 1 + ) + let panel = TerminalPanel(workspaceId: UUID()) + defer { panel.close() } + let agent = SessionRestorableAgentSnapshot( + kind: .custom("test-agent"), + sessionId: "eventual-exit-hibernation", + workingDirectory: "/tmp", + launchCommand: nil + ) + let eventualWaitStarted = AsyncStream.makeStream() + let allowEventualExit = AsyncStream.makeStream() + let controller = AgentHibernationController.shared + let result = controller.terminateScopedProcessesForHibernation( + [termination], + onTeardownCommit: { + panel.beginAgentHibernationTermination( + agent: agent, + lastActivityAt: Date(timeIntervalSince1970: 1) + ) + }, + currentProcessID: 999, + currentProcessGroupID: 999, + processIdentityProvider: { _ in identity }, + processGroupProvider: { _ in 1 }, + signalErrorProvider: { _, _ in nil } + ) + #expect(result == .committedAwaitingExit) + controller.observeCommittedTermination( + panelID: panel.id, + terminations: [termination], + waitForExit: { _ in + eventualWaitStarted.continuation.yield() + for await _ in allowEventualExit.stream { + break + } + return true + }, + onExit: { + panel.completeAgentHibernationTermination() + } + ) + let observationTask = controller + .committedTerminationObservationsByPanelID[panel.id]? + .task + var eventualWaitStartedIterator = eventualWaitStarted.stream.makeAsyncIterator() + _ = await eventualWaitStartedIterator.next() + + #expect(panel.isAgentHibernationTerminating) + #expect(panel.prepareAgentHibernationResume() == .unavailable) + + allowEventualExit.continuation.yield() + allowEventualExit.continuation.finish() + await observationTask?.value + + #expect(!panel.isAgentHibernationTerminating) + #expect(panel.isAgentHibernated) + eventualWaitStarted.continuation.finish() } @Test @@ -265,7 +368,8 @@ struct AgentHibernationProcessTerminationTests { startMicroseconds: 0 ) - let didExit = await AgentHibernationController.waitForExactProcessGenerationsToExit( + let didExit = await AgentHibernationController + .waitForExactProcessGenerationsToExitWithoutTimeout( [ .init( processID: 101, @@ -273,7 +377,6 @@ struct AgentHibernationProcessTerminationTests { processGroupID: 1 ), ], - timeout: .zero, processIdentityProvider: { _ in reusedIdentity } ) diff --git a/cmuxTests/AgentHibernationTrackingLifecycleTests.swift b/cmuxTests/AgentHibernationTrackingLifecycleTests.swift index 8524fc838088..ad7d6602ffa3 100644 --- a/cmuxTests/AgentHibernationTrackingLifecycleTests.swift +++ b/cmuxTests/AgentHibernationTrackingLifecycleTests.swift @@ -160,6 +160,46 @@ struct AgentHibernationTrackingLifecycleTests { expectTrackingWasDiscarded(for: key, controller: controller) } + @Test + func permanentPanelCloseCancelsCommittedTerminationObservation() async { + let controller = AgentHibernationController.shared + let panel = TerminalPanel(workspaceId: UUID()) + let identity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let exitEvents = AsyncStream.makeStream() + var didCompleteHibernation = false + controller.observeCommittedTermination( + panelID: panel.id, + terminations: [ + .init( + processID: 101, + processIdentity: identity, + processGroupID: 1 + ), + ], + waitForExit: { _ in + for await _ in exitEvents.stream {} + return true + }, + onExit: { + didCompleteHibernation = true + } + ) + let observationTask = controller + .committedTerminationObservationsByPanelID[panel.id]? + .task + + panel.close() + exitEvents.continuation.finish() + await observationTask?.value + + #expect(controller.committedTerminationObservationsByPanelID[panel.id] == nil) + #expect(!didCompleteHibernation) + } + private func expectTrackingWasDiscarded( for key: AgentHibernationPanelKey, controller: AgentHibernationController From 4f6d2295dd8abba1c4e0b3f7cda3c8a11f77323a Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 28 Jul 2026 20:40:59 -0700 Subject: [PATCH 09/20] fix: compile transcript revalidation closure --- .../App/AgentHibernationController+ProcessTermination.swift | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Sources/App/AgentHibernationController+ProcessTermination.swift b/Sources/App/AgentHibernationController+ProcessTermination.swift index d1b535e9b38e..fe6bd945e0ce 100644 --- a/Sources/App/AgentHibernationController+ProcessTermination.swift +++ b/Sources/App/AgentHibernationController+ProcessTermination.swift @@ -186,7 +186,9 @@ extension AgentHibernationController { index: preSignalIndex, shouldProceed: shouldProceed ), - snapshot.map(AgentHibernationTranscriptGuard.liveFileVersionStillMatches) ?? true else { + snapshot.map({ + AgentHibernationTranscriptGuard.liveFileVersionStillMatches($0) + }) ?? true else { return false } From 8e7796f44a9afcb300849d641de2399256bf5d05 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 28 Jul 2026 20:52:29 -0700 Subject: [PATCH 10/20] test: cover incomplete hibernation process identity scope --- ...ntHibernationProcessTerminationTests.swift | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/cmuxTests/AgentHibernationProcessTerminationTests.swift b/cmuxTests/AgentHibernationProcessTerminationTests.swift index 435e2b508958..c74d0522ce92 100644 --- a/cmuxTests/AgentHibernationProcessTerminationTests.swift +++ b/cmuxTests/AgentHibernationProcessTerminationTests.swift @@ -125,6 +125,33 @@ struct AgentHibernationProcessTerminationTests { #expect(terminations == nil) } + @Test + func rejectsScopeWithUnrecordedProcessIdentity() { + let workspaceID = UUID() + let panelID = UUID() + let identity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let scope = AgentHibernationController.ProcessTerminationScope( + key: AgentHibernationPanelKey(workspaceId: workspaceID, panelId: panelID), + processIDs: [101, 202], + processIdentities: [101: identity] + ) + + let terminations = AgentHibernationController.validatedScopedProcessTerminations( + for: scope, + processIdentityProvider: { _ in identity }, + processArgumentsProvider: { _ in + Self.processArguments(workspaceID: workspaceID, panelID: panelID) + }, + processGroupProvider: { _ in 1_101 } + ) + + #expect(terminations == nil) + } + @MainActor @Test func committedObservationDoesNotCompleteUntilTheExactProcessGenerationExits() async { From a751a4b2a9d87b2fd7eab4aa50f415fb5bb24f4d Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 28 Jul 2026 20:57:06 -0700 Subject: [PATCH 11/20] perf: batch Dock panel ownership cleanup --- Sources/DockSplitStore+PanelDestruction.swift | 5 +++++ Sources/DockSplitStore+Reset.swift | 4 ++-- Sources/DockSplitStore.swift | 7 ++++--- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/Sources/DockSplitStore+PanelDestruction.swift b/Sources/DockSplitStore+PanelDestruction.swift index b1d9d8c0ae37..c5cbdf596128 100644 --- a/Sources/DockSplitStore+PanelDestruction.swift +++ b/Sources/DockSplitStore+PanelDestruction.swift @@ -10,6 +10,11 @@ extension DockSplitStore { for tabID in tabIDs { surfaceIdToPanelId.removeValue(forKey: tabID) } + return discardPanelStateAndClose(panelId: panelId) + } + + @discardableResult + func discardPanelStateAndClose(panelId: UUID) -> (any Panel)? { panelCancellables[panelId]?.cancel() panelCancellables.removeValue(forKey: panelId) AppDelegate.shared?.notificationStore?.clearNotifications( diff --git a/Sources/DockSplitStore+Reset.swift b/Sources/DockSplitStore+Reset.swift index 78edb3c10ad3..838072b31a8f 100644 --- a/Sources/DockSplitStore+Reset.swift +++ b/Sources/DockSplitStore+Reset.swift @@ -10,10 +10,10 @@ extension DockSplitStore { for tabId in tabIds { _ = bonsplitController.closeTab(tabId) } collapseToSingleEmptyPane() reconcilePanels() + surfaceIdToPanelId.removeAll() for panelId in Array(panels.keys) { - discardPanelOwnershipAndClose(panelId: panelId) + discardPanelStateAndClose(panelId: panelId) } - surfaceIdToPanelId.removeAll() detachedSurfaceTransfersByPanelId.removeAll() restoredTerminalScrollbackByPanelId.removeAll() restoredAgentLifecycle.snapshotsByPanelId.removeAll() diff --git a/Sources/DockSplitStore.swift b/Sources/DockSplitStore.swift index 15a1805bfd64..6229560b90b7 100644 --- a/Sources/DockSplitStore.swift +++ b/Sources/DockSplitStore.swift @@ -602,9 +602,10 @@ final class DockSplitStore: BonsplitDelegate { func reconcilePanels() { let live = Set(bonsplitController.allTabIds) let staleTabIds = surfaceIdToPanelId.keys.filter { !live.contains($0) } - for tabId in staleTabIds { - guard let panelId = surfaceIdToPanelId.removeValue(forKey: tabId) else { continue } - discardPanelOwnershipAndClose(panelId: panelId) + let stalePanelIds = Set(staleTabIds.compactMap { surfaceIdToPanelId[$0] }) + surfaceIdToPanelId = surfaceIdToPanelId.filter { !stalePanelIds.contains($0.value) } + for panelId in stalePanelIds { + discardPanelStateAndClose(panelId: panelId) } } From 120713c6e0873c29037183291a7d4132c66c3b78 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 28 Jul 2026 21:08:25 -0700 Subject: [PATCH 12/20] fix: release restore monitors after teardown aborts --- ...rnationController+ProcessTermination.swift | 14 ++++++ .../AgentHibernationController+Teardown.swift | 19 ++++++++ .../AgentHibernationRestoreMonitorTests.swift | 45 +++++++++++++++++++ 3 files changed, 78 insertions(+) diff --git a/Sources/App/AgentHibernationController+ProcessTermination.swift b/Sources/App/AgentHibernationController+ProcessTermination.swift index fe6bd945e0ce..2932ff3b5909 100644 --- a/Sources/App/AgentHibernationController+ProcessTermination.swift +++ b/Sources/App/AgentHibernationController+ProcessTermination.swift @@ -189,6 +189,13 @@ extension AgentHibernationController { snapshot.map({ AgentHibernationTranscriptGuard.liveFileVersionStillMatches($0) }) ?? true else { + if let snapshot { + await releaseArmedRestoreMonitorAfterAbortedTeardown( + snapshot, + sessionId: record.agent.sessionId, + restoreOwnedSnapshotPaths: &restoreOwnedSnapshotPaths + ) + } return false } @@ -228,6 +235,13 @@ extension AgentHibernationController { ) switch terminationResult { case .rejected: + if let snapshot { + await releaseArmedRestoreMonitorAfterAbortedTeardown( + snapshot, + sessionId: record.agent.sessionId, + restoreOwnedSnapshotPaths: &restoreOwnedSnapshotPaths + ) + } return false case .exited: finishTeardown() diff --git a/Sources/App/AgentHibernationController+Teardown.swift b/Sources/App/AgentHibernationController+Teardown.swift index d327c0102812..4aa145caa02d 100644 --- a/Sources/App/AgentHibernationController+Teardown.swift +++ b/Sources/App/AgentHibernationController+Teardown.swift @@ -145,6 +145,25 @@ extension AgentHibernationController { } } + func releaseArmedRestoreMonitorAfterAbortedTeardown( + _ snapshot: AgentHibernationTranscriptGuard.TeardownTranscriptSnapshot, + sessionId: String?, + restoreOwnedSnapshotPaths: inout Set + ) async { + await cancelPostTeardownRestoreTaskForReplacement( + transcriptPath: snapshot.transcriptPath + ) + restoreOwnedSnapshotPaths.remove(snapshot.snapshotPath) + guard AgentHibernationTranscriptGuard.liveFileVersionStillMatches(snapshot) else { + AgentHibernationTranscriptGuard.retainSnapshotForRecovery( + snapshot, + sessionId: sessionId + ) + return + } + try? FileManager.default.removeItem(atPath: snapshot.snapshotPath) + } + private static func snapshotOutcomes( for requests: [ConfirmedTeardownRequest] ) async -> [AgentHibernationPanelKey: AgentHibernationTranscriptGuard.TeardownSnapshotOutcome] { diff --git a/cmuxTests/AgentHibernationRestoreMonitorTests.swift b/cmuxTests/AgentHibernationRestoreMonitorTests.swift index 51c9298be560..3ee0bf8d53d3 100644 --- a/cmuxTests/AgentHibernationRestoreMonitorTests.swift +++ b/cmuxTests/AgentHibernationRestoreMonitorTests.swift @@ -117,6 +117,51 @@ struct AgentHibernationRestoreMonitorTests { ) == false) } + @MainActor + @Test + func lateAbortReleasesArmedMonitorAndDisposesItsSnapshot() async throws { + let controller = AgentHibernationController.shared + defer { resetSharedHibernationState(controller) } + + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("cmux-hibernation-late-abort-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + + let live = directory.appendingPathComponent("live.jsonl") + let snapshotURL = directory.appendingPathComponent("snapshot.jsonl") + let content = #"{"type":"user","message":{"content":"protected"}}"# + "\n" + try content.write(to: live, atomically: true, encoding: .utf8) + try content.write(to: snapshotURL, atomically: true, encoding: .utf8) + let snapshot = try #require( + AgentHibernationTranscriptGuard.snapshotStillMatchesLive( + .init( + transcriptPath: live.path, + snapshotPath: snapshotURL.path + ) + ) + ) + var restoreOwnedSnapshotPaths: Set = [snapshot.snapshotPath] + #expect(controller.armPostTeardownRestoreMonitor( + snapshot: snapshot, + processIDs: [] + )) + + await controller.releaseArmedRestoreMonitorAfterAbortedTeardown( + snapshot, + sessionId: "late-abort", + restoreOwnedSnapshotPaths: &restoreOwnedSnapshotPaths + ) + + let monitorKey = AgentHibernationController.postTeardownRestoreTaskKey( + transcriptPath: live.path + ) + #expect(controller.postTeardownRestoreTasksByTranscriptPath[monitorKey] == nil) + #expect(restoreOwnedSnapshotPaths.isEmpty) + #expect(FileManager.default.fileExists(atPath: snapshotURL.path) == false) + #expect(try String(contentsOf: live, encoding: .utf8) == content) + } + @MainActor @Test func armedForfeitMonitorRestoresClobberedTranscriptAndRefusesDuplicates() async throws { From 37ce5fd980d5f1b1f6ecfc07499ef586c0e7405f Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Tue, 28 Jul 2026 22:43:05 -0700 Subject: [PATCH 13/20] test: preserve live Dock panel aliases during reconcile --- cmuxTests/DockRuntimeParityTests.swift | 29 +++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/cmuxTests/DockRuntimeParityTests.swift b/cmuxTests/DockRuntimeParityTests.swift index a8432c4327a4..ed10f8397347 100644 --- a/cmuxTests/DockRuntimeParityTests.swift +++ b/cmuxTests/DockRuntimeParityTests.swift @@ -22,12 +22,15 @@ private final class DockRuntimeParityPanel: Panel, ObservableObject { var isDirty = false private(set) var flashReasons: [WorkspaceAttentionFlashReason] = [] + private(set) var closeCount = 0 init(title: String) { displayTitle = title } - func close() {} + func close() { + closeCount += 1 + } func focus() {} func unfocus() {} @@ -61,6 +64,30 @@ private extension DockSplitStore { struct DockRuntimeParityTests { private static let socketWorker = DispatchQueue(label: "DockRuntimeParityTests.socketWorker") + @Test("Reconciling a stale tab alias preserves the live panel owner") + func reconcilingStaleTabAliasPreservesLivePanelOwner() throws { + let dock = DockSplitStore(workspaceId: UUID(), baseDirectoryProvider: { nil }) + let panel = DockRuntimeParityPanel(title: "Shared panel") + let paneID = try dock.seedRuntimeParityPanel(panel) + let liveTabID = try #require(dock.surfaceId(forPanelId: panel.id)) + let staleAliasID = try #require( + dock.bonsplitController.createTab( + title: "Stale alias", + icon: panel.displayIcon, + kind: panel.panelType.rawValue, + isDirty: false, + inPane: paneID + ) + ) + dock.surfaceIdToPanelId[staleAliasID] = panel.id + + #expect(dock.bonsplitController.closeTab(staleAliasID)) + + #expect(dock.panel(for: liveTabID) === panel) + #expect(dock.surfaceIdToPanelId[staleAliasID] == nil) + #expect(panel.closeCount == 0) + } + private func socketEnvelope( method: String, params: [String: Any] = [:] From 7d9c8ee11bcb2580e894f7a17ace60d7de3659da Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 29 Jul 2026 06:15:42 -0700 Subject: [PATCH 14/20] fix: hibernate idle agent panes under memory pressure --- ...minalSurfaceRuntimeTeardownAdmission.swift | 40 ++ ...inalSurfaceRuntimeTeardownCompletion.swift | 42 ++ ...nalSurfaceRuntimeTeardownCoordinator.swift | 119 +++- ...lSurfaceRuntimeTeardownExecutionLane.swift | 8 + ...erminalSurfaceRuntimeTeardownRequest.swift | 5 +- ...nalSurfaceRuntimeTeardownReservation.swift | 10 + ...TerminalSurfaceRuntimeTeardownTicket.swift | 46 ++ .../TerminalSurface+RuntimeLifecycle.swift | 83 ++- .../Surface/TerminalSurface.swift | 3 + ...rfaceRuntimeTeardownCoordinatorTests.swift | 106 ++- ...SurfaceTeardownCallbackLifetimeTests.swift | 64 +- Resources/Localizable.xcstrings | 51 ++ .../AgentHibernationPanelPhase.swift | 21 +- ...ntroller+CommittedTerminationCleanup.swift | 8 + ...ller+CommittedTerminationObservation.swift | 81 +++ ...bernationController+InFlightTeardown.swift | 10 - ...HibernationController+PanelLifecycle.swift | 46 +- ...ionController+ProcessExitObservation.swift | 445 ++++++++++++ ...rnationController+ProcessExitWaiting.swift | 301 ++++++++ ...bernationController+ProcessSignaling.swift | 190 +++++ ...rnationController+ProcessTermination.swift | 384 +++++----- .../AgentHibernationController+Records.swift | 26 +- ...oller+ScopedProcessTerminationResult.swift | 9 + .../AgentHibernationController+Teardown.swift | 4 +- ...rnationController+TeardownValidation.swift | 24 +- Sources/App/AgentHibernationController.swift | 9 +- Sources/App/AgentHibernationPlanner.swift | 9 +- ...gentHibernationProcessExitCompletion.swift | 42 ++ .../AgentHibernationProcessExitEpoch.swift | 19 + ...ibernationProcessSnapshotCoordinator.swift | 231 ++++++ ...nTranscriptGuard+PostTeardownRestore.swift | 12 +- Sources/CmuxTopSnapshot.swift | 69 +- Sources/DockSplitStore.swift | 5 +- .../AgentHibernationPlaceholderMode.swift | 5 + .../TerminalPanel+AgentHibernation.swift | 87 ++- Sources/Panels/TerminalPanel.swift | 1 + Sources/Panels/TerminalPanelView.swift | 92 ++- Sources/RestorableAgentSession.swift | 67 +- Sources/Workspace.swift | 18 +- cmux.xcodeproj/project.pbxproj | 52 ++ .../AgentHibernationPlannerSwiftTests.swift | 4 + ...ibernationProcessSignalBoundaryTests.swift | 343 +++++++++ ...ationProcessSnapshotCoordinatorTests.swift | 156 +++++ ...ntHibernationProcessTerminationTests.swift | 661 ++++++++++++++++-- .../AgentHibernationRestoreMonitorTests.swift | 125 ++++ ...ntHibernationTerminationFailureTests.swift | 355 ++++++++++ ...entHibernationTrackingLifecycleTests.swift | 194 ++++- cmuxTests/AgentResumeLivenessTests.swift | 6 +- ...ompletedRestoredAgentGenerationTests.swift | 6 +- 49 files changed, 4337 insertions(+), 357 deletions(-) create mode 100644 Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownAdmission.swift create mode 100644 Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCompletion.swift create mode 100644 Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownExecutionLane.swift create mode 100644 Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownReservation.swift create mode 100644 Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownTicket.swift create mode 100644 Sources/App/AgentHibernationController+CommittedTerminationCleanup.swift create mode 100644 Sources/App/AgentHibernationController+CommittedTerminationObservation.swift create mode 100644 Sources/App/AgentHibernationController+ProcessExitObservation.swift create mode 100644 Sources/App/AgentHibernationController+ProcessExitWaiting.swift create mode 100644 Sources/App/AgentHibernationController+ProcessSignaling.swift create mode 100644 Sources/App/AgentHibernationController+ScopedProcessTerminationResult.swift create mode 100644 Sources/App/AgentHibernationProcessExitCompletion.swift create mode 100644 Sources/App/AgentHibernationProcessExitEpoch.swift create mode 100644 Sources/App/AgentHibernationProcessSnapshotCoordinator.swift create mode 100644 Sources/Panels/AgentHibernationPlaceholderMode.swift create mode 100644 cmuxTests/AgentHibernationProcessSignalBoundaryTests.swift create mode 100644 cmuxTests/AgentHibernationProcessSnapshotCoordinatorTests.swift create mode 100644 cmuxTests/AgentHibernationTerminationFailureTests.swift diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownAdmission.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownAdmission.swift new file mode 100644 index 000000000000..403ba88a3186 --- /dev/null +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownAdmission.swift @@ -0,0 +1,40 @@ +import Foundation + +/// Main-actor admission state for one coordinator's bounded native-free slots. +@MainActor +final class TerminalSurfaceRuntimeTeardownAdmission { + private var availableExecutionSlots: Set + private var executionSlotByReservationID: [UUID: Int] = [:] + + nonisolated init() { + availableExecutionSlots = Set( + 0.. TerminalSurfaceRuntimeTeardownReservation? { + guard let executionSlot = availableExecutionSlots.min() else { + return nil + } + let reservation = TerminalSurfaceRuntimeTeardownReservation() + availableExecutionSlots.remove(executionSlot) + executionSlotByReservationID[reservation.id] = executionSlot + return reservation + } + + func executionSlot( + for reservation: TerminalSurfaceRuntimeTeardownReservation + ) -> Int? { + executionSlotByReservationID[reservation.id] + } + + func release(_ reservation: TerminalSurfaceRuntimeTeardownReservation) { + guard let executionSlot = executionSlotByReservationID.removeValue( + forKey: reservation.id + ) else { + return + } + availableExecutionSlots.insert(executionSlot) + } +} diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCompletion.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCompletion.swift new file mode 100644 index 000000000000..01599ac057bd --- /dev/null +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCompletion.swift @@ -0,0 +1,42 @@ +import Foundation + +/// One native-free completion shared by the enqueueing surface and teardown worker. +actor TerminalSurfaceRuntimeTeardownCompletion { + private var didFinish = false + private var waiters: [UUID: CheckedContinuation] = [:] + + func wait() async -> Bool { + if didFinish { return true } + if Task.isCancelled { return false } + let waiterID = UUID() + return await withTaskCancellationHandler { + await withCheckedContinuation { continuation in + if didFinish { + continuation.resume(returning: true) + } else if Task.isCancelled { + continuation.resume(returning: false) + } else { + waiters[waiterID] = continuation + } + } + } onCancel: { + Task { + await self.cancel(waiterID: waiterID) + } + } + } + + func finish() { + guard !didFinish else { return } + didFinish = true + let pendingWaiters = waiters.values + waiters.removeAll(keepingCapacity: false) + for waiter in pendingWaiters { + waiter.resume(returning: true) + } + } + + private func cancel(waiterID: UUID) { + waiters.removeValue(forKey: waiterID)?.resume(returning: false) + } +} diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift index 024a955b692f..1f7487e54c76 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift @@ -1,17 +1,23 @@ public import Foundation public import GhosttyKit public import CmuxTerminalCore +internal import Dispatch #if DEBUG internal import CMUXDebugLog #endif -/// Serializes native `ghostty_surface_free` calls off the close/deinit paths. +/// Coordinates native `ghostty_surface_free` calls off the close/deinit paths. /// -/// Frees run one at a time on a utility worker so re-entrant close/deinit -/// loops cannot form, with a deadline observer that reports (but never -/// blocks on) a stuck native free. The app constructs exactly one instance -/// and injects it through ``TerminalSurfaceRuntimeDependencies``. +/// Close/deinit frees run one at a time on a utility worker so re-entrant +/// teardown loops cannot form. Each admitted hibernation owns one independently +/// startable utility slot, so one stuck native join cannot strand another pane. +/// Deadline observers report, but never block on, stuck frees. The app constructs +/// exactly one instance and injects it through +/// ``TerminalSurfaceRuntimeDependencies``. public actor TerminalSurfaceRuntimeTeardownCoordinator { + /// Largest batch that can own independently startable native-free slots. + public static let maximumIsolatedHibernationTeardownCount = 2 + private let timeout: Duration = .seconds(5) #if DEBUG // Readable at internal scope in DEBUG so the debug-only extension in @@ -23,9 +29,34 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { #endif private var queuedRequests: [TerminalSurfaceRuntimeTeardownRequest] = [] private var isWorkerRunning = false + private let isolatedHibernationQueues: [DispatchQueue] + private nonisolated let isolatedHibernationAdmission = + TerminalSurfaceRuntimeTeardownAdmission() /// Creates the process's teardown coordinator. - public init() {} + public init() { + isolatedHibernationQueues = ( + 0.. TerminalSurfaceRuntimeTeardownReservation? { + isolatedHibernationAdmission.reserve() + } + + @MainActor + func cancelIsolatedHibernationTeardown( + _ reservation: TerminalSurfaceRuntimeTeardownReservation + ) { + isolatedHibernationAdmission.release(reservation) + } /// Reads a bounded screen tail away from the main actor and before any /// subsequently enqueued native free for the same surface. @@ -50,6 +81,8 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { /// main actor after the free completes. /// - freeSurface: The free operation; defaults to /// `ghostty_surface_free`. + /// - Returns: A ticket that completes after the native free and userdata releases. + @discardableResult public nonisolated func enqueueRuntimeTeardown( id: UUID, workspaceId: UUID, @@ -59,7 +92,7 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { freeSurface: @escaping @Sendable (ghostty_surface_t) -> Void = { surface in ghostty_surface_free(surface) } - ) { + ) -> TerminalSurfaceRuntimeTeardownTicket { enqueueRuntimeTeardown( id: id, workspaceId: workspaceId, @@ -97,6 +130,8 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { /// actor after the free completes. /// - freeSurface: The free operation; defaults to /// `ghostty_surface_free`. + /// - Returns: A ticket that completes after the native free and userdata releases. + @discardableResult nonisolated func enqueueRuntimeTeardown( id: UUID, workspaceId: UUID, @@ -105,10 +140,15 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { callbackContext: Unmanaged?, manualIOContext: Unmanaged?, byteTeeLease: (any TerminalByteTeeLease)?, + executionLane: TerminalSurfaceRuntimeTeardownExecutionLane = .serializedClose, + isolatedHibernationReservation: + TerminalSurfaceRuntimeTeardownReservation? = nil, freeSurface: @escaping @Sendable (ghostty_surface_t) -> Void = { surface in ghostty_surface_free(surface) } - ) { + ) -> TerminalSurfaceRuntimeTeardownTicket { + let completion = TerminalSurfaceRuntimeTeardownCompletion() + let ticket = TerminalSurfaceRuntimeTeardownTicket(completion: completion) let request = TerminalSurfaceRuntimeTeardownRequest( id: id, workspaceId: workspaceId, @@ -117,15 +157,59 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { callbackContext: callbackContext, manualIOContext: manualIOContext, byteTeeLease: byteTeeLease, - freeSurface: freeSurface + freeSurface: freeSurface, + completion: completion ) Task { - await self.enqueue(request) + await self.enqueue( + request, + executionLane: executionLane, + isolatedHibernationReservation: isolatedHibernationReservation + ) } + return ticket } - func enqueue(_ request: TerminalSurfaceRuntimeTeardownRequest) { + func enqueue( + _ request: TerminalSurfaceRuntimeTeardownRequest, + executionLane: TerminalSurfaceRuntimeTeardownExecutionLane = .serializedClose, + isolatedHibernationReservation: + TerminalSurfaceRuntimeTeardownReservation? = nil + ) async { pendingReasonsById[request.id] = request.reason + switch executionLane { + case .isolatedHibernation: + guard let isolatedHibernationReservation, + let executionSlot = await isolatedHibernationAdmission.executionSlot( + for: isolatedHibernationReservation + ), + isolatedHibernationQueues.indices.contains(executionSlot) else { + preconditionFailure( + "isolated hibernation teardown requires an active reservation" + ) + } + // Each reservation exclusively owns one queue until its native free + // returns. Ghostty locks its shared surface registry, while renderer + // and IO joins are surface-owned, so separate surfaces may tear down + // concurrently. This bounds blocked native workers at two without + // letting one stuck pane strand another admitted pane. + Task { + await self.observeTimeout(id: request.id) + } + isolatedHibernationQueues[executionSlot].async { + self.freeNativeSurface(request) + Task { + await self.isolatedHibernationAdmission.release( + isolatedHibernationReservation + ) + await self.finishFree(request) + await self.complete(id: request.id) + } + } + return + case .serializedClose: + break + } queuedRequests.append(request) if !isWorkerRunning { isWorkerRunning = true @@ -134,7 +218,8 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { Task { await self.observeTimeout(id: request.id) } - await Self.free(request) + self.freeNativeSurface(request) + await self.finishFree(request) await self.complete(id: request.id) } } @@ -149,7 +234,9 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { return queuedRequests.removeFirst() } - private nonisolated static func free(_ request: TerminalSurfaceRuntimeTeardownRequest) async { + private nonisolated func freeNativeSurface( + _ request: TerminalSurfaceRuntimeTeardownRequest + ) { #if DEBUG logDebugEvent( "surface.lifecycle.nativeFree.begin surface=\(request.surfaceToken) " + @@ -157,6 +244,11 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { ) #endif request.freeSurface(request.surface) + } + + private nonisolated func finishFree( + _ request: TerminalSurfaceRuntimeTeardownRequest + ) async { if request.callbackContext != nil || request.manualIOContext != nil || request.byteTeeLease != nil { // The request is the @unchecked Sendable transport for the // Unmanaged contexts; release through the request so the @Sendable @@ -171,6 +263,7 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { request.byteTeeLease?.release() } } + await request.completion.finish() #if DEBUG logDebugEvent( "surface.lifecycle.nativeFree.end surface=\(request.surfaceToken) " + diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownExecutionLane.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownExecutionLane.swift new file mode 100644 index 000000000000..0a2a800691d5 --- /dev/null +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownExecutionLane.swift @@ -0,0 +1,8 @@ +/// Selects the ownership boundary for a native surface free. +enum TerminalSurfaceRuntimeTeardownExecutionLane: Sendable { + /// Preserves ordering for close/deinit flows that can re-enter teardown. + case serializedClose + + /// Gives an explicitly owned hibernation join an independent bounded slot. + case isolatedHibernation +} diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownRequest.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownRequest.swift index 3395648179c5..18d82e8f78b1 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownRequest.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownRequest.swift @@ -25,6 +25,7 @@ struct TerminalSurfaceRuntimeTeardownRequest: @unchecked Sendable { let manualIOContext: Unmanaged? let byteTeeLease: (any TerminalByteTeeLease)? let freeSurface: @Sendable (ghostty_surface_t) -> Void + let completion: TerminalSurfaceRuntimeTeardownCompletion #if DEBUG let surfaceToken: String let workspaceToken: String @@ -38,7 +39,8 @@ struct TerminalSurfaceRuntimeTeardownRequest: @unchecked Sendable { callbackContext: Unmanaged?, manualIOContext: Unmanaged?, byteTeeLease: (any TerminalByteTeeLease)?, - freeSurface: @escaping @Sendable (ghostty_surface_t) -> Void + freeSurface: @escaping @Sendable (ghostty_surface_t) -> Void, + completion: TerminalSurfaceRuntimeTeardownCompletion ) { self.id = id self.workspaceId = workspaceId @@ -48,6 +50,7 @@ struct TerminalSurfaceRuntimeTeardownRequest: @unchecked Sendable { self.manualIOContext = manualIOContext self.byteTeeLease = byteTeeLease self.freeSurface = freeSurface + self.completion = completion #if DEBUG self.surfaceToken = String(id.uuidString.prefix(5)) self.workspaceToken = String(workspaceId.uuidString.prefix(5)) diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownReservation.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownReservation.swift new file mode 100644 index 000000000000..c79fd74320ee --- /dev/null +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownReservation.swift @@ -0,0 +1,10 @@ +import Foundation + +/// Exclusive admission to one failure-isolated hibernation teardown slot. +struct TerminalSurfaceRuntimeTeardownReservation: Equatable, Sendable { + let id: UUID + + init(id: UUID = UUID()) { + self.id = id + } +} diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownTicket.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownTicket.swift new file mode 100644 index 000000000000..1ed5df6ac631 --- /dev/null +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownTicket.swift @@ -0,0 +1,46 @@ +public import Foundation + +/// An awaitable handle for one serialized native surface teardown. +public struct TerminalSurfaceRuntimeTeardownTicket: Sendable { + /// Stable identity used by the surface to clear only its current teardown. + public let id: UUID + + private let completion: TerminalSurfaceRuntimeTeardownCompletion + + init( + id: UUID = UUID(), + completion: TerminalSurfaceRuntimeTeardownCompletion + ) { + self.id = id + self.completion = completion + } + + /// Waits until the native free and callback-userdata releases finish. + /// + /// A `nil` deadline is the event-driven recovery wait used after a bounded + /// foreground attempt has already reported failure. + /// + /// - Parameter timeout: Maximum wait, or `nil` to wait until completion. + /// - Returns: `true` only when teardown completed. + public func wait(timeout: Duration?) async -> Bool { + guard let timeout else { + return await completion.wait() + } + return await withTaskGroup(of: Bool.self) { group in + group.addTask { + await completion.wait() + } + group.addTask { + do { + try await ContinuousClock().sleep(for: timeout) + return false + } catch { + return false + } + } + let completed = await group.next() ?? false + group.cancelAll() + return completed + } + } +} diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift index 103ee407b562..3001a3b27e1e 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift @@ -299,8 +299,18 @@ extension TerminalSurface { /// Frees the runtime surface while keeping the model alive for an /// agent-hibernation resume. + /// + /// - Returns: `false` without changing the surface when the bounded + /// hibernation teardown lane has no capacity. + @discardableResult @MainActor - public func suspendRuntimeSurfaceForAgentHibernation(reason: String) { + public func suspendRuntimeSurfaceForAgentHibernation(reason: String) -> Bool { + guard let teardownReservation = + agentHibernationRuntimeTeardownReservation ?? + runtimeTeardown.reserveIsolatedHibernationTeardown() else { + return false + } + agentHibernationRuntimeTeardownReservation = nil _ = fontSizeLineageSnapshot() mobileViewportFontFitState = nil runtimeSurfaceSuspendedForAgentHibernation = true @@ -330,10 +340,13 @@ extension TerminalSurface { desiredFocusState = false guard let surfaceToFree else { + runtimeTeardown.cancelIsolatedHibernationTeardown( + teardownReservation + ) callbackContext?.release() manualIOContext?.release() teeLease?.release() - return + return true } #if DEBUG @@ -348,7 +361,7 @@ extension TerminalSurface { // Transport manualIOContext and teeLease through the request too: // the coordinator releases all callback userdata only after the // native free, which is what joins ghostty's IO threads. - runtimeTeardown.enqueueRuntimeTeardown( + agentHibernationRuntimeTeardownTicket = runtimeTeardown.enqueueRuntimeTeardown( id: id, workspaceId: tabId, reason: reason, @@ -356,26 +369,76 @@ extension TerminalSurface { callbackContext: callbackContext, manualIOContext: manualIOContext, byteTeeLease: teeLease, + executionLane: .isolatedHibernation, + isolatedHibernationReservation: teardownReservation, freeSurface: freeSurface ) - return + return true } #endif - Task { @MainActor in - ghostty_surface_free(surfaceToFree) - callbackContext?.release() - manualIOContext?.release() - teeLease?.release() + agentHibernationRuntimeTeardownTicket = runtimeTeardown.enqueueRuntimeTeardown( + id: id, + workspaceId: tabId, + reason: reason, + surface: surfaceToFree, + callbackContext: callbackContext, + manualIOContext: manualIOContext, + byteTeeLease: teeLease, + executionLane: .isolatedHibernation, + isolatedHibernationReservation: teardownReservation + ) + return true + } + + /// Reserves the bounded native-free lane at the final pre-signal gate. + @MainActor + public func reserveAgentHibernationRuntimeTeardown() -> Bool { + guard agentHibernationRuntimeTeardownTicket == nil else { return false } + if agentHibernationRuntimeTeardownReservation != nil { return true } + guard let reservation = + runtimeTeardown.reserveIsolatedHibernationTeardown() else { + return false + } + agentHibernationRuntimeTeardownReservation = reservation + return true + } + + /// Releases an unused reservation when the signal batch fails before commit. + @MainActor + public func cancelAgentHibernationRuntimeTeardownReservation() { + guard let reservation = agentHibernationRuntimeTeardownReservation else { + return + } + agentHibernationRuntimeTeardownReservation = nil + runtimeTeardown.cancelIsolatedHibernationTeardown(reservation) + } + + /// Waits for the old hibernated runtime generation to finish native teardown. + /// + /// - Parameter timeout: Maximum wait, or `nil` for event-driven recovery. + /// - Returns: `true` only after the old native surface and callback contexts are gone. + @MainActor + public func waitForAgentHibernationRuntimeTeardown(timeout: Duration?) async -> Bool { + guard let ticket = agentHibernationRuntimeTeardownTicket else { return true } + let completed = await ticket.wait(timeout: timeout) + if completed, agentHibernationRuntimeTeardownTicket?.id == ticket.id { + agentHibernationRuntimeTeardownTicket = nil } + return completed } /// Marks the resume side of agent hibernation and primes the next runtime /// spawn's initial input. + /// + /// - Returns: `true` when the old runtime is fully gone and resume was armed. + @discardableResult @MainActor - public func prepareAgentHibernationResume(initialInput: String?) { + public func prepareAgentHibernationResume(initialInput: String?) -> Bool { + guard agentHibernationRuntimeTeardownTicket == nil else { return false } runtimeSurfaceSuspendedForAgentHibernation = false prepareNextRuntimeInitialInput(initialInput) + return true } /// Primes the initial input for the next runtime spawn only. diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface.swift index f52e27033dca..8549f8816089 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface.swift @@ -255,6 +255,9 @@ public final class TerminalSurface: Identifiable, ObservableObject { var restoredRuntimeSurfaceStartQueued = false var requiresRestoreSpawnPacing = false var runtimeSurfaceSuspendedForAgentHibernation = false + var agentHibernationRuntimeTeardownTicket: TerminalSurfaceRuntimeTeardownTicket? + var agentHibernationRuntimeTeardownReservation: + TerminalSurfaceRuntimeTeardownReservation? var headlessStartupWindow: NSWindow? var surfaceCallbackContext: Unmanaged? var claudeCommandShim: ClaudeCommandShim? diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift index e82833142d3b..fe14022e2b63 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift @@ -1,3 +1,4 @@ +import Dispatch import Foundation import os import Testing @@ -59,13 +60,24 @@ private final class LifetimeRecordingByteTeeLease: TerminalByteTeeLease, @unchec } @Suite struct TerminalSurfaceRuntimeTeardownCoordinatorTests { + @Test func ticketDistinguishesDeadlineFromEventualCompletion() async { + let completion = TerminalSurfaceRuntimeTeardownCompletion() + let ticket = TerminalSurfaceRuntimeTeardownTicket(completion: completion) + + #expect(await ticket.wait(timeout: .zero) == false) + + await completion.finish() + + #expect(await ticket.wait(timeout: nil)) + } + @Test func enqueuedTeardownInvokesInjectedFreeWithTheSamePointer() async { let coordinator = TerminalSurfaceRuntimeTeardownCoordinator() let recorder = FreedSurfaceRecorder() let surface = UnsafeMutableRawPointer.allocate(byteCount: 8, alignment: 8) defer { surface.deallocate() } - coordinator.enqueueRuntimeTeardown( + let ticket = coordinator.enqueueRuntimeTeardown( id: UUID(), workspaceId: UUID(), reason: "test", @@ -78,6 +90,7 @@ private final class LifetimeRecordingByteTeeLease: TerminalByteTeeLease, @unchec ) await recorder.waitForFreeCount(1) + #expect(await ticket.wait(timeout: .seconds(1))) #expect(await recorder.freed == [UInt(bitPattern: surface)]) } @@ -107,6 +120,97 @@ private final class LifetimeRecordingByteTeeLease: TerminalByteTeeLease, @unchec #expect(await Set(recorder.freed) == Set(surfaces.map { UInt(bitPattern: $0) })) } + @Test func stuckHibernationFreeDoesNotStrandAnotherAdmissionOrClose() async throws { + let coordinator = TerminalSurfaceRuntimeTeardownCoordinator() + let isolatedSurface = UnsafeMutableRawPointer.allocate(byteCount: 8, alignment: 8) + let queuedIsolatedSurface = UnsafeMutableRawPointer.allocate( + byteCount: 8, + alignment: 8 + ) + let serializedSurface = UnsafeMutableRawPointer.allocate(byteCount: 8, alignment: 8) + defer { + isolatedSurface.deallocate() + queuedIsolatedSurface.deallocate() + serializedSurface.deallocate() + } + let isolatedFreeStarted = AsyncStream.makeStream() + let releaseIsolatedFree = DispatchSemaphore(value: 0) + let secondIsolatedFreeCount = OSAllocatedUnfairLock(initialState: 0) + let serializedFreeCount = OSAllocatedUnfairLock(initialState: 0) + defer { + releaseIsolatedFree.signal() + isolatedFreeStarted.continuation.finish() + } + + let isolatedReservation = try #require( + await coordinator.reserveIsolatedHibernationTeardown() + ) + let isolatedTicket = coordinator.enqueueRuntimeTeardown( + id: UUID(), + workspaceId: UUID(), + reason: "test.isolatedHibernation", + surface: isolatedSurface, + callbackContext: nil, + manualIOContext: nil, + byteTeeLease: nil, + executionLane: .isolatedHibernation, + isolatedHibernationReservation: isolatedReservation, + freeSurface: { _ in + isolatedFreeStarted.continuation.yield() + _ = releaseIsolatedFree.wait(timeout: .distantFuture) + } + ) + var isolatedFreeIterator = isolatedFreeStarted.stream.makeAsyncIterator() + _ = await isolatedFreeIterator.next() + + let secondReservation = try #require( + await coordinator.reserveIsolatedHibernationTeardown() + ) + let secondIsolatedTicket = coordinator.enqueueRuntimeTeardown( + id: UUID(), + workspaceId: UUID(), + reason: "test.secondIsolatedHibernation", + surface: queuedIsolatedSurface, + callbackContext: nil, + manualIOContext: nil, + byteTeeLease: nil, + executionLane: .isolatedHibernation, + isolatedHibernationReservation: secondReservation, + freeSurface: { _ in + secondIsolatedFreeCount.withLock { $0 += 1 } + } + ) + #expect(await coordinator.reserveIsolatedHibernationTeardown() == nil) + let serializedTicket = coordinator.enqueueRuntimeTeardown( + id: UUID(), + workspaceId: UUID(), + reason: "test.serializedClose", + surface: serializedSurface, + callbackContext: nil, + freeSurface: { _ in + serializedFreeCount.withLock { $0 += 1 } + } + ) + + #expect(await serializedTicket.wait(timeout: .seconds(1))) + #expect(await secondIsolatedTicket.wait(timeout: .seconds(1))) + #expect(serializedFreeCount.withLock { $0 } == 1) + #expect(await isolatedTicket.wait(timeout: .zero) == false) + #expect(secondIsolatedFreeCount.withLock { $0 } == 1) + + let replacementReservation = try #require( + await coordinator.reserveIsolatedHibernationTeardown() + ) + #expect(await coordinator.reserveIsolatedHibernationTeardown() == nil) + await coordinator.cancelIsolatedHibernationTeardown(replacementReservation) + releaseIsolatedFree.signal() + #expect(await isolatedTicket.wait(timeout: .seconds(1))) + let nextReservation = try #require( + await coordinator.reserveIsolatedHibernationTeardown() + ) + await coordinator.cancelIsolatedHibernationTeardown(nextReservation) + } + @Test func byteTeeCallbackOwnerIsReleasedOnlyAfterNativeFreeReturns() async { let coordinator = TerminalSurfaceRuntimeTeardownCoordinator() let recorder = TeardownLifetimeRecorder() diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceTeardownCallbackLifetimeTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceTeardownCallbackLifetimeTests.swift index 2ceb5cb14b44..a6bac1c55886 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceTeardownCallbackLifetimeTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceTeardownCallbackLifetimeTests.swift @@ -77,6 +77,64 @@ import Testing #expect(recorder.events == [.nativeFree, .teeLeaseRelease]) } + @Test func agentHibernationResumeWaitsForNativeFreeCompletion() async { + let registry = TerminalSurfaceRegistry() + let surface = makeSurface(registry: registry) + let runtimeSurface = UnsafeMutableRawPointer.allocate(byteCount: 8, alignment: 8) + registry.registerRuntimeSurface(runtimeSurface, ownerId: surface.id) + surface.installRuntimeSurfaceForTesting(runtimeSurface) + defer { runtimeSurface.deallocate() } + let freeStarted = AsyncStream.makeStream() + let allowFree = DispatchSemaphore(value: 0) + TerminalSurface.runtimeSurfaceFreeOverrideForTesting = { _ in + freeStarted.continuation.yield() + allowFree.wait() + } + defer { + allowFree.signal() + TerminalSurface.runtimeSurfaceFreeOverrideForTesting = nil + } + + surface.suspendRuntimeSurfaceForAgentHibernation(reason: "test.hibernate") + var freeStartedIterator = freeStarted.stream.makeAsyncIterator() + _ = await freeStartedIterator.next() + + #expect(!surface.prepareAgentHibernationResume(initialInput: nil)) + + allowFree.signal() + #expect(await surface.waitForAgentHibernationRuntimeTeardown(timeout: .seconds(1))) + #expect(surface.prepareAgentHibernationResume(initialInput: nil)) + freeStarted.continuation.finish() + } + + @Test func hibernationAdmissionFailurePreservesLiveRuntimeSurface() throws { + let coordinator = TerminalSurfaceRuntimeTeardownCoordinator() + let firstReservation = try #require( + coordinator.reserveIsolatedHibernationTeardown() + ) + let secondReservation = try #require( + coordinator.reserveIsolatedHibernationTeardown() + ) + defer { + coordinator.cancelIsolatedHibernationTeardown(firstReservation) + coordinator.cancelIsolatedHibernationTeardown(secondReservation) + } + + let surface = makeSurface(runtimeTeardown: coordinator) + surface.installRuntimeSurfaceForTesting(fakeRuntimeSurface()) + TerminalSurface.runtimeSurfaceFreeOverrideForTesting = { _ in } + defer { TerminalSurface.runtimeSurfaceFreeOverrideForTesting = nil } + + #expect( + surface.suspendRuntimeSurfaceForAgentHibernation( + reason: "test.admissionFailure" + ) == false + ) + #expect(surface.hasLiveSurface) + + surface.teardownSurface() + } + @Test func deinitKeepsTeeLeaseUntilCoordinatorFree() async { let recorder = TeardownOrderRecorder() var surface: TerminalSurface? = makeSurface() @@ -161,7 +219,9 @@ import Testing } private func makeSurface( - registry: any TerminalSurfaceRegistering = FakeSurfaceRegistry() + registry: any TerminalSurfaceRegistering = FakeSurfaceRegistry(), + runtimeTeardown: TerminalSurfaceRuntimeTeardownCoordinator = + TerminalSurfaceRuntimeTeardownCoordinator() ) -> TerminalSurface { let nativeView = FakeTerminalSurfaceNativeView(frame: NSRect(x: 0, y: 0, width: 800, height: 600)) let paneHost = FakeTerminalSurfacePaneHost(surfaceView: nativeView) @@ -177,7 +237,7 @@ import Testing byteTee: FakeTerminalByteTee(), rendererRealization: FakeRendererRealizationScheduler(), hibernationRecorder: FakeHibernationRecorder(), - runtimeTeardown: TerminalSurfaceRuntimeTeardownCoordinator(), + runtimeTeardown: runtimeTeardown, restoreSpawnScheduler: TerminalSurfaceRestoreSpawnScheduler(interSpawnDelay: .zero), runtimeFilesystem: TerminalSurfaceRuntimeFilesystem( claudeCommandShimTemporaryDirectory: URL(fileURLWithPath: "/tmp/cmux-terminal-tests", isDirectory: true), diff --git a/Resources/Localizable.xcstrings b/Resources/Localizable.xcstrings index b0a82a5e73a4..2ab8fa2f2b84 100644 --- a/Resources/Localizable.xcstrings +++ b/Resources/Localizable.xcstrings @@ -234622,6 +234622,40 @@ } } }, + "terminal.agentHibernation.failed": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "Agent shutdown needs attention" + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "エージェントの終了を確認してください" + } + } + } + }, + "terminal.agentHibernation.finishing": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "Finishing agent shutdown" + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "エージェントを終了しています" + } + } + } + }, "terminal.agentHibernation.lastActivity": { "extractionState": "manual", "localizations": { @@ -234656,6 +234690,23 @@ } } }, + "terminal.agentHibernation.retry": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "Retry shutdown" + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "終了を再試行" + } + } + } + }, "terminal.agentHibernation.title": { "extractionState": "manual", "localizations": { diff --git a/Sources/AgentHibernation/AgentHibernationPanelPhase.swift b/Sources/AgentHibernation/AgentHibernationPanelPhase.swift index 90a185f80164..6f85e58a78ff 100644 --- a/Sources/AgentHibernation/AgentHibernationPanelPhase.swift +++ b/Sources/AgentHibernation/AgentHibernationPanelPhase.swift @@ -13,13 +13,18 @@ struct AgentHibernationPanelState { enum AgentHibernationPanelPhase { case live case terminating(AgentHibernationPanelState) + case recovering(AgentHibernationPanelState) + case terminationFailed(AgentHibernationPanelState) case hibernated(AgentHibernationPanelState) var state: AgentHibernationPanelState? { switch self { case .live: nil - case .terminating(let state), .hibernated(let state): + case .terminating(let state), + .recovering(let state), + .terminationFailed(let state), + .hibernated(let state): state } } @@ -33,6 +38,20 @@ enum AgentHibernationPanelPhase { if case .terminating = self { return true } return false } + + var terminationFailed: Bool { + if case .terminationFailed = self { return true } + return false + } + + var isAwaitingCommit: Bool { + switch self { + case .terminating, .recovering, .terminationFailed: + true + case .live, .hibernated: + false + } + } } enum AgentHibernationResumePreparation: Equatable { diff --git a/Sources/App/AgentHibernationController+CommittedTerminationCleanup.swift b/Sources/App/AgentHibernationController+CommittedTerminationCleanup.swift new file mode 100644 index 000000000000..b16af8846f40 --- /dev/null +++ b/Sources/App/AgentHibernationController+CommittedTerminationCleanup.swift @@ -0,0 +1,8 @@ +import Foundation + +extension AgentHibernationController { + struct CommittedTerminationCleanup { + let requestID: UUID + let task: Task + } +} diff --git a/Sources/App/AgentHibernationController+CommittedTerminationObservation.swift b/Sources/App/AgentHibernationController+CommittedTerminationObservation.swift new file mode 100644 index 000000000000..6a876b8baa43 --- /dev/null +++ b/Sources/App/AgentHibernationController+CommittedTerminationObservation.swift @@ -0,0 +1,81 @@ +import Foundation + +extension AgentHibernationController { + final class CommittedTerminationObservation { + let requestID: UUID + let processExitCompletion: AgentHibernationProcessExitCompletion + var task: Task? + var retryRecovery: (@MainActor () -> Void)? + + init( + requestID: UUID, + processExitCompletion: AgentHibernationProcessExitCompletion + ) { + self.requestID = requestID + self.processExitCompletion = processExitCompletion + } + } + + @discardableResult + func registerCommittedTerminationObservation( + panelID: UUID, + requestID: UUID = UUID(), + processExitCompletion: AgentHibernationProcessExitCompletion + ) -> UUID { + if let current = committedTerminationObservationsByPanelID[panelID], + current.requestID == requestID { + return requestID + } + committedTerminationCleanupByPanelID + .removeValue(forKey: panelID)? + .task + .cancel() + if let previous = committedTerminationObservationsByPanelID.removeValue( + forKey: panelID + ) { + previous.task?.cancel() + if previous.processExitCompletion !== processExitCompletion { + Task { + await previous.processExitCompletion.finish(false) + } + } + } + committedTerminationObservationsByPanelID[panelID] = + CommittedTerminationObservation( + requestID: requestID, + processExitCompletion: processExitCompletion + ) + return requestID + } + + @discardableResult + func replaceCommittedTerminationTask( + panelID: UUID, + requestID: UUID, + with task: Task + ) -> Bool { + guard let observation = committedTerminationObservationsByPanelID[panelID], + observation.requestID == requestID else { + task.cancel() + return false + } + let previousTask = observation.task + observation.task = task + previousTask?.cancel() + return true + } + + func removeCommittedTerminationObservation( + panelID: UUID, + requestID: UUID + ) { + guard committedTerminationObservationsByPanelID[panelID]?.requestID == requestID, + let observation = committedTerminationObservationsByPanelID.removeValue( + forKey: panelID + ) else { + return + } + observation.task?.cancel() + committedTerminationCleanupByPanelID.removeValue(forKey: panelID)?.task.cancel() + } +} diff --git a/Sources/App/AgentHibernationController+InFlightTeardown.swift b/Sources/App/AgentHibernationController+InFlightTeardown.swift index 0abae83af6f9..8bc4ded312ec 100644 --- a/Sources/App/AgentHibernationController+InFlightTeardown.swift +++ b/Sources/App/AgentHibernationController+InFlightTeardown.swift @@ -6,14 +6,4 @@ extension AgentHibernationController { let trigger: AgentHibernationReclaimTrigger } - enum ScopedProcessTerminationResult: Equatable, Sendable { - case rejected - case exited - case committedAwaitingExit - } - - struct CommittedTerminationObservation { - let requestID: UUID - let task: Task - } } diff --git a/Sources/App/AgentHibernationController+PanelLifecycle.swift b/Sources/App/AgentHibernationController+PanelLifecycle.swift index 5516ffa43350..39f7271ec60f 100644 --- a/Sources/App/AgentHibernationController+PanelLifecycle.swift +++ b/Sources/App/AgentHibernationController+PanelLifecycle.swift @@ -2,7 +2,7 @@ import Foundation extension AgentHibernationController { func discardTrackingStateForClosedPanel(workspaceId: UUID, panelId: UUID) { - committedTerminationObservationsByPanelID.removeValue(forKey: panelId)?.task.cancel() + limitCommittedTerminationObservationAfterPanelClose(panelID: panelId) let key = AgentHibernationPanelKey(workspaceId: workspaceId, panelId: panelId) activityByPanel.removeValue(forKey: key) terminalInputByPanel.removeValue(forKey: key) @@ -50,6 +50,50 @@ extension AgentHibernationController { discardTransientTrackingState(for: destinationKey) } + func limitCommittedTerminationObservationAfterPanelClose( + panelID: UUID, + cleanupDelay: Duration = .seconds(30), + sleepUntilDeadline: @escaping @Sendable (Duration) async -> Bool = { duration in + do { + try await ContinuousClock().sleep(for: duration) + return true + } catch { + return false + } + } + ) { + guard let observation = committedTerminationObservationsByPanelID[panelID] else { + return + } + let observationRequestID = observation.requestID + committedTerminationCleanupByPanelID + .removeValue(forKey: panelID)? + .task + .cancel() + let cleanupID = UUID() + let cleanupTask = Task { @MainActor [weak self] in + guard await sleepUntilDeadline(cleanupDelay), + let self, + !Task.isCancelled, + self.committedTerminationCleanupByPanelID[panelID]?.requestID == + cleanupID, + self.committedTerminationObservationsByPanelID[panelID]?.requestID == + observationRequestID, + let removed = self.committedTerminationObservationsByPanelID.removeValue( + forKey: panelID + ) else { + return + } + self.committedTerminationCleanupByPanelID.removeValue(forKey: panelID) + removed.task?.cancel() + await removed.processExitCompletion.finish(false) + } + committedTerminationCleanupByPanelID[panelID] = CommittedTerminationCleanup( + requestID: cleanupID, + task: cleanupTask + ) + } + private func transferTimestamp( _ timestamps: inout [AgentHibernationPanelKey: TimeInterval], from sourceKey: AgentHibernationPanelKey, diff --git a/Sources/App/AgentHibernationController+ProcessExitObservation.swift b/Sources/App/AgentHibernationController+ProcessExitObservation.swift new file mode 100644 index 000000000000..7c880e41d0bb --- /dev/null +++ b/Sources/App/AgentHibernationController+ProcessExitObservation.swift @@ -0,0 +1,445 @@ +import Foundation + +extension AgentHibernationController { + typealias CommittedTerminationRetry = + @Sendable () async -> ScopedProcessTerminationResult + + func observeCommittedTermination( + panelID: UUID, + requestID: UUID? = nil, + terminations: [ScopedProcessTermination], + processScopeKey: AgentHibernationPanelKey? = nil, + processExitCompletion: AgentHibernationProcessExitCompletion = + AgentHibernationProcessExitCompletion(), + waitForExit: (@Sendable ([ScopedProcessTermination]) async -> Bool)? = nil, + waitForRecoveryExit: (@Sendable ([ScopedProcessTermination]) async -> Bool)? = nil, + retryTermination: CommittedTerminationRetry? = nil, + waitForRecoveryReadiness: @escaping @MainActor @Sendable () async -> Bool = { true }, + recoveryDeadline: Duration = .seconds(30), + sleepUntilRecoveryDeadline: @escaping @Sendable (Duration) async -> Bool = { + duration in + do { + // Genuine recovery deadline; cancellation aborts the wait. + try await ContinuousClock().sleep(for: duration) + return true + } catch { + return false + } + }, + onExit: @escaping @MainActor () async -> Bool, + onFailure: @escaping @MainActor () async -> Void = {}, + onRecovery: @escaping @MainActor () async -> Void, + onRecoveryFailure: @escaping @MainActor () async -> Void = {}, + onRecoveryRetry: @escaping @MainActor () -> Void = {} + ) { + let observationRequestID: UUID + if let requestID { + guard let observation = committedTerminationObservationsByPanelID[panelID], + observation.requestID == requestID, + observation.processExitCompletion === processExitCompletion else { + return + } + observationRequestID = requestID + } else { + observationRequestID = registerCommittedTerminationObservation( + panelID: panelID, + processExitCompletion: processExitCompletion + ) + } + let nextEpochProvider = processExitEpochProvider() + let task = Task { @MainActor [weak self] in + let didExit = await Self.waitForScopedProcessGenerationsToExitAfterEscalation( + terminations, + processScopeKey: processScopeKey, + waitForExit: waitForExit, + nextEpochProvider: nextEpochProvider + ) + guard let self, + self.committedTerminationObservationsByPanelID[panelID]?.requestID == + observationRequestID, + !Task.isCancelled else { + return + } + if didExit { + await processExitCompletion.finish(true) + } + guard self.committedTerminationObservationsByPanelID[panelID]?.requestID == + observationRequestID, + !Task.isCancelled else { + return + } + let didFinish = didExit ? await onExit() : false + guard self.committedTerminationObservationsByPanelID[panelID]?.requestID == + observationRequestID, + !Task.isCancelled else { + return + } + if didFinish { + self.removeCommittedTerminationObservation( + panelID: panelID, + requestID: observationRequestID + ) + return + } + await onFailure() + guard self.committedTerminationObservationsByPanelID[panelID]?.requestID == + observationRequestID, + !Task.isCancelled else { + return + } + self.replaceCommittedTerminationWithRecoveryObservation( + panelID: panelID, + requestID: observationRequestID, + terminations: terminations, + processScopeKey: processScopeKey, + processExitCompletion: processExitCompletion, + waitForRecoveryExit: waitForRecoveryExit, + retryTermination: retryTermination, + waitForRecoveryReadiness: waitForRecoveryReadiness, + recoveryDeadline: recoveryDeadline, + sleepUntilRecoveryDeadline: sleepUntilRecoveryDeadline, + nextEpochProvider: nextEpochProvider, + onRecovery: onRecovery, + onRecoveryFailure: onRecoveryFailure, + onRecoveryRetry: onRecoveryRetry + ) + } + replaceCommittedTerminationTask( + panelID: panelID, + requestID: observationRequestID, + with: task + ) + } + + func observeCommittedTerminationRecovery( + panelID: UUID, + requestID: UUID? = nil, + terminations: [ScopedProcessTermination], + processScopeKey: AgentHibernationPanelKey? = nil, + processExitCompletion: AgentHibernationProcessExitCompletion = + AgentHibernationProcessExitCompletion(), + waitForRecoveryExit: (@Sendable ([ScopedProcessTermination]) async -> Bool)? = nil, + retryTermination: CommittedTerminationRetry? = nil, + waitForRecoveryReadiness: @escaping @MainActor @Sendable () async -> Bool = { true }, + recoveryDeadline: Duration = .seconds(30), + sleepUntilRecoveryDeadline: @escaping @Sendable (Duration) async -> Bool = { + duration in + do { + // Genuine recovery deadline; cancellation aborts the wait. + try await ContinuousClock().sleep(for: duration) + return true + } catch { + return false + } + }, + onRecovery: @escaping @MainActor () async -> Void, + onRecoveryFailure: @escaping @MainActor () async -> Void = {}, + onRecoveryRetry: @escaping @MainActor () -> Void = {} + ) { + let observationRequestID: UUID + if let requestID { + guard let observation = committedTerminationObservationsByPanelID[panelID], + observation.requestID == requestID, + observation.processExitCompletion === processExitCompletion else { + return + } + observationRequestID = requestID + } else { + observationRequestID = registerCommittedTerminationObservation( + panelID: panelID, + processExitCompletion: processExitCompletion + ) + } + replaceCommittedTerminationWithRecoveryObservation( + panelID: panelID, + requestID: observationRequestID, + terminations: terminations, + processScopeKey: processScopeKey, + processExitCompletion: processExitCompletion, + waitForRecoveryExit: waitForRecoveryExit, + retryTermination: retryTermination, + waitForRecoveryReadiness: waitForRecoveryReadiness, + recoveryDeadline: recoveryDeadline, + sleepUntilRecoveryDeadline: sleepUntilRecoveryDeadline, + nextEpochProvider: processExitEpochProvider(), + onRecovery: onRecovery, + onRecoveryFailure: onRecoveryFailure, + onRecoveryRetry: onRecoveryRetry + ) + } + + func retryCommittedTerminationRecovery(panelID: UUID) { + committedTerminationObservationsByPanelID[panelID]?.retryRecovery?() + } + + private func replaceCommittedTerminationWithRecoveryObservation( + panelID: UUID, + requestID: UUID, + terminations: [ScopedProcessTermination], + processScopeKey: AgentHibernationPanelKey?, + processExitCompletion: AgentHibernationProcessExitCompletion, + waitForRecoveryExit: (@Sendable ([ScopedProcessTermination]) async -> Bool)?, + retryTermination: CommittedTerminationRetry?, + waitForRecoveryReadiness: @escaping @MainActor @Sendable () async -> Bool, + recoveryDeadline: Duration, + sleepUntilRecoveryDeadline: @escaping @Sendable (Duration) async -> Bool, + nextEpochProvider: @escaping ProcessExitEpochProvider, + onRecovery: @escaping @MainActor () async -> Void, + onRecoveryFailure: @escaping @MainActor () async -> Void, + onRecoveryRetry: @escaping @MainActor () -> Void + ) { + guard let observation = committedTerminationObservationsByPanelID[panelID], + observation.requestID == requestID else { + return + } + let recoveryTask = Task { @MainActor [weak self] in + let didRecover = await Self.waitForCommittedTerminationRecovery( + terminations: terminations, + processScopeKey: processScopeKey, + waitForRecoveryExit: waitForRecoveryExit, + waitForRecoveryReadiness: waitForRecoveryReadiness, + recoveryDeadline: recoveryDeadline, + sleepUntilRecoveryDeadline: sleepUntilRecoveryDeadline, + nextEpochProvider: nextEpochProvider + ) + guard let self, + self.committedTerminationObservationsByPanelID[panelID]?.requestID == + requestID, + !Task.isCancelled else { + return + } + guard didRecover else { + await onRecoveryFailure() + return + } + await processExitCompletion.finish(true) + guard self.committedTerminationObservationsByPanelID[panelID]?.requestID == + requestID, + !Task.isCancelled else { + return + } + await onRecovery() + guard self.committedTerminationObservationsByPanelID[panelID]?.requestID == + requestID, + !Task.isCancelled else { + return + } + self.removeCommittedTerminationObservation( + panelID: panelID, + requestID: requestID + ) + } + observation.retryRecovery = { @MainActor [weak self] in + guard let self, + self.committedTerminationObservationsByPanelID[panelID]?.requestID == + requestID else { + return + } + onRecoveryRetry() + if let retryTermination { + self.replaceCommittedTerminationWithRetryObservation( + panelID: panelID, + requestID: requestID, + terminations: terminations, + processScopeKey: processScopeKey, + processExitCompletion: processExitCompletion, + waitForRecoveryExit: waitForRecoveryExit, + retryTermination: retryTermination, + waitForRecoveryReadiness: waitForRecoveryReadiness, + recoveryDeadline: recoveryDeadline, + sleepUntilRecoveryDeadline: sleepUntilRecoveryDeadline, + nextEpochProvider: nextEpochProvider, + onRecovery: onRecovery, + onRecoveryFailure: onRecoveryFailure, + onRecoveryRetry: onRecoveryRetry + ) + return + } + self.replaceCommittedTerminationWithRecoveryObservation( + panelID: panelID, + requestID: requestID, + terminations: terminations, + processScopeKey: processScopeKey, + processExitCompletion: processExitCompletion, + waitForRecoveryExit: waitForRecoveryExit, + retryTermination: nil, + waitForRecoveryReadiness: waitForRecoveryReadiness, + recoveryDeadline: recoveryDeadline, + sleepUntilRecoveryDeadline: sleepUntilRecoveryDeadline, + nextEpochProvider: nextEpochProvider, + onRecovery: onRecovery, + onRecoveryFailure: onRecoveryFailure, + onRecoveryRetry: onRecoveryRetry + ) + } + replaceCommittedTerminationTask( + panelID: panelID, + requestID: requestID, + with: recoveryTask + ) + } + + private func replaceCommittedTerminationWithRetryObservation( + panelID: UUID, + requestID: UUID, + terminations: [ScopedProcessTermination], + processScopeKey: AgentHibernationPanelKey?, + processExitCompletion: AgentHibernationProcessExitCompletion, + waitForRecoveryExit: (@Sendable ([ScopedProcessTermination]) async -> Bool)?, + retryTermination: @escaping CommittedTerminationRetry, + waitForRecoveryReadiness: @escaping @MainActor @Sendable () async -> Bool, + recoveryDeadline: Duration, + sleepUntilRecoveryDeadline: @escaping @Sendable (Duration) async -> Bool, + nextEpochProvider: @escaping ProcessExitEpochProvider, + onRecovery: @escaping @MainActor () async -> Void, + onRecoveryFailure: @escaping @MainActor () async -> Void, + onRecoveryRetry: @escaping @MainActor () -> Void + ) { + guard let observation = committedTerminationObservationsByPanelID[panelID], + observation.requestID == requestID else { + return + } + let retryTask = Task { @MainActor [weak self] in + let terminationResult = await retryTermination() + guard let self, + self.committedTerminationObservationsByPanelID[panelID]?.requestID == + requestID, + !Task.isCancelled else { + return + } + + let didExit: Bool + switch terminationResult { + case .rejected: + await onRecoveryFailure() + return + case .exited: + didExit = true + case .committedAwaitingExit: + didExit = await Self.waitForScopedProcessGenerationsToExitAfterEscalation( + terminations, + processScopeKey: processScopeKey, + waitForExit: waitForRecoveryExit, + nextEpochProvider: nextEpochProvider + ) + } + if didExit { + await processExitCompletion.finish(true) + } + guard self.committedTerminationObservationsByPanelID[panelID]?.requestID == + requestID, + !Task.isCancelled else { + return + } + + let didRecover = await Self.waitForCommittedTerminationRecovery( + terminations: didExit ? [] : terminations, + processScopeKey: processScopeKey, + waitForRecoveryExit: didExit ? { _ in true } : waitForRecoveryExit, + waitForRecoveryReadiness: waitForRecoveryReadiness, + recoveryDeadline: recoveryDeadline, + sleepUntilRecoveryDeadline: sleepUntilRecoveryDeadline, + nextEpochProvider: nextEpochProvider + ) + guard self.committedTerminationObservationsByPanelID[panelID]?.requestID == + requestID, + !Task.isCancelled else { + return + } + guard didRecover else { + await onRecoveryFailure() + return + } + await processExitCompletion.finish(true) + await onRecovery() + guard self.committedTerminationObservationsByPanelID[panelID]?.requestID == + requestID, + !Task.isCancelled else { + return + } + self.removeCommittedTerminationObservation( + panelID: panelID, + requestID: requestID + ) + } + observation.retryRecovery = { @MainActor [weak self] in + guard let self, + self.committedTerminationObservationsByPanelID[panelID]?.requestID == + requestID else { + return + } + onRecoveryRetry() + self.replaceCommittedTerminationWithRetryObservation( + panelID: panelID, + requestID: requestID, + terminations: terminations, + processScopeKey: processScopeKey, + processExitCompletion: processExitCompletion, + waitForRecoveryExit: waitForRecoveryExit, + retryTermination: retryTermination, + waitForRecoveryReadiness: waitForRecoveryReadiness, + recoveryDeadline: recoveryDeadline, + sleepUntilRecoveryDeadline: sleepUntilRecoveryDeadline, + nextEpochProvider: nextEpochProvider, + onRecovery: onRecovery, + onRecoveryFailure: onRecoveryFailure, + onRecoveryRetry: onRecoveryRetry + ) + } + replaceCommittedTerminationTask( + panelID: panelID, + requestID: requestID, + with: retryTask + ) + } + + private nonisolated static func waitForCommittedTerminationRecovery( + terminations: [ScopedProcessTermination], + processScopeKey: AgentHibernationPanelKey?, + waitForRecoveryExit: (@Sendable ([ScopedProcessTermination]) async -> Bool)?, + waitForRecoveryReadiness: @escaping @MainActor @Sendable () async -> Bool, + recoveryDeadline: Duration, + sleepUntilRecoveryDeadline: @escaping @Sendable (Duration) async -> Bool, + nextEpochProvider: @escaping ProcessExitEpochProvider + ) async -> Bool { + await withTaskGroup(of: Bool.self) { group in + group.addTask(priority: .utility) { + guard await waitForRecoveryReadiness() else { return false } + if let waitForRecoveryExit { + return await waitForRecoveryExit(terminations) + } + return await waitForScopedProcessGenerationsToExitWithoutTimeout( + terminations, + processScopeKey: processScopeKey, + nextEpochProvider: nextEpochProvider + ) + } + group.addTask(priority: .utility) { + guard await sleepUntilRecoveryDeadline(recoveryDeadline) else { + return false + } + return false + } + let recovered = await group.next() ?? false + group.cancelAll() + return recovered + } + } + + private func processExitEpochProvider() -> ProcessExitEpochProvider { + { + [processSnapshotCoordinator] + processGroupLeaders, + processScopeKey, + ttyDevice, + exitedIdentities in + await processSnapshotCoordinator.refreshedExitEpoch( + processGroupLeaders: processGroupLeaders, + processScopeKey: processScopeKey, + ttyDevice: ttyDevice, + excluding: exitedIdentities + ) + } + } + +} diff --git a/Sources/App/AgentHibernationController+ProcessExitWaiting.swift b/Sources/App/AgentHibernationController+ProcessExitWaiting.swift new file mode 100644 index 000000000000..be45e154f29e --- /dev/null +++ b/Sources/App/AgentHibernationController+ProcessExitWaiting.swift @@ -0,0 +1,301 @@ +import Darwin +import Foundation + +extension AgentHibernationController { + typealias ProcessExitEpochProvider = @Sendable ( + [pid_t: AgentPIDProcessIdentity], + AgentHibernationPanelKey?, + Int64?, + Set + ) async -> AgentHibernationProcessExitEpoch? + + #if compiler(>=6.2) + @concurrent + #endif + nonisolated static func waitForScopedProcessGenerationsToExitAfterEscalation( + _ terminations: [ScopedProcessTermination], + processScopeKey: AgentHibernationPanelKey? = nil, + gracePeriod: Duration = .seconds(5), + postKillExitPeriod: Duration = .seconds(5), + waitForExit: (@Sendable ([ScopedProcessTermination]) async -> Bool)? = nil, + sleepUntilDeadline: @escaping @Sendable (Duration) async -> Bool = { duration in + do { + // Genuine termination deadline; cancellation aborts the wait. + try await ContinuousClock().sleep(for: duration) + return true + } catch { + return false + } + }, + processIdentityProvider: @escaping @Sendable (pid_t) -> AgentPIDProcessIdentity? = { + AgentPIDProcessIdentity(pid: $0) + }, + processGroupProvider: @escaping @Sendable (pid_t) -> pid_t = { + getpgid($0) + }, + processArgumentsProvider: @escaping @Sendable (Int) -> CmuxTopProcessArguments? = { + CmuxTopProcessSnapshot.processArgumentsAndEnvironment(for: $0) + }, + processTTYDeviceProvider: @escaping @Sendable (pid_t) -> Int64? = { + agentLiveProcessIdentity(pid: $0)?.ttyDevice + }, + signalErrorProvider: @escaping @Sendable (pid_t, Int32) -> Int32? = { target, signal in + kill(target, signal) == 0 ? nil : errno + }, + nextEpochProvider: @escaping ProcessExitEpochProvider = { _, _, _, _ in nil } + ) async -> Bool { + let ttyDevice = commonTTYDevice(in: terminations) + if processScopeKey != nil, ttyDevice == nil { + return false + } + return await withTaskGroup( + of: (didExit: Bool?, graceElapsed: Bool?).self + ) { group in + group.addTask(priority: .utility) { + if let waitForExit { + return (await waitForExit(terminations), nil) + } + return ( + await waitForScopedProcessGenerationsToExitWithoutTimeout( + terminations, + processScopeKey: processScopeKey, + nextEpochProvider: nextEpochProvider + ), + nil + ) + } + group.addTask(priority: .utility) { + (nil, await sleepUntilDeadline(gracePeriod)) + } + guard let firstEvent = await group.next() else { return false } + if let didExit = firstEvent.didExit { + group.cancelAll() + return didExit + } + guard firstEvent.graceElapsed == true else { + group.cancelAll() + return false + } + + let processGroupLeaders = processGroupLeaders(in: terminations) + guard processGroupLeaders.count == + Set(terminations.map(\.processGroupID)).filter({ $0 > 1 }).count, + let refreshedEpoch = await nextEpochProvider( + processGroupLeaders, + processScopeKey, + ttyDevice, + [] + ) else { + group.cancelAll() + return false + } + + let refreshedTerminations = refreshedEpoch.terminations + if refreshedTerminations.isEmpty { + let originalGenerationsExited = terminations.allSatisfy { termination in + processIdentityProvider(pid_t(termination.processID)) != + termination.processIdentity + } + group.cancelAll() + return originalGenerationsExited + } + guard refreshedTerminations.allSatisfy({ termination in + let processID = pid_t(termination.processID) + return processIdentityProvider(processID) == + termination.processIdentity && + processGroupProvider(processID) == termination.processGroupID + }) else { + group.cancelAll() + return false + } + if let processScopeKey { + guard refreshedTerminations.allSatisfy({ termination in + guard let ttyDevice = termination.ttyDevice else { + return false + } + return processTTYDeviceProvider( + pid_t(termination.processID) + ) == ttyDevice + }), + refreshedTerminations.allSatisfy({ termination in + processArgumentsProvider(termination.processID)? + .matchesCMUXScope( + workspaceId: processScopeKey.workspaceId, + surfaceId: processScopeKey.panelId + ) == true + }), + // Close same-generation exec and process-group races + // introduced by the uncached scope/TTY probes. + refreshedTerminations.allSatisfy({ termination in + let processID = pid_t(termination.processID) + return processIdentityProvider(processID) == + termination.processIdentity && + processGroupProvider(processID) == + termination.processGroupID + }) else { + group.cancelAll() + return false + } + } + for (processGroupID, leaderIdentity) in refreshedEpoch.processGroupLeaders { + let liveLeader = processIdentityProvider(processGroupID) + guard liveLeader == nil || liveLeader == leaderIdentity else { + continue + } + if let error = signalErrorProvider(-processGroupID, SIGKILL), + error != ESRCH { + group.cancelAll() + return false + } + } + group.addTask(priority: .utility) { + (nil, await sleepUntilDeadline(postKillExitPeriod)) + } + guard let postKillEvent = await group.next() else { return false } + if let didExit = postKillEvent.didExit { + group.cancelAll() + return didExit + } + group.cancelAll() + return false + } + } + + #if compiler(>=6.2) + @concurrent + #endif + nonisolated static func waitForScopedProcessGenerationsToExitWithoutTimeout( + _ terminations: [ScopedProcessTermination], + processScopeKey: AgentHibernationPanelKey? = nil, + processIdentityProvider: @escaping @Sendable (pid_t) -> AgentPIDProcessIdentity? = { + AgentPIDProcessIdentity(pid: $0) + }, + waitForExactEpoch: (@Sendable ([ScopedProcessTermination]) async -> Bool)? = nil, + nextEpochProvider: @escaping ProcessExitEpochProvider = { _, _, _, _ in nil } + ) async -> Bool { + let ttyDevice = commonTTYDevice(in: terminations) + if processScopeKey != nil, ttyDevice == nil { + return false + } + var processGroupLeaders = processGroupLeaders(in: terminations) + guard processGroupLeaders.count == + Set(terminations.map(\.processGroupID)).filter({ $0 > 1 }).count else { + return false + } + var epoch = terminations + var exitedIdentities: Set = [] + var refreshCount = 0 + while true { + let didExit: Bool + if let waitForExactEpoch { + didExit = await waitForExactEpoch(epoch) + } else { + didExit = await waitForExactProcessGenerationsToExitWithoutTimeout( + epoch, + processIdentityProvider: processIdentityProvider + ) + } + guard didExit, !Task.isCancelled else { return false } + exitedIdentities.formUnion(epoch.map(\.processIdentity)) + guard refreshCount < maximumProcessExitEpochRefreshCount else { + return false + } + refreshCount += 1 + guard let nextEpoch = await nextEpochProvider( + processGroupLeaders, + processScopeKey, + ttyDevice, + exitedIdentities + ) else { + return false + } + guard !nextEpoch.terminations.isEmpty else { return true } + processGroupLeaders = nextEpoch.processGroupLeaders + epoch = nextEpoch.terminations + } + } + + nonisolated static func processGroupLeaders( + in terminations: [ScopedProcessTermination] + ) -> [pid_t: AgentPIDProcessIdentity] { + Dictionary( + uniqueKeysWithValues: terminations.compactMap { termination in + let processGroupID = termination.processGroupID + guard processGroupID > 1, + termination.processID == Int(processGroupID) else { + return nil + } + return (processGroupID, termination.processIdentity) + } + ) + } + + nonisolated static func commonTTYDevice( + in terminations: [ScopedProcessTermination] + ) -> Int64? { + let ttyDevices = Set(terminations.compactMap(\.ttyDevice)) + guard ttyDevices.count == 1 else { return nil } + return ttyDevices.first + } + + #if compiler(>=6.2) + @concurrent + #endif + nonisolated static func waitForExactProcessGenerationsToExitWithoutTimeout( + _ terminations: [ScopedProcessTermination], + processIdentityProvider: @escaping @Sendable (pid_t) -> AgentPIDProcessIdentity? = { + AgentPIDProcessIdentity(pid: $0) + } + ) async -> Bool { + await withTaskGroup(of: Bool.self) { group in + for termination in terminations { + group.addTask(priority: .utility) { + await waitForExactProcessGenerationToExit( + termination, + processIdentityProvider: processIdentityProvider + ) + } + } + for await didExit in group where !didExit { + group.cancelAll() + return false + } + return true + } + } + + private nonisolated static func waitForExactProcessGenerationToExit( + _ termination: ScopedProcessTermination, + processIdentityProvider: @escaping @Sendable (pid_t) -> AgentPIDProcessIdentity? + ) async -> Bool { + let processID = pid_t(termination.processID) + guard processIdentityProvider(processID) == termination.processIdentity else { + return true + } + + let exitEvents = AsyncStream(bufferingPolicy: .bufferingNewest(1)) { continuation in + let source = DispatchSource.makeProcessSource( + identifier: processID, + eventMask: .exit, + queue: .global(qos: .utility) + ) + source.setEventHandler { + continuation.yield() + continuation.finish() + } + continuation.onTermination = { @Sendable _ in + source.cancel() + } + source.resume() + } + // Close the registration race without ever accepting a reused PID. + guard processIdentityProvider(processID) == termination.processIdentity else { + return true + } + + for await _ in exitEvents { + return true + } + return processIdentityProvider(processID) != termination.processIdentity + } +} diff --git a/Sources/App/AgentHibernationController+ProcessSignaling.swift b/Sources/App/AgentHibernationController+ProcessSignaling.swift new file mode 100644 index 000000000000..5adec24025cd --- /dev/null +++ b/Sources/App/AgentHibernationController+ProcessSignaling.swift @@ -0,0 +1,190 @@ +import Darwin +import Foundation + +extension AgentHibernationController { + @discardableResult + #if compiler(>=6.2) + @concurrent + #else + @Sendable + #endif + nonisolated static func terminateScopedProcessesForHibernation( + _ terminations: [ScopedProcessTermination], + processScopeKey: AgentHibernationPanelKey, + shouldCommit: @escaping @MainActor @Sendable () -> Bool = { true }, + onTeardownCommit: @escaping @MainActor @Sendable () -> Void = {}, + currentProcessID: pid_t = getpid(), + currentProcessGroupID: pid_t = getpgrp(), + processIdentityProvider: @escaping @Sendable (pid_t) -> AgentPIDProcessIdentity? = { + AgentPIDProcessIdentity(pid: $0) + }, + processGroupProvider: @escaping @Sendable (pid_t) -> pid_t = { getpgid($0) }, + processArgumentsProvider: @escaping @Sendable (Int) -> CmuxTopProcessArguments? = { + CmuxTopProcessSnapshot.processArgumentsAndEnvironment(for: $0) + }, + processTTYDeviceProvider: @escaping @Sendable (pid_t) -> Int64? = { + agentLiveProcessIdentity(pid: $0)?.ttyDevice + }, + signalErrorProvider: @escaping @Sendable (pid_t, Int32) -> Int32? = { + target, signal in + kill(target, signal) == 0 ? nil : errno + } + ) async -> ScopedProcessTerminationResult { + let processGroupLeaders = processGroupLeaders(in: terminations) + guard processGroupLeaders.count == + Set(terminations.map(\.processGroupID)).filter({ $0 > 1 }).count else { + return .rejected + } + return await terminateScopedProcessEpochForHibernation( + .init( + terminations: terminations, + processGroupLeaders: processGroupLeaders, + signalableProcessIdentities: Set(terminations.map(\.processIdentity)) + ), + processScopeKey: processScopeKey, + shouldCommit: shouldCommit, + onTeardownCommit: onTeardownCommit, + currentProcessID: currentProcessID, + currentProcessGroupID: currentProcessGroupID, + processIdentityProvider: processIdentityProvider, + processGroupProvider: processGroupProvider, + processArgumentsProvider: processArgumentsProvider, + processTTYDeviceProvider: processTTYDeviceProvider, + signalErrorProvider: signalErrorProvider + ) + } + + @discardableResult + #if compiler(>=6.2) + @concurrent + #else + @Sendable + #endif + nonisolated static func terminateScopedProcessEpochForHibernation( + _ epoch: AgentHibernationProcessExitEpoch, + processScopeKey: AgentHibernationPanelKey, + shouldCommit: @escaping @MainActor @Sendable () -> Bool = { true }, + onTeardownCommit: @escaping @MainActor @Sendable () -> Void = {}, + currentProcessID: pid_t = getpid(), + currentProcessGroupID: pid_t = getpgrp(), + processIdentityProvider: @escaping @Sendable (pid_t) -> AgentPIDProcessIdentity? = { + AgentPIDProcessIdentity(pid: $0) + }, + processGroupProvider: @escaping @Sendable (pid_t) -> pid_t = { getpgid($0) }, + processArgumentsProvider: @escaping @Sendable (Int) -> CmuxTopProcessArguments? = { + CmuxTopProcessSnapshot.processArgumentsAndEnvironment(for: $0) + }, + processTTYDeviceProvider: @escaping @Sendable (pid_t) -> Int64? = { + agentLiveProcessIdentity(pid: $0)?.ttyDevice + }, + signalErrorProvider: @escaping @Sendable (pid_t, Int32) -> Int32? = { + target, signal in + kill(target, signal) == 0 ? nil : errno + } + ) async -> ScopedProcessTerminationResult { + let terminations = epoch.terminations + guard !terminations.isEmpty else { + return await MainActor.run { + guard shouldCommit() else { return .rejected } + onTeardownCommit() + return .exited + } + } + guard terminations.count <= maximumScopedProcessTerminationCount, + commonTTYDevice(in: terminations) != nil else { + return .rejected + } + + let signalableTerminations = terminations.filter { + epoch.signalableProcessIdentities.contains($0.processIdentity) + } + let signalableProcessGroupIDs = Set( + signalableTerminations.map(\.processGroupID) + ) + guard !signalableProcessGroupIDs.isEmpty, + signalableProcessGroupIDs.allSatisfy({ + $0 > 1 && + $0 != currentProcessGroupID && + epoch.processGroupLeaders[$0] != nil + }), + terminations.allSatisfy({ termination in + let processID = pid_t(termination.processID) + return processID != currentProcessID && + processIdentityProvider(processID) == + termination.processIdentity && + processGroupProvider(processID) == termination.processGroupID + }) else { + return .rejected + } + + guard terminations.allSatisfy({ termination in + guard let ttyDevice = termination.ttyDevice else { + return false + } + return processTTYDeviceProvider( + pid_t(termination.processID) + ) == ttyDevice + }), + terminations.allSatisfy({ termination in + processArgumentsProvider(termination.processID)?.matchesCMUXScope( + workspaceId: processScopeKey.workspaceId, + surfaceId: processScopeKey.panelId + ) == true + }), + terminations.allSatisfy({ termination in + let processID = pid_t(termination.processID) + return processIdentityProvider(processID) == + termination.processIdentity && + processGroupProvider(processID) == termination.processGroupID + }) else { + return .rejected + } + + // The probes above can block, so they deliberately run off the main actor. + // The mutable panel gate, minimal exact kernel trust recheck, signal batch, + // and UI commit below are one synchronous main-actor boundary. The full + // process snapshot and argv/environment probes remain off-main, while no + // input/focus event or actor hop can interleave after the final decision. + return await MainActor.run { + guard terminations.allSatisfy({ termination in + let processID = pid_t(termination.processID) + guard let ttyDevice = termination.ttyDevice else { + return false + } + return processIdentityProvider(processID) == + termination.processIdentity && + processGroupProvider(processID) == termination.processGroupID && + processTTYDeviceProvider(processID) == ttyDevice + }), + signalableProcessGroupIDs.allSatisfy({ processGroupID in + guard let expectedLeader = + epoch.processGroupLeaders[processGroupID] else { + return false + } + // A process group can outlive its leader. A reused leader + // PID, however, invalidates the original group authority. + let liveLeader = processIdentityProvider(processGroupID) + return liveLeader == nil || liveLeader == expectedLeader + }), + shouldCommit() else { + return .rejected + } + + var teardownIsCommitted = false + for processGroupID in signalableProcessGroupIDs.sorted() { + if let error = signalErrorProvider(-processGroupID, SIGTERM) { + if error != ESRCH, !teardownIsCommitted { + return .rejected + } + } else { + teardownIsCommitted = true + } + } + // Every target may have exited between validation and signaling. + // That is still an irreversible commit: no authorized generation + // remains, and exact-exit observation resolves the final state. + onTeardownCommit() + return .committedAwaitingExit + } + } +} diff --git a/Sources/App/AgentHibernationController+ProcessTermination.swift b/Sources/App/AgentHibernationController+ProcessTermination.swift index 2932ff3b5909..b4f67b2a4105 100644 --- a/Sources/App/AgentHibernationController+ProcessTermination.swift +++ b/Sources/App/AgentHibernationController+ProcessTermination.swift @@ -12,6 +12,11 @@ extension AgentHibernationRecord { } extension AgentHibernationController { + /// Bounds synchronous exact-kernel validation in the final signal commit boundary. + nonisolated static let maximumScopedProcessTerminationCount = 32 + /// Bounds full process-table refreshes while discovering late process generations. + nonisolated static let maximumProcessExitEpochRefreshCount = 8 + struct ProcessTerminationScope: Sendable { let key: AgentHibernationPanelKey let processIDs: Set @@ -22,6 +27,19 @@ extension AgentHibernationController { let processID: Int let processIdentity: AgentPIDProcessIdentity let processGroupID: pid_t + let ttyDevice: Int64? + + init( + processID: Int, + processIdentity: AgentPIDProcessIdentity, + processGroupID: pid_t, + ttyDevice: Int64? = nil + ) { + self.processID = processID + self.processIdentity = processIdentity + self.processGroupID = processGroupID + self.ttyDevice = ttyDevice + } } nonisolated static func processIdentities( @@ -63,9 +81,10 @@ extension AgentHibernationController { processIdentityProvider: { AgentPIDProcessIdentity(pid: pid_t($0)) }, - processArgumentsProvider: - CmuxTopProcessSnapshot.processArgumentsAndEnvironment(for:), - processGroupProvider: { getpgid(pid_t($0)) } + processGroupProvider: { getpgid(pid_t($0)) }, + processTTYDeviceProvider: { + agentLiveProcessIdentity(pid: pid_t($0))?.ttyDevice + } ) ) } @@ -82,28 +101,29 @@ extension AgentHibernationController { nonisolated static func validatedScopedProcessTerminations( for scope: ProcessTerminationScope, processIdentityProvider: (Int) -> AgentPIDProcessIdentity?, - processArgumentsProvider: (Int) -> CmuxTopProcessArguments?, - processGroupProvider: (Int) -> pid_t + processGroupProvider: (Int) -> pid_t, + processTTYDeviceProvider: (Int) -> Int64? = { + agentLiveProcessIdentity(pid: pid_t($0))?.ttyDevice + } ) -> [ScopedProcessTermination]? { - guard Set(scope.processIdentities.keys) == scope.processIDs else { return nil } + guard scope.processIDs.count <= maximumScopedProcessTerminationCount, + Set(scope.processIdentities.keys) == scope.processIDs else { + return nil + } var terminations: [ScopedProcessTermination] = [] for processID in scope.processIDs.sorted(by: >) { guard processID > 0, processID <= Int(Int32.max), let expectedIdentity = scope.processIdentities[processID], - processIdentityProvider(processID) == expectedIdentity, - let process = processArgumentsProvider(processID), - process.matchesCMUXScope( - workspaceId: scope.key.workspaceId, - surfaceId: scope.key.panelId - ) else { + processIdentityProvider(processID) == expectedIdentity else { return nil } terminations.append( ScopedProcessTermination( processID: processID, processIdentity: expectedIdentity, - processGroupID: processGroupProvider(processID) + processGroupID: processGroupProvider(processID), + ttyDevice: processTTYDeviceProvider(processID) ) ) } @@ -133,6 +153,8 @@ extension AgentHibernationController { ) return false } + let processExitCompletion = AgentHibernationProcessExitCompletion() + let committedTerminationRequestID = UUID() if let snapshot { // Hand off any older monitor only after every other precondition passed. @@ -164,7 +186,10 @@ extension AgentHibernationController { // trigger an interrupted-exit transcript rewrite. guard armPostTeardownRestoreMonitor( snapshot: snapshot, - processIDs: record.processIDs + processIDs: record.processIDs, + awaitProcessExit: { + await processExitCompletion.wait() + } ) else { preserveSnapshotAfterAbortedTeardown( snapshot, @@ -205,36 +230,133 @@ extension AgentHibernationController { let panelID = record.key.panelId let workspaceID = record.key.workspaceId let agent = record.agent - let finishTeardown: @MainActor () -> Void = { + let finalizeTeardown: @MainActor () -> Bool = { [weak workspace = record.workspace, weak terminalPanel = record.terminalPanel] in - guard let terminalPanel else { return } + guard let terminalPanel else { return true } if let workspace, let currentPanel = workspace.panels[panelID] as? TerminalPanel, currentPanel === terminalPanel, terminalPanel.workspaceId == workspaceID, - terminalPanel.isAgentHibernationTerminating { - workspace.enterAgentHibernation( + terminalPanel.isAgentHibernationCommitPending { + return workspace.enterAgentHibernation( panelId: panelID, agent: agent, lastActivityAt: lastActivityAt ) - } else { - // A live move carries the phase on TerminalPanel. Its new owner - // gets the same resumable state without consulting the source. - terminalPanel.completeAgentHibernationTermination() } + // A live move carries the phase on TerminalPanel. Its new owner + // gets the same resumable state without consulting the source. + terminalPanel.completeAgentHibernationTermination() + return true + } + let finishTeardown: @MainActor () async -> Bool = { + [weak terminalPanel = record.terminalPanel] in + guard let terminalPanel else { return true } + guard await terminalPanel.surface.waitForAgentHibernationRuntimeTeardown( + timeout: .seconds(5) + ) else { + return false + } + return finalizeTeardown() + } + let recoverTeardown: @MainActor () async -> Void = { + [weak terminalPanel = record.terminalPanel] in + guard terminalPanel != nil else { return } + _ = finalizeTeardown() + } + let waitForRecoveryReadiness: @MainActor @Sendable () async -> Bool = { + [weak terminalPanel = record.terminalPanel] in + guard let terminalPanel else { return true } + return await terminalPanel.surface.waitForAgentHibernationRuntimeTeardown( + timeout: .seconds(30) + ) + } + let beginTerminationRecovery: @MainActor () async -> Void = { + [weak terminalPanel = record.terminalPanel] in + terminalPanel?.beginAgentHibernationTerminationRecovery() + } + let handleRecoveryFailure: @MainActor () async -> Void = { + [weak self, weak terminalPanel = record.terminalPanel] in + guard let self, let terminalPanel else { return } + terminalPanel.failAgentHibernationTermination() + terminalPanel.onRequestAgentHibernationTerminationRetry = { + [weak self, weak terminalPanel] in + guard let self, let terminalPanel else { return } + self.retryCommittedTerminationRecovery(panelID: terminalPanel.id) + } + } + let beginRecoveryRetry: @MainActor () -> Void = { + [weak terminalPanel = record.terminalPanel] in + terminalPanel?.beginAgentHibernationTerminationRecovery() + } + let finalCommitIsSafe: @MainActor @Sendable () -> Bool = { + [weak self, weak terminalPanel = record.terminalPanel] in + guard let self, let terminalPanel, + self.teardownIsStillSafe( + request, + index: preSignalIndex, + shouldProceed: shouldProceed + ), + (snapshot.map { + AgentHibernationTranscriptGuard.liveFileVersionStillMatches($0) + } ?? true) else { + return false + } + return terminalPanel.surface.reserveAgentHibernationRuntimeTeardown() + } + let processGroupLeaders = Self.processGroupLeaders( + in: scopedProcessTerminations + ) + let processScopeKey = record.key + let processSnapshotCoordinator = processSnapshotCoordinator + let retryTermination: CommittedTerminationRetry = { + [weak self, weak terminalPanel = record.terminalPanel] in + guard !scopedProcessTerminations.isEmpty else { + return .exited + } + guard let epoch = await processSnapshotCoordinator.refreshedExitEpoch( + processGroupLeaders: processGroupLeaders, + processScopeKey: processScopeKey, + ttyDevice: Self.commonTTYDevice(in: scopedProcessTerminations), + excluding: [] + ) else { + return .rejected + } + return await Self.terminateScopedProcessEpochForHibernation( + epoch, + processScopeKey: processScopeKey, + shouldCommit: { + guard let self, let terminalPanel, + terminalPanel.isAgentHibernationCommitPending, + self.committedTerminationObservationsByPanelID[panelID]? + .requestID == committedTerminationRequestID else { + return false + } + return true + } + ) } - let terminationResult = terminateScopedProcessesForHibernation( + let terminationResult = await Self.terminateScopedProcessesForHibernation( scopedProcessTerminations, + processScopeKey: record.key, + shouldCommit: finalCommitIsSafe, onTeardownCommit: { - record.terminalPanel.beginAgentHibernationTermination( - agent: record.agent, + [self, weak terminalPanel = record.terminalPanel] in + registerCommittedTerminationObservation( + panelID: panelID, + requestID: committedTerminationRequestID, + processExitCompletion: processExitCompletion + ) + terminalPanel?.beginAgentHibernationTermination( + agent: agent, lastActivityAt: lastActivityAt ) } ) switch terminationResult { case .rejected: + record.terminalPanel.surface + .cancelAgentHibernationRuntimeTeardownReservation() if let snapshot { await releaseArmedRestoreMonitorAfterAbortedTeardown( snapshot, @@ -244,12 +366,63 @@ extension AgentHibernationController { } return false case .exited: - finishTeardown() + guard committedTerminationObservationsByPanelID[panelID]?.requestID == + committedTerminationRequestID else { + return false + } + await processExitCompletion.finish(true) + guard committedTerminationObservationsByPanelID[panelID]?.requestID == + committedTerminationRequestID else { + return false + } + guard await finishTeardown() else { + guard committedTerminationObservationsByPanelID[panelID]?.requestID == + committedTerminationRequestID else { + return false + } + await beginTerminationRecovery() + guard committedTerminationObservationsByPanelID[panelID]?.requestID == + committedTerminationRequestID else { + return false + } + observeCommittedTerminationRecovery( + panelID: panelID, + requestID: committedTerminationRequestID, + terminations: [], + processScopeKey: nil, + processExitCompletion: processExitCompletion, + retryTermination: retryTermination, + waitForRecoveryReadiness: waitForRecoveryReadiness, + recoveryDeadline: .seconds(60), + onRecovery: recoverTeardown, + onRecoveryFailure: handleRecoveryFailure, + onRecoveryRetry: beginRecoveryRetry + ) + return false + } + guard committedTerminationObservationsByPanelID[panelID]?.requestID == + committedTerminationRequestID else { + return false + } + removeCommittedTerminationObservation( + panelID: panelID, + requestID: committedTerminationRequestID + ) case .committedAwaitingExit: observeCommittedTermination( panelID: panelID, + requestID: committedTerminationRequestID, terminations: scopedProcessTerminations, - onExit: finishTeardown + processScopeKey: record.key, + processExitCompletion: processExitCompletion, + retryTermination: retryTermination, + waitForRecoveryReadiness: waitForRecoveryReadiness, + recoveryDeadline: .seconds(60), + onExit: finishTeardown, + onFailure: beginTerminationRecovery, + onRecovery: recoverTeardown, + onRecoveryFailure: handleRecoveryFailure, + onRecoveryRetry: beginRecoveryRetry ) return false } @@ -258,161 +431,4 @@ extension AgentHibernationController { !record.terminalPanel.isAgentHibernationTerminating } - @discardableResult - func terminateScopedProcessesForHibernation( - _ terminations: [ScopedProcessTermination], - onTeardownCommit: @MainActor () -> Void = {}, - currentProcessID: pid_t = getpid(), - currentProcessGroupID: pid_t = getpgrp(), - processIdentityProvider: (pid_t) -> AgentPIDProcessIdentity? = { - AgentPIDProcessIdentity(pid: $0) - }, - processGroupProvider: (pid_t) -> pid_t = { getpgid($0) }, - signalErrorProvider: (pid_t, Int32) -> Int32? = { target, signal in - kill(target, signal) == 0 ? nil : errno - } - ) -> ScopedProcessTerminationResult { - guard !terminations.isEmpty else { - onTeardownCommit() - return .exited - } - guard terminations.allSatisfy({ termination in - let pid = pid_t(termination.processID) - return pid != currentProcessID && - processIdentityProvider(pid) == termination.processIdentity && - processGroupProvider(pid) == termination.processGroupID - }) else { - return .rejected - } - var teardownIsCommitted = false - let commitTeardownIfNeeded = { - guard !teardownIsCommitted else { return } - teardownIsCommitted = true - onTeardownCommit() - } - var signaledProcessGroups: Set = [] - for termination in terminations { - let pid = pid_t(termination.processID) - let processGroupID = termination.processGroupID - if processGroupID > 1, - processGroupID != currentProcessGroupID, - signaledProcessGroups.insert(processGroupID).inserted { - if let error = signalErrorProvider(-processGroupID, SIGTERM) { - if error != ESRCH, !teardownIsCommitted { - return .rejected - } - } else { - commitTeardownIfNeeded() - } - } - if let error = signalErrorProvider(pid, SIGTERM) { - if error != ESRCH, !teardownIsCommitted { - return .rejected - } - } else { - commitTeardownIfNeeded() - } - } - // Every target may have exited between validation and signaling. That is - // still a committed teardown: there is no live generation left to kill. - commitTeardownIfNeeded() - // A later signal error cannot roll back a teardown after the first signal. - // Exact-generation exit is completed by a stored per-panel observation so - // one slow process cannot block later critical-pressure reclamation batches. - return .committedAwaitingExit - } - - func observeCommittedTermination( - panelID: UUID, - terminations: [ScopedProcessTermination], - waitForExit: @escaping @Sendable ([ScopedProcessTermination]) async -> Bool = { - await AgentHibernationController - .waitForExactProcessGenerationsToExitWithoutTimeout($0) - }, - onExit: @escaping @MainActor () -> Void - ) { - committedTerminationObservationsByPanelID - .removeValue(forKey: panelID)? - .task - .cancel() - let requestID = UUID() - let task = Task { @MainActor [weak self] in - let didExit = await waitForExit(terminations) - guard let self, - self.committedTerminationObservationsByPanelID[panelID]?.requestID == - requestID else { - return - } - self.committedTerminationObservationsByPanelID.removeValue(forKey: panelID) - guard didExit, !Task.isCancelled else { return } - onExit() - } - committedTerminationObservationsByPanelID[panelID] = CommittedTerminationObservation( - requestID: requestID, - task: task - ) - } - - #if compiler(>=6.2) - @concurrent - #else - @Sendable - #endif - nonisolated static func waitForExactProcessGenerationsToExitWithoutTimeout( - _ terminations: [ScopedProcessTermination], - processIdentityProvider: @escaping @Sendable (pid_t) -> AgentPIDProcessIdentity? = { - AgentPIDProcessIdentity(pid: $0) - } - ) async -> Bool { - await withTaskGroup(of: Bool.self) { group in - for termination in terminations { - group.addTask(priority: .utility) { - await waitForExactProcessGenerationToExit( - termination, - processIdentityProvider: processIdentityProvider - ) - } - } - for await didExit in group where !didExit { - group.cancelAll() - return false - } - return true - } - } - - private nonisolated static func waitForExactProcessGenerationToExit( - _ termination: ScopedProcessTermination, - processIdentityProvider: @escaping @Sendable (pid_t) -> AgentPIDProcessIdentity? - ) async -> Bool { - let processID = pid_t(termination.processID) - guard processIdentityProvider(processID) == termination.processIdentity else { - return true - } - - let exitEvents = AsyncStream(bufferingPolicy: .bufferingNewest(1)) { continuation in - let source = DispatchSource.makeProcessSource( - identifier: processID, - eventMask: .exit, - queue: .global(qos: .utility) - ) - source.setEventHandler { - continuation.yield() - continuation.finish() - } - continuation.onTermination = { @Sendable _ in - source.cancel() - } - source.resume() - } - // Close the registration race without ever accepting a reused PID. - guard processIdentityProvider(processID) == termination.processIdentity else { - return true - } - - for await _ in exitEvents { - return true - } - return processIdentityProvider(processID) != termination.processIdentity - } } diff --git a/Sources/App/AgentHibernationController+Records.swift b/Sources/App/AgentHibernationController+Records.swift index 36c06b0b64f1..ce01215ab47e 100644 --- a/Sources/App/AgentHibernationController+Records.swift +++ b/Sources/App/AgentHibernationController+Records.swift @@ -1,5 +1,14 @@ import Foundation +extension AgentHibernationRecord { + var hasPressureSafeProcessEvidence: Bool { + hasLiveProcess && + !containsUnrelatedProcess && + !processIDs.isEmpty && + processIdentities.count == processIDs.count + } +} + extension AppDelegate { @MainActor func agentHibernationPanelIsProtected(workspace: Workspace, panelId: UUID) -> Bool { @@ -49,7 +58,11 @@ extension AppDelegate { panelId: panelId, fallback: index.lifecycle(workspaceId: workspace.id, panelId: panelId) ) - let processIDs = index.processIDs(workspaceId: workspace.id, panelId: panelId) + let processEntry = index.entry( + workspaceId: workspace.id, + panelId: panelId + ) + let panelProcessIDs = processEntry?.processIDs ?? [] records.append( AgentHibernationRecord( key: key, @@ -60,12 +73,11 @@ extension AppDelegate { hasUnconfirmedTerminalInput: terminalInputAt > lifecycleChangeAt, lastActivityAt: max(indexActivity, localActivity, createdAt), isProtected: workspaceIsVisible && visiblePanelIds.contains(panelId), - hasLiveProcess: !processIDs.isEmpty, - processIDs: processIDs, - processIdentities: index.processIdentities( - workspaceId: workspace.id, - panelId: panelId - ) + hasLiveProcess: !panelProcessIDs.isEmpty, + containsUnrelatedProcess: processEntry?.containsUnrelatedProcess ?? false, + panelProcessIDs: processEntry?.hibernationPanelProcessIDs ?? [], + processIDs: processEntry?.terminationProcessIDs ?? [], + processIdentities: processEntry?.terminationProcessIdentities ?? [:] ) ) } diff --git a/Sources/App/AgentHibernationController+ScopedProcessTerminationResult.swift b/Sources/App/AgentHibernationController+ScopedProcessTerminationResult.swift new file mode 100644 index 000000000000..1eba038e53da --- /dev/null +++ b/Sources/App/AgentHibernationController+ScopedProcessTerminationResult.swift @@ -0,0 +1,9 @@ +import Foundation + +extension AgentHibernationController { + enum ScopedProcessTerminationResult: Equatable, Sendable { + case rejected + case exited + case committedAwaitingExit + } +} diff --git a/Sources/App/AgentHibernationController+Teardown.swift b/Sources/App/AgentHibernationController+Teardown.swift index 4aa145caa02d..1109862a591f 100644 --- a/Sources/App/AgentHibernationController+Teardown.swift +++ b/Sources/App/AgentHibernationController+Teardown.swift @@ -272,7 +272,8 @@ extension AgentHibernationController { func armPostTeardownRestoreMonitor( snapshot: AgentHibernationTranscriptGuard.TeardownTranscriptSnapshot, processIDs: Set, - snapshotDisposal: AgentHibernationTranscriptGuard.PostTeardownSnapshotDisposal = .deleteWhenSafe + snapshotDisposal: AgentHibernationTranscriptGuard.PostTeardownSnapshotDisposal = .deleteWhenSafe, + awaitProcessExit: (@Sendable () async -> Bool)? = nil ) -> Bool { let transcriptPath = snapshot.transcriptPath let requestID = UUID() @@ -282,6 +283,7 @@ extension AgentHibernationController { snapshot: snapshot, processIDs: processIDs, snapshotDisposal: snapshotDisposal, + awaitProcessExit: awaitProcessExit, shouldContinue: { await MainActor.run { AgentHibernationController.shared.postTeardownRestoreTaskIsCurrent( diff --git a/Sources/App/AgentHibernationController+TeardownValidation.swift b/Sources/App/AgentHibernationController+TeardownValidation.swift index 9e1fcab0c4a6..230a826d04ae 100644 --- a/Sources/App/AgentHibernationController+TeardownValidation.swift +++ b/Sources/App/AgentHibernationController+TeardownValidation.swift @@ -16,6 +16,15 @@ extension AgentHibernationController { for: record, index: index ) + let currentProcessEntry = index.entry( + workspaceId: record.key.workspaceId, + panelId: record.key.panelId + ) + let currentHibernationPanelProcessIDs = + currentProcessEntry?.hibernationPanelProcessIDs ?? [] + let currentTerminationProcessIDs = currentProcessEntry?.terminationProcessIDs ?? [] + let currentTerminationProcessIdentities = + currentProcessEntry?.terminationProcessIdentities ?? [:] return (shouldProceed?() ?? true) && AgentHibernationTrackingGate.isEnabled() && record.isStillOwnedByOriginalWorkspace && @@ -26,14 +35,13 @@ extension AgentHibernationController { panelId: record.key.panelId ) ) && - index.processIDs( - workspaceId: record.key.workspaceId, - panelId: record.key.panelId - ) == record.processIDs && - index.processIdentities( - workspaceId: record.key.workspaceId, - panelId: record.key.panelId - ) == record.processIdentities && + ( + request.trigger != .systemMemoryPressure || + currentProcessEntry?.containsUnrelatedProcess == false + ) && + currentHibernationPanelProcessIDs == record.panelProcessIDs && + currentTerminationProcessIDs == record.processIDs && + currentTerminationProcessIdentities == record.processIdentities && TabManager.restorableAgentSnapshotFingerprint(currentAgent) == TabManager.restorableAgentSnapshotFingerprint(record.agent) && !record.terminalPanel.isAgentHibernated && diff --git a/Sources/App/AgentHibernationController.swift b/Sources/App/AgentHibernationController.swift index 50e81a9346a0..0632197944c3 100644 --- a/Sources/App/AgentHibernationController.swift +++ b/Sources/App/AgentHibernationController.swift @@ -16,6 +16,8 @@ struct AgentHibernationRecord { let lastActivityAt: TimeInterval let isProtected: Bool let hasLiveProcess: Bool + let containsUnrelatedProcess: Bool + let panelProcessIDs: Set let processIDs: Set let processIdentities: [Int: AgentPIDProcessIdentity] } @@ -41,6 +43,8 @@ final class AgentHibernationController { var postSnapshotValidationIndexTask: PostSnapshotValidationIndexTask? var teardownInFlightByPanel: [AgentHibernationPanelKey: InFlightTeardown] = [:] var committedTerminationObservationsByPanelID: [UUID: CommittedTerminationObservation] = [:] + var committedTerminationCleanupByPanelID: [UUID: CommittedTerminationCleanup] = [:] + let processSnapshotCoordinator = AgentHibernationProcessSnapshotCoordinator() var confirmations: [AgentHibernationPanelKey: Confirmation] = [:] var tailFingerprintSamples: [AgentHibernationPanelKey: TailFingerprintSample] = [:] var memoryPressureEvaluation: (id: UUID, task: Task)? @@ -274,10 +278,7 @@ final class AgentHibernationController { !record.hasUnconfirmedTerminalInput, !record.isProtected, (trigger == .systemMemoryPressure || !record.hasLiveProcess), - ( - trigger != .systemMemoryPressure || - record.processIdentities.count == record.processIDs.count - ), + trigger != .systemMemoryPressure || record.hasPressureSafeProcessEvidence, record.terminalPanel.surface.hasLiveSurface, !record.terminalPanel.isAgentHibernated else { confirmations.removeValue(forKey: record.key) diff --git a/Sources/App/AgentHibernationPlanner.swift b/Sources/App/AgentHibernationPlanner.swift index b0627fbab875..350244bbff51 100644 --- a/Sources/App/AgentHibernationPlanner.swift +++ b/Sources/App/AgentHibernationPlanner.swift @@ -1,3 +1,4 @@ +import CmuxTerminal import Foundation enum AgentHibernationReclaimTrigger: Equatable, Sendable { @@ -6,8 +7,6 @@ enum AgentHibernationReclaimTrigger: Equatable, Sendable { } enum AgentHibernationPlanner { - private static let systemMemoryPressureBatchLimit = 2 - static func selectedPanelKeys( inputs: [AgentHibernationPlannerInput], settings: AgentHibernationSettings.Values, @@ -22,7 +21,11 @@ enum AgentHibernationPlanner { excess = liveRestorable.count - settings.maxLiveTerminals case .systemMemoryPressure: // Snapshot and reclaim a small batch before the next pressure pass. - excess = min(liveRestorable.count, systemMemoryPressureBatchLimit) + excess = min( + liveRestorable.count, + TerminalSurfaceRuntimeTeardownCoordinator + .maximumIsolatedHibernationTeardownCount + ) } guard excess > 0 else { return [] } diff --git a/Sources/App/AgentHibernationProcessExitCompletion.swift b/Sources/App/AgentHibernationProcessExitCompletion.swift new file mode 100644 index 000000000000..2107290eeb72 --- /dev/null +++ b/Sources/App/AgentHibernationProcessExitCompletion.swift @@ -0,0 +1,42 @@ +import Foundation + +/// One process-exit result shared by the panel observation and transcript guard. +actor AgentHibernationProcessExitCompletion { + private var result: Bool? + private var waiters: [UUID: CheckedContinuation] = [:] + + func wait() async -> Bool { + if let result { return result } + if Task.isCancelled { return false } + let waiterID = UUID() + return await withTaskCancellationHandler { + await withCheckedContinuation { continuation in + if let result { + continuation.resume(returning: result) + } else if Task.isCancelled { + continuation.resume(returning: false) + } else { + waiters[waiterID] = continuation + } + } + } onCancel: { + Task { + await self.cancel(waiterID: waiterID) + } + } + } + + func finish(_ result: Bool) { + guard self.result == nil else { return } + self.result = result + let pendingWaiters = waiters.values + waiters.removeAll(keepingCapacity: false) + for waiter in pendingWaiters { + waiter.resume(returning: result) + } + } + + private func cancel(waiterID: UUID) { + waiters.removeValue(forKey: waiterID)?.resume(returning: false) + } +} diff --git a/Sources/App/AgentHibernationProcessExitEpoch.swift b/Sources/App/AgentHibernationProcessExitEpoch.swift new file mode 100644 index 000000000000..96503a73814b --- /dev/null +++ b/Sources/App/AgentHibernationProcessExitEpoch.swift @@ -0,0 +1,19 @@ +import Darwin +import Foundation + +/// One refreshed process generation observed after hibernation commits. +struct AgentHibernationProcessExitEpoch: Sendable { + let terminations: [AgentHibernationController.ScopedProcessTermination] + let processGroupLeaders: [pid_t: AgentPIDProcessIdentity] + let signalableProcessIdentities: Set + + init( + terminations: [AgentHibernationController.ScopedProcessTermination], + processGroupLeaders: [pid_t: AgentPIDProcessIdentity], + signalableProcessIdentities: Set = [] + ) { + self.terminations = terminations + self.processGroupLeaders = processGroupLeaders + self.signalableProcessIdentities = signalableProcessIdentities + } +} diff --git a/Sources/App/AgentHibernationProcessSnapshotCoordinator.swift b/Sources/App/AgentHibernationProcessSnapshotCoordinator.swift new file mode 100644 index 000000000000..e5d5bba3d7fc --- /dev/null +++ b/Sources/App/AgentHibernationProcessSnapshotCoordinator.swift @@ -0,0 +1,231 @@ +import Darwin +import Foundation + +/// Coalesces authoritative process-table refreshes across hibernating panels. +actor AgentHibernationProcessSnapshotCoordinator { + typealias SnapshotCapture = @Sendable () async -> CmuxTopProcessSnapshot + typealias CaptureScheduler = @Sendable () async -> Void + typealias ProcessArgumentsProvider = + @Sendable (Int) -> CmuxTopProcessArguments? + typealias ProcessIdentityProvider = + @Sendable (pid_t) -> AgentPIDProcessIdentity? + typealias ProcessGroupProvider = @Sendable (pid_t) -> pid_t + + private let captureSnapshot: SnapshotCapture + private let beforeCapture: CaptureScheduler + private let processArgumentsProvider: ProcessArgumentsProvider + private let processIdentityProvider: ProcessIdentityProvider + private let processGroupProvider: ProcessGroupProvider + private var activeSnapshotWaiters: + [UUID: CheckedContinuation] = [:] + private var queuedSnapshotWaiters: + [UUID: CheckedContinuation] = [:] + private var captureTask: Task? + private var captureHasStarted = false + + init( + beforeCapture: @escaping CaptureScheduler = { + await Task.yield() + }, + captureSnapshot: @escaping SnapshotCapture = { + await AgentHibernationProcessSnapshotCoordinator.captureFreshSnapshot() + }, + processArgumentsProvider: @escaping ProcessArgumentsProvider = { + CmuxTopProcessSnapshot.processArgumentsAndEnvironment(for: $0) + }, + processIdentityProvider: @escaping ProcessIdentityProvider = { + AgentPIDProcessIdentity(pid: $0) + }, + processGroupProvider: @escaping ProcessGroupProvider = { + getpgid($0) + } + ) { + self.beforeCapture = beforeCapture + self.captureSnapshot = captureSnapshot + self.processArgumentsProvider = processArgumentsProvider + self.processIdentityProvider = processIdentityProvider + self.processGroupProvider = processGroupProvider + } + + /// Returns one fresh snapshot shared by requests already queued for this capture epoch. + func nextSnapshot() async -> CmuxTopProcessSnapshot? { + let waiterID = UUID() + return await withTaskCancellationHandler { + await withCheckedContinuation { continuation in + guard !Task.isCancelled else { + continuation.resume(returning: nil) + return + } + queuedSnapshotWaiters[waiterID] = continuation + scheduleCaptureIfNeeded() + } + } onCancel: { + Task { + await self.cancelSnapshotWaiter(waiterID) + } + } + } + + func refreshedExitEpoch( + processGroupLeaders: [pid_t: AgentPIDProcessIdentity], + processScopeKey: AgentHibernationPanelKey?, + ttyDevice: Int64?, + excluding exitedIdentities: Set + ) async -> AgentHibernationProcessExitEpoch? { + guard let snapshot = await nextSnapshot() else { return nil } + return Self.exitEpoch( + in: snapshot, + processGroupLeaders: processGroupLeaders, + processScopeKey: processScopeKey, + ttyDevice: ttyDevice, + processArgumentsProvider: processArgumentsProvider, + processIdentityProvider: processIdentityProvider, + processGroupProvider: processGroupProvider, + excluding: exitedIdentities + ) + } + + private func scheduleCaptureIfNeeded() { + guard captureTask == nil, !queuedSnapshotWaiters.isEmpty else { return } + captureTask = Task { [weak self, beforeCapture] in + // Let all exit events already queued on the actor join one fresh capture. + await beforeCapture() + guard !Task.isCancelled else { + await self?.finishCancelledCapture() + return + } + await self?.runCapture() + } + } + + private func runCapture() async { + captureHasStarted = true + activeSnapshotWaiters = queuedSnapshotWaiters + queuedSnapshotWaiters.removeAll(keepingCapacity: true) + let snapshot = await captureSnapshot() + let waiters = Array(activeSnapshotWaiters.values) + activeSnapshotWaiters.removeAll(keepingCapacity: true) + for waiter in waiters { + waiter.resume(returning: snapshot) + } + captureHasStarted = false + captureTask = nil + scheduleCaptureIfNeeded() + } + + private func cancelSnapshotWaiter(_ waiterID: UUID) { + let waiter = activeSnapshotWaiters.removeValue(forKey: waiterID) ?? + queuedSnapshotWaiters.removeValue(forKey: waiterID) + waiter?.resume(returning: nil) + let captureHasNoWaiters = captureHasStarted + ? activeSnapshotWaiters.isEmpty + : queuedSnapshotWaiters.isEmpty + if captureHasNoWaiters { + captureTask?.cancel() + } + } + + private func finishCancelledCapture() { + captureHasStarted = false + captureTask = nil + scheduleCaptureIfNeeded() + } + + #if compiler(>=6.2) + @concurrent + #endif + private nonisolated static func captureFreshSnapshot() async -> CmuxTopProcessSnapshot { + CmuxTopProcessSnapshot.capture( + includeProcessDetails: false, + includeCMUXScope: false + ) + } + + private nonisolated static func exitEpoch( + in snapshot: CmuxTopProcessSnapshot, + processGroupLeaders: [pid_t: AgentPIDProcessIdentity], + processScopeKey: AgentHibernationPanelKey?, + ttyDevice: Int64?, + processArgumentsProvider: ProcessArgumentsProvider, + processIdentityProvider: ProcessIdentityProvider, + processGroupProvider: ProcessGroupProvider, + excluding exitedIdentities: Set + ) -> AgentHibernationProcessExitEpoch? { + var authorizedLeaders: [pid_t: AgentPIDProcessIdentity] = [:] + for (processGroupID, leaderIdentity) in processGroupLeaders { + let memberIDs = snapshot.pids(forProcessGroupID: Int(processGroupID)) + guard !memberIDs.isEmpty else { continue } + if snapshot.processesByPID[Int(processGroupID)] != nil, + processIdentityProvider(processGroupID) != leaderIdentity { + continue + } + authorizedLeaders[processGroupID] = leaderIdentity + } + + var observedProcessIDs: Set = [] + for processGroupID in authorizedLeaders.keys { + observedProcessIDs.formUnion( + snapshot.pids(forProcessGroupID: Int(processGroupID)) + ) + } + if processScopeKey != nil { + guard let ttyDevice else { return nil } + observedProcessIDs.formUnion( + snapshot.pids(forTTYDevice: ttyDevice) + ) + } + guard observedProcessIDs.count <= + AgentHibernationController.maximumScopedProcessTerminationCount else { + return nil + } + if let processScopeKey { + guard observedProcessIDs.allSatisfy({ processID in + processArgumentsProvider(processID)?.matchesCMUXScope( + workspaceId: processScopeKey.workspaceId, + surfaceId: processScopeKey.panelId + ) == true + }) else { + return nil + } + } + + var nextTerminations: [AgentPIDProcessIdentity: + AgentHibernationController.ScopedProcessTermination] = [:] + var liveAuthorizedLeaders: [pid_t: AgentPIDProcessIdentity] = [:] + var signalableProcessIdentities: Set = [] + for processID in observedProcessIDs { + guard processID > 0, + let process = snapshot.processesByPID[processID], + let rawProcessGroupID = process.processGroupID, + rawProcessGroupID > 1 else { + return nil + } + let processGroupID = pid_t(rawProcessGroupID) + guard let identity = processIdentityProvider(pid_t(processID)) else { + return nil + } + guard !exitedIdentities.contains(identity) else { continue } + guard processGroupProvider(pid_t(processID)) == processGroupID else { + return nil + } + nextTerminations[identity] = .init( + processID: processID, + processIdentity: identity, + processGroupID: processGroupID, + ttyDevice: process.ttyDevice + ) + if let leaderIdentity = authorizedLeaders[processGroupID] { + liveAuthorizedLeaders[processGroupID] = leaderIdentity + signalableProcessIdentities.insert(identity) + } + } + + return AgentHibernationProcessExitEpoch( + terminations: nextTerminations.values.sorted { + $0.processID > $1.processID + }, + processGroupLeaders: liveAuthorizedLeaders, + signalableProcessIdentities: signalableProcessIdentities + ) + } +} diff --git a/Sources/App/AgentHibernationTranscriptGuard+PostTeardownRestore.swift b/Sources/App/AgentHibernationTranscriptGuard+PostTeardownRestore.swift index 8db77762e8de..86b579db7413 100644 --- a/Sources/App/AgentHibernationTranscriptGuard+PostTeardownRestore.swift +++ b/Sources/App/AgentHibernationTranscriptGuard+PostTeardownRestore.swift @@ -21,6 +21,7 @@ extension AgentHibernationTranscriptGuard { clock: ContinuousClock = ContinuousClock(), fileManager: FileManager = .default, snapshotDisposal: PostTeardownSnapshotDisposal = .deleteWhenSafe, + awaitProcessExit: (@Sendable () async -> Bool)? = nil, shouldContinue: @Sendable () async -> Bool = { true }, shouldRestoreOnCancellation: @Sendable () async -> Bool = { true } ) async { @@ -66,7 +67,16 @@ extension AgentHibernationTranscriptGuard { } } - if !processIDs.isEmpty { + if let awaitProcessExit { + let didExit = await awaitProcessExit() + if Task.isCancelled { + await restoreBeforeStoppedReturn() + return + } + if didExit { + if await stopIfNoLongerCurrent() { return } + } + } else if !processIDs.isEmpty { let deadline = clock.now.advanced(by: .seconds(30)) while clock.now < deadline { let anyAlive = processIDs.contains { pid in diff --git a/Sources/CmuxTopSnapshot.swift b/Sources/CmuxTopSnapshot.swift index 61ebc43325e0..eeb90bbcb6eb 100644 --- a/Sources/CmuxTopSnapshot.swift +++ b/Sources/CmuxTopSnapshot.swift @@ -137,6 +137,7 @@ final class CmuxTopProcessSnapshot: @unchecked Sendable { private let childrenByParentPID: [Int: [Int]] private let pidsByTTYDevice: [Int64: [Int]] private let pidsByCMUXSurfaceID: [UUID: [Int]] + private let pidsByProcessGroupID: [Int: [Int]] private let residentMemorySources: [CmuxTopProcessMemorySource] static func capture( @@ -176,6 +177,7 @@ final class CmuxTopProcessSnapshot: @unchecked Sendable { var children: [Int: [Int]] = [:] var ttyMap: [Int64: [Int]] = [:] var cmuxSurfaceMap: [UUID: [Int]] = [:] + var processGroupMap: [Int: [Int]] = [:] for process in processMap.values { if process.parentPID > 0 { children[process.parentPID, default: []].append(process.pid) @@ -186,10 +188,14 @@ final class CmuxTopProcessSnapshot: @unchecked Sendable { if let cmuxSurfaceID = process.cmuxSurfaceID { cmuxSurfaceMap[cmuxSurfaceID, default: []].append(process.pid) } + if let processGroupID = process.processGroupID { + processGroupMap[processGroupID, default: []].append(process.pid) + } } self.childrenByParentPID = children.mapValues { $0.sorted() } self.pidsByTTYDevice = ttyMap.mapValues { $0.sorted() } self.pidsByCMUXSurfaceID = cmuxSurfaceMap.mapValues { $0.sorted() } + self.pidsByProcessGroupID = processGroupMap.mapValues { $0.sorted() } } func samplePayload() -> [String: Any] { @@ -240,13 +246,21 @@ final class CmuxTopProcessSnapshot: @unchecked Sendable { guard let device = Self.deviceIdentifier(forTTYName: ttyName) else { return [] } - return Set(pidsByTTYDevice[device] ?? []) + return pids(forTTYDevice: device) + } + + func pids(forTTYDevice ttyDevice: Int64) -> Set { + Set(pidsByTTYDevice[ttyDevice] ?? []) } func pids(forCMUXSurfaceID surfaceID: UUID) -> Set { Set(pidsByCMUXSurfaceID[surfaceID] ?? []) } + func pids(forProcessGroupID processGroupID: Int) -> Set { + Set(pidsByProcessGroupID[processGroupID] ?? []) + } + func cmuxScopedProcesses() -> [CmuxTopProcessInfo] { processesByPID.values .filter { $0.cmuxWorkspaceID != nil && $0.cmuxSurfaceID != nil } @@ -269,6 +283,59 @@ final class CmuxTopProcessSnapshot: @unchecked Sendable { return result } + func agentHibernationProcessScope( + panelProcessIDs: Set, + agentProcessIDs: Set + ) -> RestorableAgentSessionIndex.HibernationProcessScope { + let terminalDevices = Set(agentProcessIDs.compactMap { + processesByPID[$0]?.ttyDevice + }) + let terminalProcessIDs = Set(terminalDevices.flatMap { + pidsByTTYDevice[$0] ?? [] + }) + let observedPanelProcessIDs = panelProcessIDs.union(terminalProcessIDs) + let descendantProcessIDs = expandedPIDs(rootPIDs: agentProcessIDs) + var allowedProcessIDs = descendantProcessIDs + + for rootProcessID in agentProcessIDs { + var currentProcessID = rootProcessID + var visitedProcessIDs: Set = [] + while visitedProcessIDs.insert(currentProcessID).inserted, + let parentProcessID = processesByPID[currentProcessID]?.parentPID, + observedPanelProcessIDs.contains(parentProcessID) { + allowedProcessIDs.insert(parentProcessID) + currentProcessID = parentProcessID + } + } + let processGroupIDs = Set(descendantProcessIDs.compactMap { + processesByPID[$0]?.processGroupID + }).filter { $0 > 1 } + let processGroupMemberIDs = Set(processGroupIDs.flatMap { + pidsByProcessGroupID[$0] ?? [] + }) + let terminationProcessIDs = descendantProcessIDs.union(processGroupMemberIDs) + + let hasCompleteAgentRoots = + !agentProcessIDs.isEmpty && + agentProcessIDs.isSubset(of: terminationProcessIDs) + let hasTerminalEvidence = agentProcessIDs.allSatisfy { + processesByPID[$0]?.ttyDevice != nil + } + let hasCompleteProcessGroups = processGroupIDs.allSatisfy { processGroupID in + processesByPID[processGroupID]?.processGroupID == processGroupID + } + return ( + observedPanelProcessIDs, + terminationProcessIDs, + !hasCompleteAgentRoots || + !hasTerminalEvidence || + processGroupIDs.isEmpty || + !hasCompleteProcessGroups || + !processGroupMemberIDs.isSubset(of: allowedProcessIDs) || + !observedPanelProcessIDs.isSubset(of: allowedProcessIDs) + ) + } + func descendantPIDs(rootPID: Int, includeRoot: Bool = false) -> Set { guard rootPID > 0 else { return [] } diff --git a/Sources/DockSplitStore.swift b/Sources/DockSplitStore.swift index 6229560b90b7..ae2a092185cc 100644 --- a/Sources/DockSplitStore.swift +++ b/Sources/DockSplitStore.swift @@ -603,8 +603,9 @@ final class DockSplitStore: BonsplitDelegate { let live = Set(bonsplitController.allTabIds) let staleTabIds = surfaceIdToPanelId.keys.filter { !live.contains($0) } let stalePanelIds = Set(staleTabIds.compactMap { surfaceIdToPanelId[$0] }) - surfaceIdToPanelId = surfaceIdToPanelId.filter { !stalePanelIds.contains($0.value) } - for panelId in stalePanelIds { + surfaceIdToPanelId = surfaceIdToPanelId.filter { live.contains($0.key) } + let livePanelIds = Set(surfaceIdToPanelId.values) + for panelId in stalePanelIds.subtracting(livePanelIds) { discardPanelStateAndClose(panelId: panelId) } } diff --git a/Sources/Panels/AgentHibernationPlaceholderMode.swift b/Sources/Panels/AgentHibernationPlaceholderMode.swift new file mode 100644 index 000000000000..1c980b6b89d6 --- /dev/null +++ b/Sources/Panels/AgentHibernationPlaceholderMode.swift @@ -0,0 +1,5 @@ +enum AgentHibernationPlaceholderMode: Equatable { + case hibernated + case recovering + case failed +} diff --git a/Sources/Panels/TerminalPanel+AgentHibernation.swift b/Sources/Panels/TerminalPanel+AgentHibernation.swift index 838ff252b8b6..8aeae162e554 100644 --- a/Sources/Panels/TerminalPanel+AgentHibernation.swift +++ b/Sources/Panels/TerminalPanel+AgentHibernation.swift @@ -9,57 +9,112 @@ extension TerminalPanel { agentHibernationPhase.isTerminating } + var agentHibernationTerminationFailed: Bool { + agentHibernationPhase.terminationFailed + } + + var isAgentHibernationCommitPending: Bool { + agentHibernationPhase.isAwaitingCommit + } + var agentHibernationState: AgentHibernationPanelState? { agentHibernationPhase.state } + @discardableResult func enterAgentHibernation( agent: SessionRestorableAgentSnapshot, lastActivityAt: Date, hibernatedAt: Date = .now - ) { - if isAgentHibernationTerminating { + ) -> Bool { + if isAgentHibernationCommitPending { completeAgentHibernationTermination() - return + return true } - agentHibernationPhase = .hibernated(AgentHibernationPanelState( + let state = AgentHibernationPanelState( agent: agent, hibernatedAt: hibernatedAt, lastActivityAt: lastActivityAt - )) - suspendRuntimeForAgentHibernation(reason: "agentHibernation") + ) + guard suspendRuntimeForAgentHibernation(reason: "agentHibernation") else { + return false + } + agentHibernationPhase = .hibernated(state) + return true } + @discardableResult func beginAgentHibernationTermination( agent: SessionRestorableAgentSnapshot, lastActivityAt: Date, committedAt: Date = .now - ) { - guard case .live = agentHibernationPhase else { return } - agentHibernationPhase = .terminating(AgentHibernationPanelState( + ) -> Bool { + guard case .live = agentHibernationPhase else { return false } + onRequestAgentHibernationTerminationRetry = nil + let state = AgentHibernationPanelState( agent: agent, hibernatedAt: committedAt, lastActivityAt: lastActivityAt - )) - suspendRuntimeForAgentHibernation(reason: "agentHibernation.terminating") + ) + guard suspendRuntimeForAgentHibernation( + reason: "agentHibernation.terminating" + ) else { + return false + } + agentHibernationPhase = .terminating(state) + return true } func completeAgentHibernationTermination() { - guard case .terminating(let state) = agentHibernationPhase else { return } + let state: AgentHibernationPanelState + switch agentHibernationPhase { + case .terminating(let value), + .recovering(let value), + .terminationFailed(let value): + state = value + case .live, .hibernated: + return + } + onRequestAgentHibernationTerminationRetry = nil agentHibernationPhase = .hibernated(state) } + func beginAgentHibernationTerminationRecovery() { + let state: AgentHibernationPanelState + switch agentHibernationPhase { + case .terminating(let value), .terminationFailed(let value): + state = value + case .live, .recovering, .hibernated: + return + } + agentHibernationPhase = .recovering(state) + } + + func failAgentHibernationTermination() { + guard case .recovering(let state) = agentHibernationPhase else { return } + agentHibernationPhase = .terminationFailed(state) + } + func discardAgentHibernationPhaseForPermanentClose() { + onRequestAgentHibernationTerminationRetry = nil agentHibernationPhase = .live } - private func suspendRuntimeForAgentHibernation(reason: String) { + func retryAgentHibernationTermination() { + guard agentHibernationTerminationFailed else { return } + onRequestAgentHibernationTerminationRetry?() + } + + private func suspendRuntimeForAgentHibernation(reason: String) -> Bool { + guard surface.suspendRuntimeSurfaceForAgentHibernation(reason: reason) else { + return false + } unfocus() searchState = nil hostedView.setVisibleInUI(false) TerminalWindowPortalRegistry.detach(hostedView: hostedView) - surface.suspendRuntimeSurfaceForAgentHibernation(reason: reason) requestViewReattach() + return true } @discardableResult @@ -68,8 +123,10 @@ extension TerminalPanel { return .unavailable } let resumeStartupInput = state.agent.resumeStartupInput() + guard surface.prepareAgentHibernationResume(initialInput: resumeStartupInput) else { + return .unavailable + } agentHibernationPhase = .live - surface.prepareAgentHibernationResume(initialInput: resumeStartupInput) requestViewReattach() surface.requestBackgroundSurfaceStartIfNeeded() return .resumed(queuedStartupInput: resumeStartupInput != nil) diff --git a/Sources/Panels/TerminalPanel.swift b/Sources/Panels/TerminalPanel.swift index 201aa3a84bee..8211f212584e 100644 --- a/Sources/Panels/TerminalPanel.swift +++ b/Sources/Panels/TerminalPanel.swift @@ -94,6 +94,7 @@ final class TerminalPanel: Panel, ObservableObject { var onRequestWorkspacePaneFlash: ((WorkspaceAttentionFlashReason) -> Void)? var onRequestAgentHibernationResume: ((Bool) -> Bool)? + var onRequestAgentHibernationTerminationRetry: (() -> Void)? private var cancellables = Set() diff --git a/Sources/Panels/TerminalPanelView.swift b/Sources/Panels/TerminalPanelView.swift index b66669b8cacf..9287b146e4df 100644 --- a/Sources/Panels/TerminalPanelView.swift +++ b/Sources/Panels/TerminalPanelView.swift @@ -42,6 +42,24 @@ struct TerminalPanelView: View { Color(nsColor: appearance.contentBackgroundColor) .frame(maxWidth: .infinity, maxHeight: .infinity) .id("hibernation-terminating-\(panel.id.uuidString)") + case .recovering(let hibernationState): + AgentHibernationPlaceholderView( + state: hibernationState, + appearance: appearance, + mode: AgentHibernationPlaceholderMode.recovering, + onAction: nil + ) + .id("hibernation-termination-recovery-\(panel.id.uuidString)") + case .terminationFailed(let hibernationState): + AgentHibernationPlaceholderView( + state: hibernationState, + appearance: appearance, + mode: AgentHibernationPlaceholderMode.failed, + onAction: { + panel.retryAgentHibernationTermination() + } + ) + .id("hibernation-termination-failed-\(panel.id.uuidString)") case .hibernated(let hibernationState): hibernationBody(hibernationState) } @@ -60,7 +78,8 @@ struct TerminalPanelView: View { AgentHibernationPlaceholderView( state: hibernationState, appearance: appearance, - onResume: onResumeAgentHibernation + mode: AgentHibernationPlaceholderMode.hibernated, + onAction: onResumeAgentHibernation ) .id("hibernated-\(panel.id.uuidString)") .onChange(of: isVisibleInUI) { _, visible in @@ -234,7 +253,42 @@ struct TerminalPanelView: View { private struct AgentHibernationPlaceholderView: View { let state: AgentHibernationPanelState let appearance: PanelAppearance - let onResume: () -> Void + let mode: AgentHibernationPlaceholderMode + let onAction: (() -> Void)? + + private var title: String { + switch mode { + case .hibernated: + String( + localized: "terminal.agentHibernation.title", + defaultValue: "Agent hibernated" + ) + case .recovering: + String( + localized: "terminal.agentHibernation.finishing", + defaultValue: "Finishing agent shutdown" + ) + case .failed: + String( + localized: "terminal.agentHibernation.failed", + defaultValue: "Agent shutdown needs attention" + ) + } + } + + private var actionTitle: String? { + switch mode { + case .hibernated: + String(localized: "terminal.agentHibernation.resume", defaultValue: "Resume") + case .recovering: + nil + case .failed: + String( + localized: "terminal.agentHibernation.retry", + defaultValue: "Retry shutdown" + ) + } + } private var lastActivityText: String { let formatter = RelativeDateTimeFormatter() @@ -244,10 +298,24 @@ private struct AgentHibernationPlaceholderView: View { var body: some View { VStack(spacing: 14) { - CmuxSystemSymbolImage(magnified: "pause.circle", pointSize: 34, weight: .regular) + switch mode { + case .recovering: + ProgressView() + .controlSize(.small) + .accessibilityIdentifier("AgentHibernationTerminationRecoveryProgress") + case .hibernated: + CmuxSystemSymbolImage(magnified: "pause.circle", pointSize: 34, weight: .regular) + .foregroundStyle(.secondary) + case .failed: + CmuxSystemSymbolImage( + magnified: "exclamationmark.triangle", + pointSize: 34, + weight: .regular + ) .foregroundStyle(.secondary) + } VStack(spacing: 4) { - Text(String(localized: "terminal.agentHibernation.title", defaultValue: "Agent hibernated")) + Text(title) .cmuxFont(.headline) Text(state.agentDisplayName) .cmuxFont(.subheadline) @@ -261,12 +329,18 @@ private struct AgentHibernationPlaceholderView: View { .cmuxFont(.caption) .foregroundStyle(.tertiary) } - Button(String(localized: "terminal.agentHibernation.resume", defaultValue: "Resume")) { - onResume() + if let actionTitle, let onAction { + Button(actionTitle) { + onAction() + } + .buttonStyle(.borderedProminent) + .controlSize(.small) + .accessibilityIdentifier( + mode == .failed + ? "AgentHibernationTerminationRetryButton" + : "AgentHibernationResumeButton" + ) } - .buttonStyle(.borderedProminent) - .controlSize(.small) - .accessibilityIdentifier("AgentHibernationResumeButton") } .frame(maxWidth: .infinity, maxHeight: .infinity) .background(Color(nsColor: appearance.contentBackgroundColor)) diff --git a/Sources/RestorableAgentSession.swift b/Sources/RestorableAgentSession.swift index fb9507b1f6aa..bc0987686ee4 100644 --- a/Sources/RestorableAgentSession.swift +++ b/Sources/RestorableAgentSession.swift @@ -913,6 +913,10 @@ struct RestorableAgentSessionIndex: Sendable { let processIdentities: [Int: AgentPIDProcessIdentity] let agentProcessIDs: Set let agentProcessIdentities: [Int: AgentPIDProcessIdentity] + let hibernationPanelProcessIDs: Set + let terminationProcessIDs: Set + let terminationProcessIdentities: [Int: AgentPIDProcessIdentity] + let containsUnrelatedProcess: Bool } enum ProcessDetectedSessionIDSource: Equatable, Sendable { @@ -930,6 +934,12 @@ struct RestorableAgentSessionIndex: Sendable { sessionIDSource: ProcessDetectedSessionIDSource ) + typealias HibernationProcessScope = ( + panelProcessIDs: Set, + terminationProcessIDs: Set, + containsUnrelatedProcess: Bool + ) + private struct SessionKey: Hashable { let kind: RestorableAgentKind let sessionId: String @@ -1062,15 +1072,25 @@ struct RestorableAgentSessionIndex: Sendable { fileManager: FileManager = .default ) -> RestorableAgentSessionIndex { let registry = CmuxVaultAgentRegistry.load(homeDirectory: homeDirectory, fileManager: fileManager) + let processSnapshot = CmuxTopProcessSnapshot.capture(includeProcessDetails: true) let detectedSnapshots = processDetectedSnapshots( registry: registry, - fileManager: fileManager + fileManager: fileManager, + processSnapshot: processSnapshot, + capturedAt: processSnapshot.sampledAt.timeIntervalSince1970 ) + let hibernationProcessScopes = detectedSnapshots.mapValues { detected in + processSnapshot.agentHibernationProcessScope( + panelProcessIDs: detected.processIDs, + agentProcessIDs: detected.agentProcessIDs + ) + } return load( homeDirectory: homeDirectory, fileManager: fileManager, registry: registry, - detectedSnapshots: detectedSnapshots + detectedSnapshots: detectedSnapshots, + hibernationProcessScopes: hibernationProcessScopes ) } @@ -1079,6 +1099,7 @@ struct RestorableAgentSessionIndex: Sendable { fileManager: FileManager, registry: CmuxVaultAgentRegistry, detectedSnapshots: [PanelKey: ProcessDetectedSnapshotEntry], + hibernationProcessScopes: [PanelKey: HibernationProcessScope] = [:], processArgumentsProvider: (Int) -> CmuxTopProcessArguments? = { CmuxTopProcessSnapshot.processArgumentsAndEnvironment(for: $0) }, @@ -1216,7 +1237,14 @@ struct RestorableAgentSessionIndex: Sendable { processIDs: liveProcessID.map { [$0] } ?? [], processIdentities: liveProcessIdentities, agentProcessIDs: liveProcessID.map { [$0] } ?? [], - agentProcessIdentities: liveProcessIdentities + agentProcessIdentities: liveProcessIdentities, + hibernationPanelProcessIDs: liveProcessID.map { [$0] } ?? [], + terminationProcessIDs: liveProcessID.map { [$0] } ?? [], + terminationProcessIdentities: liveProcessIdentities, + // A saved hook PID proves liveness but cannot prove the + // surrounding pane is exclusive. Critical-pressure + // termination requires a fresh process-tree detection. + containsUnrelatedProcess: liveProcessID != nil ) if shouldReplaceHookEntry( existing: hookCandidatesByPanelAndKind[panelKindKey], @@ -1258,11 +1286,26 @@ struct RestorableAgentSessionIndex: Sendable { } } - func processDetectedEntry(snapshot: SessionRestorableAgentSnapshot, lifecycle: AgentHibernationLifecycleState?, updatedAt: TimeInterval, detected: ProcessDetectedSnapshotEntry) -> Entry { - let processIdentities = processIdentities( + func processDetectedEntry( + key: PanelKey, + snapshot: SessionRestorableAgentSnapshot, + lifecycle: AgentHibernationLifecycleState?, + updatedAt: TimeInterval, + detected: ProcessDetectedSnapshotEntry + ) -> Entry { + let processIdentities = Self.processIdentities( for: detected.processIDs, processIdentityProvider: processIdentityProvider ) + let hibernationScope = hibernationProcessScopes[key] ?? ( + detected.processIDs, + detected.agentProcessIDs, + !detected.processIDs.isSubset(of: detected.agentProcessIDs) + ) + let terminationProcessIdentities = Self.processIdentities( + for: hibernationScope.terminationProcessIDs, + processIdentityProvider: processIdentityProvider + ) return Entry( snapshot: snapshot, lifecycle: lifecycle, updatedAt: updatedAt, processLiveness: .running, @@ -1271,7 +1314,11 @@ struct RestorableAgentSessionIndex: Sendable { agentProcessIDs: detected.agentProcessIDs, agentProcessIdentities: processIdentities.filter { detected.agentProcessIDs.contains($0.key) - } + }, + hibernationPanelProcessIDs: hibernationScope.panelProcessIDs, + terminationProcessIDs: hibernationScope.terminationProcessIDs, + terminationProcessIdentities: terminationProcessIdentities, + containsUnrelatedProcess: hibernationScope.containsUnrelatedProcess ) } @@ -1297,7 +1344,7 @@ struct RestorableAgentSessionIndex: Sendable { detected: detected, processIdentityProvider: processIdentityProvider ) { - resolved[key] = processDetectedEntry(snapshot: panelCandidate.snapshot, lifecycle: panelCandidate.lifecycle, updatedAt: panelCandidate.updatedAt, detected: detected) + resolved[key] = processDetectedEntry(key: key, snapshot: panelCandidate.snapshot, lifecycle: panelCandidate.lifecycle, updatedAt: panelCandidate.updatedAt, detected: detected) } else if detected.sessionIDSource == .forkParentFallback, Self.forkParentFallbackMustYield(kind: detected.snapshot.kind, toExisting: resolved[key]) { // A nested fork process inside another agent's pane must not displace @@ -1309,7 +1356,7 @@ struct RestorableAgentSessionIndex: Sendable { // cwd. Prefer the hook-store identity for this stable panel/surface while still carrying // live process evidence for the restored panel. The workspace UUID can rotate during // session restore, but the surface id is intentionally reused on the normal restore path. - resolved[key] = processDetectedEntry(snapshot: panelCandidate.snapshot, lifecycle: panelCandidate.lifecycle, updatedAt: panelCandidate.updatedAt, detected: detected) + resolved[key] = processDetectedEntry(key: key, snapshot: panelCandidate.snapshot, lifecycle: panelCandidate.lifecycle, updatedAt: panelCandidate.updatedAt, detected: detected) } else if let existing = Self.matchingHookEntry( for: detected.snapshot, resolved: resolved[key], @@ -1318,9 +1365,9 @@ struct RestorableAgentSessionIndex: Sendable { SessionKey(kind: detected.snapshot.kind, sessionId: detected.snapshot.sessionId) ] ) { - resolved[key] = processDetectedEntry(snapshot: detected.snapshot, lifecycle: existing.lifecycle, updatedAt: existing.updatedAt, detected: detected) + resolved[key] = processDetectedEntry(key: key, snapshot: detected.snapshot, lifecycle: existing.lifecycle, updatedAt: existing.updatedAt, detected: detected) } else { - resolved[key] = processDetectedEntry(snapshot: detected.snapshot, lifecycle: nil, updatedAt: 0, detected: detected) + resolved[key] = processDetectedEntry(key: key, snapshot: detected.snapshot, lifecycle: nil, updatedAt: 0, detected: detected) } } diff --git a/Sources/Workspace.swift b/Sources/Workspace.swift index afc870b72c2d..a0fee569fa15 100644 --- a/Sources/Workspace.swift +++ b/Sources/Workspace.swift @@ -4750,16 +4750,24 @@ final class Workspace: Identifiable, ObservableObject { return snapshot } + @discardableResult func enterAgentHibernation( panelId: UUID, agent: SessionRestorableAgentSnapshot, lastActivityAt: Date - ) { + ) -> Bool { guard let terminalPanel = panels[panelId] as? TerminalPanel, - !terminalPanel.isAgentHibernated || terminalPanel.isAgentHibernationTerminating else { - return + !terminalPanel.isAgentHibernated || + terminalPanel.isAgentHibernationCommitPending else { + return false + } + guard agent.resumeCommand != nil, + terminalPanel.enterAgentHibernation( + agent: agent, + lastActivityAt: lastActivityAt + ) else { + return false } - guard agent.resumeCommand != nil else { return } restoredAgentSnapshotsByPanelId[panelId] = agent restoredAgentResumeStatesByPanelId[panelId] = .manualResumeAvailable invalidatedRestoredAgentFingerprintsByPanelId.removeValue(forKey: panelId) @@ -4770,7 +4778,7 @@ final class Workspace: Identifiable, ObservableObject { if !keys.isEmpty { refreshTrackedAgentPorts() } - terminalPanel.enterAgentHibernation(agent: agent, lastActivityAt: lastActivityAt) + return true } @discardableResult diff --git a/cmux.xcodeproj/project.pbxproj b/cmux.xcodeproj/project.pbxproj index fc325207a3e7..50c59762361e 100644 --- a/cmux.xcodeproj/project.pbxproj +++ b/cmux.xcodeproj/project.pbxproj @@ -50,6 +50,8 @@ C0DEF0D30000000000000001 /* AgentForkExecutableIdentityResolver.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0DEF0D30000000000000002 /* AgentForkExecutableIdentityResolver.swift */; }; C0DEF0C10000000000000001 /* AgentForkSupport.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0DEF0C10000000000000002 /* AgentForkSupport.swift */; }; C0DEF0D40000000000000001 /* AgentForkTimeoutResumeGate.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0DEF0D40000000000000002 /* AgentForkTimeoutResumeGate.swift */; }; + 8997C00E8997C00E8997C00E /* AgentHibernationController+CommittedTerminationCleanup.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D00E8997D00E8997D00E /* AgentHibernationController+CommittedTerminationCleanup.swift */; }; + 8997C00F8997C00F8997C00F /* AgentHibernationController+CommittedTerminationObservation.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D00F8997D00F8997D00F /* AgentHibernationController+CommittedTerminationObservation.swift */; }; F65760100000000000000001 /* AgentHibernationController+Confirmation.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760100000000000000002 /* AgentHibernationController+Confirmation.swift */; }; F65760110000000000000001 /* AgentHibernationController+InFlightTeardown.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760110000000000000002 /* AgentHibernationController+InFlightTeardown.swift */; }; 8997C0018997C0018997C001 /* AgentHibernationController+MemoryPressure.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D0018997D0018997D001 /* AgentHibernationController+MemoryPressure.swift */; }; @@ -57,8 +59,12 @@ F65760120000000000000001 /* AgentHibernationController+PostSnapshotValidationIndexTask.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760120000000000000002 /* AgentHibernationController+PostSnapshotValidationIndexTask.swift */; }; F65760260000000000000001 /* AgentHibernationController+PostTeardownRestoreCancellationState.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760260000000000000002 /* AgentHibernationController+PostTeardownRestoreCancellationState.swift */; }; F65760130000000000000001 /* AgentHibernationController+PostTeardownRestoreTask.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760130000000000000002 /* AgentHibernationController+PostTeardownRestoreTask.swift */; }; + 8997C0088997C0088997C008 /* AgentHibernationController+ProcessExitObservation.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D0088997D0088997D008 /* AgentHibernationController+ProcessExitObservation.swift */; }; + 8997C00A8997C00A8997C00A /* AgentHibernationController+ProcessExitWaiting.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D00A8997D00A8997D00A /* AgentHibernationController+ProcessExitWaiting.swift */; }; + 8997C00D8997C00D8997C00D /* AgentHibernationController+ProcessSignaling.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D00D8997D00D8997D00D /* AgentHibernationController+ProcessSignaling.swift */; }; 8997C0038997C0038997C003 /* AgentHibernationController+ProcessTermination.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D0038997D0038997D003 /* AgentHibernationController+ProcessTermination.swift */; }; F65760030000000000000001 /* AgentHibernationController+Records.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760030000000000000002 /* AgentHibernationController+Records.swift */; }; + 8997C0108997C0108997C010 /* AgentHibernationController+ScopedProcessTerminationResult.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D0108997D0108997D010 /* AgentHibernationController+ScopedProcessTerminationResult.swift */; }; F65760140000000000000001 /* AgentHibernationController+TailFingerprintSample.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760140000000000000002 /* AgentHibernationController+TailFingerprintSample.swift */; }; F65760060000000000000001 /* AgentHibernationController+Teardown.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760060000000000000002 /* AgentHibernationController+Teardown.swift */; }; 8997E0018997E0018997E001 /* AgentHibernationController+TeardownValidation.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997F0018997F0018997F001 /* AgentHibernationController+TeardownValidation.swift */; }; @@ -68,12 +74,19 @@ D36A00030000000000000003 /* AgentHibernationLifecycleState.swift in Sources */ = {isa = PBXBuildFile; fileRef = D36A00030000000000000002 /* AgentHibernationLifecycleState.swift */; }; 8997C0028997C0028997C002 /* AgentHibernationMemoryPressureResponder.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D0028997D0028997D002 /* AgentHibernationMemoryPressureResponder.swift */; }; 8997E0028997E0028997E002 /* AgentHibernationPanelPhase.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997F0028997F0028997F002 /* AgentHibernationPanelPhase.swift */; }; + 8997C00C8997C00C8997C00C /* AgentHibernationPlaceholderMode.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D00C8997D00C8997D00C /* AgentHibernationPlaceholderMode.swift */; }; F65760160000000000000001 /* AgentHibernationPlanner.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760160000000000000002 /* AgentHibernationPlanner.swift */; }; F65760170000000000000001 /* AgentHibernationPlannerInput.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760170000000000000002 /* AgentHibernationPlannerInput.swift */; }; F65760010000000000000001 /* AgentHibernationPlannerSwiftTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760010000000000000002 /* AgentHibernationPlannerSwiftTests.swift */; }; D72260010000000000000001 /* AgentHibernationPortalVisibilityTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D72260010000000000000002 /* AgentHibernationPortalVisibilityTests.swift */; }; + 8997C0098997C0098997C009 /* AgentHibernationProcessExitCompletion.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D0098997D0098997D009 /* AgentHibernationProcessExitCompletion.swift */; }; + 8997C0118997C0118997C011 /* AgentHibernationProcessExitEpoch.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D0118997D0118997D011 /* AgentHibernationProcessExitEpoch.swift */; }; + 8997C0138997C0138997C013 /* AgentHibernationProcessSignalBoundaryTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D0138997D0138997D013 /* AgentHibernationProcessSignalBoundaryTests.swift */; }; + 8997C00B8997C00B8997C00B /* AgentHibernationProcessSnapshotCoordinator.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D00B8997D00B8997D00B /* AgentHibernationProcessSnapshotCoordinator.swift */; }; + 8997C0128997C0128997C012 /* AgentHibernationProcessSnapshotCoordinatorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D0128997D0128997D012 /* AgentHibernationProcessSnapshotCoordinatorTests.swift */; }; 8997C0048997C0048997C004 /* AgentHibernationProcessTerminationTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D0048997D0048997D004 /* AgentHibernationProcessTerminationTests.swift */; }; F65760240000000000000001 /* AgentHibernationRestoreMonitorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760240000000000000002 /* AgentHibernationRestoreMonitorTests.swift */; }; + 8997C0078997C0078997C007 /* AgentHibernationTerminationFailureTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D0078997D0078997D007 /* AgentHibernationTerminationFailureTests.swift */; }; F65760020000000000000001 /* AgentHibernationTestHelpers.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760020000000000000002 /* AgentHibernationTestHelpers.swift */; }; D36A00020000000000000001 /* AgentHibernationTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D36A00020000000000000002 /* AgentHibernationTests.swift */; }; 8997C0058997C0058997C005 /* AgentHibernationTrackingLifecycleTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D0058997D0058997D005 /* AgentHibernationTrackingLifecycleTests.swift */; }; @@ -2547,6 +2560,8 @@ C0DE71B10000000000000001 /* AppDelegate+AgentChatNotifications.swift in Sources C0DEF0D30000000000000002 /* AgentForkExecutableIdentityResolver.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentForkExecutableIdentityResolver.swift; sourceTree = ""; }; C0DEF0C10000000000000002 /* AgentForkSupport.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentForkSupport.swift; sourceTree = ""; }; C0DEF0D40000000000000002 /* AgentForkTimeoutResumeGate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentForkTimeoutResumeGate.swift; sourceTree = ""; }; + 8997D00E8997D00E8997D00E /* AgentHibernationController+CommittedTerminationCleanup.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+CommittedTerminationCleanup.swift"; sourceTree = ""; }; + 8997D00F8997D00F8997D00F /* AgentHibernationController+CommittedTerminationObservation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+CommittedTerminationObservation.swift"; sourceTree = ""; }; F65760100000000000000002 /* AgentHibernationController+Confirmation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+Confirmation.swift"; sourceTree = ""; }; F65760110000000000000002 /* AgentHibernationController+InFlightTeardown.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+InFlightTeardown.swift"; sourceTree = ""; }; 8997D0018997D0018997D001 /* AgentHibernationController+MemoryPressure.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+MemoryPressure.swift"; sourceTree = ""; }; @@ -2554,8 +2569,12 @@ C0DE71B10000000000000001 /* AppDelegate+AgentChatNotifications.swift in Sources F65760120000000000000002 /* AgentHibernationController+PostSnapshotValidationIndexTask.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+PostSnapshotValidationIndexTask.swift"; sourceTree = ""; }; F65760260000000000000002 /* AgentHibernationController+PostTeardownRestoreCancellationState.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+PostTeardownRestoreCancellationState.swift"; sourceTree = ""; }; F65760130000000000000002 /* AgentHibernationController+PostTeardownRestoreTask.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+PostTeardownRestoreTask.swift"; sourceTree = ""; }; + 8997D0088997D0088997D008 /* AgentHibernationController+ProcessExitObservation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+ProcessExitObservation.swift"; sourceTree = ""; }; + 8997D00A8997D00A8997D00A /* AgentHibernationController+ProcessExitWaiting.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+ProcessExitWaiting.swift"; sourceTree = ""; }; + 8997D00D8997D00D8997D00D /* AgentHibernationController+ProcessSignaling.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+ProcessSignaling.swift"; sourceTree = ""; }; 8997D0038997D0038997D003 /* AgentHibernationController+ProcessTermination.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+ProcessTermination.swift"; sourceTree = ""; }; F65760030000000000000002 /* AgentHibernationController+Records.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+Records.swift"; sourceTree = ""; }; + 8997D0108997D0108997D010 /* AgentHibernationController+ScopedProcessTerminationResult.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+ScopedProcessTerminationResult.swift"; sourceTree = ""; }; F65760140000000000000002 /* AgentHibernationController+TailFingerprintSample.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+TailFingerprintSample.swift"; sourceTree = ""; }; F65760060000000000000002 /* AgentHibernationController+Teardown.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+Teardown.swift"; sourceTree = ""; }; 8997F0018997F0018997F001 /* AgentHibernationController+TeardownValidation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+TeardownValidation.swift"; sourceTree = ""; }; @@ -2564,12 +2583,19 @@ C0DE71B10000000000000001 /* AppDelegate+AgentChatNotifications.swift in Sources D36A00030000000000000002 /* AgentHibernationLifecycleState.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernation/AgentHibernationLifecycleState.swift; sourceTree = ""; }; 8997D0028997D0028997D002 /* AgentHibernationMemoryPressureResponder.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = App/AgentHibernationMemoryPressureResponder.swift; sourceTree = ""; }; 8997F0028997F0028997F002 /* AgentHibernationPanelPhase.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernation/AgentHibernationPanelPhase.swift; sourceTree = ""; }; + 8997D00C8997D00C8997D00C /* AgentHibernationPlaceholderMode.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Panels/AgentHibernationPlaceholderMode.swift; sourceTree = ""; }; F65760160000000000000002 /* AgentHibernationPlanner.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationPlanner.swift"; sourceTree = ""; }; F65760170000000000000002 /* AgentHibernationPlannerInput.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationPlannerInput.swift"; sourceTree = ""; }; F65760010000000000000002 /* AgentHibernationPlannerSwiftTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernationPlannerSwiftTests.swift; sourceTree = ""; }; D72260010000000000000002 /* AgentHibernationPortalVisibilityTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernationPortalVisibilityTests.swift; sourceTree = ""; }; + 8997D0098997D0098997D009 /* AgentHibernationProcessExitCompletion.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationProcessExitCompletion.swift"; sourceTree = ""; }; + 8997D0118997D0118997D011 /* AgentHibernationProcessExitEpoch.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationProcessExitEpoch.swift"; sourceTree = ""; }; + 8997D0138997D0138997D013 /* AgentHibernationProcessSignalBoundaryTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernationProcessSignalBoundaryTests.swift; sourceTree = ""; }; + 8997D00B8997D00B8997D00B /* AgentHibernationProcessSnapshotCoordinator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationProcessSnapshotCoordinator.swift"; sourceTree = ""; }; + 8997D0128997D0128997D012 /* AgentHibernationProcessSnapshotCoordinatorTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernationProcessSnapshotCoordinatorTests.swift; sourceTree = ""; }; 8997D0048997D0048997D004 /* AgentHibernationProcessTerminationTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernationProcessTerminationTests.swift; sourceTree = ""; }; F65760240000000000000002 /* AgentHibernationRestoreMonitorTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernationRestoreMonitorTests.swift; sourceTree = ""; }; + 8997D0078997D0078997D007 /* AgentHibernationTerminationFailureTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernationTerminationFailureTests.swift; sourceTree = ""; }; F65760020000000000000002 /* AgentHibernationTestHelpers.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernationTestHelpers.swift; sourceTree = ""; }; D36A00020000000000000002 /* AgentHibernationTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernationTests.swift; sourceTree = ""; }; 8997D0058997D0058997D005 /* AgentHibernationTrackingLifecycleTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernationTrackingLifecycleTests.swift; sourceTree = ""; }; @@ -5313,11 +5339,20 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = E9014000000000000000000C /* AgentSessionAutoRetrySettings.swift */, D5671002D5671002D5671002 /* TerminalScrollSpeedSettings.swift */, F65760100000000000000002 /* AgentHibernationController+Confirmation.swift */, + 8997D00E8997D00E8997D00E /* AgentHibernationController+CommittedTerminationCleanup.swift */, + 8997D00F8997D00F8997D00F /* AgentHibernationController+CommittedTerminationObservation.swift */, F65760110000000000000002 /* AgentHibernationController+InFlightTeardown.swift */, F65760120000000000000002 /* AgentHibernationController+PostSnapshotValidationIndexTask.swift */, F65760130000000000000002 /* AgentHibernationController+PostTeardownRestoreTask.swift */, F65760260000000000000002 /* AgentHibernationController+PostTeardownRestoreCancellationState.swift */, 8997D0038997D0038997D003 /* AgentHibernationController+ProcessTermination.swift */, + 8997D0088997D0088997D008 /* AgentHibernationController+ProcessExitObservation.swift */, + 8997D00A8997D00A8997D00A /* AgentHibernationController+ProcessExitWaiting.swift */, + 8997D00D8997D00D8997D00D /* AgentHibernationController+ProcessSignaling.swift */, + 8997D0108997D0108997D010 /* AgentHibernationController+ScopedProcessTerminationResult.swift */, + 8997D0098997D0098997D009 /* AgentHibernationProcessExitCompletion.swift */, + 8997D0118997D0118997D011 /* AgentHibernationProcessExitEpoch.swift */, + 8997D00B8997D00B8997D00B /* AgentHibernationProcessSnapshotCoordinator.swift */, 8997D0068997D0068997D006 /* AgentHibernationController+PanelLifecycle.swift */, 8997F0018997F0018997F001 /* AgentHibernationController+TeardownValidation.swift */, F65760030000000000000002 /* AgentHibernationController+Records.swift */, @@ -6159,6 +6194,7 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = D7632B14A1B2C3D4E5F60718 /* WKWebView+CmuxPrintOperation.swift */, A500RG00 /* ReactGrab.swift */, A5001413 /* TerminalPanelView.swift */, + 8997D00C8997D00C8997D00C /* AgentHibernationPlaceholderMode.swift */, C71500010000000000000001 /* SessionContentWidthModifier.swift */, C71500030000000000000001 /* SessionContentWidthPresentation.swift */, C71500050000000000000001 /* TerminalPaneBackgroundView.swift */, @@ -6827,8 +6863,11 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = A9E040000000000000000001 /* CodexAppServerSessionTests.swift */, F65760010000000000000002 /* AgentHibernationPlannerSwiftTests.swift */, 8997D0048997D0048997D004 /* AgentHibernationProcessTerminationTests.swift */, + 8997D0138997D0138997D013 /* AgentHibernationProcessSignalBoundaryTests.swift */, + 8997D0128997D0128997D012 /* AgentHibernationProcessSnapshotCoordinatorTests.swift */, 8997D0058997D0058997D005 /* AgentHibernationTrackingLifecycleTests.swift */, F65760240000000000000002 /* AgentHibernationRestoreMonitorTests.swift */, + 8997D0078997D0078997D007 /* AgentHibernationTerminationFailureTests.swift */, F65760040000000000000002 /* AgentHibernationTranscriptGuardTests.swift */, F65760080000000000000002 /* AgentHibernationTranscriptGuardScanTests.swift */, F65760250000000000000002 /* AgentHibernationTranscriptSnapshotRaceTests.swift */, @@ -7897,6 +7936,8 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = C0DEF0D30000000000000001 /* AgentForkExecutableIdentityResolver.swift in Sources */, C0DEF0C10000000000000001 /* AgentForkSupport.swift in Sources */, C0DEF0D40000000000000001 /* AgentForkTimeoutResumeGate.swift in Sources */, + 8997C00E8997C00E8997C00E /* AgentHibernationController+CommittedTerminationCleanup.swift in Sources */, + 8997C00F8997C00F8997C00F /* AgentHibernationController+CommittedTerminationObservation.swift in Sources */, F65760100000000000000001 /* AgentHibernationController+Confirmation.swift in Sources */, F65760110000000000000001 /* AgentHibernationController+InFlightTeardown.swift in Sources */, 8997C0018997C0018997C001 /* AgentHibernationController+MemoryPressure.swift in Sources */, @@ -7904,8 +7945,12 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = F65760120000000000000001 /* AgentHibernationController+PostSnapshotValidationIndexTask.swift in Sources */, F65760260000000000000001 /* AgentHibernationController+PostTeardownRestoreCancellationState.swift in Sources */, F65760130000000000000001 /* AgentHibernationController+PostTeardownRestoreTask.swift in Sources */, + 8997C0088997C0088997C008 /* AgentHibernationController+ProcessExitObservation.swift in Sources */, + 8997C00A8997C00A8997C00A /* AgentHibernationController+ProcessExitWaiting.swift in Sources */, + 8997C00D8997C00D8997C00D /* AgentHibernationController+ProcessSignaling.swift in Sources */, 8997C0038997C0038997C003 /* AgentHibernationController+ProcessTermination.swift in Sources */, F65760030000000000000001 /* AgentHibernationController+Records.swift in Sources */, + 8997C0108997C0108997C010 /* AgentHibernationController+ScopedProcessTerminationResult.swift in Sources */, F65760140000000000000001 /* AgentHibernationController+TailFingerprintSample.swift in Sources */, F65760060000000000000001 /* AgentHibernationController+Teardown.swift in Sources */, 8997E0018997E0018997E001 /* AgentHibernationController+TeardownValidation.swift in Sources */, @@ -7914,8 +7959,12 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = D36A00030000000000000001 /* AgentHibernationLifecycleState.swift in Sources */, 8997C0028997C0028997C002 /* AgentHibernationMemoryPressureResponder.swift in Sources */, 8997E0028997E0028997E002 /* AgentHibernationPanelPhase.swift in Sources */, + 8997C00C8997C00C8997C00C /* AgentHibernationPlaceholderMode.swift in Sources */, F65760160000000000000001 /* AgentHibernationPlanner.swift in Sources */, F65760170000000000000001 /* AgentHibernationPlannerInput.swift in Sources */, + 8997C0098997C0098997C009 /* AgentHibernationProcessExitCompletion.swift in Sources */, + 8997C0118997C0118997C011 /* AgentHibernationProcessExitEpoch.swift in Sources */, + 8997C00B8997C00B8997C00B /* AgentHibernationProcessSnapshotCoordinator.swift in Sources */, F65760090000000000000001 /* AgentHibernationTranscriptGuard+ClaudeWorkflow.swift in Sources */, F65760220000000000000001 /* AgentHibernationTranscriptGuard+PostTeardownRestore.swift in Sources */, F65760230000000000000001 /* AgentHibernationTranscriptGuard+StableFileComparison.swift in Sources */, @@ -9591,8 +9640,11 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = A9E020000000000000000005 /* AgentExecutableResolverTests.swift in Sources */, F65760010000000000000001 /* AgentHibernationPlannerSwiftTests.swift in Sources */, D72260010000000000000001 /* AgentHibernationPortalVisibilityTests.swift in Sources */, + 8997C0138997C0138997C013 /* AgentHibernationProcessSignalBoundaryTests.swift in Sources */, + 8997C0128997C0128997C012 /* AgentHibernationProcessSnapshotCoordinatorTests.swift in Sources */, 8997C0048997C0048997C004 /* AgentHibernationProcessTerminationTests.swift in Sources */, F65760240000000000000001 /* AgentHibernationRestoreMonitorTests.swift in Sources */, + 8997C0078997C0078997C007 /* AgentHibernationTerminationFailureTests.swift in Sources */, F65760020000000000000001 /* AgentHibernationTestHelpers.swift in Sources */, D36A00020000000000000001 /* AgentHibernationTests.swift in Sources */, 8997C0058997C0058997C005 /* AgentHibernationTrackingLifecycleTests.swift in Sources */, diff --git a/cmuxTests/AgentHibernationPlannerSwiftTests.swift b/cmuxTests/AgentHibernationPlannerSwiftTests.swift index 51e482463eed..919ae3990b79 100644 --- a/cmuxTests/AgentHibernationPlannerSwiftTests.swift +++ b/cmuxTests/AgentHibernationPlannerSwiftTests.swift @@ -134,6 +134,8 @@ struct AgentHibernationPlannerSwiftTests { lastActivityAt: 0, isProtected: false, hasLiveProcess: false, + containsUnrelatedProcess: false, + panelProcessIDs: [], processIDs: [], processIdentities: [:] ) @@ -466,6 +468,8 @@ struct AgentHibernationPlannerSwiftTests { lastActivityAt: 100, isProtected: false, hasLiveProcess: false, + containsUnrelatedProcess: false, + panelProcessIDs: [], processIDs: [], processIdentities: [:] ) diff --git a/cmuxTests/AgentHibernationProcessSignalBoundaryTests.swift b/cmuxTests/AgentHibernationProcessSignalBoundaryTests.swift new file mode 100644 index 000000000000..c873d4c270bb --- /dev/null +++ b/cmuxTests/AgentHibernationProcessSignalBoundaryTests.swift @@ -0,0 +1,343 @@ +import Darwin +import Foundation +import os +import Testing + +#if canImport(cmux_DEV) +@testable import cmux_DEV +#elseif canImport(cmux) +@testable import cmux +#endif + +struct AgentHibernationProcessSignalBoundaryTests { + private nonisolated static let signalScopeKey = AgentHibernationPanelKey( + workspaceId: UUID(), + panelId: UUID() + ) + private nonisolated static let signalScopeArguments = CmuxTopProcessArguments( + arguments: ["/usr/bin/test-agent"], + environment: [ + "CMUX_WORKSPACE_ID": signalScopeKey.workspaceId.uuidString, + "CMUX_SURFACE_ID": signalScopeKey.panelId.uuidString, + ] + ) + + @MainActor + @Test + func rejectsScopeOrTTYDriftWithoutSendingSignals() async { + let firstIdentity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let secondIdentity = AgentPIDProcessIdentity( + pid: 202, + startSeconds: 20, + startMicroseconds: 2 + ) + let signaledTargets = OSAllocatedUnfairLock(initialState: [pid_t]()) + let wrongScopeArguments = CmuxTopProcessArguments( + arguments: ["/usr/bin/test-agent"], + environment: [ + "CMUX_WORKSPACE_ID": UUID().uuidString, + "CMUX_SURFACE_ID": UUID().uuidString, + ] + ) + + let wrongScopeResult = await AgentHibernationController + .terminateScopedProcessesForHibernation( + [ + .init( + processID: 101, + processIdentity: firstIdentity, + processGroupID: 101, + ttyDevice: 123 + ), + ], + processScopeKey: Self.signalScopeKey, + currentProcessID: 999, + currentProcessGroupID: 999, + processIdentityProvider: { _ in firstIdentity }, + processGroupProvider: { _ in 101 }, + processArgumentsProvider: { _ in wrongScopeArguments }, + processTTYDeviceProvider: { _ in 123 }, + signalErrorProvider: { target, _ in + signaledTargets.withLock { $0.append(target) } + return nil + } + ) + let ttyDriftResult = await AgentHibernationController + .terminateScopedProcessesForHibernation( + [ + .init( + processID: 101, + processIdentity: firstIdentity, + processGroupID: 101, + ttyDevice: 123 + ), + .init( + processID: 202, + processIdentity: secondIdentity, + processGroupID: 202, + ttyDevice: 123 + ), + ], + processScopeKey: Self.signalScopeKey, + currentProcessID: 999, + currentProcessGroupID: 999, + processIdentityProvider: { pid in + pid == 101 ? firstIdentity : secondIdentity + }, + processGroupProvider: { $0 }, + processArgumentsProvider: { _ in Self.signalScopeArguments }, + processTTYDeviceProvider: { pid in + pid == 101 ? 123 : 456 + }, + signalErrorProvider: { target, _ in + signaledTargets.withLock { $0.append(target) } + return nil + } + ) + + #expect(wrongScopeResult == .rejected) + #expect(ttyDriftResult == .rejected) + #expect(signaledTargets.withLock { $0 }.isEmpty) + } + + @MainActor + @Test + func rejectsTTYChangeBeforeFinalCommit() async { + let identity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let ttyProbeCount = OSAllocatedUnfairLock(initialState: 0) + let signaledTargets = OSAllocatedUnfairLock(initialState: [pid_t]()) + + let result = await AgentHibernationController + .terminateScopedProcessesForHibernation( + [ + .init( + processID: 101, + processIdentity: identity, + processGroupID: 101, + ttyDevice: 123 + ), + ], + processScopeKey: Self.signalScopeKey, + currentProcessID: 999, + currentProcessGroupID: 999, + processIdentityProvider: { _ in identity }, + processGroupProvider: { _ in 101 }, + processArgumentsProvider: { _ in Self.signalScopeArguments }, + processTTYDeviceProvider: { _ in + let probe = ttyProbeCount.withLock { + $0 += 1 + return $0 + } + return probe == 1 ? 123 : 456 + }, + signalErrorProvider: { target, _ in + signaledTargets.withLock { $0.append(target) } + return nil + } + ) + + #expect(result == .rejected) + #expect(ttyProbeCount.withLock { $0 } == 2) + #expect(signaledTargets.withLock { $0 }.isEmpty) + } + + @MainActor + @Test + func rejectsMultipleRecordedTTYsBeforeSendingSignals() async { + let firstIdentity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let secondIdentity = AgentPIDProcessIdentity( + pid: 202, + startSeconds: 20, + startMicroseconds: 2 + ) + let didProbe = OSAllocatedUnfairLock(initialState: false) + let signaledTargets = OSAllocatedUnfairLock(initialState: [pid_t]()) + + let result = await AgentHibernationController + .terminateScopedProcessesForHibernation( + [ + .init( + processID: 101, + processIdentity: firstIdentity, + processGroupID: 101, + ttyDevice: 123 + ), + .init( + processID: 202, + processIdentity: secondIdentity, + processGroupID: 202, + ttyDevice: 456 + ), + ], + processScopeKey: Self.signalScopeKey, + currentProcessID: 999, + currentProcessGroupID: 999, + processIdentityProvider: { _ in + didProbe.withLock { $0 = true } + return firstIdentity + }, + signalErrorProvider: { target, _ in + signaledTargets.withLock { $0.append(target) } + return nil + } + ) + + #expect(result == .rejected) + #expect(didProbe.withLock { $0 } == false) + #expect(signaledTargets.withLock { $0 }.isEmpty) + } + + @MainActor + @Test + func rejectsUnboundedSignalAuthorityBeforeKernelProbes() async { + let processCount = + AgentHibernationController.maximumScopedProcessTerminationCount + 1 + let terminations = (1...processCount).map { offset in + let processID = 100 + offset + return AgentHibernationController.ScopedProcessTermination( + processID: processID, + processIdentity: .init( + pid: pid_t(processID), + startSeconds: Int64(processID), + startMicroseconds: 1 + ), + processGroupID: pid_t(processID), + ttyDevice: 123 + ) + } + let didProbe = OSAllocatedUnfairLock(initialState: false) + let signaledTargets = OSAllocatedUnfairLock(initialState: [pid_t]()) + + let result = await AgentHibernationController + .terminateScopedProcessesForHibernation( + terminations, + processScopeKey: Self.signalScopeKey, + processIdentityProvider: { _ in + didProbe.withLock { $0 = true } + return nil + }, + signalErrorProvider: { target, _ in + signaledTargets.withLock { $0.append(target) } + return nil + } + ) + + #expect(result == .rejected) + #expect(didProbe.withLock { $0 } == false) + #expect(signaledTargets.withLock { $0 }.isEmpty) + } + + @Test + func rejectsUnboundedTerminationScopeBeforeKernelProbes() { + let processCount = + AgentHibernationController.maximumScopedProcessTerminationCount + 1 + let processIDs = Set(1...processCount) + let identities = Dictionary( + uniqueKeysWithValues: processIDs.map { processID in + ( + processID, + AgentPIDProcessIdentity( + pid: pid_t(processID), + startSeconds: Int64(processID), + startMicroseconds: 1 + ) + ) + } + ) + let didProbe = OSAllocatedUnfairLock(initialState: false) + + let terminations = AgentHibernationController + .validatedScopedProcessTerminations( + for: .init( + key: Self.signalScopeKey, + processIDs: processIDs, + processIdentities: identities + ), + processIdentityProvider: { _ in + didProbe.withLock { $0 = true } + return nil + }, + processGroupProvider: { _ in + didProbe.withLock { $0 = true } + return 0 + } + ) + + #expect(terminations == nil) + #expect(didProbe.withLock { $0 } == false) + } + + @Test + func boundsLateGenerationRefreshChurn() async { + let rootIdentity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let refreshCount = OSAllocatedUnfairLock(initialState: 0) + let waitedEpochCount = OSAllocatedUnfairLock(initialState: 0) + + let didExit = await AgentHibernationController + .waitForScopedProcessGenerationsToExitWithoutTimeout( + [ + .init( + processID: 101, + processIdentity: rootIdentity, + processGroupID: 101, + ttyDevice: 123 + ), + ], + processScopeKey: Self.signalScopeKey, + waitForExactEpoch: { _ in + waitedEpochCount.withLock { $0 += 1 } + return true + }, + nextEpochProvider: { processGroupLeaders, scopeKey, ttyDevice, _ in + #expect(scopeKey == Self.signalScopeKey) + #expect(ttyDevice == 123) + let refresh = refreshCount.withLock { + $0 += 1 + return $0 + } + let processID = 200 + refresh + return AgentHibernationProcessExitEpoch( + terminations: [ + .init( + processID: processID, + processIdentity: .init( + pid: pid_t(processID), + startSeconds: Int64(processID), + startMicroseconds: 1 + ), + processGroupID: 101, + ttyDevice: 123 + ), + ], + processGroupLeaders: processGroupLeaders + ) + } + ) + + #expect(didExit == false) + #expect( + refreshCount.withLock { $0 } == + AgentHibernationController.maximumProcessExitEpochRefreshCount + ) + #expect( + waitedEpochCount.withLock { $0 } == + AgentHibernationController.maximumProcessExitEpochRefreshCount + 1 + ) + } +} diff --git a/cmuxTests/AgentHibernationProcessSnapshotCoordinatorTests.swift b/cmuxTests/AgentHibernationProcessSnapshotCoordinatorTests.swift new file mode 100644 index 000000000000..c564d26702c6 --- /dev/null +++ b/cmuxTests/AgentHibernationProcessSnapshotCoordinatorTests.swift @@ -0,0 +1,156 @@ +import Darwin +import Foundation +import os +import Testing + +#if canImport(cmux_DEV) +@testable import cmux_DEV +#elseif canImport(cmux) +@testable import cmux +#endif + +@Suite(.serialized) +struct AgentHibernationProcessSnapshotCoordinatorTests { + @Test + func coalescesOneQueuedRefreshEpoch() async throws { + let captureScheduled = AsyncStream.makeStream() + let allowCapture = AsyncStream.makeStream() + let captureCount = OSAllocatedUnfairLock(initialState: 0) + let snapshot = CmuxTopProcessSnapshot( + processes: [], + sampledAt: .now, + includesProcessDetails: false + ) + let coordinator = AgentHibernationProcessSnapshotCoordinator( + beforeCapture: { + captureScheduled.continuation.yield() + for await _ in allowCapture.stream { return } + }, + captureSnapshot: { + captureCount.withLock { $0 += 1 } + return snapshot + } + ) + let first = Task { await coordinator.nextSnapshot() } + var captureScheduledIterator = captureScheduled.stream.makeAsyncIterator() + _ = await captureScheduledIterator.next() + let second = Task { await coordinator.nextSnapshot() } + await Task.yield() + + allowCapture.continuation.yield() + allowCapture.continuation.finish() + let firstSnapshot = try #require(await first.value) + let secondSnapshot = try #require(await second.value) + + #expect(firstSnapshot === snapshot) + #expect(secondSnapshot === snapshot) + #expect(captureCount.withLock { $0 } == 1) + captureScheduled.continuation.finish() + } + + @Test + func cancelledWaiterDoesNotAwaitOrDuplicateCapture() async throws { + let captureScheduled = AsyncStream.makeStream() + let allowCapture = AsyncStream.makeStream() + let captureCount = OSAllocatedUnfairLock(initialState: 0) + let snapshot = CmuxTopProcessSnapshot( + processes: [], + sampledAt: .now, + includesProcessDetails: false + ) + let coordinator = AgentHibernationProcessSnapshotCoordinator( + beforeCapture: { + captureScheduled.continuation.yield() + for await _ in allowCapture.stream { return } + }, + captureSnapshot: { + captureCount.withLock { $0 += 1 } + return snapshot + } + ) + let cancelledRequest = Task { await coordinator.nextSnapshot() } + var captureScheduledIterator = captureScheduled.stream.makeAsyncIterator() + _ = await captureScheduledIterator.next() + + cancelledRequest.cancel() + #expect(await cancelledRequest.value == nil) + #expect(captureCount.withLock { $0 } == 0) + + allowCapture.continuation.yield() + allowCapture.continuation.finish() + let nextSnapshot = try #require(await coordinator.nextSnapshot()) + + #expect(nextSnapshot === snapshot) + #expect(captureCount.withLock { $0 } == 1) + captureScheduled.continuation.finish() + } + + @Test + func rejectsLateFanoutBeforePerCandidateProbes() async { + let scopeKey = AgentHibernationPanelKey( + workspaceId: UUID(), + panelId: UUID() + ) + let ttyDevice = Int64(123) + let processGroupID = 101 + let processes = (101...133).map { processID in + CmuxTopProcessInfo( + pid: processID, + parentPID: 1, + name: "test", + path: nil, + ttyDevice: ttyDevice, + cmuxWorkspaceID: nil, + cmuxSurfaceID: nil, + cmuxAttributionReason: nil, + processGroupID: processGroupID, + terminalProcessGroupID: processGroupID, + cpuPercent: 0, + residentBytes: 0, + virtualBytes: 0, + threadCount: 1 + ) + } + let snapshot = CmuxTopProcessSnapshot( + processes: processes, + sampledAt: .now, + includesProcessDetails: false, + includesCMUXScope: false + ) + let leaderIdentity = AgentPIDProcessIdentity( + pid: pid_t(processGroupID), + startSeconds: 10, + startMicroseconds: 1 + ) + let identityProbeIDs = OSAllocatedUnfairLock(initialState: [pid_t]()) + let argumentProbeCount = OSAllocatedUnfairLock(initialState: 0) + let processGroupProbeCount = OSAllocatedUnfairLock(initialState: 0) + let coordinator = AgentHibernationProcessSnapshotCoordinator( + captureSnapshot: { snapshot }, + processArgumentsProvider: { _ in + argumentProbeCount.withLock { $0 += 1 } + return nil + }, + processIdentityProvider: { processID in + identityProbeIDs.withLock { $0.append(processID) } + return processID == pid_t(processGroupID) ? leaderIdentity : nil + }, + processGroupProvider: { _ in + processGroupProbeCount.withLock { $0 += 1 } + return pid_t(processGroupID) + } + ) + + let epoch = await coordinator.refreshedExitEpoch( + processGroupLeaders: [pid_t(processGroupID): leaderIdentity], + processScopeKey: scopeKey, + ttyDevice: ttyDevice, + excluding: [] + ) + + #expect(epoch == nil) + #expect(identityProbeIDs.withLock { $0 } == [pid_t(processGroupID)]) + #expect(argumentProbeCount.withLock { $0 } == 0) + #expect(processGroupProbeCount.withLock { $0 } == 0) + } +} diff --git a/cmuxTests/AgentHibernationProcessTerminationTests.swift b/cmuxTests/AgentHibernationProcessTerminationTests.swift index c74d0522ce92..bc856c55634b 100644 --- a/cmuxTests/AgentHibernationProcessTerminationTests.swift +++ b/cmuxTests/AgentHibernationProcessTerminationTests.swift @@ -1,5 +1,6 @@ import Darwin import Foundation +import os import Testing #if canImport(cmux_DEV) @@ -9,6 +10,18 @@ import Testing #endif struct AgentHibernationProcessTerminationTests { + private nonisolated static let signalScopeKey = AgentHibernationPanelKey( + workspaceId: UUID(), + panelId: UUID() + ) + private nonisolated static let signalScopeArguments = CmuxTopProcessArguments( + arguments: ["/usr/bin/test-agent"], + environment: [ + "CMUX_WORKSPACE_ID": signalScopeKey.workspaceId.uuidString, + "CMUX_SURFACE_ID": signalScopeKey.panelId.uuidString, + ] + ) + private actor CompletionRecorder { private(set) var value: Bool? @@ -18,7 +31,415 @@ struct AgentHibernationProcessTerminationTests { } @Test - func validatesExactProcessGenerationAndCmuxScope() throws { + func processScopeRejectsPanelScopedSiblingOutsideAgentSubtree() { + let workspaceID = UUID() + let panelID = UUID() + let ttyDevice = Int64(0x123) + let process: (Int, Int, Bool) -> CmuxTopProcessInfo = { + processID, parentProcessID, isEnvironmentScoped in + CmuxTopProcessInfo( + pid: processID, parentPID: parentProcessID, name: "test", path: nil, + ttyDevice: ttyDevice, + cmuxWorkspaceID: isEnvironmentScoped ? workspaceID : nil, + cmuxSurfaceID: isEnvironmentScoped ? panelID : nil, + cmuxAttributionReason: isEnvironmentScoped ? "environment" : nil, + processGroupID: nil, + terminalProcessGroupID: nil, cpuPercent: 0, residentBytes: 0, + virtualBytes: 0, threadCount: 1 + ) + } + let snapshot = CmuxTopProcessSnapshot( + processes: [ + process(10, 1, true), + process(20, 10, true), + process(21, 20, false), + process(30, 10, false), + ], + sampledAt: .now, + includesProcessDetails: true + ) + + let scope = snapshot.agentHibernationProcessScope( + panelProcessIDs: [10, 20], + agentProcessIDs: [20] + ) + + #expect(scope.terminationProcessIDs == [20, 21]) + #expect(scope.containsUnrelatedProcess) + } + + @Test + func freshExitEpochProbesOnlyTTYAndAuthorizedGroupCandidates() async throws { + let ttyDevice = Int64(0x123) + let process: (Int, Int64, Int) -> CmuxTopProcessInfo = { + processID, processTTYDevice, processGroupID in + CmuxTopProcessInfo( + pid: processID, + parentPID: 1, + name: "test", + path: nil, + ttyDevice: processTTYDevice, + cmuxWorkspaceID: nil, + cmuxSurfaceID: nil, + cmuxAttributionReason: nil, + processGroupID: processGroupID, + terminalProcessGroupID: processGroupID, + cpuPercent: 0, + residentBytes: 0, + virtualBytes: 0, + threadCount: 1 + ) + } + let snapshot = CmuxTopProcessSnapshot( + processes: [ + process(101, ttyDevice, 101), + process(202, ttyDevice, 202), + process(303, 0x999, 303), + ], + sampledAt: .now, + includesProcessDetails: false, + includesCMUXScope: false + ) + let rootIdentity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let lateIdentity = AgentPIDProcessIdentity( + pid: 202, + startSeconds: 20, + startMicroseconds: 2 + ) + let unrelatedIdentity = AgentPIDProcessIdentity( + pid: 303, + startSeconds: 30, + startMicroseconds: 3 + ) + let identities = [ + pid_t(101): rootIdentity, + pid_t(202): lateIdentity, + pid_t(303): unrelatedIdentity, + ] + let probedProcessIDs = OSAllocatedUnfairLock(initialState: Set()) + let coordinator = AgentHibernationProcessSnapshotCoordinator( + captureSnapshot: { snapshot }, + processArgumentsProvider: { processID in + probedProcessIDs.withLock { $0.insert(processID) } + return Self.signalScopeArguments + }, + processIdentityProvider: { identities[$0] }, + processGroupProvider: { $0 } + ) + + let epoch = try #require( + await coordinator.refreshedExitEpoch( + processGroupLeaders: [101: rootIdentity], + processScopeKey: Self.signalScopeKey, + ttyDevice: ttyDevice, + excluding: [] + ) + ) + + #expect(Set(epoch.terminations.map(\.processID)) == [101, 202]) + #expect(epoch.signalableProcessIdentities == [rootIdentity]) + #expect(probedProcessIDs.withLock { $0 } == [101, 202]) + } + + @MainActor + @Test + func terminationSignalsValidatedProcessGroupWithoutRedundantPIDSignal() async { + let identity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let signaledTargets = OSAllocatedUnfairLock(initialState: [pid_t]()) + + let result = await AgentHibernationController + .terminateScopedProcessesForHibernation( + [ + .init( + processID: 101, + processIdentity: identity, + processGroupID: 101, + ttyDevice: 123 + ), + ], + processScopeKey: Self.signalScopeKey, + currentProcessID: 999, + currentProcessGroupID: 999, + processIdentityProvider: { _ in identity }, + processGroupProvider: { _ in 101 }, + processArgumentsProvider: { _ in Self.signalScopeArguments }, + processTTYDeviceProvider: { _ in 123 }, + signalErrorProvider: { target, _ in + signaledTargets.withLock { $0.append(target) } + return nil + } + ) + + #expect(result == .committedAwaitingExit) + #expect(signaledTargets.withLock { $0 } == [-101]) + + let allowExit = AsyncStream.makeStream() + let escalatedTargets = OSAllocatedUnfairLock(initialState: [pid_t]()) + let deadlineCallCount = OSAllocatedUnfairLock(initialState: 0) + let postKillDeadline = AsyncStream.makeStream() + let lateIdentity = AgentPIDProcessIdentity( + pid: 202, + startSeconds: 20, + startMicroseconds: 2 + ) + let didExit = await AgentHibernationController + .waitForScopedProcessGenerationsToExitAfterEscalation( + [.init(processID: 101, processIdentity: identity, processGroupID: 101)], + waitForExit: { _ in + for await _ in allowExit.stream { return true } + return false + }, + sleepUntilDeadline: { _ in + let call = deadlineCallCount.withLock { + $0 += 1 + return $0 + } + guard call > 1 else { return true } + for await _ in postKillDeadline.stream { return true } + return false + }, + processIdentityProvider: { processID in + switch processID { + case 101: identity + case 202: lateIdentity + default: nil + } + }, + processGroupProvider: { $0 }, + signalErrorProvider: { target, signal in + escalatedTargets.withLock { $0.append(target) } + #expect(signal == SIGKILL) + allowExit.continuation.yield() + allowExit.continuation.finish() + return nil + }, + nextEpochProvider: { processGroupLeaders, _, _, _ in + AgentHibernationProcessExitEpoch( + terminations: [ + .init( + processID: 101, + processIdentity: identity, + processGroupID: 101 + ), + .init( + processID: 202, + processIdentity: lateIdentity, + processGroupID: 202 + ), + ], + processGroupLeaders: processGroupLeaders, + signalableProcessIdentities: [identity] + ) + } + ) + #expect(didExit) + #expect(escalatedTargets.withLock { $0 } == [-101]) + #expect(!escalatedTargets.withLock { $0 }.contains(202)) + #expect(!escalatedTargets.withLock { $0 }.contains(-202)) + postKillDeadline.continuation.finish() + } + + @MainActor + @Test + func mutableCommitGateRunsAfterBlockingProcessProbes() async { + let identity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let didProbe = OSAllocatedUnfairLock(initialState: false) + let signaledTargets = OSAllocatedUnfairLock(initialState: [pid_t]()) + + let result = await AgentHibernationController + .terminateScopedProcessesForHibernation( + [ + .init( + processID: 101, + processIdentity: identity, + processGroupID: 101, + ttyDevice: 123 + ), + ], + processScopeKey: Self.signalScopeKey, + shouldCommit: { + #expect(didProbe.withLock { $0 }) + return false + }, + currentProcessID: 999, + currentProcessGroupID: 999, + processIdentityProvider: { _ in identity }, + processGroupProvider: { _ in 101 }, + processArgumentsProvider: { _ in + didProbe.withLock { $0 = true } + return Self.signalScopeArguments + }, + processTTYDeviceProvider: { _ in 123 }, + signalErrorProvider: { target, _ in + signaledTargets.withLock { $0.append(target) } + return nil + } + ) + + #expect(result == .rejected) + #expect(signaledTargets.withLock { $0 }.isEmpty) + } + + @Test + func sigkillEscalationRejectsFreshScopeDrift() async { + let identity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let termination = AgentHibernationController.ScopedProcessTermination( + processID: 101, + processIdentity: identity, + processGroupID: 101, + ttyDevice: 123 + ) + let exitWait = AsyncStream.makeStream() + let signaledTargets = OSAllocatedUnfairLock(initialState: [pid_t]()) + let wrongScopeArguments = CmuxTopProcessArguments( + arguments: ["/usr/bin/test-agent"], + environment: [ + "CMUX_WORKSPACE_ID": UUID().uuidString, + "CMUX_SURFACE_ID": UUID().uuidString, + ] + ) + + let didExit = await AgentHibernationController + .waitForScopedProcessGenerationsToExitAfterEscalation( + [termination], + processScopeKey: Self.signalScopeKey, + gracePeriod: .zero, + postKillExitPeriod: .zero, + waitForExit: { _ in + for await _ in exitWait.stream {} + return false + }, + sleepUntilDeadline: { _ in true }, + processIdentityProvider: { _ in identity }, + processGroupProvider: { _ in 101 }, + processArgumentsProvider: { _ in wrongScopeArguments }, + processTTYDeviceProvider: { _ in 123 }, + signalErrorProvider: { target, _ in + signaledTargets.withLock { $0.append(target) } + return nil + }, + nextEpochProvider: { processGroupLeaders, _, _, _ in + AgentHibernationProcessExitEpoch( + terminations: [termination], + processGroupLeaders: processGroupLeaders, + signalableProcessIdentities: [identity] + ) + } + ) + + #expect(didExit == false) + #expect(signaledTargets.withLock { $0 }.isEmpty) + exitWait.continuation.finish() + } + + @MainActor + @Test + func signalBatchFinishesBeforeCommittedUITransition() async { + let firstIdentity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let secondIdentity = AgentPIDProcessIdentity( + pid: 202, + startSeconds: 20, + startMicroseconds: 2 + ) + let events = OSAllocatedUnfairLock(initialState: [String]()) + + let result = await AgentHibernationController + .terminateScopedProcessesForHibernation( + [ + .init( + processID: 101, + processIdentity: firstIdentity, + processGroupID: 101, + ttyDevice: 123 + ), + .init( + processID: 202, + processIdentity: secondIdentity, + processGroupID: 202, + ttyDevice: 123 + ), + ], + processScopeKey: Self.signalScopeKey, + onTeardownCommit: { + events.withLock { $0.append("commit") } + }, + currentProcessID: 999, + currentProcessGroupID: 999, + processIdentityProvider: { pid in + pid == 101 ? firstIdentity : secondIdentity + }, + processGroupProvider: { $0 }, + processArgumentsProvider: { _ in Self.signalScopeArguments }, + processTTYDeviceProvider: { _ in 123 }, + signalErrorProvider: { target, _ in + events.withLock { $0.append("signal:\(target)") } + return nil + } + ) + + #expect(result == .committedAwaitingExit) + #expect(events.withLock { $0 } == ["signal:-101", "signal:-202", "commit"]) + } + + @Test + func processScopeIncludesRelatedWrapperGroupLeader() { + let workspaceID = UUID() + let panelID = UUID() + let ttyDevice = Int64(0x123) + let process: (Int, Int, Int) -> CmuxTopProcessInfo = { + processID, parentProcessID, processGroupID in + CmuxTopProcessInfo( + pid: processID, parentPID: parentProcessID, name: "test", path: nil, + ttyDevice: ttyDevice, + cmuxWorkspaceID: workspaceID, + cmuxSurfaceID: panelID, + cmuxAttributionReason: "environment", + processGroupID: processGroupID, + terminalProcessGroupID: processGroupID, cpuPercent: 0, residentBytes: 0, + virtualBytes: 0, threadCount: 1 + ) + } + let snapshot = CmuxTopProcessSnapshot( + processes: [ + process(100, 1, 100), + process(101, 100, 100), + process(102, 101, 100), + ], + sampledAt: .now, + includesProcessDetails: true + ) + + let scope = snapshot.agentHibernationProcessScope( + panelProcessIDs: [100, 101, 102], + agentProcessIDs: [101] + ) + + #expect(scope.terminationProcessIDs == [100, 101, 102]) + #expect(scope.containsUnrelatedProcess == false) + } + + @Test + func validatesExactProcessGeneration() throws { let workspaceID = UUID() let panelID = UUID() let firstIdentity = AgentPIDProcessIdentity( @@ -42,9 +463,6 @@ struct AgentHibernationProcessTerminationTests { AgentHibernationController.validatedScopedProcessTerminations( for: scope, processIdentityProvider: { identities[$0] }, - processArgumentsProvider: { _ in - Self.processArguments(workspaceID: workspaceID, panelID: panelID) - }, processGroupProvider: { pid_t($0 + 1_000) } ) ) @@ -89,36 +507,6 @@ struct AgentHibernationProcessTerminationTests { startMicroseconds: 0 ) }, - processArgumentsProvider: { _ in - Self.processArguments(workspaceID: workspaceID, panelID: panelID) - }, - processGroupProvider: { _ in 1_101 } - ) - - #expect(terminations == nil) - } - - @Test - func rejectsProcessOutsidePaneScope() { - let workspaceID = UUID() - let panelID = UUID() - let identity = AgentPIDProcessIdentity( - pid: 101, - startSeconds: 10, - startMicroseconds: 1 - ) - let scope = AgentHibernationController.ProcessTerminationScope( - key: AgentHibernationPanelKey(workspaceId: workspaceID, panelId: panelID), - processIDs: [101], - processIdentities: [101: identity] - ) - - let terminations = AgentHibernationController.validatedScopedProcessTerminations( - for: scope, - processIdentityProvider: { _ in identity }, - processArgumentsProvider: { _ in - Self.processArguments(workspaceID: workspaceID, panelID: UUID()) - }, processGroupProvider: { _ in 1_101 } ) @@ -143,9 +531,6 @@ struct AgentHibernationProcessTerminationTests { let terminations = AgentHibernationController.validatedScopedProcessTerminations( for: scope, processIdentityProvider: { _ in identity }, - processArgumentsProvider: { _ in - Self.processArguments(workspaceID: workspaceID, panelID: panelID) - }, processGroupProvider: { _ in 1_101 } ) @@ -163,7 +548,8 @@ struct AgentHibernationProcessTerminationTests { let termination = AgentHibernationController.ScopedProcessTermination( processID: 101, processIdentity: identity, - processGroupID: 1 + processGroupID: 101, + ttyDevice: 123 ) let waitStarted = AsyncStream.makeStream() let allowExit = AsyncStream.makeStream() @@ -182,7 +568,9 @@ struct AgentHibernationProcessTerminationTests { }, onExit: { didComplete = true - } + return true + }, + onRecovery: {} ) let task = controller.committedTerminationObservationsByPanelID[panelID]?.task var waitStartedIterator = waitStarted.stream.makeAsyncIterator() @@ -199,7 +587,7 @@ struct AgentHibernationProcessTerminationTests { @MainActor @Test - func signalFailureOtherThanMissingProcessAbortsBeforeCommit() { + func signalFailureOtherThanMissingProcessAbortsBeforeCommit() async { let identity = AgentPIDProcessIdentity( pid: 101, startSeconds: 10, @@ -208,16 +596,19 @@ struct AgentHibernationProcessTerminationTests { let termination = AgentHibernationController.ScopedProcessTermination( processID: 101, processIdentity: identity, - processGroupID: 1 + processGroupID: 101 ) - let result = AgentHibernationController.shared + let result = await AgentHibernationController .terminateScopedProcessesForHibernation( [termination], + processScopeKey: Self.signalScopeKey, currentProcessID: 999, currentProcessGroupID: 999, processIdentityProvider: { _ in identity }, - processGroupProvider: { _ in 1 }, + processGroupProvider: { _ in 101 }, + processArgumentsProvider: { _ in Self.signalScopeArguments }, + processTTYDeviceProvider: { _ in 123 }, signalErrorProvider: { _, _ in EPERM } ) @@ -241,28 +632,35 @@ struct AgentHibernationProcessTerminationTests { AgentHibernationController.ScopedProcessTermination( processID: 101, processIdentity: firstIdentity, - processGroupID: 1 + processGroupID: 101, + ttyDevice: 123 ), AgentHibernationController.ScopedProcessTermination( processID: 202, processIdentity: secondIdentity, - processGroupID: 1 + processGroupID: 202, + ttyDevice: 123 ), ] let waitRecorder = CompletionRecorder() let controller = AgentHibernationController.shared - let result = controller + let result = await AgentHibernationController .terminateScopedProcessesForHibernation( terminations, + processScopeKey: Self.signalScopeKey, currentProcessID: 999, currentProcessGroupID: 999, processIdentityProvider: { pid in pid == 101 ? firstIdentity : secondIdentity }, - processGroupProvider: { _ in 1 }, + processGroupProvider: { pid in + pid + }, + processArgumentsProvider: { _ in Self.signalScopeArguments }, + processTTYDeviceProvider: { _ in 123 }, signalErrorProvider: { target, _ in - target == 101 ? nil : EPERM + target == -101 || target == 101 ? nil : EPERM } ) let panelID = UUID() @@ -273,7 +671,8 @@ struct AgentHibernationProcessTerminationTests { await waitRecorder.record(observedTerminations == terminations) return observedTerminations == terminations }, - onExit: {} + onExit: { true }, + onRecovery: {} ) let observationTask = controller .committedTerminationObservationsByPanelID[panelID]? @@ -322,7 +721,8 @@ struct AgentHibernationProcessTerminationTests { let termination = AgentHibernationController.ScopedProcessTermination( processID: 101, processIdentity: identity, - processGroupID: 1 + processGroupID: 101, + ttyDevice: 123 ) let panel = TerminalPanel(workspaceId: UUID()) defer { panel.close() } @@ -335,10 +735,11 @@ struct AgentHibernationProcessTerminationTests { let eventualWaitStarted = AsyncStream.makeStream() let allowEventualExit = AsyncStream.makeStream() let controller = AgentHibernationController.shared - let result = controller.terminateScopedProcessesForHibernation( + let result = await AgentHibernationController.terminateScopedProcessesForHibernation( [termination], - onTeardownCommit: { - panel.beginAgentHibernationTermination( + processScopeKey: Self.signalScopeKey, + onTeardownCommit: { [weak panel] in + panel?.beginAgentHibernationTermination( agent: agent, lastActivityAt: Date(timeIntervalSince1970: 1) ) @@ -346,7 +747,9 @@ struct AgentHibernationProcessTerminationTests { currentProcessID: 999, currentProcessGroupID: 999, processIdentityProvider: { _ in identity }, - processGroupProvider: { _ in 1 }, + processGroupProvider: { _ in 101 }, + processArgumentsProvider: { _ in Self.signalScopeArguments }, + processTTYDeviceProvider: { _ in 123 }, signalErrorProvider: { _, _ in nil } ) #expect(result == .committedAwaitingExit) @@ -362,6 +765,10 @@ struct AgentHibernationProcessTerminationTests { }, onExit: { panel.completeAgentHibernationTermination() + return true + }, + onRecovery: { + panel.completeAgentHibernationTermination() } ) let observationTask = controller @@ -410,16 +817,142 @@ struct AgentHibernationProcessTerminationTests { #expect(didExit) } - private static func processArguments( - workspaceID: UUID, - panelID: UUID - ) -> CmuxTopProcessArguments { - CmuxTopProcessArguments( - arguments: ["/usr/bin/agent"], - environment: [ - "CMUX_WORKSPACE_ID": workspaceID.uuidString, - "CMUX_SURFACE_ID": panelID.uuidString, - ] + @Test + func scopedExitWaitIncludesLateProcessInAuthorizedGroup() async { + let processScopeKey = AgentHibernationPanelKey( + workspaceId: UUID(), + panelId: UUID() ) + let rootIdentity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let lateIdentity = AgentPIDProcessIdentity( + pid: 202, + startSeconds: 20, + startMicroseconds: 2 + ) + let nextEpochCount = OSAllocatedUnfairLock(initialState: 0) + let waitCompleted = OSAllocatedUnfairLock(initialState: false) + let lateEpochStarted = AsyncStream.makeStream() + let allowLateExit = AsyncStream.makeStream() + let wait = Task { + let didExit = await AgentHibernationController + .waitForScopedProcessGenerationsToExitWithoutTimeout( + [ + .init( + processID: 101, + processIdentity: rootIdentity, + processGroupID: 101, + ttyDevice: 123 + ), + ], + processScopeKey: processScopeKey, + waitForExactEpoch: { epoch in + guard epoch.contains(where: { + $0.processIdentity == lateIdentity + }) else { + return true + } + lateEpochStarted.continuation.yield() + for await _ in allowLateExit.stream { return true } + return false + }, + nextEpochProvider: { + processGroupLeaders, + receivedScopeKey, + receivedTTYDevice, + exitedIdentities in + #expect(receivedScopeKey == processScopeKey) + #expect(receivedTTYDevice == 123) + let call = nextEpochCount.withLock { + $0 += 1 + return $0 + } + guard call == 1 else { + #expect(processGroupLeaders[101] == rootIdentity) + #expect(exitedIdentities == [rootIdentity, lateIdentity]) + return AgentHibernationProcessExitEpoch( + terminations: [], + processGroupLeaders: [:] + ) + } + #expect(exitedIdentities == [rootIdentity]) + return AgentHibernationProcessExitEpoch( + terminations: [ + .init( + processID: 202, + processIdentity: lateIdentity, + processGroupID: 101 + ), + ], + processGroupLeaders: [101: rootIdentity] + ) + } + ) + waitCompleted.withLock { $0 = true } + return didExit + } + var lateEpochIterator = lateEpochStarted.stream.makeAsyncIterator() + _ = await lateEpochIterator.next() + + #expect(waitCompleted.withLock { $0 } == false) + + allowLateExit.continuation.yield() + allowLateExit.continuation.finish() + #expect(await wait.value) + lateEpochStarted.continuation.finish() + } + + @Test + func scopedExitWaitObservesLateProcessWithoutAuthorizingItsGroup() async { + let rootIdentity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let lateIdentity = AgentPIDProcessIdentity( + pid: 202, + startSeconds: 20, + startMicroseconds: 2 + ) + + let refreshCount = OSAllocatedUnfairLock(initialState: 0) + let didExit = await AgentHibernationController + .waitForScopedProcessGenerationsToExitWithoutTimeout( + [ + .init( + processID: 101, + processIdentity: rootIdentity, + processGroupID: 101 + ), + ], + waitForExactEpoch: { _ in true }, + nextEpochProvider: { processGroupLeaders, _, _, _ in + let refresh = refreshCount.withLock { + $0 += 1 + return $0 + } + guard refresh == 1 else { + return AgentHibernationProcessExitEpoch( + terminations: [], + processGroupLeaders: processGroupLeaders + ) + } + AgentHibernationProcessExitEpoch( + terminations: [ + .init( + processID: 202, + processIdentity: lateIdentity, + processGroupID: 202 + ), + ], + processGroupLeaders: processGroupLeaders + ) + } + ) + + #expect(didExit) } } diff --git a/cmuxTests/AgentHibernationRestoreMonitorTests.swift b/cmuxTests/AgentHibernationRestoreMonitorTests.swift index 3ee0bf8d53d3..8dc8a4186379 100644 --- a/cmuxTests/AgentHibernationRestoreMonitorTests.swift +++ b/cmuxTests/AgentHibernationRestoreMonitorTests.swift @@ -271,6 +271,83 @@ struct AgentHibernationRestoreMonitorTests { #expect(try String(contentsOf: retained, encoding: .utf8) == snapshotContent) } + @Test + func committedMonitorDefersRestoreChecksUntilExactProcessExit() async throws { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("cmux-hibernation-exact-exit-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + + let live = directory.appendingPathComponent("live.jsonl") + let snapshot = directory.appendingPathComponent("snapshot.jsonl") + let metadataStub = #"{"type":"last-prompt","prompt":"continue"}"# + "\n" + let snapshotContent = #"{"type":"user","message":{"content":"protected"}}"# + "\n" + try metadataStub.write(to: live, atomically: true, encoding: .utf8) + try snapshotContent.write(to: snapshot, atomically: true, encoding: .utf8) + let waitStarted = AsyncStream.makeStream() + let allowExit = AsyncStream.makeStream() + + let task = Task { + await AgentHibernationTranscriptGuard.runPostTeardownRestoreChecks( + snapshot: .init(transcriptPath: live.path, snapshotPath: snapshot.path), + processIDs: [101], + initialRetryDelaysNanoseconds: [0], + backstopDelaysSeconds: [], + awaitProcessExit: { + waitStarted.continuation.yield() + for await _ in allowExit.stream { + break + } + return true + } + ) + } + var waitStartedIterator = waitStarted.stream.makeAsyncIterator() + _ = await waitStartedIterator.next() + + #expect(try String(contentsOf: live, encoding: .utf8) == metadataStub) + #expect(FileManager.default.fileExists(atPath: snapshot.path)) + + allowExit.continuation.yield() + allowExit.continuation.finish() + await task.value + #expect(try String(contentsOf: live, encoding: .utf8).hasPrefix(snapshotContent)) + #expect(FileManager.default.fileExists(atPath: snapshot.path) == false) + waitStarted.continuation.finish() + } + + @Test + func cappedExitObservationStillRunsTranscriptRecoveryChecks() async throws { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent( + "cmux-hibernation-capped-exit-\(UUID().uuidString)", + isDirectory: true + ) + try FileManager.default.createDirectory( + at: directory, + withIntermediateDirectories: true + ) + defer { try? FileManager.default.removeItem(at: directory) } + + let live = directory.appendingPathComponent("live.jsonl") + let snapshot = directory.appendingPathComponent("snapshot.jsonl") + let metadataStub = #"{"type":"last-prompt","prompt":"continue"}"# + "\n" + let snapshotContent = #"{"type":"user","message":{"content":"protected"}}"# + "\n" + try metadataStub.write(to: live, atomically: true, encoding: .utf8) + try snapshotContent.write(to: snapshot, atomically: true, encoding: .utf8) + + await AgentHibernationTranscriptGuard.runPostTeardownRestoreChecks( + snapshot: .init(transcriptPath: live.path, snapshotPath: snapshot.path), + processIDs: [101], + initialRetryDelaysNanoseconds: [0], + backstopDelaysSeconds: [], + awaitProcessExit: { false } + ) + + #expect(try String(contentsOf: live, encoding: .utf8).hasPrefix(snapshotContent)) + #expect(FileManager.default.fileExists(atPath: snapshot.path) == false) + } + @MainActor @Test func monitorKeyResolvesSymlinkedTranscriptPathAliases() throws { @@ -317,6 +394,54 @@ struct AgentHibernationRestoreMonitorTests { #expect(FileManager.default.fileExists(atPath: snapshot.path)) } + @MainActor + @Test + func committedTerminationRecoveryKeepsRestoreMonitorArmedUntilExactExit() async throws { + let controller = AgentHibernationController.shared + defer { resetSharedHibernationState(controller) } + + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("cmux-hibernation-termination-failure-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + + let live = directory.appendingPathComponent("live.jsonl") + let snapshotURL = directory.appendingPathComponent("snapshot.jsonl") + let protectedContent = #"{"type":"user","message":{"content":"protected"}}"# + "\n" + let clobberedContent = #"{"type":"last-prompt","prompt":"continue"}"# + "\n" + try protectedContent.write(to: live, atomically: true, encoding: .utf8) + try protectedContent.write(to: snapshotURL, atomically: true, encoding: .utf8) + let snapshot = AgentHibernationTranscriptGuard.TeardownTranscriptSnapshot( + transcriptPath: live.path, + snapshotPath: snapshotURL.path + ) + let processExit = AsyncStream.makeStream() + #expect(controller.armPostTeardownRestoreMonitor( + snapshot: snapshot, + processIDs: [101], + awaitProcessExit: { + for await _ in processExit.stream { return true } + return false + } + )) + let monitor = try #require( + controller.postTeardownRestoreTasksByTranscriptPath.values.first?.task + ) + + try clobberedContent.write(to: live, atomically: true, encoding: .utf8) + + #expect(controller.postTeardownRestoreTasksByTranscriptPath.isEmpty == false) + #expect(FileManager.default.fileExists(atPath: snapshotURL.path)) + + processExit.continuation.yield() + processExit.continuation.finish() + await monitor.value + + #expect(controller.postTeardownRestoreTasksByTranscriptPath.isEmpty) + #expect(try String(contentsOf: live, encoding: .utf8).hasPrefix(protectedContent)) + #expect(FileManager.default.fileExists(atPath: snapshotURL.path) == false) + } + @MainActor private func restoreTask( live: URL, diff --git a/cmuxTests/AgentHibernationTerminationFailureTests.swift b/cmuxTests/AgentHibernationTerminationFailureTests.swift new file mode 100644 index 000000000000..6c75d99bfdbd --- /dev/null +++ b/cmuxTests/AgentHibernationTerminationFailureTests.swift @@ -0,0 +1,355 @@ +import Darwin +import Foundation +import os +import Testing + +#if canImport(cmux_DEV) +@testable import cmux_DEV +#elseif canImport(cmux) +@testable import cmux +#endif + +@Suite(.serialized) +struct AgentHibernationTerminationFailureTests { + @Test(arguments: [Int32?.none, Int32(EPERM)]) + func escalationFailureOrPostKillDeadlineFailsClosed(signalError: Int32?) async { + let identity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let processExit = AsyncStream.makeStream() + + let didExit = await AgentHibernationController + .waitForScopedProcessGenerationsToExitAfterEscalation( + [.init(processID: 101, processIdentity: identity, processGroupID: 1)], + gracePeriod: .zero, + postKillExitPeriod: .zero, + waitForExit: { _ in + for await _ in processExit.stream { return true } + return false + }, + sleepUntilDeadline: { _ in true }, + processIdentityProvider: { _ in identity }, + signalErrorProvider: { _, _ in signalError } + ) + + #expect(didExit == false) + processExit.continuation.finish() + } + + @Test + func emptyRefreshDoesNotProveOriginalGenerationExited() async { + let identity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let processExit = AsyncStream.makeStream() + + let didExit = await AgentHibernationController + .waitForScopedProcessGenerationsToExitAfterEscalation( + [ + .init( + processID: 101, + processIdentity: identity, + processGroupID: 101, + ttyDevice: 123 + ), + ], + gracePeriod: .zero, + waitForExit: { _ in + for await _ in processExit.stream { return true } + return false + }, + sleepUntilDeadline: { _ in true }, + processIdentityProvider: { _ in identity }, + processGroupProvider: { _ in 101 }, + nextEpochProvider: { processGroupLeaders, _, _, _ in + AgentHibernationProcessExitEpoch( + terminations: [], + processGroupLeaders: processGroupLeaders + ) + } + ) + + #expect(didExit == false) + processExit.continuation.finish() + } + + @MainActor + @Test + func failedCommittedTerminationRecoversAfterExactExit() async { + let panel = TerminalPanel(workspaceId: UUID()) + defer { panel.close() } + let agent = SessionRestorableAgentSnapshot( + kind: .custom("test-agent"), + sessionId: "failed-termination", + workingDirectory: "/tmp", + launchCommand: nil + ) + panel.beginAgentHibernationTermination( + agent: agent, + lastActivityAt: Date(timeIntervalSince1970: 1) + ) + let controller = AgentHibernationController.shared + let recoveryExit = AsyncStream.makeStream() + controller.observeCommittedTermination( + panelID: panel.id, + terminations: [], + waitForExit: { _ in false }, + waitForRecoveryExit: { _ in + for await _ in recoveryExit.stream { return true } + return false + }, + onExit: { true }, + onFailure: { + panel.beginAgentHibernationTerminationRecovery() + }, + onRecovery: { + panel.completeAgentHibernationTermination() + } + ) + let observation = controller.committedTerminationObservationsByPanelID[panel.id]?.task + + await observation?.value + + #expect(panel.agentHibernationTerminationFailed == false) + #expect(panel.isAgentHibernationCommitPending) + #expect(panel.prepareAgentHibernationResume() == .unavailable) + let recovery = controller.committedTerminationObservationsByPanelID[panel.id]?.task + #expect(recovery != nil) + + recoveryExit.continuation.yield() + recoveryExit.continuation.finish() + await recovery?.value + + #expect(panel.agentHibernationTerminationFailed == false) + #expect(controller.committedTerminationObservationsByPanelID[panel.id] == nil) + } + + @MainActor + @Test + func failedTerminationKeepsRecoveryUntilNativeTeardownCompletes() async { + let panel = TerminalPanel(workspaceId: UUID()) + defer { panel.close() } + panel.beginAgentHibernationTermination( + agent: .init( + kind: .custom("test-agent"), + sessionId: "delayed-native-teardown", + workingDirectory: "/tmp", + launchCommand: nil + ), + lastActivityAt: Date(timeIntervalSince1970: 1) + ) + let controller = AgentHibernationController.shared + let nativeTeardownCompleted = AsyncStream.makeStream() + controller.observeCommittedTermination( + panelID: panel.id, + terminations: [], + waitForExit: { _ in false }, + waitForRecoveryExit: { _ in true }, + waitForRecoveryReadiness: { + for await _ in nativeTeardownCompleted.stream { return true } + return false + }, + onExit: { true }, + onFailure: { + panel.beginAgentHibernationTerminationRecovery() + }, + onRecovery: { + panel.completeAgentHibernationTermination() + } + ) + let initialObservation = + controller.committedTerminationObservationsByPanelID[panel.id]?.task + await initialObservation?.value + + #expect(panel.agentHibernationTerminationFailed == false) + #expect(panel.isAgentHibernationCommitPending) + let recovery = controller.committedTerminationObservationsByPanelID[panel.id]?.task + #expect(recovery != nil) + + nativeTeardownCompleted.continuation.yield() + nativeTeardownCompleted.continuation.finish() + await recovery?.value + + #expect(panel.isAgentHibernated) + #expect(controller.committedTerminationObservationsByPanelID[panel.id] == nil) + } + + @MainActor + @Test + func failedNoProcessRecoveryRetrySkipsExitWaitAndSucceeds() async { + let panel = TerminalPanel(workspaceId: UUID()) + defer { panel.close() } + panel.beginAgentHibernationTermination( + agent: .init( + kind: .custom("test-agent"), + sessionId: "retryable-recovery", + workingDirectory: "/tmp", + launchCommand: nil + ), + lastActivityAt: Date(timeIntervalSince1970: 1) + ) + let controller = AgentHibernationController.shared + let readinessAttempts = OSAllocatedUnfairLock(initialState: 0) + let terminationRetryCount = OSAllocatedUnfairLock(initialState: 0) + controller.observeCommittedTermination( + panelID: panel.id, + terminations: [], + waitForExit: { _ in false }, + waitForRecoveryExit: { _ in true }, + retryTermination: { + terminationRetryCount.withLock { $0 += 1 } + return .exited + }, + waitForRecoveryReadiness: { + readinessAttempts.withLock { + $0 += 1 + return $0 > 1 + } + }, + onExit: { true }, + onFailure: { + panel.beginAgentHibernationTerminationRecovery() + }, + onRecovery: { + panel.completeAgentHibernationTermination() + }, + onRecoveryFailure: { + panel.failAgentHibernationTermination() + }, + onRecoveryRetry: { + panel.beginAgentHibernationTerminationRecovery() + } + ) + let initial = controller.committedTerminationObservationsByPanelID[panel.id]?.task + await initial?.value + let failedRecovery = + controller.committedTerminationObservationsByPanelID[panel.id]?.task + await failedRecovery?.value + + #expect(panel.agentHibernationTerminationFailed) + #expect(controller.committedTerminationObservationsByPanelID[panel.id] != nil) + + controller.retryCommittedTerminationRecovery(panelID: panel.id) + let retry = controller.committedTerminationObservationsByPanelID[panel.id]?.task + await retry?.value + + #expect(panel.isAgentHibernated) + #expect(controller.committedTerminationObservationsByPanelID[panel.id] == nil) + #expect(terminationRetryCount.withLock { $0 } == 1) + } + + @MainActor + @Test + func recoveryDeadlineExposesRetryableFailure() async { + let panel = TerminalPanel(workspaceId: UUID()) + defer { panel.close() } + panel.beginAgentHibernationTermination( + agent: .init( + kind: .custom("test-agent"), + sessionId: "recovery-deadline", + workingDirectory: "/tmp", + launchCommand: nil + ), + lastActivityAt: Date(timeIntervalSince1970: 1) + ) + let controller = AgentHibernationController.shared + let readiness = AsyncStream.makeStream() + controller.observeCommittedTermination( + panelID: panel.id, + terminations: [], + waitForExit: { _ in false }, + waitForRecoveryReadiness: { + for await _ in readiness.stream { return true } + return false + }, + recoveryDeadline: .zero, + sleepUntilRecoveryDeadline: { _ in true }, + onExit: { true }, + onFailure: { + panel.beginAgentHibernationTerminationRecovery() + }, + onRecovery: { + panel.completeAgentHibernationTermination() + }, + onRecoveryFailure: { + panel.failAgentHibernationTermination() + } + ) + let initial = controller.committedTerminationObservationsByPanelID[panel.id]?.task + await initial?.value + let recovery = controller.committedTerminationObservationsByPanelID[panel.id]?.task + await recovery?.value + + #expect(panel.agentHibernationTerminationFailed) + #expect(controller.committedTerminationObservationsByPanelID[panel.id] != nil) + readiness.continuation.finish() + } + + @MainActor + @Test + func recoveredFailureRunsWorkspaceHibernationCommit() throws { + let workspace = Workspace() + let panelID = try #require(workspace.focusedPanelId) + let panel = try #require(workspace.panels[panelID] as? TerminalPanel) + defer { panel.close() } + let agent = SessionRestorableAgentSnapshot( + kind: .codex, + sessionId: "recovered-workspace-commit", + workingDirectory: "/tmp", + launchCommand: nil + ) + try #require(agent.resumeCommand != nil) + panel.beginAgentHibernationTermination( + agent: agent, + lastActivityAt: Date(timeIntervalSince1970: 1) + ) + panel.beginAgentHibernationTerminationRecovery() + panel.failAgentHibernationTermination() + workspace.restoredAgentResumeStatesByPanelId[panelID] = .completedAgentExit + + workspace.enterAgentHibernation( + panelId: panelID, + agent: agent, + lastActivityAt: Date(timeIntervalSince1970: 1) + ) + + #expect(panel.agentHibernationTerminationFailed == false) + #expect(panel.isAgentHibernated) + #expect(workspace.restoredAgentSnapshotsByPanelId[panelID]?.sessionId == agent.sessionId) + #expect(workspace.restoredAgentResumeStatesByPanelId[panelID] == .manualResumeAvailable) + } + + @MainActor + @Test + func unknownProcessAbsenceIsUnsafeForPressureTeardown() throws { + let workspace = Workspace() + let panelID = try #require(workspace.focusedPanelId) + let panel = try #require(workspace.panels[panelID] as? TerminalPanel) + let record = AgentHibernationRecord( + key: .init(workspaceId: workspace.id, panelId: panelID), + workspace: workspace, + terminalPanel: panel, + agent: .init( + kind: .custom("test-agent"), + sessionId: "unknown-process-evidence", + workingDirectory: "/tmp", + launchCommand: nil + ), + lifecycle: .idle, + hasUnconfirmedTerminalInput: false, + lastActivityAt: 0, + isProtected: false, + hasLiveProcess: false, + containsUnrelatedProcess: false, + panelProcessIDs: [], + processIDs: [], + processIdentities: [:] + ) + + #expect(record.hasPressureSafeProcessEvidence == false) + } +} diff --git a/cmuxTests/AgentHibernationTrackingLifecycleTests.swift b/cmuxTests/AgentHibernationTrackingLifecycleTests.swift index ad7d6602ffa3..7e6924ba6c7b 100644 --- a/cmuxTests/AgentHibernationTrackingLifecycleTests.swift +++ b/cmuxTests/AgentHibernationTrackingLifecycleTests.swift @@ -161,7 +161,7 @@ struct AgentHibernationTrackingLifecycleTests { } @Test - func permanentPanelCloseCancelsCommittedTerminationObservation() async { + func permanentPanelCloseKeepsTranscriptExitObservationUntilExit() async { let controller = AgentHibernationController.shared let panel = TerminalPanel(workspaceId: UUID()) let identity = AgentPIDProcessIdentity( @@ -186,7 +186,9 @@ struct AgentHibernationTrackingLifecycleTests { }, onExit: { didCompleteHibernation = true - } + return true + }, + onRecovery: {} ) let observationTask = controller .committedTerminationObservationsByPanelID[panel.id]? @@ -196,8 +198,196 @@ struct AgentHibernationTrackingLifecycleTests { exitEvents.continuation.finish() await observationTask?.value + #expect(controller.committedTerminationObservationsByPanelID[panel.id] == nil) + #expect(didCompleteHibernation) + } + + @Test + func permanentPanelCloseBoundsTranscriptExitObservation() async { + let controller = AgentHibernationController.shared + let panel = TerminalPanel(workspaceId: UUID()) + let identity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let exitEvents = AsyncStream.makeStream() + let processExitCompletion = AgentHibernationProcessExitCompletion() + var didCompleteHibernation = false + controller.observeCommittedTermination( + panelID: panel.id, + terminations: [ + .init( + processID: 101, + processIdentity: identity, + processGroupID: 1 + ), + ], + processExitCompletion: processExitCompletion, + waitForExit: { _ in + for await _ in exitEvents.stream {} + return true + }, + onExit: { + didCompleteHibernation = true + return true + }, + onRecovery: {} + ) + + panel.close() + controller.limitCommittedTerminationObservationAfterPanelClose( + panelID: panel.id, + cleanupDelay: .zero, + sleepUntilDeadline: { _ in true } + ) + let didExit = await processExitCompletion.wait() + + #expect(!didExit) #expect(controller.committedTerminationObservationsByPanelID[panel.id] == nil) #expect(!didCompleteHibernation) + exitEvents.continuation.finish() + } + + @Test + func panelCloseCleanupFollowsObservationIntoRecovery() async throws { + let controller = AgentHibernationController.shared + let panelID = UUID() + let identity = AgentPIDProcessIdentity( + pid: 101, + startSeconds: 10, + startMicroseconds: 1 + ) + let initialWaitStarted = AsyncStream.makeStream() + let allowInitialFailure = AsyncStream.makeStream() + let nativeReadiness = AsyncStream.makeStream() + let allowCleanup = AsyncStream.makeStream() + let processExitCompletion = AgentHibernationProcessExitCompletion() + controller.observeCommittedTermination( + panelID: panelID, + terminations: [ + .init( + processID: 101, + processIdentity: identity, + processGroupID: 1 + ), + ], + processExitCompletion: processExitCompletion, + waitForExit: { _ in + initialWaitStarted.continuation.yield() + for await _ in allowInitialFailure.stream { return false } + return false + }, + waitForRecoveryReadiness: { + for await _ in nativeReadiness.stream { return true } + return false + }, + onExit: { true }, + onRecovery: {} + ) + let initialObservation = try #require( + controller.committedTerminationObservationsByPanelID[panelID] + ) + let initialObservationTask = try #require(initialObservation.task) + var initialWaitIterator = initialWaitStarted.stream.makeAsyncIterator() + _ = await initialWaitIterator.next() + + controller.limitCommittedTerminationObservationAfterPanelClose( + panelID: panelID, + cleanupDelay: .zero, + sleepUntilDeadline: { _ in + for await _ in allowCleanup.stream { return true } + return false + } + ) + let cleanupTask = try #require( + controller.committedTerminationCleanupByPanelID[panelID]?.task + ) + allowInitialFailure.continuation.yield() + allowInitialFailure.continuation.finish() + await initialObservationTask.value + + let recoveryObservation = try #require( + controller.committedTerminationObservationsByPanelID[panelID] + ) + #expect(recoveryObservation.requestID == initialObservation.requestID) + + allowCleanup.continuation.yield() + allowCleanup.continuation.finish() + await cleanupTask.value + + #expect(controller.committedTerminationObservationsByPanelID[panelID] == nil) + #expect(await processExitCompletion.wait() == false) + initialWaitStarted.continuation.finish() + nativeReadiness.continuation.finish() + } + + @Test + func panelCloseCleanupOwnsCommitBeforeNativeRecoveryCanSuspend() async throws { + let controller = AgentHibernationController.shared + let workspaceID = UUID() + let panelID = UUID() + let requestID = UUID() + let processExitCompletion = AgentHibernationProcessExitCompletion() + let allowCleanup = AsyncStream.makeStream() + let allowLateRecovery = AsyncStream.makeStream() + var didRecover = false + + controller.registerCommittedTerminationObservation( + panelID: panelID, + requestID: requestID, + processExitCompletion: processExitCompletion + ) + #expect( + controller.committedTerminationObservationsByPanelID[panelID]? + .requestID == requestID + ) + + controller.discardTrackingStateForClosedPanel( + workspaceId: workspaceID, + panelId: panelID + ) + #expect(controller.committedTerminationCleanupByPanelID[panelID] != nil) + controller.limitCommittedTerminationObservationAfterPanelClose( + panelID: panelID, + cleanupDelay: .zero, + sleepUntilDeadline: { _ in + for await _ in allowCleanup.stream { return true } + return false + } + ) + let cleanupTask = try #require( + controller.committedTerminationCleanupByPanelID[panelID]?.task + ) + + let lateRecoveryTask = Task { @MainActor in + for await _ in allowLateRecovery.stream { + controller.observeCommittedTerminationRecovery( + panelID: panelID, + requestID: requestID, + terminations: [], + processExitCompletion: processExitCompletion, + onRecovery: { + didRecover = true + } + ) + return + } + } + + allowCleanup.continuation.yield() + allowCleanup.continuation.finish() + await cleanupTask.value + + #expect(controller.committedTerminationObservationsByPanelID[panelID] == nil) + #expect(await processExitCompletion.wait() == false) + + allowLateRecovery.continuation.yield() + allowLateRecovery.continuation.finish() + await lateRecoveryTask.value + + #expect(controller.committedTerminationObservationsByPanelID[panelID] == nil) + #expect(!didRecover) } private func expectTrackingWasDiscarded( diff --git a/cmuxTests/AgentResumeLivenessTests.swift b/cmuxTests/AgentResumeLivenessTests.swift index 5aa4ea3bd769..63018b0c7702 100644 --- a/cmuxTests/AgentResumeLivenessTests.swift +++ b/cmuxTests/AgentResumeLivenessTests.swift @@ -30,7 +30,11 @@ struct AgentResumeLivenessTests { processIDs: processIDs, processIdentities: [:], agentProcessIDs: processIDs, - agentProcessIdentities: [:] + agentProcessIdentities: [:], + hibernationPanelProcessIDs: processIDs, + terminationProcessIDs: processIDs, + terminationProcessIdentities: [:], + containsUnrelatedProcess: false ) } diff --git a/cmuxTests/CompletedRestoredAgentGenerationTests.swift b/cmuxTests/CompletedRestoredAgentGenerationTests.swift index e8a00adc2aec..d144a278b21f 100644 --- a/cmuxTests/CompletedRestoredAgentGenerationTests.swift +++ b/cmuxTests/CompletedRestoredAgentGenerationTests.swift @@ -237,7 +237,11 @@ struct CompletedRestoredAgentGenerationTests { processIDs: [Int(identity.pid)], processIdentities: [Int(identity.pid): identity], agentProcessIDs: [Int(identity.pid)], - agentProcessIdentities: [Int(identity.pid): identity] + agentProcessIdentities: [Int(identity.pid): identity], + hibernationPanelProcessIDs: [Int(identity.pid)], + terminationProcessIDs: [Int(identity.pid)], + terminationProcessIdentities: [Int(identity.pid): identity], + containsUnrelatedProcess: false ) } From bb29d1470a89aeb5eb247748537631678ce485ee Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 29 Jul 2026 06:33:17 -0700 Subject: [PATCH 15/20] test: repair merged main CI guards --- ...pDelegateEqualizeSplitsShortcutTests.swift | 23 +++++++++---------- web/tests/client-config-env.test.ts | 2 ++ web/tests/subrouter-accounts-route.test.ts | 10 ++------ 3 files changed, 15 insertions(+), 20 deletions(-) diff --git a/cmuxTests/AppDelegateEqualizeSplitsShortcutTests.swift b/cmuxTests/AppDelegateEqualizeSplitsShortcutTests.swift index 8e3fba2e1b0f..0ef079913887 100644 --- a/cmuxTests/AppDelegateEqualizeSplitsShortcutTests.swift +++ b/cmuxTests/AppDelegateEqualizeSplitsShortcutTests.swift @@ -804,7 +804,7 @@ final class AppDelegateEqualizeSplitsShortcutTests { } @Test - func testDefaultWorkspaceTerminalFontSizeDrainScheduleIsOneShotAndCancellable() + func testWorkspaceTerminalFontSizeDrainScheduleContractIsOneShotAndCancellable() async { var firedCount = 0 let fired = expectation( @@ -819,21 +819,20 @@ final class AppDelegateEqualizeSplitsShortcutTests { await waitWhileSuspended(for: [fired], timeout: 1) completedCancellation() - try? await Task.sleep( - nanoseconds: 50_000_000 - ) + completedCancellation() XCTAssertEqual(firedCount, 1) var cancelledFireCount = 0 - let pendingCancellation = - WorkspaceTerminalFontSizeCoordinator - .scheduleDefaultDrain(after: 0.05) { - cancelledFireCount += 1 - } + // Drive cancellation through the coordinator's injected scheduler + // instead of waiting for a real deadline. + let scheduler = ManualWorkspaceFontSizeDrainScheduler() + let pendingCancellation = scheduler.schedule( + delay: 0.05 + ) { + cancelledFireCount += 1 + } pendingCancellation() - try? await Task.sleep( - nanoseconds: 100_000_000 - ) + scheduler.fire(at: 0) XCTAssertEqual(cancelledFireCount, 0) } diff --git a/web/tests/client-config-env.test.ts b/web/tests/client-config-env.test.ts index d5644f936b87..71d89ff18b0d 100644 --- a/web/tests/client-config-env.test.ts +++ b/web/tests/client-config-env.test.ts @@ -10,6 +10,8 @@ const requiredEnv = { STACK_SECRET_SERVER_KEY: "stack-secret", NEXT_PUBLIC_STACK_PROJECT_ID: "stack-project", NEXT_PUBLIC_STACK_PUBLISHABLE_CLIENT_KEY: "stack-public", + SUBROUTER_ENFORCE_STACK_PERMISSIONS: "0", + SUBROUTER_ALLOWED_TEAM_IDS: "*", }; const requiredIrohProductionEnv = { diff --git a/web/tests/subrouter-accounts-route.test.ts b/web/tests/subrouter-accounts-route.test.ts index 68fb209be207..7b832fa0311b 100644 --- a/web/tests/subrouter-accounts-route.test.ts +++ b/web/tests/subrouter-accounts-route.test.ts @@ -85,12 +85,7 @@ describe("subrouter accounts route", () => { const operation = withSubrouterAuthorizationDeadline( async () => await new Promise(() => {}), ); - const result = await Promise.race([ - operation.catch((error: unknown) => error), - new Promise<"still-pending">((resolve) => - setTimeout(() => resolve("still-pending"), 100) - ), - ]); + const result = await operation.catch((error: unknown) => error); expect(result).toBeInstanceOf(SubrouterAuthorizationTimeoutError); }); @@ -998,9 +993,8 @@ describe("subrouter accounts route", () => { { length: 12 }, () => teamsRoute.GET(request("/api/subrouter/teams")), ); - await new Promise((resolve) => setTimeout(resolve, 50)); - releasePermissions(); const responses = await Promise.all(routes); + releasePermissions(); expect(responses.every((response) => response.status === 503)).toBe(true); expect(maxActive).toBeLessThanOrEqual(8); From d212472f2e5cc34170c3361b416289736cebf656 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 29 Jul 2026 06:41:19 -0700 Subject: [PATCH 16/20] refactor: split agent hibernation panel types --- .../AgentHibernationPanelPhase.swift | 27 ------------------- .../AgentHibernationPanelState.swift | 11 ++++++++ .../AgentHibernationResumePreparation.swift | 14 ++++++++++ cmux.xcodeproj/project.pbxproj | 8 ++++++ 4 files changed, 33 insertions(+), 27 deletions(-) create mode 100644 Sources/AgentHibernation/AgentHibernationPanelState.swift create mode 100644 Sources/AgentHibernation/AgentHibernationResumePreparation.swift diff --git a/Sources/AgentHibernation/AgentHibernationPanelPhase.swift b/Sources/AgentHibernation/AgentHibernationPanelPhase.swift index 6f85e58a78ff..dc8dca798c9c 100644 --- a/Sources/AgentHibernation/AgentHibernationPanelPhase.swift +++ b/Sources/AgentHibernation/AgentHibernationPanelPhase.swift @@ -1,15 +1,3 @@ -import Foundation - -struct AgentHibernationPanelState { - let agent: SessionRestorableAgentSnapshot - let hibernatedAt: Date - let lastActivityAt: Date - - var agentDisplayName: String { - agent.agentDisplayName - } -} - enum AgentHibernationPanelPhase { case live case terminating(AgentHibernationPanelState) @@ -53,18 +41,3 @@ enum AgentHibernationPanelPhase { } } } - -enum AgentHibernationResumePreparation: Equatable { - case unavailable - case resumed(queuedStartupInput: Bool) - - var didResume: Bool { - if case .resumed = self { return true } - return false - } - - var queuedStartupInput: Bool { - if case .resumed(let queuedStartupInput) = self { return queuedStartupInput } - return false - } -} diff --git a/Sources/AgentHibernation/AgentHibernationPanelState.swift b/Sources/AgentHibernation/AgentHibernationPanelState.swift new file mode 100644 index 000000000000..e5baf4bba86d --- /dev/null +++ b/Sources/AgentHibernation/AgentHibernationPanelState.swift @@ -0,0 +1,11 @@ +import Foundation + +struct AgentHibernationPanelState { + let agent: SessionRestorableAgentSnapshot + let hibernatedAt: Date + let lastActivityAt: Date + + var agentDisplayName: String { + agent.agentDisplayName + } +} diff --git a/Sources/AgentHibernation/AgentHibernationResumePreparation.swift b/Sources/AgentHibernation/AgentHibernationResumePreparation.swift new file mode 100644 index 000000000000..090d9a6ea42a --- /dev/null +++ b/Sources/AgentHibernation/AgentHibernationResumePreparation.swift @@ -0,0 +1,14 @@ +enum AgentHibernationResumePreparation: Equatable { + case unavailable + case resumed(queuedStartupInput: Bool) + + var didResume: Bool { + if case .resumed = self { return true } + return false + } + + var queuedStartupInput: Bool { + if case .resumed(let queuedStartupInput) = self { return queuedStartupInput } + return false + } +} diff --git a/cmux.xcodeproj/project.pbxproj b/cmux.xcodeproj/project.pbxproj index 0b6a9e912091..a1cacd40409a 100644 --- a/cmux.xcodeproj/project.pbxproj +++ b/cmux.xcodeproj/project.pbxproj @@ -74,6 +74,7 @@ D36A00030000000000000003 /* AgentHibernationLifecycleState.swift in Sources */ = {isa = PBXBuildFile; fileRef = D36A00030000000000000002 /* AgentHibernationLifecycleState.swift */; }; 8997C0028997C0028997C002 /* AgentHibernationMemoryPressureResponder.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D0028997D0028997D002 /* AgentHibernationMemoryPressureResponder.swift */; }; 8997E0028997E0028997E002 /* AgentHibernationPanelPhase.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997F0028997F0028997F002 /* AgentHibernationPanelPhase.swift */; }; + 9090E0019090E0019090E001 /* AgentHibernationPanelState.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9090F0019090F0019090F001 /* AgentHibernationPanelState.swift */; }; 8997C00C8997C00C8997C00C /* AgentHibernationPlaceholderMode.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D00C8997D00C8997D00C /* AgentHibernationPlaceholderMode.swift */; }; F65760160000000000000001 /* AgentHibernationPlanner.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760160000000000000002 /* AgentHibernationPlanner.swift */; }; F65760170000000000000001 /* AgentHibernationPlannerInput.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760170000000000000002 /* AgentHibernationPlannerInput.swift */; }; @@ -86,6 +87,7 @@ 8997C0128997C0128997C012 /* AgentHibernationProcessSnapshotCoordinatorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D0128997D0128997D012 /* AgentHibernationProcessSnapshotCoordinatorTests.swift */; }; 8997C0048997C0048997C004 /* AgentHibernationProcessTerminationTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D0048997D0048997D004 /* AgentHibernationProcessTerminationTests.swift */; }; F65760240000000000000001 /* AgentHibernationRestoreMonitorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760240000000000000002 /* AgentHibernationRestoreMonitorTests.swift */; }; + 9090E0029090E0029090E002 /* AgentHibernationResumePreparation.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9090F0029090F0029090F002 /* AgentHibernationResumePreparation.swift */; }; 8997C0078997C0078997C007 /* AgentHibernationTerminationFailureTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D0078997D0078997D007 /* AgentHibernationTerminationFailureTests.swift */; }; F65760020000000000000001 /* AgentHibernationTestHelpers.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760020000000000000002 /* AgentHibernationTestHelpers.swift */; }; D36A00020000000000000001 /* AgentHibernationTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D36A00020000000000000002 /* AgentHibernationTests.swift */; }; @@ -2639,6 +2641,7 @@ C0DE71B10000000000000001 /* AppDelegate+AgentChatNotifications.swift in Sources D36A00030000000000000002 /* AgentHibernationLifecycleState.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernation/AgentHibernationLifecycleState.swift; sourceTree = ""; }; 8997D0028997D0028997D002 /* AgentHibernationMemoryPressureResponder.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = App/AgentHibernationMemoryPressureResponder.swift; sourceTree = ""; }; 8997F0028997F0028997F002 /* AgentHibernationPanelPhase.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernation/AgentHibernationPanelPhase.swift; sourceTree = ""; }; + 9090F0019090F0019090F001 /* AgentHibernationPanelState.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernation/AgentHibernationPanelState.swift; sourceTree = ""; }; 8997D00C8997D00C8997D00C /* AgentHibernationPlaceholderMode.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Panels/AgentHibernationPlaceholderMode.swift; sourceTree = ""; }; F65760160000000000000002 /* AgentHibernationPlanner.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationPlanner.swift"; sourceTree = ""; }; F65760170000000000000002 /* AgentHibernationPlannerInput.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationPlannerInput.swift"; sourceTree = ""; }; @@ -2651,6 +2654,7 @@ C0DE71B10000000000000001 /* AppDelegate+AgentChatNotifications.swift in Sources 8997D0128997D0128997D012 /* AgentHibernationProcessSnapshotCoordinatorTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernationProcessSnapshotCoordinatorTests.swift; sourceTree = ""; }; 8997D0048997D0048997D004 /* AgentHibernationProcessTerminationTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernationProcessTerminationTests.swift; sourceTree = ""; }; F65760240000000000000002 /* AgentHibernationRestoreMonitorTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernationRestoreMonitorTests.swift; sourceTree = ""; }; + 9090F0029090F0029090F002 /* AgentHibernationResumePreparation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernation/AgentHibernationResumePreparation.swift; sourceTree = ""; }; 8997D0078997D0078997D007 /* AgentHibernationTerminationFailureTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernationTerminationFailureTests.swift; sourceTree = ""; }; F65760020000000000000002 /* AgentHibernationTestHelpers.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernationTestHelpers.swift; sourceTree = ""; }; D36A00020000000000000002 /* AgentHibernationTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernationTests.swift; sourceTree = ""; }; @@ -5940,6 +5944,8 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = 0A1107110000000000000023 /* SessionRestorableAgentSnapshot+Commands.swift */, D36A00030000000000000002 /* AgentHibernationLifecycleState.swift */, 8997F0028997F0028997F002 /* AgentHibernationPanelPhase.swift */, + 9090F0019090F0019090F001 /* AgentHibernationPanelState.swift */, + 9090F0029090F0029090F002 /* AgentHibernationResumePreparation.swift */, F0ACC0E0000000000000002 /* CachedAgentProcessIdentityValidator.swift */, B3575000000000000000000B /* SessionIndexModels.swift */, F0ACC0DE0000000000000008 /* SharedLiveAgentIndexLoader.swift */, @@ -8127,12 +8133,14 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = D36A00030000000000000001 /* AgentHibernationLifecycleState.swift in Sources */, 8997C0028997C0028997C002 /* AgentHibernationMemoryPressureResponder.swift in Sources */, 8997E0028997E0028997E002 /* AgentHibernationPanelPhase.swift in Sources */, + 9090E0019090E0019090E001 /* AgentHibernationPanelState.swift in Sources */, 8997C00C8997C00C8997C00C /* AgentHibernationPlaceholderMode.swift in Sources */, F65760160000000000000001 /* AgentHibernationPlanner.swift in Sources */, F65760170000000000000001 /* AgentHibernationPlannerInput.swift in Sources */, 8997C0098997C0098997C009 /* AgentHibernationProcessExitCompletion.swift in Sources */, 8997C0118997C0118997C011 /* AgentHibernationProcessExitEpoch.swift in Sources */, 8997C00B8997C00B8997C00B /* AgentHibernationProcessSnapshotCoordinator.swift in Sources */, + 9090E0029090E0029090E002 /* AgentHibernationResumePreparation.swift in Sources */, F65760090000000000000001 /* AgentHibernationTranscriptGuard+ClaudeWorkflow.swift in Sources */, F65760220000000000000001 /* AgentHibernationTranscriptGuard+PostTeardownRestore.swift in Sources */, F65760230000000000000001 /* AgentHibernationTranscriptGuard+StableFileComparison.swift in Sources */, From 5037f250f890b2ee2686a639294d2e0a8114b880 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 29 Jul 2026 06:47:52 -0700 Subject: [PATCH 17/20] test: cover hibernation cleanup fallback failures --- ...rfaceRuntimeTeardownCoordinatorTests.swift | 31 +++++++++++- cmux.xcodeproj/project.pbxproj | 4 ++ ...entHibernationPanelCloseCleanupTests.swift | 47 +++++++++++++++++++ ...ntHibernationProcessTerminationTests.swift | 3 +- 4 files changed, 82 insertions(+), 3 deletions(-) create mode 100644 cmuxTests/AgentHibernationPanelCloseCleanupTests.swift diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift index fe14022e2b63..25973959cc17 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift @@ -166,6 +166,7 @@ private final class LifetimeRecordingByteTeeLease: TerminalByteTeeLease, @unchec let secondReservation = try #require( await coordinator.reserveIsolatedHibernationTeardown() ) + #expect(await coordinator.reserveIsolatedHibernationTeardown() == nil) let secondIsolatedTicket = coordinator.enqueueRuntimeTeardown( id: UUID(), workspaceId: UUID(), @@ -180,7 +181,6 @@ private final class LifetimeRecordingByteTeeLease: TerminalByteTeeLease, @unchec secondIsolatedFreeCount.withLock { $0 += 1 } } ) - #expect(await coordinator.reserveIsolatedHibernationTeardown() == nil) let serializedTicket = coordinator.enqueueRuntimeTeardown( id: UUID(), workspaceId: UUID(), @@ -211,6 +211,35 @@ private final class LifetimeRecordingByteTeeLease: TerminalByteTeeLease, @unchec await coordinator.cancelIsolatedHibernationTeardown(nextReservation) } + @Test func staleIsolatedReservationFallsBackToSerializedFree() async throws { + let coordinator = TerminalSurfaceRuntimeTeardownCoordinator() + let surface = UnsafeMutableRawPointer.allocate(byteCount: 8, alignment: 8) + defer { surface.deallocate() } + let freeCount = OSAllocatedUnfairLock(initialState: 0) + let staleReservation = try #require( + await coordinator.reserveIsolatedHibernationTeardown() + ) + await coordinator.cancelIsolatedHibernationTeardown(staleReservation) + + let ticket = coordinator.enqueueRuntimeTeardown( + id: UUID(), + workspaceId: UUID(), + reason: "test.staleIsolatedReservation", + surface: surface, + callbackContext: nil, + manualIOContext: nil, + byteTeeLease: nil, + executionLane: .isolatedHibernation, + isolatedHibernationReservation: staleReservation, + freeSurface: { _ in + freeCount.withLock { $0 += 1 } + } + ) + + #expect(await ticket.wait(timeout: .seconds(1))) + #expect(freeCount.withLock { $0 } == 1) + } + @Test func byteTeeCallbackOwnerIsReleasedOnlyAfterNativeFreeReturns() async { let coordinator = TerminalSurfaceRuntimeTeardownCoordinator() let recorder = TeardownLifetimeRecorder() diff --git a/cmux.xcodeproj/project.pbxproj b/cmux.xcodeproj/project.pbxproj index a1cacd40409a..a34b08c33da5 100644 --- a/cmux.xcodeproj/project.pbxproj +++ b/cmux.xcodeproj/project.pbxproj @@ -73,6 +73,7 @@ D36A00030000000000000001 /* AgentHibernationLifecycleState.swift in Sources */ = {isa = PBXBuildFile; fileRef = D36A00030000000000000002 /* AgentHibernationLifecycleState.swift */; }; D36A00030000000000000003 /* AgentHibernationLifecycleState.swift in Sources */ = {isa = PBXBuildFile; fileRef = D36A00030000000000000002 /* AgentHibernationLifecycleState.swift */; }; 8997C0028997C0028997C002 /* AgentHibernationMemoryPressureResponder.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D0028997D0028997D002 /* AgentHibernationMemoryPressureResponder.swift */; }; + 9090E0039090E0039090E003 /* AgentHibernationPanelCloseCleanupTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9090F0039090F0039090F003 /* AgentHibernationPanelCloseCleanupTests.swift */; }; 8997E0028997E0028997E002 /* AgentHibernationPanelPhase.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997F0028997F0028997F002 /* AgentHibernationPanelPhase.swift */; }; 9090E0019090E0019090E001 /* AgentHibernationPanelState.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9090F0019090F0019090F001 /* AgentHibernationPanelState.swift */; }; 8997C00C8997C00C8997C00C /* AgentHibernationPlaceholderMode.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D00C8997D00C8997D00C /* AgentHibernationPlaceholderMode.swift */; }; @@ -2640,6 +2641,7 @@ C0DE71B10000000000000001 /* AppDelegate+AgentChatNotifications.swift in Sources D36A00010000000000000002 /* AgentHibernationController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = App/AgentHibernationController.swift; sourceTree = ""; }; D36A00030000000000000002 /* AgentHibernationLifecycleState.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernation/AgentHibernationLifecycleState.swift; sourceTree = ""; }; 8997D0028997D0028997D002 /* AgentHibernationMemoryPressureResponder.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = App/AgentHibernationMemoryPressureResponder.swift; sourceTree = ""; }; + 9090F0039090F0039090F003 /* AgentHibernationPanelCloseCleanupTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernationPanelCloseCleanupTests.swift; sourceTree = ""; }; 8997F0028997F0028997F002 /* AgentHibernationPanelPhase.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernation/AgentHibernationPanelPhase.swift; sourceTree = ""; }; 9090F0019090F0019090F001 /* AgentHibernationPanelState.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernation/AgentHibernationPanelState.swift; sourceTree = ""; }; 8997D00C8997D00C8997D00C /* AgentHibernationPlaceholderMode.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Panels/AgentHibernationPlaceholderMode.swift; sourceTree = ""; }; @@ -7035,6 +7037,7 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = A9E050000000000000000001 /* AgentSessionWebRendererTests.swift */, A9E040000000000000000001 /* CodexAppServerSessionTests.swift */, F65760010000000000000002 /* AgentHibernationPlannerSwiftTests.swift */, + 9090F0039090F0039090F003 /* AgentHibernationPanelCloseCleanupTests.swift */, 8997D0048997D0048997D004 /* AgentHibernationProcessTerminationTests.swift */, 8997D0138997D0138997D013 /* AgentHibernationProcessSignalBoundaryTests.swift */, 8997D0128997D0128997D012 /* AgentHibernationProcessSnapshotCoordinatorTests.swift */, @@ -9869,6 +9872,7 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = C7A51D100000000000000002 /* AgentChatSessionRegistryObservationReviewRegressionTests.swift in Sources */, C7A51D000000000000000002 /* AgentChatSessionRegistryObservationTests.swift in Sources */, A9E020000000000000000005 /* AgentExecutableResolverTests.swift in Sources */, + 9090E0039090E0039090E003 /* AgentHibernationPanelCloseCleanupTests.swift in Sources */, F65760010000000000000001 /* AgentHibernationPlannerSwiftTests.swift in Sources */, D72260010000000000000001 /* AgentHibernationPortalVisibilityTests.swift in Sources */, 8997C0138997C0138997C013 /* AgentHibernationProcessSignalBoundaryTests.swift in Sources */, diff --git a/cmuxTests/AgentHibernationPanelCloseCleanupTests.swift b/cmuxTests/AgentHibernationPanelCloseCleanupTests.swift new file mode 100644 index 000000000000..3f3fa8cbf620 --- /dev/null +++ b/cmuxTests/AgentHibernationPanelCloseCleanupTests.swift @@ -0,0 +1,47 @@ +import Foundation +import Testing + +#if canImport(cmux_DEV) +@testable import cmux_DEV +#elseif canImport(cmux) +@testable import cmux +#endif + +@MainActor +@Suite(.serialized) +struct AgentHibernationPanelCloseCleanupTests { + @Test + func rejectedCleanupDeadlineReleasesCleanupOwnership() async throws { + let controller = AgentHibernationController.shared + let panelID = UUID() + let requestID = UUID() + let processExitCompletion = AgentHibernationProcessExitCompletion() + controller.registerCommittedTerminationObservation( + panelID: panelID, + requestID: requestID, + processExitCompletion: processExitCompletion + ) + defer { + controller.removeCommittedTerminationObservation( + panelID: panelID, + requestID: requestID + ) + } + + controller.limitCommittedTerminationObservationAfterPanelClose( + panelID: panelID, + cleanupDelay: .zero, + sleepUntilDeadline: { _ in false } + ) + let cleanupTask = try #require( + controller.committedTerminationCleanupByPanelID[panelID]?.task + ) + await cleanupTask.value + + #expect(controller.committedTerminationCleanupByPanelID[panelID] == nil) + #expect( + controller.committedTerminationObservationsByPanelID[panelID]? + .requestID == requestID + ) + } +} diff --git a/cmuxTests/AgentHibernationProcessTerminationTests.swift b/cmuxTests/AgentHibernationProcessTerminationTests.swift index bc856c55634b..b27c04fe5275 100644 --- a/cmuxTests/AgentHibernationProcessTerminationTests.swift +++ b/cmuxTests/AgentHibernationProcessTerminationTests.swift @@ -9,6 +9,7 @@ import Testing @testable import cmux #endif +@Suite(.serialized) struct AgentHibernationProcessTerminationTests { private nonisolated static let signalScopeKey = AgentHibernationPanelKey( workspaceId: UUID(), @@ -242,8 +243,6 @@ struct AgentHibernationProcessTerminationTests { ) #expect(didExit) #expect(escalatedTargets.withLock { $0 } == [-101]) - #expect(!escalatedTargets.withLock { $0 }.contains(202)) - #expect(!escalatedTargets.withLock { $0 }.contains(-202)) postKillDeadline.continuation.finish() } From 812930e65549eebb66a73b80c3548a3b8537ec68 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 29 Jul 2026 06:51:54 -0700 Subject: [PATCH 18/20] fix: make hibernation cleanup fallbacks bounded --- ...nalSurfaceRuntimeTeardownCoordinator.swift | 52 ++++++++++--------- ...HibernationController+PanelLifecycle.swift | 14 +++-- ...ibernationProcessSnapshotCoordinator.swift | 4 ++ ...ationProcessSnapshotCoordinatorTests.swift | 10 +++- 4 files changed, 51 insertions(+), 29 deletions(-) diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift index 1f7487e54c76..af2ba25d4dbf 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift @@ -179,34 +179,36 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { pendingReasonsById[request.id] = request.reason switch executionLane { case .isolatedHibernation: - guard let isolatedHibernationReservation, - let executionSlot = await isolatedHibernationAdmission.executionSlot( - for: isolatedHibernationReservation - ), - isolatedHibernationQueues.indices.contains(executionSlot) else { - preconditionFailure( - "isolated hibernation teardown requires an active reservation" - ) - } - // Each reservation exclusively owns one queue until its native free - // returns. Ghostty locks its shared surface registry, while renderer - // and IO joins are surface-owned, so separate surfaces may tear down - // concurrently. This bounds blocked native workers at two without - // letting one stuck pane strand another admitted pane. - Task { - await self.observeTimeout(id: request.id) - } - isolatedHibernationQueues[executionSlot].async { - self.freeNativeSurface(request) + if let isolatedHibernationReservation, + let executionSlot = await isolatedHibernationAdmission.executionSlot( + for: isolatedHibernationReservation + ), + isolatedHibernationQueues.indices.contains(executionSlot) { + // Each reservation exclusively owns one queue until its native free + // returns. Ghostty locks its shared surface registry, while renderer + // and IO joins are surface-owned, so separate surfaces may tear down + // concurrently. This bounds blocked native workers at two without + // letting one stuck pane strand another admitted pane. Task { - await self.isolatedHibernationAdmission.release( - isolatedHibernationReservation - ) - await self.finishFree(request) - await self.complete(id: request.id) + await self.observeTimeout(id: request.id) + } + isolatedHibernationQueues[executionSlot].async { + self.freeNativeSurface(request) + Task { + await self.isolatedHibernationAdmission.release( + isolatedHibernationReservation + ) + await self.finishFree(request) + await self.complete(id: request.id) + } } + return + } + if let isolatedHibernationReservation { + await isolatedHibernationAdmission.release( + isolatedHibernationReservation + ) } - return case .serializedClose: break } diff --git a/Sources/App/AgentHibernationController+PanelLifecycle.swift b/Sources/App/AgentHibernationController+PanelLifecycle.swift index 39f7271ec60f..e324e7fda7b9 100644 --- a/Sources/App/AgentHibernationController+PanelLifecycle.swift +++ b/Sources/App/AgentHibernationController+PanelLifecycle.swift @@ -72,8 +72,17 @@ extension AgentHibernationController { .cancel() let cleanupID = UUID() let cleanupTask = Task { @MainActor [weak self] in - guard await sleepUntilDeadline(cleanupDelay), - let self, + let didReachDeadline = await sleepUntilDeadline(cleanupDelay) + guard let self else { return } + defer { + if self.committedTerminationCleanupByPanelID[panelID]?.requestID == + cleanupID { + self.committedTerminationCleanupByPanelID.removeValue( + forKey: panelID + ) + } + } + guard didReachDeadline, !Task.isCancelled, self.committedTerminationCleanupByPanelID[panelID]?.requestID == cleanupID, @@ -84,7 +93,6 @@ extension AgentHibernationController { ) else { return } - self.committedTerminationCleanupByPanelID.removeValue(forKey: panelID) removed.task?.cancel() await removed.processExitCompletion.finish(false) } diff --git a/Sources/App/AgentHibernationProcessSnapshotCoordinator.swift b/Sources/App/AgentHibernationProcessSnapshotCoordinator.swift index e5d5bba3d7fc..4eff3906cb4c 100644 --- a/Sources/App/AgentHibernationProcessSnapshotCoordinator.swift +++ b/Sources/App/AgentHibernationProcessSnapshotCoordinator.swift @@ -23,6 +23,10 @@ actor AgentHibernationProcessSnapshotCoordinator { private var captureTask: Task? private var captureHasStarted = false + var queuedSnapshotWaiterCount: Int { + queuedSnapshotWaiters.count + } + init( beforeCapture: @escaping CaptureScheduler = { await Task.yield() diff --git a/cmuxTests/AgentHibernationProcessSnapshotCoordinatorTests.swift b/cmuxTests/AgentHibernationProcessSnapshotCoordinatorTests.swift index c564d26702c6..1f70902226f6 100644 --- a/cmuxTests/AgentHibernationProcessSnapshotCoordinatorTests.swift +++ b/cmuxTests/AgentHibernationProcessSnapshotCoordinatorTests.swift @@ -35,7 +35,15 @@ struct AgentHibernationProcessSnapshotCoordinatorTests { var captureScheduledIterator = captureScheduled.stream.makeAsyncIterator() _ = await captureScheduledIterator.next() let second = Task { await coordinator.nextSnapshot() } - await Task.yield() + let clock = ContinuousClock() + let registrationDeadline = clock.now.advanced(by: .seconds(1)) + while clock.now < registrationDeadline { + if await coordinator.queuedSnapshotWaiterCount == 2 { + break + } + await Task.yield() + } + #expect(await coordinator.queuedSnapshotWaiterCount == 2) allowCapture.continuation.yield() allowCapture.continuation.finish() From e04c7f4f8bb104141909ceec812f1c7416dc5393 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 29 Jul 2026 07:10:48 -0700 Subject: [PATCH 19/20] Fix hibernation process test compilation --- cmuxTests/AgentHibernationProcessTerminationTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmuxTests/AgentHibernationProcessTerminationTests.swift b/cmuxTests/AgentHibernationProcessTerminationTests.swift index b27c04fe5275..1c645a80e6d9 100644 --- a/cmuxTests/AgentHibernationProcessTerminationTests.swift +++ b/cmuxTests/AgentHibernationProcessTerminationTests.swift @@ -939,7 +939,7 @@ struct AgentHibernationProcessTerminationTests { processGroupLeaders: processGroupLeaders ) } - AgentHibernationProcessExitEpoch( + return AgentHibernationProcessExitEpoch( terminations: [ .init( processID: 202, From 235449d7be315470473aa69fad0a1b4f412c470c Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 29 Jul 2026 07:37:55 -0700 Subject: [PATCH 20/20] Eliminate Swift concurrency warnings --- ...ionController+ProcessExitObservation.swift | 7 ++++++- ...rnationController+ProcessTermination.swift | 21 ++++++++++--------- Sources/AppDelegate.swift | 2 +- Sources/GhosttyTerminalView.swift | 4 +++- ...nalFontConfigurationReloadReconciler.swift | 2 +- ...pDelegateEqualizeSplitsShortcutTests.swift | 4 ++-- 6 files changed, 24 insertions(+), 16 deletions(-) diff --git a/Sources/App/AgentHibernationController+ProcessExitObservation.swift b/Sources/App/AgentHibernationController+ProcessExitObservation.swift index 7c880e41d0bb..18faa21fe1dc 100644 --- a/Sources/App/AgentHibernationController+ProcessExitObservation.swift +++ b/Sources/App/AgentHibernationController+ProcessExitObservation.swift @@ -335,7 +335,9 @@ extension AgentHibernationController { let didRecover = await Self.waitForCommittedTerminationRecovery( terminations: didExit ? [] : terminations, processScopeKey: processScopeKey, - waitForRecoveryExit: didExit ? { _ in true } : waitForRecoveryExit, + waitForRecoveryExit: didExit + ? Self.completedRecoveryExitWait + : waitForRecoveryExit, waitForRecoveryReadiness: waitForRecoveryReadiness, recoveryDeadline: recoveryDeadline, sleepUntilRecoveryDeadline: sleepUntilRecoveryDeadline, @@ -442,4 +444,7 @@ extension AgentHibernationController { } } + private nonisolated static let completedRecoveryExitWait: + @Sendable ([ScopedProcessTermination]) async -> Bool = { _ in true } + } diff --git a/Sources/App/AgentHibernationController+ProcessTermination.swift b/Sources/App/AgentHibernationController+ProcessTermination.swift index b4f67b2a4105..0db0d2eb295a 100644 --- a/Sources/App/AgentHibernationController+ProcessTermination.swift +++ b/Sources/App/AgentHibernationController+ProcessTermination.swift @@ -309,8 +309,17 @@ extension AgentHibernationController { ) let processScopeKey = record.key let processSnapshotCoordinator = processSnapshotCoordinator - let retryTermination: CommittedTerminationRetry = { + let retryTerminationIsSafe: @MainActor @Sendable () -> Bool = { [weak self, weak terminalPanel = record.terminalPanel] in + guard let self, let terminalPanel, + terminalPanel.isAgentHibernationCommitPending, + self.committedTerminationObservationsByPanelID[panelID]? + .requestID == committedTerminationRequestID else { + return false + } + return true + } + let retryTermination: CommittedTerminationRetry = { guard !scopedProcessTerminations.isEmpty else { return .exited } @@ -325,15 +334,7 @@ extension AgentHibernationController { return await Self.terminateScopedProcessEpochForHibernation( epoch, processScopeKey: processScopeKey, - shouldCommit: { - guard let self, let terminalPanel, - terminalPanel.isAgentHibernationCommitPending, - self.committedTerminationObservationsByPanelID[panelID]? - .requestID == committedTerminationRequestID else { - return false - } - return true - } + shouldCommit: retryTerminationIsSafe ) } let terminationResult = await Self.terminateScopedProcessesForHibernation( diff --git a/Sources/AppDelegate.swift b/Sources/AppDelegate.swift index 215e44fb2365..f6ac33a4366e 100644 --- a/Sources/AppDelegate.swift +++ b/Sources/AppDelegate.swift @@ -12771,7 +12771,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate, UNUserNotificationCent queue: .main ) { [weak self] _ in GhosttyConfig.invalidateLoadCache() - MainActor.assumeIsolated { + _ = MainActor.assumeIsolated { self?.reloadConfiguration( source: "globalFontMagnificationDidChange", reloadSettingsFromFile: false diff --git a/Sources/GhosttyTerminalView.swift b/Sources/GhosttyTerminalView.swift index ad456c071f37..e7489c3538f1 100644 --- a/Sources/GhosttyTerminalView.swift +++ b/Sources/GhosttyTerminalView.swift @@ -1156,7 +1156,9 @@ class GhosttyApp { object: nil, queue: .main ) { [weak self] _ in - self?.reloadConfiguration(source: "settings.terminal.copyOnSelect") + _ = MainActor.assumeIsolated { + self?.reloadConfiguration(source: "settings.terminal.copyOnSelect") + } }) #endif diff --git a/Sources/TerminalFontConfigurationReloadReconciler.swift b/Sources/TerminalFontConfigurationReloadReconciler.swift index 1421ca41db85..e3b6d84692b3 100644 --- a/Sources/TerminalFontConfigurationReloadReconciler.swift +++ b/Sources/TerminalFontConfigurationReloadReconciler.swift @@ -8,7 +8,7 @@ final class TerminalFontConfigurationReloadReconciler { typealias CaptureNextWork = @MainActor () -> ReconciliationWork? typealias Scheduler = - @MainActor (@escaping @MainActor () -> Void) -> Void + @MainActor @Sendable (@escaping @MainActor @Sendable () -> Void) -> Void nonisolated static let defaultMaximumSurfaceVisitsPerDrain = 8 nonisolated static let defaultMaximumAttemptsPerWork = 3 diff --git a/cmuxTests/AppDelegateEqualizeSplitsShortcutTests.swift b/cmuxTests/AppDelegateEqualizeSplitsShortcutTests.swift index 0ef079913887..3eef18d97cac 100644 --- a/cmuxTests/AppDelegateEqualizeSplitsShortcutTests.swift +++ b/cmuxTests/AppDelegateEqualizeSplitsShortcutTests.swift @@ -8044,10 +8044,10 @@ private final class ManualWorkspaceFontSizeDrainScheduler { @MainActor private final class ManualTerminalFontConfigurationReloadScheduler { - private var actions: [@MainActor () -> Void] = [] + private var actions: [@MainActor @Sendable () -> Void] = [] func schedule( - action: @escaping @MainActor () -> Void + action: @escaping @MainActor @Sendable () -> Void ) { actions.append(action) }