Hibernate idle agents before critical memory pressure panics

CLI/cmux.swift

@@ -15397,7 +15397,7 @@ struct CMUXCLI {
             return """
             Usage: cmux agent-hibernation <on|off> [--json]
 
-            Enable or disable Agent Hibernation.
+            Enable or disable routine Agent Hibernation.
             Configure idle and live-terminal limits from Settings or cmux settings JSON.
             """
         case "restore-session":

Packages/macOS/CmuxSettingsUI/Sources/CmuxSettingsUI/Sections/TerminalSection.swift

@@ -378,7 +378,7 @@ public struct TerminalSection: View {
                 String(localized: "settings.terminal.agentHibernation", defaultValue: "Agent Hibernation"),
                 subtitle: hibernation.current
                     ? String(localized: "settings.terminal.agentHibernation.subtitleOn", defaultValue: "Idle background agent terminals can be suspended when the live-terminal limit is exceeded.")
-                    : String(localized: "settings.terminal.agentHibernation.subtitleOff", defaultValue: "Agent terminals stay live until you close them or quit cmux.")
+                    : String(localized: "settings.terminal.agentHibernation.subtitleOff", defaultValue: "Scheduled hibernation is off. During critical memory pressure, cmux may still hibernate safe idle background agents.")
             ) {
                 Toggle("", isOn: Binding(get: { hibernation.current }, set: { hibernation.set($0) }))
                     .labelsHidden()

Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownAdmission.swift

@@ -0,0 +1,40 @@
+import Foundation
+
+/// Main-actor admission state for one coordinator's bounded native-free slots.
+@MainActor
+final class TerminalSurfaceRuntimeTeardownAdmission {
+    private var availableExecutionSlots: Set<Int>
+    private var executionSlotByReservationID: [UUID: Int] = [:]
+
+    nonisolated init() {
+        availableExecutionSlots = Set(
+            0..<TerminalSurfaceRuntimeTeardownCoordinator
+                .maximumIsolatedHibernationTeardownCount
+        )
+    }
+
+    func reserve() -> TerminalSurfaceRuntimeTeardownReservation? {
+        guard let executionSlot = availableExecutionSlots.min() else {
+            return nil
+        }
+        let reservation = TerminalSurfaceRuntimeTeardownReservation()
+        availableExecutionSlots.remove(executionSlot)
+        executionSlotByReservationID[reservation.id] = executionSlot
+        return reservation
+    }
+
+    func executionSlot(
+        for reservation: TerminalSurfaceRuntimeTeardownReservation
+    ) -> Int? {
+        executionSlotByReservationID[reservation.id]
+    }
+
+    func release(_ reservation: TerminalSurfaceRuntimeTeardownReservation) {
+        guard let executionSlot = executionSlotByReservationID.removeValue(
+            forKey: reservation.id
+        ) else {
+            return
+        }
+        availableExecutionSlots.insert(executionSlot)
+    }
+}

Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCompletion.swift

@@ -0,0 +1,42 @@
+import Foundation
+
+/// One native-free completion shared by the enqueueing surface and teardown worker.
+actor TerminalSurfaceRuntimeTeardownCompletion {
+    private var didFinish = false
+    private var waiters: [UUID: CheckedContinuation<Bool, Never>] = [:]
+
+    func wait() async -> Bool {
+        if didFinish { return true }
+        if Task.isCancelled { return false }
+        let waiterID = UUID()
+        return await withTaskCancellationHandler {
+            await withCheckedContinuation { continuation in
+                if didFinish {
+                    continuation.resume(returning: true)
+                } else if Task.isCancelled {
+                    continuation.resume(returning: false)
+                } else {
+                    waiters[waiterID] = continuation
+                }
+            }
+        } onCancel: {
+            Task {
+                await self.cancel(waiterID: waiterID)
+            }
+        }
+    }
+
+    func finish() {
+        guard !didFinish else { return }
+        didFinish = true
+        let pendingWaiters = waiters.values
+        waiters.removeAll(keepingCapacity: false)
+        for waiter in pendingWaiters {
+            waiter.resume(returning: true)
+        }
+    }
+
+    private func cancel(waiterID: UUID) {
+        waiters.removeValue(forKey: waiterID)?.resume(returning: false)
+    }
+}

Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift

@@ -1,17 +1,23 @@
 public import Foundation
 public import GhosttyKit
 public import CmuxTerminalCore
+internal import Dispatch
 #if DEBUG
 internal import CMUXDebugLog
 #endif
 
-/// Serializes native `ghostty_surface_free` calls off the close/deinit paths.
+/// Coordinates native `ghostty_surface_free` calls off the close/deinit paths.
 ///
-/// Frees run one at a time on a utility worker so re-entrant close/deinit
-/// loops cannot form, with a deadline observer that reports (but never
-/// blocks on) a stuck native free. The app constructs exactly one instance
-/// and injects it through ``TerminalSurfaceRuntimeDependencies``.
+/// Close/deinit frees run one at a time on a utility worker so re-entrant
+/// teardown loops cannot form. Each admitted hibernation owns one independently
+/// startable utility slot, so one stuck native join cannot strand another pane.
+/// Deadline observers report, but never block on, stuck frees. The app constructs
+/// exactly one instance and injects it through
+/// ``TerminalSurfaceRuntimeDependencies``.
 public actor TerminalSurfaceRuntimeTeardownCoordinator {
+    /// Largest batch that can own independently startable native-free slots.
+    public static let maximumIsolatedHibernationTeardownCount = 2
+
     private let timeout: Duration = .seconds(5)
 #if DEBUG
     // Readable at internal scope in DEBUG so the debug-only extension in
@@ -23,9 +29,34 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator {
 #endif
     private var queuedRequests: [TerminalSurfaceRuntimeTeardownRequest] = []
     private var isWorkerRunning = false
+    private let isolatedHibernationQueues: [DispatchQueue]
+    private nonisolated let isolatedHibernationAdmission =
+        TerminalSurfaceRuntimeTeardownAdmission()
 
     /// Creates the process's teardown coordinator.
-    public init() {}
+    public init() {
+        isolatedHibernationQueues = (
+            0..<Self.maximumIsolatedHibernationTeardownCount
+        ).map { executionSlot in
+            DispatchQueue(
+                label: "com.cmux.terminal-surface-hibernation-teardown.\(executionSlot)",
+                qos: .utility
+            )
+        }
+    }
+
+    @MainActor
+    func reserveIsolatedHibernationTeardown()
+        -> TerminalSurfaceRuntimeTeardownReservation? {
+        isolatedHibernationAdmission.reserve()
+    }
+
+    @MainActor
+    func cancelIsolatedHibernationTeardown(
+        _ reservation: TerminalSurfaceRuntimeTeardownReservation
+    ) {
+        isolatedHibernationAdmission.release(reservation)
+    }
 
     /// Reads a bounded screen tail away from the main actor and before any
     /// subsequently enqueued native free for the same surface.
@@ -50,6 +81,8 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator {
     ///     main actor after the free completes.
     ///   - freeSurface: The free operation; defaults to
     ///     `ghostty_surface_free`.
+    /// - Returns: A ticket that completes after the native free and userdata releases.
+    @discardableResult
     public nonisolated func enqueueRuntimeTeardown(
         id: UUID,
         workspaceId: UUID,
@@ -59,7 +92,7 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator {
         freeSurface: @escaping @Sendable (ghostty_surface_t) -> Void = { surface in
             ghostty_surface_free(surface)
         }
-    ) {
+    ) -> TerminalSurfaceRuntimeTeardownTicket {
         enqueueRuntimeTeardown(
             id: id,
             workspaceId: workspaceId,
@@ -97,6 +130,8 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator {
     ///     actor after the free completes.
     ///   - freeSurface: The free operation; defaults to
     ///     `ghostty_surface_free`.
+    /// - Returns: A ticket that completes after the native free and userdata releases.
+    @discardableResult
     nonisolated func enqueueRuntimeTeardown(
         id: UUID,
         workspaceId: UUID,
@@ -105,10 +140,15 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator {
         callbackContext: Unmanaged<GhosttySurfaceCallbackContext>?,
         manualIOContext: Unmanaged<TerminalManualIOWriteBox>?,
         byteTeeLease: (any TerminalByteTeeLease)?,
+        executionLane: TerminalSurfaceRuntimeTeardownExecutionLane = .serializedClose,
+        isolatedHibernationReservation:
+            TerminalSurfaceRuntimeTeardownReservation? = nil,
         freeSurface: @escaping @Sendable (ghostty_surface_t) -> Void = { surface in
             ghostty_surface_free(surface)
         }
-    ) {
+    ) -> TerminalSurfaceRuntimeTeardownTicket {
+        let completion = TerminalSurfaceRuntimeTeardownCompletion()
+        let ticket = TerminalSurfaceRuntimeTeardownTicket(completion: completion)
         let request = TerminalSurfaceRuntimeTeardownRequest(
             id: id,
             workspaceId: workspaceId,
@@ -117,15 +157,61 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator {
             callbackContext: callbackContext,
             manualIOContext: manualIOContext,
             byteTeeLease: byteTeeLease,
-            freeSurface: freeSurface
+            freeSurface: freeSurface,
+            completion: completion
         )
         Task {
-            await self.enqueue(request)
+            await self.enqueue(
+                request,
+                executionLane: executionLane,
+                isolatedHibernationReservation: isolatedHibernationReservation
+            )
         }
+        return ticket
     }
 
-    func enqueue(_ request: TerminalSurfaceRuntimeTeardownRequest) {
+    func enqueue(
+        _ request: TerminalSurfaceRuntimeTeardownRequest,
+        executionLane: TerminalSurfaceRuntimeTeardownExecutionLane = .serializedClose,
+        isolatedHibernationReservation:
+            TerminalSurfaceRuntimeTeardownReservation? = nil
+    ) async {
         pendingReasonsById[request.id] = request.reason
+        switch executionLane {
+        case .isolatedHibernation:
+            if let isolatedHibernationReservation,
+               let executionSlot = await isolatedHibernationAdmission.executionSlot(
+                   for: isolatedHibernationReservation
+               ),
+               isolatedHibernationQueues.indices.contains(executionSlot) {
+                // Each reservation exclusively owns one queue until its native free
+                // returns. Ghostty locks its shared surface registry, while renderer
+                // and IO joins are surface-owned, so separate surfaces may tear down
+                // concurrently. This bounds blocked native workers at two without
+                // letting one stuck pane strand another admitted pane.
+                Task {
+                    await self.observeTimeout(id: request.id)
+                }
+                isolatedHibernationQueues[executionSlot].async {
+                    self.freeNativeSurface(request)
+                    Task {
+                        await self.isolatedHibernationAdmission.release(
+                            isolatedHibernationReservation
+                        )
+                        await self.finishFree(request)
+                        await self.complete(id: request.id)
+                    }
+                }
+                return
+            }
+            if let isolatedHibernationReservation {
+                await isolatedHibernationAdmission.release(
+                    isolatedHibernationReservation
+                )
+            }
+        case .serializedClose:
+            break
+        }
         queuedRequests.append(request)
         if !isWorkerRunning {
             isWorkerRunning = true
@@ -134,7 +220,8 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator {
                     Task {
                         await self.observeTimeout(id: request.id)
                     }
-                    await Self.free(request)
+                    self.freeNativeSurface(request)
+                    await self.finishFree(request)
                     await self.complete(id: request.id)
                 }
             }
@@ -149,14 +236,21 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator {
         return queuedRequests.removeFirst()
     }
 
-    private nonisolated static func free(_ request: TerminalSurfaceRuntimeTeardownRequest) async {
+    private nonisolated func freeNativeSurface(
+        _ request: TerminalSurfaceRuntimeTeardownRequest
+    ) {
 #if DEBUG
         logDebugEvent(
             "surface.lifecycle.nativeFree.begin surface=\(request.surfaceToken) " +
             "workspace=\(request.workspaceToken) reason=\(request.reason)"
         )
 #endif
         request.freeSurface(request.surface)
+    }
+
+    private nonisolated func finishFree(
+        _ request: TerminalSurfaceRuntimeTeardownRequest
+    ) async {
         if request.callbackContext != nil || request.manualIOContext != nil || request.byteTeeLease != nil {
             // The request is the @unchecked Sendable transport for the
             // Unmanaged contexts; release through the request so the @Sendable
@@ -171,6 +265,7 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator {
                 request.byteTeeLease?.release()
             }
         }
+        await request.completion.finish()
 #if DEBUG
         logDebugEvent(
             "surface.lifecycle.nativeFree.end surface=\(request.surfaceToken) " +

Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownExecutionLane.swift

@@ -0,0 +1,8 @@
+/// Selects the ownership boundary for a native surface free.
+enum TerminalSurfaceRuntimeTeardownExecutionLane: Sendable {
+    /// Preserves ordering for close/deinit flows that can re-enter teardown.
+    case serializedClose
+
+    /// Gives an explicitly owned hibernation join an independent bounded slot.
+    case isolatedHibernation
+}

Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownRequest.swift

@@ -25,6 +25,7 @@ struct TerminalSurfaceRuntimeTeardownRequest: @unchecked Sendable {
     let manualIOContext: Unmanaged<TerminalManualIOWriteBox>?
     let byteTeeLease: (any TerminalByteTeeLease)?
     let freeSurface: @Sendable (ghostty_surface_t) -> Void
+    let completion: TerminalSurfaceRuntimeTeardownCompletion
 #if DEBUG
     let surfaceToken: String
     let workspaceToken: String
@@ -38,7 +39,8 @@ struct TerminalSurfaceRuntimeTeardownRequest: @unchecked Sendable {
         callbackContext: Unmanaged<GhosttySurfaceCallbackContext>?,
         manualIOContext: Unmanaged<TerminalManualIOWriteBox>?,
         byteTeeLease: (any TerminalByteTeeLease)?,
-        freeSurface: @escaping @Sendable (ghostty_surface_t) -> Void
+        freeSurface: @escaping @Sendable (ghostty_surface_t) -> Void,
+        completion: TerminalSurfaceRuntimeTeardownCompletion
     ) {
         self.id = id
         self.workspaceId = workspaceId
@@ -48,6 +50,7 @@ struct TerminalSurfaceRuntimeTeardownRequest: @unchecked Sendable {
         self.manualIOContext = manualIOContext
         self.byteTeeLease = byteTeeLease
         self.freeSurface = freeSurface
+        self.completion = completion
 #if DEBUG
         self.surfaceToken = String(id.uuidString.prefix(5))
         self.workspaceToken = String(workspaceId.uuidString.prefix(5))

Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownReservation.swift

@@ -0,0 +1,10 @@
+import Foundation
+
+/// Exclusive admission to one failure-isolated hibernation teardown slot.
+struct TerminalSurfaceRuntimeTeardownReservation: Equatable, Sendable {
+    let id: UUID
+
+    init(id: UUID = UUID()) {
+        self.id = id
+    }
+}