diff --git a/Sources/App/AgentHibernationController+ProcessTermination.swift b/Sources/App/AgentHibernationController+ProcessTermination.swift index f870f3f7a26..59f55a9fa77 100644 --- a/Sources/App/AgentHibernationController+ProcessTermination.swift +++ b/Sources/App/AgentHibernationController+ProcessTermination.swift @@ -145,8 +145,9 @@ extension AgentHibernationController { snapshot = value case .nothingToProtect: snapshot = nil - case .unableToProtect: - // Forfeit hibernation rather than risk issue #6565 transcript loss. + case .unableToProtect, .backgroundWorkPending: + // Forfeit hibernation rather than risk issue #6565 transcript loss, + // or killing background work the agent is still waiting on. unableToProtectByPanel[record.key] = UnableToProtectMarker( fingerprint: request.confirmationFingerprint, lastActivityAt: request.effectiveLastActivityAt, diff --git a/Sources/App/AgentHibernationController+Records.swift b/Sources/App/AgentHibernationController+Records.swift index dc2a18f69c8..01503e4c9ef 100644 --- a/Sources/App/AgentHibernationController+Records.swift +++ b/Sources/App/AgentHibernationController+Records.swift @@ -12,6 +12,22 @@ extension AgentHibernationRecord { Set(processIdentities.keys) == processIDs } + /// Background work the transcript records before this belongs to an earlier + /// agent process: the start of the oldest process in the agent's scope. With + /// no live process left, every recorded launch died with it. + var backgroundWorkNotBefore: Date? { + guard processLiveness == .running else { return .distantFuture } + guard let earliest = processIdentities.values.min(by: { + ($0.startSeconds, $0.startMicroseconds) < ($1.startSeconds, $1.startMicroseconds) + }) else { + return nil + } + return Date( + timeIntervalSince1970: TimeInterval(earliest.startSeconds) + + TimeInterval(earliest.startMicroseconds) / 1_000_000 + ) + } + /// Reclaim may terminate a live process only with complete scope evidence. var processSafetyAllowsHibernation: Bool { switch processLiveness { diff --git a/Sources/App/AgentHibernationController+Teardown.swift b/Sources/App/AgentHibernationController+Teardown.swift index 63d2371fec9..860a6170f6d 100644 --- a/Sources/App/AgentHibernationController+Teardown.swift +++ b/Sources/App/AgentHibernationController+Teardown.swift @@ -167,7 +167,9 @@ extension AgentHibernationController { private static func snapshotOutcomes( for requests: [ConfirmedTeardownRequest] ) async -> [AgentHibernationPanelKey: AgentHibernationTranscriptGuard.TeardownSnapshotOutcome] { - let agents = requests.map { ($0.record.key, $0.record.agent) } + let agents = requests.map { + ($0.record.key, $0.record.agent, $0.record.backgroundWorkNotBefore) + } return await withTaskGroup( of: (AgentHibernationPanelKey, AgentHibernationTranscriptGuard.TeardownSnapshotOutcome).self, returning: [AgentHibernationPanelKey: AgentHibernationTranscriptGuard.TeardownSnapshotOutcome].self @@ -175,20 +177,28 @@ extension AgentHibernationController { var nextAgentIndex = 0 let initialTaskCount = min(Self.maxConcurrentTeardownSnapshotTasks, agents.count) for _ in 0..` naming the same `` with a ``. + /// Launches before `notBefore` (the agent process start) belong to an earlier + /// process whose background work died with it, so they are ignored. + static func transcriptHasUnfinishedBackgroundWork( + atPath path: String, + notBefore: Date? = nil, + maxTailBytes: Int = Self.backgroundWorkScanTailBytes + ) -> Bool { + guard let data = tailData(atPath: path, maxBytes: maxTailBytes) else { return false } + return unfinishedBackgroundLaunchIDs(inTranscriptTail: data, notBefore: notBefore).isEmpty == false + } + + static func unfinishedBackgroundLaunchIDs( + inTranscriptTail data: Data, + notBefore: Date? + ) -> Set { + var launches: [BackgroundLaunch] = [] + var finishedIDs: Set = [] + let launchMarkers = ["\"backgroundTaskId\"", "\"async_launched\"", "\"timeoutMs\"", "\"task_id\""] + .map { Data($0.utf8) } + let notificationMarker = Data("".utf8) + let timestamps = TranscriptTimestampParser() + for line in data.split(separator: 10, omittingEmptySubsequences: true) { + let hasLaunch = launchMarkers.contains { line.range(of: $0) != nil } + let hasNotification = line.range(of: notificationMarker) != nil + guard hasLaunch || hasNotification, + let object = try? JSONSerialization.jsonObject(with: Data(line)) as? [String: Any] else { + continue + } + if hasLaunch, let lineLaunches = backgroundLaunches(in: object) { + let launchedBeforeAgent = notBefore.flatMap { notBefore in + timestamps.date(object["timestamp"]).map { $0 < notBefore } + } ?? false + if !launchedBeforeAgent { + launches.append(contentsOf: lineLaunches) + } + } + if hasLaunch, let stoppedTaskID = stoppedTaskID(in: object) { + finishedIDs.insert(stoppedTaskID) + } + if hasNotification { + for text in taskNotificationCarrierTexts(in: object) { + finishedIDs.formUnion(finishedTaskIDs(inNotificationText: text)) + } + } + } + return Set(launches.filter { launch in + !finishedIDs.contains(launch.toolUseID) && + !(launch.taskID.map(finishedIDs.contains) ?? false) + }.map(\.toolUseID)) + } + + private struct BackgroundLaunch { + let toolUseID: String + let taskID: String? + } + + /// Launches recorded by a user line whose tool result started background work. + private static func backgroundLaunches(in object: [String: Any]) -> [BackgroundLaunch]? { + guard let result = object["toolUseResult"] as? [String: Any] else { return nil } + let taskID: String? + if let id = nonEmptyString(result["backgroundTaskId"]) { + taskID = id + } else if let id = nonEmptyString(result["taskId"]), + result["timeoutMs"] != nil || result["persistent"] != nil { + // Monitor. Other tools also return a `taskId` (todo updates), so the + // Monitor-only keys are required. + taskID = id + } else if (result["isAsync"] as? Bool) == true, + (result["status"] as? String) == "async_launched" { + taskID = nonEmptyString(result["agentId"]) + } else { + return nil + } + guard let message = object["message"] as? [String: Any], + let content = message["content"] as? [[String: Any]] else { + return nil + } + let launches = content.compactMap { block -> BackgroundLaunch? in + guard block["type"] as? String == "tool_result", + let id = nonEmptyString(block["tool_use_id"]) else { + return nil + } + return BackgroundLaunch(toolUseID: id, taskID: taskID) + } + return launches.isEmpty ? nil : launches + } + + /// The task a TaskStop result reports as stopped. A stopped task does not + /// always leave a terminal notification behind. + private static func stoppedTaskID(in object: [String: Any]) -> String? { + guard let result = object["toolUseResult"] as? [String: Any], + result["task_type"] != nil else { + return nil + } + return nonEmptyString(result["task_id"]) + } + + /// The places Claude records a delivered or queued task notification. Tool + /// results that merely quote notification text are deliberately not read. + private static func taskNotificationCarrierTexts(in object: [String: Any]) -> [String] { + switch object["type"] as? String { + case "queue-operation": + return [object["content"] as? String].compactMap { $0 } + case "attachment": + let attachment = object["attachment"] as? [String: Any] + return [attachment?["prompt"] as? String].compactMap { $0 } + case "user": + let message = object["message"] as? [String: Any] + if let text = message?["content"] as? String { + return [text] + } + let blocks = message?["content"] as? [[String: Any]] ?? [] + return blocks.compactMap { block in + block["type"] as? String == "text" ? block["text"] as? String : nil + } + default: + return [] + } + } + + /// Tool-use and task ids named by terminal task notifications in `text`. + /// One notification can close several tasks ("4 background agents + /// stopped"). Monitor event notifications carry no `` and do not end + /// the task. + static func finishedTaskIDs(inNotificationText text: String) -> [String] { + var ids: [String] = [] + var remainder = text[...] + while let start = remainder.range(of: "") { + let afterStart = remainder[start.upperBound...] + let end = afterStart.range(of: "") + let body = end.map { afterStart[..<$0.lowerBound] } ?? afterStart + if body.contains("") { + ids.append(contentsOf: tagValues("tool-use-id", in: body)) + ids.append(contentsOf: tagValues("task-id", in: body)) + } + remainder = end.map { afterStart[$0.upperBound...] } ?? afterStart[afterStart.endIndex...] + } + return ids + } + + private static func tagValues(_ tag: String, in text: Substring) -> [String] { + var values: [String] = [] + var remainder = text + while let open = remainder.range(of: "<\(tag)>"), + let close = remainder[open.upperBound...].range(of: "") { + let value = remainder[open.upperBound.. String? { + guard let string = value as? String, !string.isEmpty else { return nil } + return string + } + + private struct TranscriptTimestampParser { + private let fractional: ISO8601DateFormatter = { + let formatter = ISO8601DateFormatter() + formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds] + return formatter + }() + private let whole = ISO8601DateFormatter() + + func date(_ value: Any?) -> Date? { + guard let string = value as? String else { return nil } + return fractional.date(from: string) ?? whole.date(from: string) + } + } + + /// The last `maxBytes` of the file, starting at a line boundary. + private static func tailData(atPath path: String, maxBytes: Int) -> Data? { + guard let handle = FileHandle(forReadingAtPath: path) else { return nil } + defer { try? handle.close() } + guard let size = try? handle.seekToEnd() else { return nil } + let start = size > UInt64(maxBytes) ? size - UInt64(maxBytes) : 0 + do { + try handle.seek(toOffset: start) + guard var data = try handle.readToEnd() else { return Data() } + if start > 0, let newline = data.firstIndex(of: 10) { + data = data[data.index(after: newline)...] + } + return Data(data) + } catch { + return nil + } + } +} diff --git a/Sources/App/AgentHibernationTranscriptGuard+TeardownSnapshotOutcome.swift b/Sources/App/AgentHibernationTranscriptGuard+TeardownSnapshotOutcome.swift index 5bc13c9df61..b7334b90239 100644 --- a/Sources/App/AgentHibernationTranscriptGuard+TeardownSnapshotOutcome.swift +++ b/Sources/App/AgentHibernationTranscriptGuard+TeardownSnapshotOutcome.swift @@ -5,5 +5,8 @@ extension AgentHibernationTranscriptGuard { case snapshot(TeardownTranscriptSnapshot) case nothingToProtect case unableToProtect + /// The transcript shows background work (a background Bash command, a + /// Monitor, or an async Agent) that has not reported completion. + case backgroundWorkPending } } diff --git a/Sources/App/AgentHibernationTranscriptGuard.swift b/Sources/App/AgentHibernationTranscriptGuard.swift index 37bccf56805..44cf3e43fbe 100644 --- a/Sources/App/AgentHibernationTranscriptGuard.swift +++ b/Sources/App/AgentHibernationTranscriptGuard.swift @@ -77,6 +77,7 @@ enum AgentHibernationTranscriptGuard { panelKey: AgentHibernationPanelKey? = nil, homeDirectory: String = NSHomeDirectory(), snapshotDirectory: URL? = nil, + backgroundWorkNotBefore: Date? = nil, fileManager: FileManager = .default ) -> TeardownSnapshotOutcome { guard agent.kind == .claude else { return .nothingToProtect } @@ -91,6 +92,11 @@ enum AgentHibernationTranscriptGuard { return .unableToProtect } + // Terminating the agent would kill its background shells and subagents. + if transcriptHasUnfinishedBackgroundWork(atPath: transcriptPath, notBefore: backgroundWorkNotBefore) { + return .backgroundWorkPending + } + if !transcriptHasConversationTurns(atPath: transcriptPath, fileManager: fileManager) { return transcriptContainsOnlyNonProtectiveMetadata(atPath: transcriptPath, fileManager: fileManager) ? .nothingToProtect diff --git a/Sources/CmuxTopSnapshot+AgentBackgroundWork.swift b/Sources/CmuxTopSnapshot+AgentBackgroundWork.swift new file mode 100644 index 00000000000..7121ab16b8c --- /dev/null +++ b/Sources/CmuxTopSnapshot+AgentBackgroundWork.swift @@ -0,0 +1,47 @@ +import CmuxFoundation +import Foundation + +extension CmuxTopProcessSnapshot { + /// Children an agent starts within this many seconds of its own start (MCP + /// servers, launch hooks) are its baseline and do not count as work. + static let agentBackgroundWorkBaselineSeconds: Int64 = 30 + + private static let shellProcessNames: Set = [ + "sh", "bash", "zsh", "fish", "dash", "ksh", "mksh", "tcsh", "csh", "nu", "pwsh", "elvish", "xonsh", + ] + + /// Direct children of `agentRootPIDs` that are shells the agent started + /// after its launch baseline. Claude runs every tool command in a shell it + /// spawns itself, so a live late child shell is a background command, a + /// Monitor or watch loop, or a subagent's command still doing work, and + /// terminating the agent would kill it. Shells deeper in the tree (an MCP + /// server's `sh -c` under node) are not agent work and do not count. + /// Unknown start times never count; the scope's identity checks already + /// refuse panes without complete identities. + func agentBackgroundWorkProcessIDs( + agentRootPIDs: Set, + descendantProcessIDs: Set + ) -> Set { + let rootStarts = agentRootPIDs.compactMap { processesByPID[$0]?.processIdentity?.startSeconds } + guard let agentStart = rootStarts.min() else { return [] } + let baselineEnd = agentStart + Self.agentBackgroundWorkBaselineSeconds + return descendantProcessIDs.filter { processID in + guard !agentRootPIDs.contains(processID), + let process = processesByPID[processID], + agentRootPIDs.contains(process.parentPID), + let startSeconds = process.processIdentity?.startSeconds, + startSeconds > baselineEnd else { + return false + } + return Self.isShellProcessName(process.name) + } + } + + static func isShellProcessName(_ name: String) -> Bool { + var base = (name as NSString).lastPathComponent + if base.hasPrefix("-") { + base.removeFirst() + } + return shellProcessNames.contains(base) + } +} diff --git a/Sources/CmuxTopSnapshot.swift b/Sources/CmuxTopSnapshot.swift index 726cc5c042a..52c00d6298f 100644 --- a/Sources/CmuxTopSnapshot.swift +++ b/Sources/CmuxTopSnapshot.swift @@ -425,10 +425,15 @@ final class CmuxTopProcessSnapshot: @unchecked Sendable { let hasCompleteProcessGroups = processGroupIDs.allSatisfy { processGroupID in processesByPID[processGroupID]?.processGroupID == processGroupID } + let hasLiveBackgroundWork = !agentBackgroundWorkProcessIDs( + agentRootPIDs: boundedAgentRoots, + descendantProcessIDs: descendantProcessIDs + ).isEmpty return ( observedPanelProcessIDs, terminationProcessIDs, - !hasCompleteAgentRoots || + hasLiveBackgroundWork || + !hasCompleteAgentRoots || !hasTerminalEvidence || !hasCompleteTerminationTTYEvidence || processGroupIDs.isEmpty || diff --git a/cmux.xcodeproj/project.pbxproj b/cmux.xcodeproj/project.pbxproj index 7a78a77cf35..1a2f81e51e8 100644 --- a/cmux.xcodeproj/project.pbxproj +++ b/cmux.xcodeproj/project.pbxproj @@ -62,6 +62,7 @@ C0DEF0D30000000000000001 /* AgentForkExecutableIdentityResolver.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0DEF0D30000000000000002 /* AgentForkExecutableIdentityResolver.swift */; }; C0DEF0C10000000000000001 /* AgentForkSupport.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0DEF0C10000000000000002 /* AgentForkSupport.swift */; }; C0DEF0D40000000000000001 /* AgentForkTimeoutResumeGate.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0DEF0D40000000000000002 /* AgentForkTimeoutResumeGate.swift */; }; + D686A8C1DBD4EABD91766211 /* AgentHibernationBackgroundWorkTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C4C4607FC6F461CC8B35343E /* AgentHibernationBackgroundWorkTests.swift */; }; 8997C00E8997C00E8997C00E /* AgentHibernationController+CommittedTerminationCleanup.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D00E8997D00E8997D00E /* AgentHibernationController+CommittedTerminationCleanup.swift */; }; 8997C00F8997C00F8997C00F /* AgentHibernationController+CommittedTerminationObservation.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D00F8997D00F8997D00F /* AgentHibernationController+CommittedTerminationObservation.swift */; }; F65760100000000000000001 /* AgentHibernationController+Confirmation.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760100000000000000002 /* AgentHibernationController+Confirmation.swift */; }; @@ -110,6 +111,7 @@ F65760020000000000000001 /* AgentHibernationTestHelpers.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760020000000000000002 /* AgentHibernationTestHelpers.swift */; }; D36A00020000000000000001 /* AgentHibernationTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D36A00020000000000000002 /* AgentHibernationTests.swift */; }; 8997C0058997C0058997C005 /* AgentHibernationTrackingLifecycleTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8997D0058997D0058997D005 /* AgentHibernationTrackingLifecycleTests.swift */; }; + E7D4DED363EEF8E2E15BE0B8 /* AgentHibernationTranscriptGuard+BackgroundWork.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8F695E641DFA8B05C8818231 /* AgentHibernationTranscriptGuard+BackgroundWork.swift */; }; F65760090000000000000001 /* AgentHibernationTranscriptGuard+ClaudeWorkflow.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760090000000000000002 /* AgentHibernationTranscriptGuard+ClaudeWorkflow.swift */; }; F65760220000000000000001 /* AgentHibernationTranscriptGuard+PostTeardownRestore.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760220000000000000002 /* AgentHibernationTranscriptGuard+PostTeardownRestore.swift */; }; F65760230000000000000001 /* AgentHibernationTranscriptGuard+StableFileComparison.swift in Sources */ = {isa = PBXBuildFile; fileRef = F65760230000000000000002 /* AgentHibernationTranscriptGuard+StableFileComparison.swift */; }; @@ -1428,6 +1430,7 @@ C7A512000000000000000002 /* CmuxTopProcessSnapshotCache.swift in Sources */ = {isa = PBXBuildFile; fileRef = C7A512000000000000000001 /* CmuxTopProcessSnapshotCache.swift */; }; C7A509000000000000000003 /* CmuxTopProcessSnapshotCaptureCoordinatorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C7A509000000000000000004 /* CmuxTopProcessSnapshotCaptureCoordinatorTests.swift */; }; C7A50F000000000000000002 /* CmuxTopProcessTreeTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C7A50F000000000000000001 /* CmuxTopProcessTreeTests.swift */; }; + E95CE22E4D2DD0A4EAF1C5A4 /* CmuxTopSnapshot+AgentBackgroundWork.swift in Sources */ = {isa = PBXBuildFile; fileRef = 862267A5286E6B91B81484C1 /* CmuxTopSnapshot+AgentBackgroundWork.swift */; }; C7A501000000000000000002 /* CmuxTopSnapshot.swift in Sources */ = {isa = PBXBuildFile; fileRef = C7A501000000000000000001 /* CmuxTopSnapshot.swift */; }; C7A508000000000000000002 /* CmuxTopSnapshotScopeCache.swift in Sources */ = {isa = PBXBuildFile; fileRef = C7A508000000000000000001 /* CmuxTopSnapshotScopeCache.swift */; }; C7A5090000000000000005A2 /* CmuxTopSnapshotScopeCacheTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C7A5090000000000000005A1 /* CmuxTopSnapshotScopeCacheTests.swift */; }; @@ -4373,6 +4376,7 @@ C0DEF0D30000000000000002 /* AgentForkExecutableIdentityResolver.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentForkExecutableIdentityResolver.swift; sourceTree = ""; }; C0DEF0C10000000000000002 /* AgentForkSupport.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentForkSupport.swift; sourceTree = ""; }; C0DEF0D40000000000000002 /* AgentForkTimeoutResumeGate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentForkTimeoutResumeGate.swift; sourceTree = ""; }; + C4C4607FC6F461CC8B35343E /* AgentHibernationBackgroundWorkTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AgentHibernationBackgroundWorkTests.swift"; sourceTree = ""; }; 8997D00E8997D00E8997D00E /* AgentHibernationController+CommittedTerminationCleanup.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+CommittedTerminationCleanup.swift"; sourceTree = ""; }; 8997D00F8997D00F8997D00F /* AgentHibernationController+CommittedTerminationObservation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+CommittedTerminationObservation.swift"; sourceTree = ""; }; F65760100000000000000002 /* AgentHibernationController+Confirmation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationController+Confirmation.swift"; sourceTree = ""; }; @@ -4420,6 +4424,7 @@ F65760020000000000000002 /* AgentHibernationTestHelpers.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernationTestHelpers.swift; sourceTree = ""; }; D36A00020000000000000002 /* AgentHibernationTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernationTests.swift; sourceTree = ""; }; 8997D0058997D0058997D005 /* AgentHibernationTrackingLifecycleTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentHibernationTrackingLifecycleTests.swift; sourceTree = ""; }; + 8F695E641DFA8B05C8818231 /* AgentHibernationTranscriptGuard+BackgroundWork.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationTranscriptGuard+BackgroundWork.swift"; sourceTree = ""; }; F65760090000000000000002 /* AgentHibernationTranscriptGuard+ClaudeWorkflow.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationTranscriptGuard+ClaudeWorkflow.swift"; sourceTree = ""; }; F65760220000000000000002 /* AgentHibernationTranscriptGuard+PostTeardownRestore.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationTranscriptGuard+PostTeardownRestore.swift"; sourceTree = ""; }; F65760230000000000000002 /* AgentHibernationTranscriptGuard+StableFileComparison.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "App/AgentHibernationTranscriptGuard+StableFileComparison.swift"; sourceTree = ""; }; @@ -5597,6 +5602,7 @@ C7A512000000000000000001 /* CmuxTopProcessSnapshotCache.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CmuxTopProcessSnapshotCache.swift; sourceTree = ""; }; C7A509000000000000000004 /* CmuxTopProcessSnapshotCaptureCoordinatorTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CmuxTopProcessSnapshotCaptureCoordinatorTests.swift; sourceTree = ""; }; C7A50F000000000000000001 /* CmuxTopProcessTreeTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CmuxTopProcessTreeTests.swift; sourceTree = ""; }; + 862267A5286E6B91B81484C1 /* CmuxTopSnapshot+AgentBackgroundWork.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CmuxTopSnapshot+AgentBackgroundWork.swift"; sourceTree = ""; }; C7A501000000000000000001 /* CmuxTopSnapshot.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CmuxTopSnapshot.swift; sourceTree = ""; }; C7A508000000000000000001 /* CmuxTopSnapshotScopeCache.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CmuxTopSnapshotScopeCache.swift; sourceTree = ""; }; C7A5090000000000000005A1 /* CmuxTopSnapshotScopeCacheTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CmuxTopSnapshotScopeCacheTests.swift; sourceTree = ""; }; @@ -11216,6 +11222,8 @@ 2DA313712597F71E6015ABBC /* TerminalSSHSessionDetectionTimeoutGate.swift */, 7611B3AEEB664825A7DD8733 /* TerminalSSHSessionDetector+Async.swift */, 61BF07E91B28C5740AD10DD9 /* TerminalSSHSessionDetector+ProcessInfo.swift */, + 8F695E641DFA8B05C8818231 /* AgentHibernationTranscriptGuard+BackgroundWork.swift */, + 862267A5286E6B91B81484C1 /* CmuxTopSnapshot+AgentBackgroundWork.swift */, D56DBDC9509F1BAAAF0D3358 /* CmuxEventSequenceStore.swift */, 3840C3CAEE266F282EE96A37 /* SidebarReorderInteractionState.swift */, 01AFEB5B2588D6391DC94892 /* WorkspaceActivityReorderController.swift */, @@ -12633,6 +12641,7 @@ 311C986A1E7430418708AAA6 /* MachinesListStatusToolbarRowTests.swift */, 1C38B9AFEEE4FAE6089C250F /* CloudTreeRowToolTipTests.swift */, 6F27EA9972F21B98FB4A7312 /* SurfaceCatalogObservationTests.swift */, + C4C4607FC6F461CC8B35343E /* AgentHibernationBackgroundWorkTests.swift */, A29C9B70624D86CFFD27EFEE /* CmuxEventSequenceStoreTests.swift */, BCAE531397347899AA9F4E4E /* CloudCursorlessSnapshotTests.swift */, FF5E22859B10B563032FECE7 /* CloudLinkRetryBackoffTests.swift */, @@ -13516,6 +13525,7 @@ 8997C00B8997C00B8997C00B /* AgentHibernationProcessSnapshotCoordinator.swift in Sources */, 9090E0029090E0029090E002 /* AgentHibernationResumePreparation.swift in Sources */, A12123000000000000000003 /* AgentHibernationSessionEndResolution.swift in Sources */, + E7D4DED363EEF8E2E15BE0B8 /* AgentHibernationTranscriptGuard+BackgroundWork.swift in Sources */, F65760090000000000000001 /* AgentHibernationTranscriptGuard+ClaudeWorkflow.swift in Sources */, F65760220000000000000001 /* AgentHibernationTranscriptGuard+PostTeardownRestore.swift in Sources */, F65760230000000000000001 /* AgentHibernationTranscriptGuard+StableFileComparison.swift in Sources */, @@ -14050,6 +14060,7 @@ 6616B19B5A18036FCB0C24E9 /* CmuxTopProcessSampler.swift in Sources */, 0A1107110000000000000018 /* CmuxTopProcessSnapshot+PromptAgentDetection.swift in Sources */, C7A512000000000000000002 /* CmuxTopProcessSnapshotCache.swift in Sources */, + E95CE22E4D2DD0A4EAF1C5A4 /* CmuxTopSnapshot+AgentBackgroundWork.swift in Sources */, C7A501000000000000000002 /* CmuxTopSnapshot.swift in Sources */, C7A508000000000000000002 /* CmuxTopSnapshotScopeCache.swift in Sources */, D9F800000000000000000001 /* CmuxTuiRemoteRouting.swift in Sources */, @@ -16278,6 +16289,7 @@ 88A8F9C77DFA3ECF80805D22 /* AgentChatTranscriptServiceLifetimeTests.swift in Sources */, 93800001AA11BB22CC33EE01 /* AgentDeliveryTTYBindingTests.swift in Sources */, A9E020000000000000000005 /* AgentExecutableResolverTests.swift in Sources */, + D686A8C1DBD4EABD91766211 /* AgentHibernationBackgroundWorkTests.swift in Sources */, 8740E0018740E0018740E001 /* AgentHibernationEvaluationSchedulingTests.swift in Sources */, 9090E0039090E0039090E003 /* AgentHibernationPanelCloseCleanupTests.swift in Sources */, F65760010000000000000001 /* AgentHibernationPlannerSwiftTests.swift in Sources */, diff --git a/cmuxTests/AgentHibernationBackgroundWorkTests.swift b/cmuxTests/AgentHibernationBackgroundWorkTests.swift new file mode 100644 index 00000000000..6d35f9f3e55 --- /dev/null +++ b/cmuxTests/AgentHibernationBackgroundWorkTests.swift @@ -0,0 +1,265 @@ +import CmuxFoundation +import Foundation +import Testing + +#if canImport(cmux_DEV) +@testable import cmux_DEV +#elseif canImport(cmux) +@testable import cmux +#endif + +/// Hibernation must never terminate an agent that is still waiting on +/// background work: a `run_in_background` shell, a Monitor, or an async subagent. +@Suite +struct AgentHibernationBackgroundWorkTests { + private static let agentStart: Int64 = 1_800_000_000 + + // MARK: Process scope + + @Test + func lateShellUnderTheAgentMakesTheScopeUnsafe() { + let scope = processSnapshot(including: [ + .init(pid: 300, parent: 101, group: 300, name: "zsh", startOffset: 600), + .init(pid: 301, parent: 300, group: 300, name: "sleep", startOffset: 601), + ]).agentHibernationProcessScope(panelProcessIDs: [101], agentProcessIDs: [101]) + + #expect(scope.terminationProcessIDs.contains(300)) + #expect(scope.containsUnrelatedProcess) + } + + @Test + func launchBaselineChildrenKeepTheScopeSafe() { + let scope = processSnapshot(including: []) + .agentHibernationProcessScope(panelProcessIDs: [101], agentProcessIDs: [101]) + + #expect(scope.terminationProcessIDs == [101, 102]) + #expect(scope.containsUnrelatedProcess == false) + } + + @Test + func lateNonShellChildDoesNotCountAsBackgroundWork() { + let scope = processSnapshot(including: [ + .init(pid: 310, parent: 101, group: 101, name: "node", startOffset: 900), + ]).agentHibernationProcessScope(panelProcessIDs: [101], agentProcessIDs: [101]) + + #expect(scope.containsUnrelatedProcess == false) + } + + @Test + func lateShellUnderAnMCPServerDoesNotCountAsBackgroundWork() { + let scope = processSnapshot(including: [ + .init(pid: 320, parent: 102, group: 101, name: "sh", startOffset: 900), + ]).agentHibernationProcessScope(panelProcessIDs: [101], agentProcessIDs: [101]) + + #expect(scope.containsUnrelatedProcess == false) + } + + // MARK: Transcript + + @Test + func unfinishedBackgroundBashBlocksTheTeardownSnapshot() throws { + let fixture = try TranscriptFixture() + defer { fixture.remove() } + try fixture.write([ + Self.userTurn, + Self.backgroundBashLaunch(toolUseID: "toolu_bash", taskID: "bshell1"), + ]) + + guard case .backgroundWorkPending = fixture.snapshotBeforeTeardown() else { + Issue.record("A pane with an unfinished background shell must not be torn down") + return + } + } + + @Test + func completedBackgroundBashAllowsTheTeardownSnapshot() throws { + let fixture = try TranscriptFixture() + defer { fixture.remove() } + try fixture.write([ + Self.userTurn, + Self.backgroundBashLaunch(toolUseID: "toolu_bash", taskID: "bshell1"), + Self.queuedNotification(toolUseID: "toolu_bash", taskID: "bshell1", status: "completed"), + ]) + + guard case .snapshot = fixture.snapshotBeforeTeardown() else { + Issue.record("Finished background work must not block hibernation") + return + } + } + + @Test + func unfinishedAsyncAgentBlocksTheTeardownSnapshot() throws { + let fixture = try TranscriptFixture() + defer { fixture.remove() } + try fixture.write([ + Self.userTurn, + Self.asyncAgentLaunch(toolUseID: "toolu_agent", agentID: "a1234567890abcdef"), + Self.monitorEventNotification(taskID: "a1234567890abcdef"), + ]) + + guard case .backgroundWorkPending = fixture.snapshotBeforeTeardown() else { + Issue.record("A pane with a running async subagent must not be torn down") + return + } + } + + @Test + func batchedNotificationWithOnlyTaskIDsFinishesEveryNamedTask() { + let lines = [ + Self.asyncAgentLaunch(toolUseID: "toolu_a", agentID: "a1111111111111111"), + Self.asyncAgentLaunch(toolUseID: "toolu_b", agentID: "a2222222222222222"), + #"{"type":"user","timestamp":"2026-09-28T08:09:00.000Z","message":{"role":"user","content":"\na1111111111111111\na2222222222222222\nstopped\n2 background agents stopped\n"}}"#, + ] + + #expect(Self.unfinished(lines).isEmpty) + } + + @Test + func launchesBeforeTheAgentProcessStartedAreIgnored() { + let lines = [Self.backgroundBashLaunch(toolUseID: "toolu_old", taskID: "bold", timestamp: "2026-09-28T07:00:00.000Z")] + let agentStartedAt = ISO8601DateFormatter().date(from: "2026-09-28T07:30:00Z") + + #expect(Self.unfinished(lines, notBefore: agentStartedAt).isEmpty) + #expect(Self.unfinished(lines) == ["toolu_old"]) + } + + @Test + func monitorEventsAndQuotedNotificationsDoNotFinishATask() { + let quotedNotification = #"{"type":"user","timestamp":"2026-09-28T08:02:00.000Z","message":{"role":"user","content":[{"tool_use_id":"toolu_cat","type":"tool_result","content":"\nbmon1\ntoolu_monitor\ncompleted\n"}]},"toolUseResult":{"stdout":"","stderr":""}}"# + let lines = [ + #"{"type":"user","timestamp":"2026-09-28T08:00:07.000Z","message":{"role":"user","content":[{"tool_use_id":"toolu_monitor","type":"tool_result","content":"Monitor started"}]},"toolUseResult":{"taskId":"bmon1","timeoutMs":900000,"persistent":false}}"#, + Self.monitorEventNotification(taskID: "bmon1"), + quotedNotification, + ] + + #expect(Self.unfinished(lines) == ["toolu_monitor"]) + #expect(Self.unfinished(lines + [Self.queuedNotification(toolUseID: "toolu_monitor", taskID: "bmon1", status: "completed")]).isEmpty) + } + + @Test + func todoTaskIDsAreNotBackgroundLaunches() { + let todoUpdate = #"{"type":"user","timestamp":"2026-09-28T08:00:08.000Z","message":{"role":"user","content":[{"tool_use_id":"toolu_todo","type":"tool_result","content":"Updated task #3"}]},"toolUseResult":{"success":true,"taskId":"3","updatedFields":["status"]}}"# + + #expect(Self.unfinished([todoUpdate]).isEmpty) + } + + @Test + func taskStopFinishesTheStoppedTask() { + let lines = [ + Self.backgroundBashLaunch(toolUseID: "toolu_server", taskID: "bserver1"), + #"{"type":"user","timestamp":"2026-09-28T08:03:00.000Z","message":{"role":"user","content":[{"tool_use_id":"toolu_stop","type":"tool_result","content":"Successfully stopped task: bserver1"}]},"toolUseResult":{"message":"Successfully stopped task: bserver1","task_id":"bserver1","task_type":"local_bash","command":"./server"}}"#, + ] + + #expect(Self.unfinished(lines).isEmpty) + } + + private static func unfinished(_ lines: [String], notBefore: Date? = nil) -> Set { + AgentHibernationTranscriptGuard.unfinishedBackgroundLaunchIDs( + inTranscriptTail: Data(lines.joined(separator: "\n").utf8), + notBefore: notBefore + ) + } + + // MARK: Fixtures + + private struct Process { + let pid: Int + let parent: Int + let group: Int + let name: String + let startOffset: Int64 + } + + /// Agent 101 (`claude`, group leader) with an MCP server started at launch. + private func processSnapshot(including extra: [Process]) -> CmuxTopProcessSnapshot { + let workspaceID = UUID() + let panelID = UUID() + let base = [ + Process(pid: 101, parent: 1, group: 101, name: "claude", startOffset: 0), + Process(pid: 102, parent: 101, group: 101, name: "node", startOffset: 2), + ] + let processes = (base + extra).map { process in + CmuxTopProcessInfo( + pid: process.pid, + processIdentity: AgentPIDProcessIdentity( + pid: pid_t(process.pid), + startSeconds: Self.agentStart + process.startOffset, + startMicroseconds: 0 + ), + parentPID: process.parent, name: process.name, path: nil, + ttyDevice: 0x123, + cmuxWorkspaceID: workspaceID, + cmuxSurfaceID: panelID, + cmuxAttributionReason: "environment", + processGroupID: process.group, + terminalProcessGroupID: 101, cpuPercent: 0, residentBytes: 0, + virtualBytes: 0, threadCount: 1 + ) + } + return CmuxTopProcessSnapshot(processes: processes, sampledAt: .now, includesProcessDetails: true) + } + + struct TranscriptFixture { + let home: URL + let cwd = "/tmp/background-work-repo" + let sessionId = "background-work-session" + + init() throws { + home = FileManager.default.temporaryDirectory + .appendingPathComponent("cmux-hibernation-background-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: home, withIntermediateDirectories: true) + } + + var transcriptURL: URL { + home + .appendingPathComponent(".claude/projects", isDirectory: true) + .appendingPathComponent(RestorableAgentSessionIndex.encodeClaudeProjectDir(cwd), isDirectory: true) + .appendingPathComponent("\(sessionId).jsonl", isDirectory: false) + } + + func write(_ lines: [String]) throws { + try FileManager.default.createDirectory( + at: transcriptURL.deletingLastPathComponent(), + withIntermediateDirectories: true + ) + try (lines.joined(separator: "\n") + "\n").write(to: transcriptURL, atomically: true, encoding: .utf8) + } + + func snapshotBeforeTeardown() -> AgentHibernationTranscriptGuard.TeardownSnapshotOutcome { + AgentHibernationTranscriptGuard.snapshotBeforeTeardown( + agent: SessionRestorableAgentSnapshot( + kind: .claude, + sessionId: sessionId, + workingDirectory: cwd, + launchCommand: nil + ), + homeDirectory: home.path, + snapshotDirectory: home.appendingPathComponent("snapshots", isDirectory: true) + ) + } + + func remove() { + try? FileManager.default.removeItem(at: home) + } + } + + static let userTurn = #"{"type":"user","timestamp":"2026-09-28T08:00:00.000Z","message":{"role":"user","content":"run the tests in the background"}}"# + + static func backgroundBashLaunch(toolUseID: String, taskID: String, timestamp: String = "2026-09-28T08:00:05.000Z") -> String { + #"{"type":"user","timestamp":"\#(timestamp)","message":{"role":"user","content":[{"tool_use_id":"\#(toolUseID)","type":"tool_result","content":"Command running in background with ID: \#(taskID)."}]},"toolUseResult":{"stdout":"","stderr":"","interrupted":false,"isImage":false,"noOutputExpected":false,"backgroundTaskId":"\#(taskID)"}}"# + } + + static func asyncAgentLaunch(toolUseID: String, agentID: String) -> String { + #"{"type":"user","timestamp":"2026-09-28T08:00:06.000Z","message":{"role":"user","content":[{"tool_use_id":"\#(toolUseID)","type":"tool_result","content":[{"type":"text","text":"Async agent launched successfully."}]}]},"toolUseResult":{"isAsync":true,"status":"async_launched","agentId":"\#(agentID)","description":"review"}}"# + } + + static func queuedNotification(toolUseID: String?, taskID: String, status: String) -> String { + let toolUse = toolUseID.map { #"\n\#($0)"# } ?? "" + return #"{"type":"queue-operation","operation":"enqueue","timestamp":"2026-09-28T08:05:00.000Z","content":"\n\#(taskID)\#(toolUse)\n\#(status)\ndone\n"}"# + } + + /// A Monitor-style event: it names the task but carries no ``. + static func monitorEventNotification(taskID: String) -> String { + #"{"type":"queue-operation","operation":"enqueue","timestamp":"2026-09-28T08:01:00.000Z","content":"\n\#(taskID)\nMonitor event\nline\n"}"# + } +} diff --git a/docs/agent-hooks.md b/docs/agent-hooks.md index 0dd5c19625e..6e246fce709 100644 --- a/docs/agent-hooks.md +++ b/docs/agent-hooks.md @@ -84,6 +84,7 @@ For routine hibernation, a live terminal is only a candidate when all of these h - the terminal is in the background (its panel is not currently visible) - you have more live restorable agent terminals than the live-terminal limit (`maxLiveTerminals`, default `12`) - the terminal has had no output, input, or lifecycle change for at least the idle window (`idleSeconds`, default `5`) +- the agent has no background work still running: no shell it started after launch is alive, and its Claude transcript shows no unfinished `run_in_background` command, Monitor, or async subagent The live-terminal limit is the first gate. Under the limit, nothing hibernates no matter how long it sits idle. Once you are over the limit, cmux frees only the oldest-idle background terminals, just enough to get back under the limit. Visible terminals are never touched.