Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -145,8 +145,9 @@ extension AgentHibernationController {
snapshot = value
case .nothingToProtect:
snapshot = nil
case .unableToProtect:
// Forfeit hibernation rather than risk issue #6565 transcript loss.
case .unableToProtect, .backgroundWorkPending:
// Forfeit hibernation rather than risk issue #6565 transcript loss,
// or killing background work the agent is still waiting on.
unableToProtectByPanel[record.key] = UnableToProtectMarker(
fingerprint: request.confirmationFingerprint,
lastActivityAt: request.effectiveLastActivityAt,
Expand Down
16 changes: 16 additions & 0 deletions Sources/App/AgentHibernationController+Records.swift
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,22 @@ extension AgentHibernationRecord {
Set(processIdentities.keys) == processIDs
}

/// Background work the transcript records before this belongs to an earlier
/// agent process: the start of the oldest process in the agent's scope. With
/// no live process left, every recorded launch died with it.
var backgroundWorkNotBefore: Date? {
guard processLiveness == .running else { return .distantFuture }
guard let earliest = processIdentities.values.min(by: {
($0.startSeconds, $0.startMicroseconds) < ($1.startSeconds, $1.startMicroseconds)
}) else {
return nil
}
return Date(
timeIntervalSince1970: TimeInterval(earliest.startSeconds) +
TimeInterval(earliest.startMicroseconds) / 1_000_000
)
}

/// Reclaim may terminate a live process only with complete scope evidence.
var processSafetyAllowsHibernation: Bool {
switch processLiveness {
Expand Down
20 changes: 15 additions & 5 deletions Sources/App/AgentHibernationController+Teardown.swift
Original file line number Diff line number Diff line change
Expand Up @@ -167,28 +167,38 @@ extension AgentHibernationController {
private static func snapshotOutcomes(
for requests: [ConfirmedTeardownRequest]
) async -> [AgentHibernationPanelKey: AgentHibernationTranscriptGuard.TeardownSnapshotOutcome] {
let agents = requests.map { ($0.record.key, $0.record.agent) }
let agents = requests.map {
($0.record.key, $0.record.agent, $0.record.backgroundWorkNotBefore)
}
return await withTaskGroup(
of: (AgentHibernationPanelKey, AgentHibernationTranscriptGuard.TeardownSnapshotOutcome).self,
returning: [AgentHibernationPanelKey: AgentHibernationTranscriptGuard.TeardownSnapshotOutcome].self
) { group in
var nextAgentIndex = 0
let initialTaskCount = min(Self.maxConcurrentTeardownSnapshotTasks, agents.count)
for _ in 0..<initialTaskCount {
let (key, agent) = agents[nextAgentIndex]
let (key, agent, startedAt) = agents[nextAgentIndex]
nextAgentIndex += 1
group.addTask(priority: .utility) {
(key, AgentHibernationTranscriptGuard.snapshotBeforeTeardown(agent: agent, panelKey: key))
(key, AgentHibernationTranscriptGuard.snapshotBeforeTeardown(
agent: agent,
panelKey: key,
backgroundWorkNotBefore: startedAt
))
}
}
var outcomes: [AgentHibernationPanelKey: AgentHibernationTranscriptGuard.TeardownSnapshotOutcome] = [:]
while let (key, outcome) = await group.next() {
outcomes[key] = outcome
guard nextAgentIndex < agents.count else { continue }
let (nextKey, nextAgent) = agents[nextAgentIndex]
let (nextKey, nextAgent, nextStartedAt) = agents[nextAgentIndex]
nextAgentIndex += 1
group.addTask(priority: .utility) {
(nextKey, AgentHibernationTranscriptGuard.snapshotBeforeTeardown(agent: nextAgent, panelKey: nextKey))
(nextKey, AgentHibernationTranscriptGuard.snapshotBeforeTeardown(
agent: nextAgent,
panelKey: nextKey,
backgroundWorkNotBefore: nextStartedAt
))
}
}
return outcomes
Expand Down
206 changes: 206 additions & 0 deletions Sources/App/AgentHibernationTranscriptGuard+BackgroundWork.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,206 @@
import Foundation

extension AgentHibernationTranscriptGuard {
/// How much of the transcript tail is scanned for background launches. A
/// launch older than this window is not seen here; the process-scope check
/// still keeps panes with live background shells awake.
static let backgroundWorkScanTailBytes = 32 * 1024 * 1024

/// Whether the Claude transcript records background work that has not
/// reported completion: a `run_in_background` Bash command, a Monitor, or an
/// async Agent. Claude writes the launch as a tool result carrying
/// `toolUseResult.backgroundTaskId`, `toolUseResult.taskId`, or
/// `toolUseResult.isAsync` with status `async_launched`, and the completion as
/// a `<task-notification>` naming the same `<tool-use-id>` with a `<status>`.
/// Launches before `notBefore` (the agent process start) belong to an earlier
/// process whose background work died with it, so they are ignored.
static func transcriptHasUnfinishedBackgroundWork(
atPath path: String,
notBefore: Date? = nil,
maxTailBytes: Int = Self.backgroundWorkScanTailBytes
) -> Bool {
guard let data = tailData(atPath: path, maxBytes: maxTailBytes) else { return false }
return unfinishedBackgroundLaunchIDs(inTranscriptTail: data, notBefore: notBefore).isEmpty == false
}

static func unfinishedBackgroundLaunchIDs(
inTranscriptTail data: Data,
notBefore: Date?
) -> Set<String> {
var launches: [BackgroundLaunch] = []
var finishedIDs: Set<String> = []
let launchMarkers = ["\"backgroundTaskId\"", "\"async_launched\"", "\"timeoutMs\"", "\"task_id\""]
.map { Data($0.utf8) }
let notificationMarker = Data("<task-notification>".utf8)
let timestamps = TranscriptTimestampParser()
for line in data.split(separator: 10, omittingEmptySubsequences: true) {
let hasLaunch = launchMarkers.contains { line.range(of: $0) != nil }
let hasNotification = line.range(of: notificationMarker) != nil
guard hasLaunch || hasNotification,
let object = try? JSONSerialization.jsonObject(with: Data(line)) as? [String: Any] else {
continue
}
if hasLaunch, let lineLaunches = backgroundLaunches(in: object) {
let launchedBeforeAgent = notBefore.flatMap { notBefore in
timestamps.date(object["timestamp"]).map { $0 < notBefore }
} ?? false
if !launchedBeforeAgent {
launches.append(contentsOf: lineLaunches)
}
}
if hasLaunch, let stoppedTaskID = stoppedTaskID(in: object) {
finishedIDs.insert(stoppedTaskID)
}
if hasNotification {
for text in taskNotificationCarrierTexts(in: object) {
finishedIDs.formUnion(finishedTaskIDs(inNotificationText: text))
}
}
}
return Set(launches.filter { launch in
!finishedIDs.contains(launch.toolUseID) &&
!(launch.taskID.map(finishedIDs.contains) ?? false)
}.map(\.toolUseID))
}

private struct BackgroundLaunch {
let toolUseID: String
let taskID: String?
}

/// Launches recorded by a user line whose tool result started background work.
private static func backgroundLaunches(in object: [String: Any]) -> [BackgroundLaunch]? {
guard let result = object["toolUseResult"] as? [String: Any] else { return nil }
let taskID: String?
if let id = nonEmptyString(result["backgroundTaskId"]) {
taskID = id
} else if let id = nonEmptyString(result["taskId"]),
result["timeoutMs"] != nil || result["persistent"] != nil {
// Monitor. Other tools also return a `taskId` (todo updates), so the
// Monitor-only keys are required.
taskID = id
} else if (result["isAsync"] as? Bool) == true,
(result["status"] as? String) == "async_launched" {
taskID = nonEmptyString(result["agentId"])
} else {
return nil
}
guard let message = object["message"] as? [String: Any],
let content = message["content"] as? [[String: Any]] else {
return nil
}
let launches = content.compactMap { block -> BackgroundLaunch? in
guard block["type"] as? String == "tool_result",
let id = nonEmptyString(block["tool_use_id"]) else {
return nil
}
return BackgroundLaunch(toolUseID: id, taskID: taskID)
}
return launches.isEmpty ? nil : launches
}

/// The task a TaskStop result reports as stopped. A stopped task does not
/// always leave a terminal notification behind.
private static func stoppedTaskID(in object: [String: Any]) -> String? {
guard let result = object["toolUseResult"] as? [String: Any],
result["task_type"] != nil else {
return nil
}
return nonEmptyString(result["task_id"])
}

/// The places Claude records a delivered or queued task notification. Tool
/// results that merely quote notification text are deliberately not read.
private static func taskNotificationCarrierTexts(in object: [String: Any]) -> [String] {
switch object["type"] as? String {
case "queue-operation":
return [object["content"] as? String].compactMap { $0 }
case "attachment":
let attachment = object["attachment"] as? [String: Any]
return [attachment?["prompt"] as? String].compactMap { $0 }
case "user":
let message = object["message"] as? [String: Any]
if let text = message?["content"] as? String {
return [text]
}
let blocks = message?["content"] as? [[String: Any]] ?? []
return blocks.compactMap { block in
block["type"] as? String == "text" ? block["text"] as? String : nil
}
default:
return []
}
}

/// Tool-use and task ids named by terminal task notifications in `text`.
/// One notification can close several tasks ("4 background agents
/// stopped"). Monitor event notifications carry no `<status>` and do not end
/// the task.
static func finishedTaskIDs(inNotificationText text: String) -> [String] {
var ids: [String] = []
var remainder = text[...]
while let start = remainder.range(of: "<task-notification>") {
let afterStart = remainder[start.upperBound...]
let end = afterStart.range(of: "</task-notification>")
let body = end.map { afterStart[..<$0.lowerBound] } ?? afterStart
if body.contains("<status>") {
ids.append(contentsOf: tagValues("tool-use-id", in: body))
ids.append(contentsOf: tagValues("task-id", in: body))
}
remainder = end.map { afterStart[$0.upperBound...] } ?? afterStart[afterStart.endIndex...]
}
return ids
}

private static func tagValues(_ tag: String, in text: Substring) -> [String] {
var values: [String] = []
var remainder = text
while let open = remainder.range(of: "<\(tag)>"),
let close = remainder[open.upperBound...].range(of: "</\(tag)>") {
let value = remainder[open.upperBound..<close.lowerBound]
.trimmingCharacters(in: .whitespacesAndNewlines)
if !value.isEmpty {
values.append(value)
}
remainder = remainder[close.upperBound...]
}
return values
}

private static func nonEmptyString(_ value: Any?) -> String? {
guard let string = value as? String, !string.isEmpty else { return nil }
return string
}

private struct TranscriptTimestampParser {
private let fractional: ISO8601DateFormatter = {
let formatter = ISO8601DateFormatter()
formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds]
return formatter
}()
private let whole = ISO8601DateFormatter()

func date(_ value: Any?) -> Date? {
guard let string = value as? String else { return nil }
return fractional.date(from: string) ?? whole.date(from: string)
}
}

/// The last `maxBytes` of the file, starting at a line boundary.
private static func tailData(atPath path: String, maxBytes: Int) -> Data? {
guard let handle = FileHandle(forReadingAtPath: path) else { return nil }
defer { try? handle.close() }
guard let size = try? handle.seekToEnd() else { return nil }
let start = size > UInt64(maxBytes) ? size - UInt64(maxBytes) : 0
do {
try handle.seek(toOffset: start)
guard var data = try handle.readToEnd() else { return Data() }
if start > 0, let newline = data.firstIndex(of: 10) {
data = data[data.index(after: newline)...]
}
return Data(data)
} catch {
return nil
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -5,5 +5,8 @@ extension AgentHibernationTranscriptGuard {
case snapshot(TeardownTranscriptSnapshot)
case nothingToProtect
case unableToProtect
/// The transcript shows background work (a background Bash command, a
/// Monitor, or an async Agent) that has not reported completion.
case backgroundWorkPending
}
}
6 changes: 6 additions & 0 deletions Sources/App/AgentHibernationTranscriptGuard.swift
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ enum AgentHibernationTranscriptGuard {
panelKey: AgentHibernationPanelKey? = nil,
homeDirectory: String = NSHomeDirectory(),
snapshotDirectory: URL? = nil,
backgroundWorkNotBefore: Date? = nil,
fileManager: FileManager = .default
) -> TeardownSnapshotOutcome {
guard agent.kind == .claude else { return .nothingToProtect }
Expand All @@ -91,6 +92,11 @@ enum AgentHibernationTranscriptGuard {
return .unableToProtect
}

// Terminating the agent would kill its background shells and subagents.
if transcriptHasUnfinishedBackgroundWork(atPath: transcriptPath, notBefore: backgroundWorkNotBefore) {
return .backgroundWorkPending
}

if !transcriptHasConversationTurns(atPath: transcriptPath, fileManager: fileManager) {
return transcriptContainsOnlyNonProtectiveMetadata(atPath: transcriptPath, fileManager: fileManager)
? .nothingToProtect
Expand Down
47 changes: 47 additions & 0 deletions Sources/CmuxTopSnapshot+AgentBackgroundWork.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
import CmuxFoundation
import Foundation

extension CmuxTopProcessSnapshot {
/// Children an agent starts within this many seconds of its own start (MCP
/// servers, launch hooks) are its baseline and do not count as work.
static let agentBackgroundWorkBaselineSeconds: Int64 = 30

private static let shellProcessNames: Set<String> = [
"sh", "bash", "zsh", "fish", "dash", "ksh", "mksh", "tcsh", "csh", "nu", "pwsh", "elvish", "xonsh",
]

/// Direct children of `agentRootPIDs` that are shells the agent started
/// after its launch baseline. Claude runs every tool command in a shell it
/// spawns itself, so a live late child shell is a background command, a
/// Monitor or watch loop, or a subagent's command still doing work, and
/// terminating the agent would kill it. Shells deeper in the tree (an MCP
/// server's `sh -c` under node) are not agent work and do not count.
/// Unknown start times never count; the scope's identity checks already
/// refuse panes without complete identities.
func agentBackgroundWorkProcessIDs(
agentRootPIDs: Set<Int>,
descendantProcessIDs: Set<Int>
) -> Set<Int> {
let rootStarts = agentRootPIDs.compactMap { processesByPID[$0]?.processIdentity?.startSeconds }
guard let agentStart = rootStarts.min() else { return [] }
let baselineEnd = agentStart + Self.agentBackgroundWorkBaselineSeconds
return descendantProcessIDs.filter { processID in
guard !agentRootPIDs.contains(processID),
let process = processesByPID[processID],
agentRootPIDs.contains(process.parentPID),
let startSeconds = process.processIdentity?.startSeconds,
startSeconds > baselineEnd else {
return false
}
return Self.isShellProcessName(process.name)
}
}

static func isShellProcessName(_ name: String) -> Bool {
var base = (name as NSString).lastPathComponent
if base.hasPrefix("-") {
base.removeFirst()
}
return shellProcessNames.contains(base)
}
}
7 changes: 6 additions & 1 deletion Sources/CmuxTopSnapshot.swift
Original file line number Diff line number Diff line change
Expand Up @@ -425,10 +425,15 @@ final class CmuxTopProcessSnapshot: @unchecked Sendable {
let hasCompleteProcessGroups = processGroupIDs.allSatisfy { processGroupID in
processesByPID[processGroupID]?.processGroupID == processGroupID
}
let hasLiveBackgroundWork = !agentBackgroundWorkProcessIDs(
agentRootPIDs: boundedAgentRoots,
descendantProcessIDs: descendantProcessIDs
).isEmpty
return (
observedPanelProcessIDs,
terminationProcessIDs,
!hasCompleteAgentRoots ||
hasLiveBackgroundWork ||
!hasCompleteAgentRoots ||
!hasTerminalEvidence ||
!hasCompleteTerminationTTYEvidence ||
processGroupIDs.isEmpty ||
Expand Down
Loading
Loading