carry of #28037: show Cloud machine transitions and failed changes

Packages/macOS/CmuxNext/Sources/CmuxNextApp/AgentTabs+Wiring.swift

@@ -128,7 +128,7 @@ extension AgentTabStore {
                 case .running, .provisioning, .paused:
                     return AgentPaneMoveTarget(id: machine.machineID, label: machine.machine.title,
                                                status: machine.machine.status.rawValue)
-                case .failed, .destroyed, .unknown:
+                case .pausing, .starting, .deleting, .failed, .destroyed, .unknown:
                     return nil
                 }
             }

Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudAppOps.swift

@@ -54,11 +54,15 @@ extension CloudMachine {
     /// the Cloud app server answers it.
     init?(next value: JSONValue) {
         guard let id = value["id"]?.stringValue, !id.isEmpty else { return nil }
+        // The in-between states stay their own (cloud-machine-ops.ts CloudMachineStatus): a
+        // pausing machine is not paused yet, and a deleting one is not gone.
         let status: Status = switch value["status"]?.stringValue {
-        case "provisioning", "starting": .provisioning
+        case "provisioning": .provisioning
+        case "starting": .starting
         case "running": .running
-        case "pausing", "paused": .paused
-        case "deleting": .destroyed
+        case "pausing": .pausing
+        case "paused": .paused
+        case "deleting": .deleting
         case "failed": .failed
         default: .unknown
         }
@@ -69,11 +73,32 @@ extension CloudMachine {
         let minutes = (value["idlePauseMinutes"]?.doubleValue).flatMap { Int(exactly: $0) }
         let idlePause = Self.idlePauseDiagnostics(value["idlePause"])
         self.init(id: id, provider: "cmux-next", status: status, displayName: value["name"]?.stringValue,
-                  pauseReason: value["pauseReason"]?.stringValue.flatMap(PauseReason.init(rawValue:)), idlePauseMinutes: minutes,
-                  idlePause: idlePause)
+                  pauseReason: Self.nextPauseReason(value["pause_reason"] ?? value["pauseReason"]), idlePauseMinutes: minutes,
+                  idlePause: idlePause, lastChangeFailure: Self.changeFailure(value["error"]))
+        serverRevision = value["revision"]?.stringValue
         createdBy = value["creator"]?.stringValue
     }
 
+    /// The backend's `pause_reason` (`pauseReason` from an older app server): why cmux paused the
+    /// machine by itself. `idle` and the cost backstop `no_report` are both cmux's own idle pause,
+    /// which an attach resumes; a VM stopped from inside (`provider_stopped`, `provider_paused`)
+    /// and a person's pause (null) carry no reason the app shows.
+    static func nextPauseReason(_ value: JSONValue?) -> PauseReason? {
+        switch value?.stringValue {
+        case "idle", "no_report": .idle
+        case "user": .user
+        default: nil
+        }
+    }
+
+    /// The backend's `error` (`{code, message, at}`): why the last pause, resume or resize failed
+    /// after the server answered it. Nil for null or a shape this client does not read.
+    static func changeFailure(_ value: JSONValue?) -> ChangeFailure? {
+        guard let value, case let .object(fields) = value, let code = fields["code"]?.stringValue, !code.isEmpty else { return nil }
+        let at = fields["at"]?.doubleValue.map { Date(timeIntervalSince1970: $0 / 1000) }
+        return ChangeFailure(code: code, message: fields["message"]?.stringValue ?? "", at: at)
+    }
+
     private static func idlePauseDiagnostics(_ value: JSONValue?) -> IdlePauseDiagnostics? {
         guard let value, case let .object(fields) = value,
               let minutes = fields["minutes"]?.doubleValue.flatMap({ Int(exactly: $0) }),

Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudMachineSession.swift

@@ -53,7 +53,17 @@ final class CloudMachineSession {
     /// The app-link connect hop in flight: a pause or a disconnect cancels
     /// it instead of waiting up to `CloudAppLinks.connectTimeout`.
     @ObservationIgnored private var connectHop: Task<Void, Never>?
+    /// A transition overtook a connect after its carrier answered. If that transition
+    /// rolls back, reconnect the interrupted live session unless another change replaced it.
+    private struct InterruptedMutation {
+        let id: UUID
+        var rolledBack = false
+    }
+    @ObservationIgnored private var interruptedMutation: InterruptedMutation?
     @ObservationIgnored private var disconnected = false
+    /// Whether this session was removed permanently because its machine disappeared.
+    /// A delete transition must leave this false until a list confirms removal.
+    var isPermanentlyDisconnected: Bool { disconnected }
     /// Repairs an empty workspace on this machine (never on another).
     @ObservationIgnored let emptyWorkspaces: EmptyWorkspaceRepair
     /// Resumes this machine through the Resume Machine path
@@ -95,13 +105,40 @@ final class CloudMachineSession {
         emptyWorkspaces = EmptyWorkspaceRepair(daemon: daemon)
     }
 
+    /// Apply an authoritative record and reconcile the transport. A connection attempted while
+    /// the provider was starting may fail before the VM is ready; running supplies a new retry signal.
+    func applyRecord(_ record: CloudMachine) {
+        let previous = machine.status
+        let wasResuming = machine.pendingChange == .starting
+        machine = record
+        // The error may have arrived before acknowledgement, while the mutation remained
+        // pending. Its settlement, not a change in displayed error text, authorizes recovery.
+        if let failedMutationID = record.failedMutationID {
+            mutationFailed(failedMutationID)
+        }
+        if !record.status.isLive, !record.shownStatus.isChanging { interruptedMutation = nil }
+        if previous.isLive, !record.status.isLive {
+            suspend()
+        } else if !previous.isLive, record.status.isLive {
+            connect()
+        } else if wasResuming, record.pendingChange == nil, record.status.isLive, daemon.connection == nil {
+            // A read can report running while the resume request still awaits acknowledgement.
+            // That read cannot connect through the pending overlay. Clearing it is a second
+            // live edge even when the raw status already reads running.
+            connect()
+        } else if previous == .starting, record.status == .running, daemon.connection == nil {
+            retryNow()
+        }
+        recoverInterruptedMutation()
+    }
+
     /// Connects (a click passes origin `user`; launch and a resumed machine
     /// connect as `script`). Serializes lifecycle hops so a late pause cannot
     /// kill a resumed link. An app link connects only when it is not up: a
     /// click never drops a healthy connection, and only a click starts a
     /// paused machine (`cloud.machine.connect` starts it).
     func connect(origin: CloudLinkOrigin = .script) {
-        guard !disconnected else { return }
+        guard !disconnected, !machine.shownStatus.isChanging else { return }
         if appLink != nil {
             guard machine.status.isLive || origin == .user, !appConnecting, appTicket == nil || linkEnded != nil else { return }
             appConnecting = true
@@ -115,13 +152,21 @@ final class CloudMachineSession {
         // task-owner: one lifecycle hop; each later hop waits for this one
         linkTransition = Task {
             await previous?.value
+            guard !disconnected, !machine.shownStatus.isChanging else {
+                noteMutationInterruption()
+                appConnecting = false
+                return
+            }
             if let appLink {
                 await connectApp(appLink, origin: origin)
                 return
             }
             guard let link else { return }
             await link.resume()
-            guard !disconnected, machine.status.isLive else { return }
+            guard !disconnected, machine.status.isLive, !machine.shownStatus.isChanging else {
+                noteMutationInterruption()
+                return
+            }
             // A link start that a pause overtook changes nothing (`noteLink`).
             let suspendsBefore = suspends
             daemon.start(remote: { [weak self] in
@@ -147,8 +192,9 @@ final class CloudMachineSession {
     /// (its connect starts a paused machine for a click); a live legacy link
     /// reconnects by itself, so nothing else changes.
     func attach(origin: CloudLinkOrigin = .user) {
+        guard !machine.shownStatus.isChanging else { return }
         if appLink != nil { return connect(origin: origin) }
-        guard machine.status == .paused, !resuming, !disconnected, let resumer else { return }
+        guard machine.shownStatus == .paused, !resuming, !disconnected, let resumer else { return }
         resuming = true
         resumeFailure = nil
         // task-owner: one resume per attach; a second attach while it runs waits on it
@@ -164,7 +210,9 @@ final class CloudMachineSession {
             // The resume's refresh connects a record that turned live; a
             // refresh that failed leaves the record paused, so connect here.
             guard !disconnected, !machine.status.isLive else { return }
-            machine.status = .running
+            // Keep the raw record paused until reconciliation supplies the live record. The
+            // resume mutation may still be pending, and connect() must not be blocked by a
+            // synthetic running status that makes the later paused -> running edge disappear.
             connect()
         }
     }
@@ -175,6 +223,7 @@ final class CloudMachineSession {
     private func connectApp(_ appLink: CloudLinkSession, origin: CloudLinkOrigin) async {
         defer {
             appConnecting = false
+            recoverInterruptedMutation()
             if reconnectAfterHop {
                 reconnectAfterHop = false
                 appTicket = nil
@@ -193,14 +242,29 @@ final class CloudMachineSession {
             ticket = try await appLink.connect(origin: origin)
         } catch {
             // A pause or a disconnect cancelled this connect: not a failure.
-            guard !disconnected, suspends == suspendsBefore, !Task.isCancelled else { return }
+            if disconnected || suspends != suspendsBefore || Task.isCancelled || machine.shownStatus.isChanging {
+                noteMutationInterruption()
+                await appLink.end(reason: "connect cancelled")
+                linkPhase = .idle
+                return
+            }
             // An ended link on a paused machine reads as the pause; a failed
             // connect that was to start it says why (`CloudMachineStageInput`).
             if machine.status == .paused { resumeFailure = CloudAppLinks.endedMessage(error) }
             showEnded(error)
             return
         }
-        guard !disconnected, suspends == suspendsBefore, !Task.isCancelled else { return }
+        if disconnected || suspends != suspendsBefore || Task.isCancelled || machine.shownStatus.isChanging {
+            // The transition overtook this connect after the carrier answered. End the
+            // ticket/transport so the carrier cannot remain attached without a daemon.
+            noteMutationInterruption()
+            await appLink.end(reason: "connect cancelled")
+            appTicket = nil
+            daemon.shutdownConnection()
+            linkPhase = .idle
+            return
+        }
+        interruptedMutation = nil
         appTicket = ticket.id
         linkPhase = .up
         let localIdentity = localIdentity
@@ -222,6 +286,27 @@ final class CloudMachineSession {
         })
     }
 
+    private func noteMutationInterruption() {
+        guard !disconnected, let id = machine.pendingMutationID else { return }
+        interruptedMutation = InterruptedMutation(id: id)
+    }
+
+    /// Only the operation that interrupted this connection can authorize its recovery. If
+    /// another operation is still pending, keep that authorization until it settles too.
+    func mutationFailed(_ id: UUID) {
+        if interruptedMutation?.id == id { interruptedMutation?.rolledBack = true }
+        // An unrelated rollback cannot authorize recovery, but may remove the last barrier
+        // after the matching operation already rolled back while another one was pending.
+        recoverInterruptedMutation()
+    }
+
+    private func recoverInterruptedMutation() {
+        guard interruptedMutation?.rolledBack == true, !disconnected,
+              machine.status.isLive, !machine.shownStatus.isChanging, !appConnecting else { return }
+        interruptedMutation = nil
+        if daemon.connection == nil { connect() }
+    }
+
     /// The current connect's link ended: drop its connection and show why.
     /// A stale ticket (an older connect's connection) changes nothing.
     @discardableResult
@@ -281,22 +366,11 @@ final class CloudMachineSession {
     /// models it paused: the session keeps its live status, pause reason and
     /// running start, and takes the record's other fields.
     func listed(_ record: CloudMachine, connectsAtRead: Int) {
-        let wasLive = machine.status.isLive
-        if wasLive, !record.status.isLive {
-            guard connects == connectsAtRead else {
-                var kept = record
-                (kept.status, kept.pauseReason, kept.runningSince) = (machine.status, machine.pauseReason, machine.runningSince)
-                machine = kept
-                return
-            }
-            machine = record
-            suspend()
-            return
-        }
-        machine = record
-        if !wasLive, record.status.isLive {
-            connect()
+        var accepted = record
+        if machine.status.isLive, !record.status.isLive, connects != connectsAtRead {
+            (accepted.status, accepted.pauseReason, accepted.runningSince) = (machine.status, machine.pauseReason, machine.runningSince)
         }
+        applyRecord(accepted)
     }
 
     /// Reads the record again before an attach decides, when it may be
@@ -321,9 +395,9 @@ final class CloudMachineSession {
     var stageInput: CloudMachineStageInput {
         var connected = false
         if case .connected = daemon.store.connectionState { connected = daemon.connection != nil }
-        return CloudMachineStageInput(creation: nil, machineStatus: machine.status, link: linkPhase, daemonConnected: connected,
+        return CloudMachineStageInput(creation: nil, machineStatus: machine.shownStatus, link: linkPhase, daemonConnected: connected,
                                       daemonLoaded: daemon.store.isLoaded, daemonFailure: daemonFailure, linkEnded: linkEnded,
-                                      pauseReason: machine.pauseReason, resuming: resuming, resumeFailure: resumeFailure,
+                                      pauseReason: machine.pauseReason, resuming: resuming, resumeFailure: resumeFailure, changeFailure: machine.lastChangeFailure?.text,
                                       linkDropped: linkDropped)
     }
 
@@ -335,8 +409,8 @@ final class CloudMachineSession {
     /// no status (decoded as provisioning) and the list is not polled, so the
     /// record alone said "provisioning" after the terminal worked.
     var effectiveStatus: CloudMachine.Status {
-        guard machine.status == .provisioning || machine.status == .unknown, case .connected = daemon.store.connectionState else {
-            return machine.status
+        guard machine.shownStatus == .provisioning || machine.shownStatus == .unknown, case .connected = daemon.store.connectionState else {
+            return machine.shownStatus
         }
         return .running
     }
@@ -434,6 +508,14 @@ final class CloudMachineSession {
         }
     }
 
+    /// Keeps a session reusable while an accepted delete is still settling. The provider can
+    /// reject the delete after the server has acknowledged it, in which case the next machine
+    /// list must be able to reconnect this same session when it returns to running.
+    func retainForDeleteTransition() {
+        machine.status = .deleting
+        suspend()
+    }
+
     /// Drops the daemon connection while keeping the link reusable after a
     /// provider pause/resume transition.
     func suspend() {