Cloud legacy link: bound and log every start step, show the failing step with Retry

Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudMachineSession.swift

@@ -5,6 +5,7 @@ import CmuxNextSidebar
 import CmuxNextWorkspaceRepair
 import Foundation
 import Observation
+import os
 import CmuxNextWakeups
 
 /// One Cloud machine: its `/api/vm` record, its link, and the daemon
@@ -41,6 +42,10 @@ final class CloudMachineSession {
     /// up, or the last start's failure. Drives the booting and connecting
     /// stages; never set from elapsed time.
     private(set) var linkPhase: CloudLinkPhase = .idle
+    /// The legacy link's last start failure, with the step it stopped at (``CloudLinkStartError``);
+    /// nil once a start comes up, and always nil for the app link. A wait on the machine ends at once
+    /// on a ``CloudLinkStartError/definitive`` one and says the step either way.
+    private(set) var legacyLinkFailure: CloudLinkStartError?
     /// The ticket of the app link's live connect; nil while none is live.
     @ObservationIgnored private var appTicket: UInt64?
     /// An app-link connect hop is queued or running.
@@ -89,6 +94,8 @@ final class CloudMachineSession {
     /// For an app link, a click's connect on a paused machine is its resume.
     private(set) var resumeFailure: String?
 
+    private static let logger = Logger(subsystem: "com.cmuxterm.app.next", category: "cloud.link")
+
     /// - Parameter mainWork: The app's main-thread scheduler (the local daemon's), for the machine's event batches.
     convenience init(machine: CloudMachine, link: CloudMachineLink, mainWork: MainWorkScheduler) {
         self.init(machine: machine, link: link, appLink: nil, localIdentity: { nil }, mainWork: mainWork)
@@ -146,12 +153,18 @@ final class CloudMachineSession {
     /// click never drops a healthy connection, and only a click starts a
     /// paused machine (`cloud.machine.connect` starts it).
     func connect(origin: CloudLinkOrigin = .script) {
-        guard !disconnected, !machine.shownStatus.isChanging else { return }
+        guard !disconnected, !machine.shownStatus.isChanging else {
+            if appLink == nil { Self.logger.notice("legacy connect \(self.machineID, privacy: .public): skipped (\(self.disconnected ? "disconnected" : "changing: \(self.machine.shownStatus.rawValue)", privacy: .public))") }
+            return
+        }
         if appLink != nil {
             guard machine.status.isLive || origin == .user, !appConnecting, appTicket == nil || linkEnded != nil else { return }
             appConnecting = true
         } else {
-            guard machine.status.isLive else { return }
+            guard machine.status.isLive else {
+                Self.logger.notice("legacy connect \(self.machineID, privacy: .public): skipped (status \(self.machine.status.rawValue, privacy: .public))")
+                return
+            }
         }
         connects += 1
         // A resume that failed earlier says nothing about this connect.
@@ -173,19 +186,23 @@ final class CloudMachineSession {
             guard let link else { return }
             await link.resume()
             guard !disconnected, machine.status.isLive, !machine.shownStatus.isChanging else {
+                Self.logger.notice("legacy connect \(self.machineID, privacy: .public): dropped after its hop (status \(self.machine.shownStatus.rawValue, privacy: .public))")
                 noteMutationInterruption()
                 return
             }
             // A link start that a pause overtook changes nothing (`noteLink`).
             let suspendsBefore = suspends
+            Self.logger.notice("legacy connect \(self.machineID, privacy: .public): daemon \(self.daemon.isStarting ? "already starting" : "start", privacy: .public)")
             daemon.start(remote: { [weak self] in
                 await self?.noteLinkStarting(since: suspendsBefore)
                 do {
                     let path = try await link.socketPath()
                     await self?.noteLink(.up, since: suspendsBefore)
                     return path
                 } catch {
-                    if !(error is CancellationError) { await self?.noteLink(.failed(CloudStrings.errorText(error)), since: suspendsBefore) }
+                    if !(error is CancellationError) {
+                        await self?.noteLink(.failed(CloudStrings.errorText(error)), since: suspendsBefore, failure: error as? CloudLinkStartError)
+                    }
                     throw error
                 }
             })
@@ -203,7 +220,13 @@ final class CloudMachineSession {
     func attach(origin: CloudLinkOrigin = .user) {
         guard !machine.shownStatus.isChanging else { return }
         if appLink != nil { return connect(origin: origin) }
-        guard machine.shownStatus == .paused, !resuming, !disconnected, let resumer else { return }
+        // A live legacy machine with no connection: its link may be waiting out a backoff after a
+        // failed start (or never started); the attach retries it now instead of doing nothing.
+        if machine.shownStatus != .paused {
+            if machine.status.isLive, daemon.connection == nil, !disconnected { retryNow() }
+            return
+        }
+        guard !resuming, !disconnected, let resumer else { return }
         resuming = true
         resumeFailure = nil
         // task-owner: one resume per attach; a second attach while it runs waits on it
@@ -365,9 +388,10 @@ final class CloudMachineSession {
     /// moved since it began) changes nothing, so a late failure cannot turn
     /// a paused machine into Failed. A failure reads the machine list again
     /// (at most every 30 s): the machine may have been paused behind the app.
-    private func noteLink(_ phase: CloudLinkPhase, since suspendsBefore: Int) {
+    private func noteLink(_ phase: CloudLinkPhase, since suspendsBefore: Int, failure: CloudLinkStartError? = nil) {
         guard !disconnected, suspends == suspendsBefore else { return }
         linkPhase = phase
+        if case .up = phase { legacyLinkFailure = nil } else if let failure { legacyLinkFailure = failure }
         if case .failed = phase, let refresher {
             // task-owner: one throttled list read; the reconcile it runs updates this record
             Task { await refresher(false) }
@@ -459,6 +483,14 @@ final class CloudMachineSession {
         retryNow()
     }
 
+    /// A new wait for this machine (a start, a move, its Retry) begins: a legacy link failure from an
+    /// earlier attempt is not this attempt's answer, so it is cleared and the link dialed again now.
+    func beginLegacyAttempt() {
+        guard link != nil, legacyLinkFailure != nil else { return }
+        legacyLinkFailure = nil
+        if machine.status.isLive { retryNow() }
+    }
+
     /// Retry on the record as it stands (the caller just read it): resume a paused legacy machine,
     /// else connect an app link, wake a legacy daemon that is starting, or connect it again.
     func retryNow() {

Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudMove.swift

@@ -293,19 +293,34 @@ struct CloudMove {
             try CloudHandlers.requireAttachableState(session)
             if session.machine.status == .paused { session.attach(origin: CloudHandlers.connectOrigin(for: invocation)) } else { session.connect(origin: CloudHandlers.connectOrigin(for: invocation)) }
         }
+        session.beginLegacyAttempt()
         let daemon = session.daemon
-        // A failed resume ends the wait at once, with its reason. An app link that ended is dialed again every 30 s within it.
+        // A failed resume ends the wait at once, with its reason, and so does a legacy link start that no
+        // retry can help (a refusal: signed out, a plan, the server's 4xx). Within the wait a link that
+        // ended is dialed again every 30 s: an app link's connect, or a legacy link's backoff woken now.
         guard await CloudWait.holds(tries: 6, each: .seconds(30), redial: {
             if session.appLink != nil, session.linkEnded != nil, session.machine.status == .running { session.retryNow() }
+            if session.link != nil, session.legacyLinkFailure != nil, session.machine.status.isLive { session.retryNow() }
         }, until: {
-            session.resumeFailure != nil || (daemon.connection != nil && AgentTabStore.registryHost(of: daemon) != nil)
+            session.resumeFailure != nil || session.legacyLinkFailure?.definitive == true
+                || (daemon.connection != nil && AgentTabStore.registryHost(of: daemon) != nil)
         }) else {
-            throw Failure(message: CloudMoveStrings.notConnected(session.machine.title), retry: .move(machineID: session.machineID))
+            throw Failure(message: notConnected(session), retry: .move(machineID: session.machineID))
         }
         if let failure = session.resumeFailure { throw Failure(message: failure, retry: .move(machineID: session.machineID)) }
+        if daemon.connection == nil || AgentTabStore.registryHost(of: daemon) == nil {
+            throw Failure(message: notConnected(session), retry: .move(machineID: session.machineID))
+        }
         return session
     }
 
+    /// Why `session` did not connect, in the person's words: the legacy link's step that failed and its
+    /// reason when one is known, else the plain "didn't connect".
+    static func notConnected(_ session: CloudMachineSession) -> String {
+        guard let failure = session.legacyLinkFailure else { return CloudMoveStrings.notConnected(session.machine.title) }
+        return CloudMoveStrings.notConnected(session.machine.title, step: failure.step, reason: failure.reason)
+    }
+
     /// Shows the moved chat on `machine`: a new Cloud workspace with a terminal
     /// in `folder`, then the chat's tab bound to `session` through the
     /// machine's store, selected; filed into the window, and shown when the run

Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudMoveStrings.swift

@@ -48,6 +48,19 @@ enum CloudMoveStrings {
     static func notConnected(_ machine: String) -> String {
         String(format: String(localized: "cloud.move.notConnected", defaultValue: "%@ didn't connect. Retry once it's online.", table: "Cloud", bundle: .module), machine)
     }
+    /// A legacy link start that stopped at `step` with `reason` (``CloudLinkStartError``).
+    static func notConnected(_ machine: String, step: CloudLinkStep, reason: String) -> String {
+        String(format: String(localized: "cloud.move.notConnectedAtStep", defaultValue: "%1$@ didn't connect: %2$@ failed (%3$@).", table: "Cloud", bundle: .module),
+               machine, linkStep(step), reason)
+    }
+    static func linkStep(_ step: CloudLinkStep) -> String {
+        switch step {
+        case .attachEndpoint: String(localized: "cloud.move.linkStep.attachEndpoint", defaultValue: "Asking cmux Cloud for the machine's address", table: "Cloud", bundle: .module)
+        case .hub: String(localized: "cloud.move.linkStep.hub", defaultValue: "Starting this Mac's tunnel to cmux Cloud", table: "Cloud", bundle: .module)
+        case .approve: String(localized: "cloud.move.linkStep.approve", defaultValue: "Approving this Mac on the machine", table: "Cloud", bundle: .module)
+        case .remoteConnect: String(localized: "cloud.move.linkStep.remoteConnect", defaultValue: "Opening the link to the machine", table: "Cloud", bundle: .module)
+        }
+    }
     static func syncFailed(_ machine: String, _ detail: String) -> String {
         String(format: String(localized: "cloud.move.syncFailed", defaultValue: "Couldn't copy the folder to %1$@: %2$@", table: "Cloud", bundle: .module),
                machine, detail)

Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudStartChat.swift

@@ -64,7 +64,19 @@ struct CloudStartChat {
             let message = (error as? CloudMove.Failure)?.message ?? (error as? ActionFailure)?.message
                 ?? CloudMoveStrings.moveFailed(CloudStrings.errorText(error))
             services.cloud.logger.error("cloud start chat failed: \(message, privacy: .public)")
-            CloudMove.toast(services, toastID, message, duration: .seconds(12))
+            // A failure a retry can help (the machine didn't connect, a copy that timed out) offers Retry: the
+            // same start again under its key, so a machine already made or a folder already copied is reused.
+            var retry: (@MainActor () -> Void)?
+            if (error as? CloudMove.Failure)?.retry != nil {
+                let tabID = tab.id
+                retry = { [weak services] in
+                    // The tab is found again by id: a tab closed since then retries nothing.
+                    guard let services, let (tab, _) = services.locateTab(tabID) else { return }
+                    // task-owner: one retried start; it ends with its own toast
+                    _ = Task { _ = await CloudStartChat(services: services).reply(start, tab: tab) }
+                }
+            }
+            CloudMove.toast(services, toastID, message, duration: .seconds(retry == nil ? 12 : 30), retry: retry)
             return AgentPaneReply.failure(code: "cloud_start_failed", message: message)
         }
     }

Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudStartSession.swift

@@ -365,13 +365,15 @@ struct CloudStartSession {
             let origin: CloudLinkOrigin = byPerson ? .user : .script
             if session.machine.status == .paused { session.attach(origin: origin) } else { session.connect(origin: origin) }
         }
+        session.beginLegacyAttempt()
         let daemon = session.daemon
         let up = await CloudWait.holds(within: Self.connectWait, until: {
-            session.resumeFailure != nil || (daemon.connection != nil && AgentTabStore.registryHost(of: daemon) != nil)
+            session.resumeFailure != nil || session.legacyLinkFailure?.definitive == true
+                || (daemon.connection != nil && AgentTabStore.registryHost(of: daemon) != nil)
         })
         // A sign-out, team switch or cancellation during the wait is that, not a machine that didn't connect.
         try Self.checked(account, now: services.cloud.account, request.asked(machine: session.machine.title))
-        guard up, session.resumeFailure == nil else { throw failure() }
+        guard up, session.resumeFailure == nil, daemon.connection != nil else { throw failure() }
     }
 
     /// The machine takes the person's harness login before the session starts, as a composer start's does (+Login).