From 5a7d4377531b9901024d067d5512e5889c0a861c Mon Sep 17 00:00:00 2001 From: Leo Li Date: Mon, 2 Nov 2026 22:42:06 -0800 Subject: [PATCH 1/7] Cloud legacy link: every start step has a deadline and a log line; a failed connect says the step and why The NEXT nightly's legacy Cloud link start went silent: no hub, no remote connect, no log line, and the Cloud chat start gave up after 3 minutes with "didn't connect". Each step of a legacy link start (attach-endpoint, the WireGuard hub, approve, remote connect) now runs under its own deadline with a notice-level began/done/failed line, and a failure keeps its step. A Cloud start's wait ends at once on a refusal and its toast says the step and reason with Retry; within the wait a legacy link's backoff is woken every 30 s; a manual connect retries a live legacy machine that has no connection. Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-3bab30503f0b9a7059e5 --- .../Cloud/CloudMachineSession.swift | 19 ++- .../Sources/CmuxNextApp/Cloud/CloudMove.swift | 20 ++- .../CmuxNextApp/Cloud/CloudMoveStrings.swift | 13 ++ .../CmuxNextApp/Cloud/CloudStartChat.swift | 8 +- .../CmuxNextApp/Cloud/CloudStartSession.swift | 5 +- .../CmuxNextCloud/API/CloudAPIClient.swift | 8 +- .../CmuxNextCloud/Link/CloudLinkStart.swift | 144 ++++++++++++++++++ .../CmuxNextCloud/Link/CloudMachineLink.swift | 55 ++++++- .../CmuxNextCloud/Tunnel/CloudTunnelHub.swift | 19 ++- 9 files changed, 274 insertions(+), 17 deletions(-) create mode 100644 Packages/macOS/CmuxNext/Sources/CmuxNextCloud/Link/CloudLinkStart.swift diff --git a/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudMachineSession.swift b/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudMachineSession.swift index 06910f3aa26e..b931ab0a1931 100644 --- a/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudMachineSession.swift +++ b/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudMachineSession.swift @@ -41,6 +41,10 @@ final class CloudMachineSession { /// up, or the last start's failure. Drives the booting and connecting /// stages; never set from elapsed time. private(set) var linkPhase: CloudLinkPhase = .idle + /// The legacy link's last start failure, with the step it stopped at (``CloudLinkStartError``); + /// nil once a start comes up, and always nil for the app link. A wait on the machine ends at once + /// on a ``CloudLinkStartError/definitive`` one and says the step either way. + private(set) var legacyLinkFailure: CloudLinkStartError? /// The ticket of the app link's live connect; nil while none is live. @ObservationIgnored private var appTicket: UInt64? /// An app-link connect hop is queued or running. @@ -185,7 +189,9 @@ final class CloudMachineSession { await self?.noteLink(.up, since: suspendsBefore) return path } catch { - if !(error is CancellationError) { await self?.noteLink(.failed(CloudStrings.errorText(error)), since: suspendsBefore) } + if !(error is CancellationError) { + await self?.noteLink(.failed(CloudStrings.errorText(error)), since: suspendsBefore, failure: error as? CloudLinkStartError) + } throw error } }) @@ -203,7 +209,13 @@ final class CloudMachineSession { func attach(origin: CloudLinkOrigin = .user) { guard !machine.shownStatus.isChanging else { return } if appLink != nil { return connect(origin: origin) } - guard machine.shownStatus == .paused, !resuming, !disconnected, let resumer else { return } + // A live legacy machine with no connection: its link may be waiting out a backoff after a + // failed start (or never started); the attach retries it now instead of doing nothing. + if machine.shownStatus != .paused { + if machine.status.isLive, daemon.connection == nil, !disconnected { retryNow() } + return + } + guard !resuming, !disconnected, let resumer else { return } resuming = true resumeFailure = nil // task-owner: one resume per attach; a second attach while it runs waits on it @@ -365,9 +377,10 @@ final class CloudMachineSession { /// moved since it began) changes nothing, so a late failure cannot turn /// a paused machine into Failed. A failure reads the machine list again /// (at most every 30 s): the machine may have been paused behind the app. - private func noteLink(_ phase: CloudLinkPhase, since suspendsBefore: Int) { + private func noteLink(_ phase: CloudLinkPhase, since suspendsBefore: Int, failure: CloudLinkStartError? = nil) { guard !disconnected, suspends == suspendsBefore else { return } linkPhase = phase + if case .up = phase { legacyLinkFailure = nil } else if let failure { legacyLinkFailure = failure } if case .failed = phase, let refresher { // task-owner: one throttled list read; the reconcile it runs updates this record Task { await refresher(false) } diff --git a/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudMove.swift b/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudMove.swift index 03bcb31f53cd..07292998cbef 100644 --- a/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudMove.swift +++ b/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudMove.swift @@ -294,18 +294,32 @@ struct CloudMove { if session.machine.status == .paused { session.attach(origin: CloudHandlers.connectOrigin(for: invocation)) } else { session.connect(origin: CloudHandlers.connectOrigin(for: invocation)) } } let daemon = session.daemon - // A failed resume ends the wait at once, with its reason. An app link that ended is dialed again every 30 s within it. + // A failed resume ends the wait at once, with its reason, and so does a legacy link start that no + // retry can help (a refusal: signed out, a plan, the server's 4xx). Within the wait a link that + // ended is dialed again every 30 s: an app link's connect, or a legacy link's backoff woken now. guard await CloudWait.holds(tries: 6, each: .seconds(30), redial: { if session.appLink != nil, session.linkEnded != nil, session.machine.status == .running { session.retryNow() } + if session.link != nil, session.legacyLinkFailure != nil, session.machine.status.isLive { session.retryNow() } }, until: { - session.resumeFailure != nil || (daemon.connection != nil && AgentTabStore.registryHost(of: daemon) != nil) + session.resumeFailure != nil || session.legacyLinkFailure?.definitive == true + || (daemon.connection != nil && AgentTabStore.registryHost(of: daemon) != nil) }) else { - throw Failure(message: CloudMoveStrings.notConnected(session.machine.title), retry: .move(machineID: session.machineID)) + throw Failure(message: notConnected(session), retry: .move(machineID: session.machineID)) } if let failure = session.resumeFailure { throw Failure(message: failure, retry: .move(machineID: session.machineID)) } + if daemon.connection == nil || AgentTabStore.registryHost(of: daemon) == nil { + throw Failure(message: notConnected(session), retry: .move(machineID: session.machineID)) + } return session } + /// Why `session` did not connect, in the person's words: the legacy link's step that failed and its + /// reason when one is known, else the plain "didn't connect". + static func notConnected(_ session: CloudMachineSession) -> String { + guard let failure = session.legacyLinkFailure else { return CloudMoveStrings.notConnected(session.machine.title) } + return CloudMoveStrings.notConnected(session.machine.title, step: failure.step, reason: failure.reason) + } + /// Shows the moved chat on `machine`: a new Cloud workspace with a terminal /// in `folder`, then the chat's tab bound to `session` through the /// machine's store, selected; filed into the window, and shown when the run diff --git a/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudMoveStrings.swift b/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudMoveStrings.swift index 2007cb2d4a16..8d2de7e368a7 100644 --- a/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudMoveStrings.swift +++ b/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudMoveStrings.swift @@ -48,6 +48,19 @@ enum CloudMoveStrings { static func notConnected(_ machine: String) -> String { String(format: String(localized: "cloud.move.notConnected", defaultValue: "%@ didn't connect. Retry once it's online.", table: "Cloud", bundle: .module), machine) } + /// A legacy link start that stopped at `step` with `reason` (``CloudLinkStartError``). + static func notConnected(_ machine: String, step: CloudLinkStep, reason: String) -> String { + String(format: String(localized: "cloud.move.notConnectedAtStep", defaultValue: "%1$@ didn't connect: %2$@ failed (%3$@).", table: "Cloud", bundle: .module), + machine, linkStep(step), reason) + } + static func linkStep(_ step: CloudLinkStep) -> String { + switch step { + case .attachEndpoint: String(localized: "cloud.move.linkStep.attachEndpoint", defaultValue: "Asking cmux Cloud for the machine's address", table: "Cloud", bundle: .module) + case .hub: String(localized: "cloud.move.linkStep.hub", defaultValue: "Starting this Mac's tunnel to cmux Cloud", table: "Cloud", bundle: .module) + case .approve: String(localized: "cloud.move.linkStep.approve", defaultValue: "Approving this Mac on the machine", table: "Cloud", bundle: .module) + case .remoteConnect: String(localized: "cloud.move.linkStep.remoteConnect", defaultValue: "Opening the link to the machine", table: "Cloud", bundle: .module) + } + } static func syncFailed(_ machine: String, _ detail: String) -> String { String(format: String(localized: "cloud.move.syncFailed", defaultValue: "Couldn't copy the folder to %1$@: %2$@", table: "Cloud", bundle: .module), machine, detail) diff --git a/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudStartChat.swift b/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudStartChat.swift index f6bf708c240e..98df331bc064 100644 --- a/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudStartChat.swift +++ b/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudStartChat.swift @@ -64,7 +64,13 @@ struct CloudStartChat { let message = (error as? CloudMove.Failure)?.message ?? (error as? ActionFailure)?.message ?? CloudMoveStrings.moveFailed(CloudStrings.errorText(error)) services.cloud.logger.error("cloud start chat failed: \(message, privacy: .public)") - CloudMove.toast(services, toastID, message, duration: .seconds(12)) + // A failure a retry can help (the machine didn't connect, a copy that timed out) offers Retry: the + // same start again under its key, so a machine already made or a folder already copied is reused. + let retry: (@MainActor () -> Void)? = (error as? CloudMove.Failure)?.retry == nil ? nil : { [services] in + // task-owner: one retried start; it ends with its own toast + Task { _ = await CloudStartChat(services: services).reply(start, tab: tab) } + } + CloudMove.toast(services, toastID, message, duration: .seconds(retry == nil ? 12 : 30), retry: retry) return AgentPaneReply.failure(code: "cloud_start_failed", message: message) } } diff --git a/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudStartSession.swift b/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudStartSession.swift index 1002a90e9a0e..e20eaa064b61 100644 --- a/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudStartSession.swift +++ b/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudStartSession.swift @@ -367,11 +367,12 @@ struct CloudStartSession { } let daemon = session.daemon let up = await CloudWait.holds(within: Self.connectWait, until: { - session.resumeFailure != nil || (daemon.connection != nil && AgentTabStore.registryHost(of: daemon) != nil) + session.resumeFailure != nil || session.legacyLinkFailure?.definitive == true + || (daemon.connection != nil && AgentTabStore.registryHost(of: daemon) != nil) }) // A sign-out, team switch or cancellation during the wait is that, not a machine that didn't connect. try Self.checked(account, now: services.cloud.account, request.asked(machine: session.machine.title)) - guard up, session.resumeFailure == nil else { throw failure() } + guard up, session.resumeFailure == nil, daemon.connection != nil else { throw failure() } } /// The machine takes the person's harness login before the session starts, as a composer start's does (+Login). diff --git a/Packages/macOS/CmuxNext/Sources/CmuxNextCloud/API/CloudAPIClient.swift b/Packages/macOS/CmuxNext/Sources/CmuxNextCloud/API/CloudAPIClient.swift index 932c52d5cdbd..0cf8e962924b 100644 --- a/Packages/macOS/CmuxNext/Sources/CmuxNextCloud/API/CloudAPIClient.swift +++ b/Packages/macOS/CmuxNext/Sources/CmuxNextCloud/API/CloudAPIClient.swift @@ -533,6 +533,12 @@ nonisolated final class SandboxDeadline: Sendable { private let state = Mutex(State()) static func race(_ bound: Duration, _ work: @escaping @Sendable () async throws -> T) async throws -> T { + try await race(bound, timeout: { CloudAPIError.timedOut("/api/runs") }, work) + } + + /// ``race(_:_:)`` with the error the bound throws (`timeout`): a Cloud link step names itself. + static func race(_ bound: Duration, timeout: @escaping @Sendable () -> any Error, + _ work: @escaping @Sendable () async throws -> T) async throws -> T { let gate = SandboxDeadline() return try await withTaskCancellationHandler { try await withCheckedThrowingContinuation { continuation in @@ -548,7 +554,7 @@ nonisolated final class SandboxDeadline: Sendable { let timer = Task { // wakeup-allow: one-shot deadline for a sandbox run (stops at the bound or the answer) try? await Task.sleep(for: bound) - gate.finish(.failure(CloudAPIError.timedOut("/api/runs"))) + gate.finish(.failure(timeout())) } gate.onFinish { job.cancel(); timer.cancel() } } diff --git a/Packages/macOS/CmuxNext/Sources/CmuxNextCloud/Link/CloudLinkStart.swift b/Packages/macOS/CmuxNext/Sources/CmuxNextCloud/Link/CloudLinkStart.swift new file mode 100644 index 000000000000..3554aa24e25f --- /dev/null +++ b/Packages/macOS/CmuxNext/Sources/CmuxNextCloud/Link/CloudLinkStart.swift @@ -0,0 +1,144 @@ +public import Foundation +import CmuxNextCompat + +/// One step of a legacy Cloud link's start (``CloudMachineLink``): each has a deadline and a +/// log line, so a start that does not come up says where it stopped and why. +public enum CloudLinkStep: String, Sendable, Hashable, CaseIterable { + /// `POST /api/vm/{id}/attach-endpoint`. + case attachEndpoint = "attach-endpoint" + /// This Mac's WireGuard hub (`POST /api/vm/tunnel`, then `cmux-tui wg hub`). + case hub + /// `POST /api/vm/{id}/cmux-remote/approve` for a new enrollment. + case approve + /// `cmux-tui remote connect` until its first `connected` line. + case remoteConnect = "remote-connect" + + /// How long the step may take. A deadline is the step's own calls' bounds plus slack, so it + /// only fires on work that stopped answering (attach 30 s, enroll 60 s + hub 45 s, approve + /// 60 s, the link's first line 60 s). + public var deadline: Duration { + switch self { + case .attachEndpoint: .seconds(45) + case .hub: .seconds(120) + case .approve: .seconds(75) + case .remoteConnect: .seconds(75) + } + } + + /// What the step does, in the words a failure shows. + public var text: String { + switch self { + case .attachEndpoint: "asking cmux Cloud for the machine's address" + case .hub: "starting this Mac's tunnel to cmux Cloud" + case .approve: "approving this Mac on the machine" + case .remoteConnect: "opening the link to the machine" + } + } +} + +/// A legacy link start that stopped at ``step``: the step's own error, or its deadline. +public struct CloudLinkStartError: Error, Sendable, Equatable, CustomStringConvertible, LocalizedError { + public let step: CloudLinkStep + public let reason: String + /// No retry of the same start can help (a refusal the server or this Mac gave: signed out, + /// a plan, a 4xx other than a conflict or rate limit, a missing `cmux-tui`). A deadline or a + /// transport error is not: the machine may still be booting. + public let definitive: Bool + public let timedOut: Bool + + public init(step: CloudLinkStep, reason: String, definitive: Bool = false, timedOut: Bool = false) { + self.step = step + self.reason = reason + self.definitive = definitive + self.timedOut = timedOut + } + + /// `error` from `step`, classified. + public init(step: CloudLinkStep, error: any Error) { + if let already = error as? CloudLinkStartError { + self = already + return + } + self.init(step: step, reason: Self.reason(error), definitive: Self.isDefinitive(error)) + } + + /// The step's deadline passed. + public static func deadline(_ step: CloudLinkStep) -> CloudLinkStartError { + CloudLinkStartError(step: step, reason: "no answer in \(step.deadline.components.seconds) s", timedOut: true) + } + + public var description: String { "Couldn't connect while \(step.text): \(reason)" } + public var errorDescription: String? { description } + + static func reason(_ error: any Error) -> String { + if let api = error as? CloudAPIError { return api.text() } + if let localized = (error as? any LocalizedError)?.errorDescription { return localized } + return String(describing: error) + } + + static func isDefinitive(_ error: any Error) -> Bool { + switch error { + case let api as CloudAPIError: + switch api { + case .notSignedIn, .planRequired, .noBackend: return true + case .http(let status, _, _, _, _): return (400..<500).contains(status) && ![408, 409, 423, 425, 429].contains(status) + case .timedOut, .transport, .decoding: return false + } + case let child as ChildProcessError: + if case .missingBinary = child { return true } + return false + case is CloudTunnelHub.HubError: + return true + default: + return false + } + } +} + +/// Where a legacy link's start is, for the app's failure text and its wait (``CloudMachineLink/progress``): +/// the step running now and when it began, and the last start's failure. Read from any thread. +public nonisolated final class CloudLinkProgress: Sendable { + public struct Snapshot: Sendable, Equatable { + public var step: CloudLinkStep? + public var since: ContinuousClock.Instant? + public var failure: CloudLinkStartError? + } + + private let state = Mutex(Snapshot()) + + init() {} + + public var snapshot: Snapshot { state.withLock { $0 } } + + func began(_ step: CloudLinkStep) { + state.withLock { $0.step = step; $0.since = .now } + } + + /// The start came up: no step runs and no failure stands. + func succeeded() { + state.withLock { $0 = Snapshot() } + } + + func failed(_ failure: CloudLinkStartError) { + state.withLock { $0.step = nil; $0.since = nil; $0.failure = failure } + } + + /// The start ended for another reason (a pause, a stop): its step no longer runs. + func ended() { + state.withLock { $0.step = nil; $0.since = nil } + } +} + +/// Runs one link step under its deadline. The bound holds even when the work ignores +/// cancellation: the caller is answered at the bound and the work is cancelled, never awaited +/// (``SandboxDeadline``), so a step that stopped answering can never hold the start forever. +func runLinkStep(_ step: CloudLinkStep, within bound: Duration? = nil, + _ work: @escaping @Sendable () async throws -> T) async throws -> T { + do { + return try await SandboxDeadline.race(bound ?? step.deadline, timeout: { CloudLinkStartError.deadline(step) }, work) + } catch is CancellationError { + throw CancellationError() + } catch { + throw CloudLinkStartError(step: step, error: error) + } +} diff --git a/Packages/macOS/CmuxNext/Sources/CmuxNextCloud/Link/CloudMachineLink.swift b/Packages/macOS/CmuxNext/Sources/CmuxNextCloud/Link/CloudMachineLink.swift index a63a0522ee00..471ab5e7b38f 100644 --- a/Packages/macOS/CmuxNext/Sources/CmuxNextCloud/Link/CloudMachineLink.swift +++ b/Packages/macOS/CmuxNext/Sources/CmuxNextCloud/Link/CloudMachineLink.swift @@ -16,6 +16,9 @@ public actor CloudMachineLink { public nonisolated let machineID: String /// Whether the running link redials, from its later snapshots. public nonisolated let redial = CloudLinkRedial() + /// The start's step and its last failure (``CloudLinkProgress``): what a start that did not come + /// up says, and whether waiting on it can still help. + public nonisolated let progress = CloudLinkProgress() /// ``callerOwnsVM`` for the main actor (Settings > Machines' row), on the same link. public nonisolated let ownership = CloudLinkOwnership() private let api: CloudAPIClient @@ -72,6 +75,7 @@ public actor CloudMachineLink { private func invalidate() { generation += 1 + progress.ended() starting?.cancel() starting = nil child?.terminate() @@ -156,7 +160,45 @@ public actor CloudMachineLink { /// PID of the running link process, for diagnostics. public var pid: Int32? { child?.pid } + /// One start: every step has a deadline and a log line (``CloudLinkStep``), and a failure, the + /// step's or its deadline's, is logged with the step and kept in ``progress``, so a start never + /// waits silently (2026-11-02: a nightly's Cloud chat start waited 3 minutes with no line). private func start(generation: Int) async throws -> String { + let began = ContinuousClock.now + logger.notice("link start for \(self.machineID, privacy: .public)") + do { + let path = try await startSteps(generation: generation) + progress.succeeded() + logger.notice("link up for \(self.machineID, privacy: .public) in \(Self.seconds(since: began), privacy: .public)") + return path + } catch is CancellationError { + progress.ended() + logger.notice("link start for \(self.machineID, privacy: .public) ended (superseded) after \(Self.seconds(since: began), privacy: .public)") + throw CancellationError() + } catch { + let failure = CloudLinkStartError(step: progress.snapshot.step ?? .remoteConnect, error: error) + progress.failed(failure) + logger.error("link for \(self.machineID, privacy: .public) failed at \(failure.step.rawValue, privacy: .public) after \(Self.seconds(since: began), privacy: .public): \(failure.reason, privacy: .public)") + throw failure + } + } + + static func seconds(since instant: ContinuousClock.Instant) -> String { + let elapsed = instant.duration(to: .now) + return String(format: "%.1f s", Double(elapsed.components.seconds) + Double(elapsed.components.attoseconds) / 1e18) + } + + /// Runs `step` under its deadline with a began/done log line. + private func step(_ step: CloudLinkStep, _ work: @escaping @Sendable () async throws -> T) async throws -> T { + progress.began(step) + let began = ContinuousClock.now + logger.notice("link \(self.machineID, privacy: .public): \(step.rawValue, privacy: .public) began") + let value = try await runLinkStep(step, work) + logger.notice("link \(self.machineID, privacy: .public): \(step.rawValue, privacy: .public) done in \(Self.seconds(since: began), privacy: .public)") + return value + } + + private func startSteps(generation: Int) async throws -> String { try checkGeneration(generation) child?.terminate() child = nil @@ -166,9 +208,10 @@ public actor CloudMachineLink { let deviceFingerprint = paths.loadLinkDeviceFingerprint(machineID: machineID) // Lifecycle transitions can run during every await. A stale start // must never publish a socket or replace a newer link. - let endpoint = try await api.attachEndpoint(machineID, deviceFingerprint: deviceFingerprint) + let (api, hub, machineID) = (api, hub, machineID) + let endpoint = try await step(.attachEndpoint) { try await api.attachEndpoint(machineID, deviceFingerprint: deviceFingerprint) } try checkGeneration(generation) - let hubEndpoint = try await hub.endpoint() + let hubEndpoint = try await step(.hub) { try await hub.endpoint() } try checkGeneration(generation) unlink(paths.linkSocket(machineID: machineID)) let inviteFile = endpoint.invitation.map { _ in @@ -197,11 +240,13 @@ public actor CloudMachineLink { arguments += ["--wireguard-hub", hubEndpoint.socket, "--wireguard-hub-token-stdin"] var environment = ProcessInfo.processInfo.environment environment["CMUX_REMOTE_STATE_DIR"] = paths.linkState.path + progress.began(.remoteConnect) guard FileManager.default.isExecutableFile(atPath: binary.path) else { throw ChildProcessError.missingBinary(binary.path) } let child = ChildProcess(executable: binary, arguments: arguments, environment: environment, input: Data((hubEndpoint.clientToken + "\n").utf8)) try child.start() self.child = child + logger.notice("link \(self.machineID, privacy: .public): remote-connect spawned pid \(child.pid ?? 0)") do { let socketTask = Task { try await child.firstLine(within: .seconds(60), label: "cmux-tui remote connect") { line in @@ -211,11 +256,12 @@ public actor CloudMachineLink { } defer { socketTask.cancel() } if let invitation = endpoint.invitation { - let approval = try await api.approveCmuxRemoteEnrollment(machineID, invitationID: invitation.invitationId) + let approval = try await step(.approve) { try await api.approveCmuxRemoteEnrollment(machineID, invitationID: invitation.invitationId) } guard approval.approved, approval.state == "approved" || approval.state == "already_enrolled" else { socketTask.cancel() throw CloudAPIError.decoding("/api/vm/\(machineID)/cmux-remote/approve") } + progress.began(.remoteConnect) if let fingerprint = approval.deviceFingerprint, !fingerprint.isEmpty { try paths.writeSecret(fingerprint + "\n", to: paths.linkDeviceFingerprint(machineID: machineID)) } @@ -227,7 +273,7 @@ public actor CloudMachineLink { let redial = redial await MainActor.run { redial.isRedialing = false } watchSnapshots(child) - logger.info("link up for \(self.machineID, privacy: .public) pid \(child.pid ?? 0)") + logger.notice("link \(self.machineID, privacy: .public): remote-connect up, pid \(child.pid ?? 0)") return path } catch { if endpoint.invitation == nil, !endpoint.trustedCarrier, deviceFingerprint != nil { @@ -237,7 +283,6 @@ public actor CloudMachineLink { } child.terminate() if self.child === child { self.child = nil } - logger.error("link for \(self.machineID, privacy: .public) failed: \(String(describing: error), privacy: .public)") throw error } } diff --git a/Packages/macOS/CmuxNext/Sources/CmuxNextCloud/Tunnel/CloudTunnelHub.swift b/Packages/macOS/CmuxNext/Sources/CmuxNextCloud/Tunnel/CloudTunnelHub.swift index 3bf67a8956e7..ffe20cd79d5f 100644 --- a/Packages/macOS/CmuxNext/Sources/CmuxNextCloud/Tunnel/CloudTunnelHub.swift +++ b/Packages/macOS/CmuxNext/Sources/CmuxNextCloud/Tunnel/CloudTunnelHub.swift @@ -62,9 +62,13 @@ public actor CloudTunnelHub { /// ``maxRestarts`` in ``restartWindow``. private var restarts: [ContinuousClock.Instant] = [] static let maxRestarts = 3 + /// The enroll's whole bound: its request's 60 s plus its token read. + static let enrollDeadline: Duration = .seconds(75) static let restartWindow: Duration = .seconds(600) private let logger = Logger(subsystem: "com.cmuxterm.app.next", category: "cloud.hub") + static func seconds(since instant: ContinuousClock.Instant) -> String { CloudMachineLink.seconds(since: instant) } + /// 32 bytes from the system's cryptographic generator, hex encoded. static func makeClientToken() -> String { var generator = SystemRandomNumberGenerator() @@ -86,8 +90,10 @@ public actor CloudTunnelHub { case .running(let endpoint) where child?.isRunning == true: return endpoint case .starting(let task): + logger.notice("wireguard hub: joining the start in flight") return try await task.value case .idle, .running: + logger.notice("wireguard hub: starting") let generation = generation let task = Task { try await self.start(generation: generation) } phase = .starting(task) @@ -186,7 +192,14 @@ public actor CloudTunnelHub { architecture: "arm64", appVersion: Bundle.main.object(forInfoDictionaryKey: "CFBundleShortVersionString") as? String ?? "next" ) - let enrollment = try await api.enrollTunnel(request) + let began = ContinuousClock.now + let api = api + // The enroll's request has its own deadline; this one also covers its sign-in token read, so a + // read that never answers can't hold every link's start behind this one (one shared hub). + let enrollment = try await SandboxDeadline.race(Self.enrollDeadline, timeout: { CloudAPIError.timedOut("/api/vm/tunnel") }) { + try await api.enrollTunnel(request) + } + logger.notice("wireguard hub: enrolled in \(Self.seconds(since: began), privacy: .public)") guard generation == self.generation else { throw CancellationError() } // The config holds the private key: it goes to the hub on stdin, never into a file. // Earlier builds wrote it next to the key; that copy goes. @@ -199,6 +212,7 @@ public actor CloudTunnelHub { ], input: Data((clientToken + "\n" + config).utf8)) try child.start() self.child = child + logger.notice("wireguard hub: spawned pid \(child.pid ?? 0)") let ready: CloudHubEndpoint do { let clientToken = clientToken @@ -209,6 +223,7 @@ public actor CloudTunnelHub { } catch { child.terminate() if self.child === child { self.child = nil } + logger.error("wireguard hub: no ready line: \(String(describing: error), privacy: .public)") throw error } guard generation == self.generation else { @@ -225,7 +240,7 @@ public actor CloudTunnelHub { } phase = .running(ready) watch(child, generation: generation) - logger.info("wireguard hub ready (tunnel \(enrollment.tunnelId, privacy: .public))") + logger.notice("wireguard hub ready (tunnel \(enrollment.tunnelId, privacy: .public))") return ready } } From 80b1c3e44bd6743c65d2b04a836e9e7d16eeb1d1 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Mon, 2 Nov 2026 22:49:28 -0800 Subject: [PATCH 2/7] Cloud legacy link: step failure strings in every language; tests for the step deadlines Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-3bab30503f0b9a7059e5 --- .../CmuxNextApp/Resources/Cloud.xcstrings | 655 ++++++++++++++++++ .../CloudLinkStartTests.swift | 81 +++ 2 files changed, 736 insertions(+) create mode 100644 Packages/macOS/CmuxNext/Tests/CmuxNextCloudTests/CloudLinkStartTests.swift diff --git a/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Resources/Cloud.xcstrings b/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Resources/Cloud.xcstrings index 16eba45c48ae..9f72d2b3e38a 100644 --- a/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Resources/Cloud.xcstrings +++ b/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Resources/Cloud.xcstrings @@ -7676,6 +7676,661 @@ } } }, + "cloud.move.notConnectedAtStep": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "%1$@ didn't connect: %2$@ failed (%3$@)." + } + }, + "ar": { + "stringUnit": { + "state": "translated", + "value": "لم يتصل %1$@: فشل %2$@ (%3$@)." + } + }, + "bs": { + "stringUnit": { + "state": "translated", + "value": "%1$@ se nije povezao: %2$@ nije uspjelo (%3$@)." + } + }, + "da": { + "stringUnit": { + "state": "translated", + "value": "%1$@ oprettede ikke forbindelse: %2$@ mislykkedes (%3$@)." + } + }, + "de": { + "stringUnit": { + "state": "translated", + "value": "%1$@ hat keine Verbindung hergestellt: %2$@ ist fehlgeschlagen (%3$@)." + } + }, + "es": { + "stringUnit": { + "state": "translated", + "value": "%1$@ no se conectó: %2$@ falló (%3$@)." + } + }, + "fr": { + "stringUnit": { + "state": "translated", + "value": "%1$@ ne s'est pas connecté : %2$@ a échoué (%3$@)." + } + }, + "it": { + "stringUnit": { + "state": "translated", + "value": "%1$@ non si è connesso: %2$@ non riuscito (%3$@)." + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "%1$@ に接続できませんでした: %2$@ に失敗しました (%3$@)。" + } + }, + "km": { + "stringUnit": { + "state": "translated", + "value": "%1$@ មិនបានភ្ជាប់ទេ៖ %2$@ បានបរាជ័យ (%3$@)។" + } + }, + "ko": { + "stringUnit": { + "state": "translated", + "value": "%1$@에 연결하지 못했습니다: %2$@ 실패 (%3$@)." + } + }, + "nb": { + "stringUnit": { + "state": "translated", + "value": "%1$@ koblet ikke til: %2$@ mislyktes (%3$@)." + } + }, + "pl": { + "stringUnit": { + "state": "translated", + "value": "%1$@ nie połączył się: %2$@ nie powiodło się (%3$@)." + } + }, + "pt-BR": { + "stringUnit": { + "state": "translated", + "value": "%1$@ não conectou: %2$@ falhou (%3$@)." + } + }, + "ru": { + "stringUnit": { + "state": "translated", + "value": "%1$@ не подключился: %2$@ — ошибка (%3$@)." + } + }, + "th": { + "stringUnit": { + "state": "translated", + "value": "%1$@ ไม่ได้เชื่อมต่อ: %2$@ ล้มเหลว (%3$@)" + } + }, + "tr": { + "stringUnit": { + "state": "translated", + "value": "%1$@ bağlanmadı: %2$@ başarısız oldu (%3$@)." + } + }, + "uk": { + "stringUnit": { + "state": "translated", + "value": "%1$@ не підключився: %2$@ — помилка (%3$@)." + } + }, + "vi": { + "stringUnit": { + "state": "translated", + "value": "%1$@ không kết nối được: %2$@ thất bại (%3$@)." + } + }, + "zh-Hans": { + "stringUnit": { + "state": "translated", + "value": "%1$@ 未连接：%2$@ 失败（%3$@）。" + } + }, + "zh-Hant": { + "stringUnit": { + "state": "translated", + "value": "%1$@ 未連線：%2$@ 失敗（%3$@）。" + } + } + } + }, + "cloud.move.linkStep.attachEndpoint": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "Asking cmux Cloud for the machine's address" + } + }, + "ar": { + "stringUnit": { + "state": "translated", + "value": "طلب عنوان الجهاز من cmux Cloud" + } + }, + "bs": { + "stringUnit": { + "state": "translated", + "value": "Traženje adrese mašine od cmux Clouda" + } + }, + "da": { + "stringUnit": { + "state": "translated", + "value": "Spørger cmux Cloud om maskinens adresse" + } + }, + "de": { + "stringUnit": { + "state": "translated", + "value": "Abfrage der Maschinenadresse bei cmux Cloud" + } + }, + "es": { + "stringUnit": { + "state": "translated", + "value": "Pedir a cmux Cloud la dirección de la máquina" + } + }, + "fr": { + "stringUnit": { + "state": "translated", + "value": "Demande de l'adresse de la machine à cmux Cloud" + } + }, + "it": { + "stringUnit": { + "state": "translated", + "value": "Richiesta a cmux Cloud dell'indirizzo della macchina" + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "cmux Cloud へのマシンのアドレスの問い合わせ" + } + }, + "km": { + "stringUnit": { + "state": "translated", + "value": "ការសួរ cmux Cloud រកអាសយដ្ឋានម៉ាស៊ីន" + } + }, + "ko": { + "stringUnit": { + "state": "translated", + "value": "cmux Cloud에 머신 주소 요청" + } + }, + "nb": { + "stringUnit": { + "state": "translated", + "value": "Spør cmux Cloud om maskinens adresse" + } + }, + "pl": { + "stringUnit": { + "state": "translated", + "value": "Pobieranie adresu maszyny z cmux Cloud" + } + }, + "pt-BR": { + "stringUnit": { + "state": "translated", + "value": "Pedir ao cmux Cloud o endereço da máquina" + } + }, + "ru": { + "stringUnit": { + "state": "translated", + "value": "Запрос адреса машины у cmux Cloud" + } + }, + "th": { + "stringUnit": { + "state": "translated", + "value": "การขอที่อยู่ของเครื่องจาก cmux Cloud" + } + }, + "tr": { + "stringUnit": { + "state": "translated", + "value": "Makinenin adresini cmux Cloud'dan isteme" + } + }, + "uk": { + "stringUnit": { + "state": "translated", + "value": "Запит адреси машини в cmux Cloud" + } + }, + "vi": { + "stringUnit": { + "state": "translated", + "value": "Hỏi cmux Cloud địa chỉ của máy" + } + }, + "zh-Hans": { + "stringUnit": { + "state": "translated", + "value": "向 cmux Cloud 获取机器地址" + } + }, + "zh-Hant": { + "stringUnit": { + "state": "translated", + "value": "向 cmux Cloud 取得機器位址" + } + } + } + }, + "cloud.move.linkStep.hub": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "Starting this Mac's tunnel to cmux Cloud" + } + }, + "ar": { + "stringUnit": { + "state": "translated", + "value": "بدء نفق هذا الـ Mac إلى cmux Cloud" + } + }, + "bs": { + "stringUnit": { + "state": "translated", + "value": "Pokretanje tunela ovog Maca do cmux Clouda" + } + }, + "da": { + "stringUnit": { + "state": "translated", + "value": "Start af denne Macs tunnel til cmux Cloud" + } + }, + "de": { + "stringUnit": { + "state": "translated", + "value": "Start des Tunnels dieses Macs zu cmux Cloud" + } + }, + "es": { + "stringUnit": { + "state": "translated", + "value": "Iniciar el túnel de este Mac a cmux Cloud" + } + }, + "fr": { + "stringUnit": { + "state": "translated", + "value": "Démarrage du tunnel de ce Mac vers cmux Cloud" + } + }, + "it": { + "stringUnit": { + "state": "translated", + "value": "Avvio del tunnel di questo Mac verso cmux Cloud" + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "この Mac から cmux Cloud へのトンネルの開始" + } + }, + "km": { + "stringUnit": { + "state": "translated", + "value": "ការចាប់ផ្ដើមផ្លូវរូងរបស់ Mac នេះទៅ cmux Cloud" + } + }, + "ko": { + "stringUnit": { + "state": "translated", + "value": "이 Mac의 cmux Cloud 터널 시작" + } + }, + "nb": { + "stringUnit": { + "state": "translated", + "value": "Start av denne Macens tunnel til cmux Cloud" + } + }, + "pl": { + "stringUnit": { + "state": "translated", + "value": "Uruchamianie tunelu tego Maca do cmux Cloud" + } + }, + "pt-BR": { + "stringUnit": { + "state": "translated", + "value": "Iniciar o túnel deste Mac para o cmux Cloud" + } + }, + "ru": { + "stringUnit": { + "state": "translated", + "value": "Запуск туннеля этого Mac к cmux Cloud" + } + }, + "th": { + "stringUnit": { + "state": "translated", + "value": "การเริ่มอุโมงค์ของ Mac เครื่องนี้ไปยัง cmux Cloud" + } + }, + "tr": { + "stringUnit": { + "state": "translated", + "value": "Bu Mac'in cmux Cloud tünelini başlatma" + } + }, + "uk": { + "stringUnit": { + "state": "translated", + "value": "Запуск тунелю цього Mac до cmux Cloud" + } + }, + "vi": { + "stringUnit": { + "state": "translated", + "value": "Khởi động đường hầm của máy Mac này tới cmux Cloud" + } + }, + "zh-Hans": { + "stringUnit": { + "state": "translated", + "value": "启动此 Mac 到 cmux Cloud 的隧道" + } + }, + "zh-Hant": { + "stringUnit": { + "state": "translated", + "value": "啟動此 Mac 到 cmux Cloud 的通道" + } + } + } + }, + "cloud.move.linkStep.approve": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "Approving this Mac on the machine" + } + }, + "ar": { + "stringUnit": { + "state": "translated", + "value": "اعتماد هذا الـ Mac على الجهاز" + } + }, + "bs": { + "stringUnit": { + "state": "translated", + "value": "Odobravanje ovog Maca na mašini" + } + }, + "da": { + "stringUnit": { + "state": "translated", + "value": "Godkendelse af denne Mac på maskinen" + } + }, + "de": { + "stringUnit": { + "state": "translated", + "value": "Freigabe dieses Macs auf der Maschine" + } + }, + "es": { + "stringUnit": { + "state": "translated", + "value": "Aprobar este Mac en la máquina" + } + }, + "fr": { + "stringUnit": { + "state": "translated", + "value": "Approbation de ce Mac sur la machine" + } + }, + "it": { + "stringUnit": { + "state": "translated", + "value": "Approvazione di questo Mac sulla macchina" + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "マシンでのこの Mac の承認" + } + }, + "km": { + "stringUnit": { + "state": "translated", + "value": "ការអនុម័ត Mac នេះនៅលើម៉ាស៊ីន" + } + }, + "ko": { + "stringUnit": { + "state": "translated", + "value": "머신에서 이 Mac 승인" + } + }, + "nb": { + "stringUnit": { + "state": "translated", + "value": "Godkjenning av denne Macen på maskinen" + } + }, + "pl": { + "stringUnit": { + "state": "translated", + "value": "Zatwierdzanie tego Maca na maszynie" + } + }, + "pt-BR": { + "stringUnit": { + "state": "translated", + "value": "Aprovar este Mac na máquina" + } + }, + "ru": { + "stringUnit": { + "state": "translated", + "value": "Подтверждение этого Mac на машине" + } + }, + "th": { + "stringUnit": { + "state": "translated", + "value": "การอนุมัติ Mac เครื่องนี้บนเครื่อง" + } + }, + "tr": { + "stringUnit": { + "state": "translated", + "value": "Bu Mac'i makinede onaylama" + } + }, + "uk": { + "stringUnit": { + "state": "translated", + "value": "Схвалення цього Mac на машині" + } + }, + "vi": { + "stringUnit": { + "state": "translated", + "value": "Phê duyệt máy Mac này trên máy" + } + }, + "zh-Hans": { + "stringUnit": { + "state": "translated", + "value": "在机器上批准此 Mac" + } + }, + "zh-Hant": { + "stringUnit": { + "state": "translated", + "value": "在機器上核准此 Mac" + } + } + } + }, + "cloud.move.linkStep.remoteConnect": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "Opening the link to the machine" + } + }, + "ar": { + "stringUnit": { + "state": "translated", + "value": "فتح الاتصال بالجهاز" + } + }, + "bs": { + "stringUnit": { + "state": "translated", + "value": "Otvaranje veze s mašinom" + } + }, + "da": { + "stringUnit": { + "state": "translated", + "value": "Åbning af forbindelsen til maskinen" + } + }, + "de": { + "stringUnit": { + "state": "translated", + "value": "Öffnen der Verbindung zur Maschine" + } + }, + "es": { + "stringUnit": { + "state": "translated", + "value": "Abrir el enlace con la máquina" + } + }, + "fr": { + "stringUnit": { + "state": "translated", + "value": "Ouverture du lien vers la machine" + } + }, + "it": { + "stringUnit": { + "state": "translated", + "value": "Apertura del collegamento alla macchina" + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "マシンへのリンクのオープン" + } + }, + "km": { + "stringUnit": { + "state": "translated", + "value": "ការបើកតំណទៅម៉ាស៊ីន" + } + }, + "ko": { + "stringUnit": { + "state": "translated", + "value": "머신 연결 열기" + } + }, + "nb": { + "stringUnit": { + "state": "translated", + "value": "Åpning av forbindelsen til maskinen" + } + }, + "pl": { + "stringUnit": { + "state": "translated", + "value": "Otwieranie połączenia z maszyną" + } + }, + "pt-BR": { + "stringUnit": { + "state": "translated", + "value": "Abrir o link com a máquina" + } + }, + "ru": { + "stringUnit": { + "state": "translated", + "value": "Открытие соединения с машиной" + } + }, + "th": { + "stringUnit": { + "state": "translated", + "value": "การเปิดลิงก์ไปยังเครื่อง" + } + }, + "tr": { + "stringUnit": { + "state": "translated", + "value": "Makineye bağlantıyı açma" + } + }, + "uk": { + "stringUnit": { + "state": "translated", + "value": "Відкриття з'єднання з машиною" + } + }, + "vi": { + "stringUnit": { + "state": "translated", + "value": "Mở liên kết tới máy" + } + }, + "zh-Hans": { + "stringUnit": { + "state": "translated", + "value": "打开到机器的链接" + } + }, + "zh-Hant": { + "stringUnit": { + "state": "translated", + "value": "開啟到機器的連結" + } + } + } + }, "cloud.move.retry": { "extractionState": "manual", "localizations": { diff --git a/Packages/macOS/CmuxNext/Tests/CmuxNextCloudTests/CloudLinkStartTests.swift b/Packages/macOS/CmuxNext/Tests/CmuxNextCloudTests/CloudLinkStartTests.swift new file mode 100644 index 000000000000..bf81915c4a4b --- /dev/null +++ b/Packages/macOS/CmuxNext/Tests/CmuxNextCloudTests/CloudLinkStartTests.swift @@ -0,0 +1,81 @@ +@testable import CmuxNextCloud +import Foundation +import Testing + +/// A legacy Cloud link start's steps (``CloudLinkStep``): each answers within its deadline, even when its +/// work never does, and a failure names the step and says whether a retry can help. 2026-11-02: a nightly's +/// Cloud chat start waited 3 minutes with no hub, no link process and no log line. +@Suite struct CloudLinkStartTests { + @Test func aStepWhoseWorkNeverAnswersFailsAtItsDeadlineNamingTheStep() async { + let began = ContinuousClock.now + do { + _ = try await runLinkStep(.hub, within: .milliseconds(100)) { () async throws -> Int in + // Work that ignores cancellation (a token read that never returns). + await withCheckedContinuation { (_: CheckedContinuation) in } + return 1 + } + Issue.record("the step answered") + } catch let failure as CloudLinkStartError { + #expect(failure.step == .hub) + #expect(failure.timedOut) + #expect(!failure.definitive, "a deadline is not a refusal: the machine may still be booting") + } catch { + Issue.record("unexpected \(error)") + } + #expect(began.duration(to: .now) < .seconds(5)) + } + + @Test func aStepsOwnErrorKeepsItsStepAndReason() async { + do { + _ = try await runLinkStep(.attachEndpoint) { () async throws -> Int in + throw CloudAPIError.http(status: 403, code: "vm_forbidden", message: "This machine belongs to another team.") + } + Issue.record("the step answered") + } catch let failure as CloudLinkStartError { + #expect(failure.step == .attachEndpoint) + #expect(failure.reason == "This machine belongs to another team.") + #expect(failure.definitive) + #expect(failure.description.contains("asking cmux Cloud for the machine's address")) + } catch { + Issue.record("unexpected \(error)") + } + } + + @Test func onlyARefusalIsDefinitive() { + let definitive: [any Error] = [ + CloudAPIError.notSignedIn, + CloudAPIError.http(status: 404, code: nil, message: nil), + CloudAPIError.planRequired(message: nil, upgradeURL: nil), + CloudTunnelHub.HubError.revoked, + ChildProcessError.missingBinary("/x/cmux-tui"), + ] + let retryable: [any Error] = [ + CloudAPIError.timedOut("/api/vm/tunnel"), + CloudAPIError.transport("offline"), + CloudAPIError.http(status: 409, code: "vm_not_ready", message: nil), + CloudAPIError.http(status: 429, code: nil, message: nil), + CloudAPIError.http(status: 503, code: nil, message: nil), + ChildProcessError.exited(label: "cmux-tui remote connect", stderr: "timed out"), + ] + for error in definitive { #expect(CloudLinkStartError(step: .hub, error: error).definitive, "\(error)") } + for error in retryable { #expect(!CloudLinkStartError(step: .hub, error: error).definitive, "\(error)") } + } + + @Test func aStartErrorPassesThroughUnchanged() { + let inner = CloudLinkStartError.deadline(.approve) + #expect(CloudLinkStartError(step: .remoteConnect, error: inner) == inner) + } + + @Test func progressKeepsTheRunningStepAndTheLastFailure() { + let progress = CloudLinkProgress() + progress.began(.attachEndpoint) + #expect(progress.snapshot.step == .attachEndpoint) + progress.failed(.deadline(.hub)) + #expect(progress.snapshot.step == nil) + #expect(progress.snapshot.failure?.step == .hub) + progress.began(.attachEndpoint) + #expect(progress.snapshot.failure?.step == .hub, "a new start keeps the last failure until it comes up") + progress.succeeded() + #expect(progress.snapshot == CloudLinkProgress.Snapshot()) + } +} From 8dbdf314e5be01d0b2d168ab32ce92f00f48118a Mon Sep 17 00:00:00 2001 From: Leo Li Date: Mon, 2 Nov 2026 22:49:53 -0800 Subject: [PATCH 3/7] fix: the Retry closure returns Void Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-3bab30503f0b9a7059e5 --- .../CmuxNext/Sources/CmuxNextApp/Cloud/CloudStartChat.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudStartChat.swift b/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudStartChat.swift index 98df331bc064..43f2189f00c9 100644 --- a/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudStartChat.swift +++ b/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudStartChat.swift @@ -68,7 +68,7 @@ struct CloudStartChat { // same start again under its key, so a machine already made or a folder already copied is reused. let retry: (@MainActor () -> Void)? = (error as? CloudMove.Failure)?.retry == nil ? nil : { [services] in // task-owner: one retried start; it ends with its own toast - Task { _ = await CloudStartChat(services: services).reply(start, tab: tab) } + _ = Task { _ = await CloudStartChat(services: services).reply(start, tab: tab) } } CloudMove.toast(services, toastID, message, duration: .seconds(retry == nil ? 12 : 30), retry: retry) return AgentPaneReply.failure(code: "cloud_start_failed", message: message) From 21801b4fe0ba4066d1d98fb5472b0bb8071f832d Mon Sep 17 00:00:00 2001 From: Leo Li Date: Mon, 2 Nov 2026 22:52:13 -0800 Subject: [PATCH 4/7] Cloud legacy link: the session logs each connect it skips, drops or starts Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-3bab30503f0b9a7059e5 --- .../CmuxNextApp/Cloud/CloudMachineSession.swift | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudMachineSession.swift b/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudMachineSession.swift index b931ab0a1931..868897a9b6c1 100644 --- a/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudMachineSession.swift +++ b/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudMachineSession.swift @@ -5,6 +5,7 @@ import CmuxNextSidebar import CmuxNextWorkspaceRepair import Foundation import Observation +import os import CmuxNextWakeups /// One Cloud machine: its `/api/vm` record, its link, and the daemon @@ -93,6 +94,8 @@ final class CloudMachineSession { /// For an app link, a click's connect on a paused machine is its resume. private(set) var resumeFailure: String? + private static let logger = Logger(subsystem: "com.cmuxterm.app.next", category: "cloud.link") + /// - Parameter mainWork: The app's main-thread scheduler (the local daemon's), for the machine's event batches. convenience init(machine: CloudMachine, link: CloudMachineLink, mainWork: MainWorkScheduler) { self.init(machine: machine, link: link, appLink: nil, localIdentity: { nil }, mainWork: mainWork) @@ -150,12 +153,18 @@ final class CloudMachineSession { /// click never drops a healthy connection, and only a click starts a /// paused machine (`cloud.machine.connect` starts it). func connect(origin: CloudLinkOrigin = .script) { - guard !disconnected, !machine.shownStatus.isChanging else { return } + guard !disconnected, !machine.shownStatus.isChanging else { + if appLink == nil { Self.logger.notice("legacy connect \(self.machineID, privacy: .public): skipped (\(self.disconnected ? "disconnected" : "changing: \(self.machine.shownStatus.rawValue)", privacy: .public))") } + return + } if appLink != nil { guard machine.status.isLive || origin == .user, !appConnecting, appTicket == nil || linkEnded != nil else { return } appConnecting = true } else { - guard machine.status.isLive else { return } + guard machine.status.isLive else { + Self.logger.notice("legacy connect \(self.machineID, privacy: .public): skipped (status \(self.machine.status.rawValue, privacy: .public))") + return + } } connects += 1 // A resume that failed earlier says nothing about this connect. @@ -177,11 +186,13 @@ final class CloudMachineSession { guard let link else { return } await link.resume() guard !disconnected, machine.status.isLive, !machine.shownStatus.isChanging else { + Self.logger.notice("legacy connect \(self.machineID, privacy: .public): dropped after its hop (status \(self.machine.shownStatus.rawValue, privacy: .public))") noteMutationInterruption() return } // A link start that a pause overtook changes nothing (`noteLink`). let suspendsBefore = suspends + Self.logger.notice("legacy connect \(self.machineID, privacy: .public): daemon \(self.daemon.isStarting ? "already starting" : "start", privacy: .public)") daemon.start(remote: { [weak self] in await self?.noteLinkStarting(since: suspendsBefore) do { From ddda1456f4c4253b244034c9b7476e6481d180e5 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Mon, 2 Nov 2026 23:14:06 -0800 Subject: [PATCH 5/7] fix: the toast's Retry finds the tab again by id (Sendable closure) Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-3bab30503f0b9a7059e5 --- .../CmuxNext/Sources/CmuxNextApp/Cloud/CloudStartChat.swift | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudStartChat.swift b/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudStartChat.swift index 43f2189f00c9..2067bfa11605 100644 --- a/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudStartChat.swift +++ b/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudStartChat.swift @@ -66,7 +66,10 @@ struct CloudStartChat { services.cloud.logger.error("cloud start chat failed: \(message, privacy: .public)") // A failure a retry can help (the machine didn't connect, a copy that timed out) offers Retry: the // same start again under its key, so a machine already made or a folder already copied is reused. - let retry: (@MainActor () -> Void)? = (error as? CloudMove.Failure)?.retry == nil ? nil : { [services] in + let tabID = tab.id + let retry: (@MainActor @Sendable () -> Void)? = (error as? CloudMove.Failure)?.retry == nil ? nil : { [weak services] in + // The tab is found again by id: a tab closed since then retries nothing. + guard let services, let (tab, _) = services.locateTab(tabID) else { return } // task-owner: one retried start; it ends with its own toast _ = Task { _ = await CloudStartChat(services: services).reply(start, tab: tab) } } From df24848bc3bdd0e81f8d3a4ba624eae52d1bdca2 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Mon, 2 Nov 2026 23:18:46 -0800 Subject: [PATCH 6/7] fix: the toast's Retry closure without a ternary Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-3bab30503f0b9a7059e5 --- .../CmuxNextApp/Cloud/CloudStartChat.swift | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudStartChat.swift b/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudStartChat.swift index 2067bfa11605..889951965a5e 100644 --- a/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudStartChat.swift +++ b/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudStartChat.swift @@ -66,12 +66,15 @@ struct CloudStartChat { services.cloud.logger.error("cloud start chat failed: \(message, privacy: .public)") // A failure a retry can help (the machine didn't connect, a copy that timed out) offers Retry: the // same start again under its key, so a machine already made or a folder already copied is reused. - let tabID = tab.id - let retry: (@MainActor @Sendable () -> Void)? = (error as? CloudMove.Failure)?.retry == nil ? nil : { [weak services] in - // The tab is found again by id: a tab closed since then retries nothing. - guard let services, let (tab, _) = services.locateTab(tabID) else { return } - // task-owner: one retried start; it ends with its own toast - _ = Task { _ = await CloudStartChat(services: services).reply(start, tab: tab) } + var retry: (@MainActor () -> Void)? + if (error as? CloudMove.Failure)?.retry != nil { + let tabID = tab.id + retry = { [weak services] in + // The tab is found again by id: a tab closed since then retries nothing. + guard let services, let (tab, _) = services.locateTab(tabID) else { return } + // task-owner: one retried start; it ends with its own toast + _ = Task { _ = await CloudStartChat(services: services).reply(start, tab: tab) } + } } CloudMove.toast(services, toastID, message, duration: .seconds(retry == nil ? 12 : 30), retry: retry) return AgentPaneReply.failure(code: "cloud_start_failed", message: message) From 6d44327f56621f08c50f53ea67a7d6640d968a5c Mon Sep 17 00:00:00 2001 From: Leo Li Date: Tue, 3 Nov 2026 02:00:55 -0800 Subject: [PATCH 7/7] fix: a new Cloud wait or its Retry clears the earlier legacy link failure Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-3bab30503f0b9a7059e5 --- .../Sources/CmuxNextApp/Cloud/CloudMachineSession.swift | 8 ++++++++ .../CmuxNext/Sources/CmuxNextApp/Cloud/CloudMove.swift | 1 + .../Sources/CmuxNextApp/Cloud/CloudStartSession.swift | 1 + 3 files changed, 10 insertions(+) diff --git a/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudMachineSession.swift b/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudMachineSession.swift index 868897a9b6c1..002e4a85d875 100644 --- a/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudMachineSession.swift +++ b/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudMachineSession.swift @@ -483,6 +483,14 @@ final class CloudMachineSession { retryNow() } + /// A new wait for this machine (a start, a move, its Retry) begins: a legacy link failure from an + /// earlier attempt is not this attempt's answer, so it is cleared and the link dialed again now. + func beginLegacyAttempt() { + guard link != nil, legacyLinkFailure != nil else { return } + legacyLinkFailure = nil + if machine.status.isLive { retryNow() } + } + /// Retry on the record as it stands (the caller just read it): resume a paused legacy machine, /// else connect an app link, wake a legacy daemon that is starting, or connect it again. func retryNow() { diff --git a/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudMove.swift b/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudMove.swift index 07292998cbef..9f8f0053fbd6 100644 --- a/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudMove.swift +++ b/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudMove.swift @@ -293,6 +293,7 @@ struct CloudMove { try CloudHandlers.requireAttachableState(session) if session.machine.status == .paused { session.attach(origin: CloudHandlers.connectOrigin(for: invocation)) } else { session.connect(origin: CloudHandlers.connectOrigin(for: invocation)) } } + session.beginLegacyAttempt() let daemon = session.daemon // A failed resume ends the wait at once, with its reason, and so does a legacy link start that no // retry can help (a refusal: signed out, a plan, the server's 4xx). Within the wait a link that diff --git a/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudStartSession.swift b/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudStartSession.swift index e20eaa064b61..d3100b56e76c 100644 --- a/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudStartSession.swift +++ b/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudStartSession.swift @@ -365,6 +365,7 @@ struct CloudStartSession { let origin: CloudLinkOrigin = byPerson ? .user : .script if session.machine.status == .paused { session.attach(origin: origin) } else { session.connect(origin: origin) } } + session.beginLegacyAttempt() let daemon = session.daemon let up = await CloudWait.holds(within: Self.connectWait, until: { session.resumeFailure != nil || session.legacyLinkFailure?.definitive == true