cmux NEXT Cloud: terminals and VM-port browser tabs come back working

Packages/macOS/CmuxNext/Sources/CmuxNextApp/AppServices+BrowserTabMachines.swift

@@ -3,6 +3,7 @@ import CmuxNextActions
 import CmuxNextBrowser
 import CmuxNextBrowserTabs
 import CmuxNextDaemon
+import CmuxNextRemoteLocalhostService
 import Foundation
 
 extension AppServices {
@@ -82,6 +83,20 @@ extension AppServices {
             localhost?.closeClient(of: daemon)
             self?.cloudVmHost.forget(daemon.machineID)
         }
+        // A Cloud proxy tab restored after a relaunch fetches its machine's route again.
+        cache.pageRequests.proxiedTabs.restore = { [weak self] machine in
+            guard let self else { return nil }
+            return await BrowserTabOpen.restoredStore(machine, services: self)
+        }
+        // A machine back after its link was down: its localhost pages that showed "not connected" load again.
+        localhost.reloadStrandedPages(keys: { [BrowserTabOpen.machineKey(machine: $0.machineID)] }) { [weak self] machineKey, addresses in
+            guard let self else { return }
+            for entry in cache.browsers.values {
+                guard let page = entry.tab as? CEFTab, page.machineStore?.machineKey == machineKey, let url = page.state.url,
+                      StrandedPageReload.matches(url, addresses: addresses) else { continue }
+                page.reload()
+            }
+        }
         browserTabs.isIncognitoPane = { [weak self] pane in
             guard let self, let workspace = machines.daemon(forPane: pane).store.workspace(containing: pane.handle)?.id else { return false }
             return windows.isIncognito(workspace: workspace)

Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudService.swift

@@ -479,6 +479,8 @@ final class CloudService {
     /// after the account's machines were dropped (sign-out, another account),
     /// or after this app added or removed a machine, is stale.
     @ObservationIgnored private var listGeneration = 0
+    /// A failed list read again on a capped backoff (``CloudListRetry``).
+    @ObservationIgnored private let listRetry = CloudListRetry()
     /// Orders every read of machine records (a list, Machine Status's one record) by when it was asked.
     @ObservationIgnored private var recordReads = 0
     /// Per machine, the read whose record it holds: an older read's answer that lands later changes nothing.
@@ -534,6 +536,7 @@ final class CloudService {
             planSizes = CloudWorkspaceSetupWizard.SizeOption.options(limits: limits, machines: list)
             if let limits { planObserved(limits.planId) }
             lastError = nil
+            listRetry.reset()
             reconcile(list, read: read, connectsAtRead: connectsAtRead)
             changes.listed(machines.cloud.map(\.machine)) { [weak self] in _ = await self?.refresh() }
             hasLoadedMachines = true
@@ -546,6 +549,7 @@ final class CloudService {
             guard listGeneration == generation, auth.isSignedIn, listAccount == account, unavailableReason == nil, !Task.isCancelled else { return false }
             lastError = CloudStrings.errorText(error)
             logger.error("machine list failed: \(String(describing: error), privacy: .public)")
+            listRetry.failed { [weak self] in _ = await self?.refresh() }
             return false
         }
     }
@@ -569,7 +573,7 @@ final class CloudService {
     /// Whether the store holds a list the server answered, within `age`, for the user and team
     /// signed in now: a failed or superseded refresh, or another team's list, is not one.
     func hasCurrentList(within age: Duration) -> Bool {
-        Self.listIsCurrent(loadedAt: hasLoadedMachines ? listLoadedAt : nil, loadedFor: loadedAccount, account: listAccount, now: .now, within: age)
+        CloudListRetry.listIsCurrent(loadedAt: hasLoadedMachines ? listLoadedAt : nil, loadedFor: loadedAccount, account: listAccount, now: .now, within: age)
     }
 
     /// Takes a read ticket before sending. A one-record request names its machine; a list does not.
@@ -707,6 +711,7 @@ final class CloudService {
         changes.stop()
         // A list in flight belongs to the machines dropped here.
         listGeneration += 1
+        listRetry.reset()
         for creation in creations.all { creations.end(creation) }
         for session in machines.cloud { machines.remove(session.machineID)?.disconnect() }
         recordReadApplied = [:]

Packages/macOS/CmuxNext/Sources/CmuxNextApp/RemoteLocalhost/BrowserTabOpen.swift

@@ -2,8 +2,8 @@ import CmuxNextBrowser
 import typealias CmuxNextDaemon.SurfaceID
 import CmuxNextPages
 import CmuxNextRemoteLocalhost
+import CmuxNextRemoteLocalhostService
 import CmuxNextSettings
-import CryptoKit
 import Foundation
 
 /// `browser.tab.open` (a host action of the Cloud page, through
@@ -19,95 +19,21 @@ enum BrowserTabOpen {
     static let action = "browser.tab.open"
     /// The only page that may hand over a proxy (the Cloud page in the app).
     static let callerPage = PageDescriptor.cloud.id
-    /// The store key is `machineKey(machine:)`: 16 lowercase hex digits of
-    /// SHA-256 over this prefix and the machine id. The prefix keeps it apart
-    /// from remote-localhost keys (SHA-256 of a daemon `registry_id`).
-    static let machineKeyPrefix = "cmux.cloud.browser-store.v1:"
+    typealias Refusal = CloudBrowserRequest.Refusal
+    typealias Request = CloudBrowserRequest
+    static let machineKeyPrefix = CloudBrowserRequest.machineKeyPrefix
 
-    enum Refusal: Error, Equatable {
-        /// A machine store opens only in Chromium (`engine: "cef"`).
-        case proxyRequiresCEF
-        /// The machine store or proxy is missing or not the local HTTP proxy on 127.0.0.1.
-        case proxyInvalid
-        /// The caller may not hand over a proxy (only the Cloud page may).
-        case proxyOriginRefused
-        /// Chromium is not available, or failed to make the tab.
-        case cefUnavailable
-        /// The URL is not an http(s) loopback URL.
-        case invalidURL
-        /// No window to open the tab in.
-        case noWindow
+    static func machineKey(machine: String) -> String { CloudBrowserRequest.machineKey(machine: machine) }
 
-        var code: String {
-            switch self {
-            case .proxyRequiresCEF: "cmux.browser.proxy_requires_cef"
-            case .proxyInvalid: "cmux.browser.proxy_invalid"
-            case .proxyOriginRefused: "cmux.browser.proxy_origin_refused"
-            case .cefUnavailable: "cmux.browser.cef_unavailable"
-            case .invalidURL: "cmux.browser.invalid_url"
-            case .noWindow: "cmux.browser.no_window"
-            }
-        }
-
-        var pageError: PageError { PageError(code: code, message: code) }
-    }
-
-    struct Request: Equatable {
-        var url: URL
-        var machine: String
-        var machineName: String
-        var incarnation: UInt64
-        var generation: UInt64
-        var proxyPort: UInt16
-        var credential: ProxyCredential
-    }
-
-    static func machineKey(machine: String) -> String {
-        SHA256.hash(data: Data((machineKeyPrefix + machine).utf8)).prefix(8).map { String(format: "%02x", $0) }.joined()
-    }
-
-    /// Checks `args` from page `page` (set by the page host, never by the page).
+    /// The calling page id comes from the page host, never the request.
     static func request(args: JSONValue, page: String) throws(Refusal) -> Request {
         guard page == callerPage else { throw .proxyOriginRefused }
-        return try checked(args)
+        return try Request.checked(args)
     }
 
-    /// `cmux cloud open-browser` (CloudHandlers+Browser.swift): `route` is the `cloud.browser.open`
-    /// answer the app asked for itself, so no page hands it over and the caller page check does not
-    /// apply. The route gets every other check of a page's request.
+    /// The app requested this route itself; only its payload needs validation.
     static func request(route: JSONValue, machineName: String) throws(Refusal) -> Request {
-        try checked(.object([
-            "engine": .string("cef"), "url": route["url"] ?? .null,
-            "incarnation": route["incarnation"] ?? .null, "generation": route["generation"] ?? .null,
-            "machineStore": .object(["machine": route["machine"] ?? .null, "machineName": .string(machineName), "proxy": route["proxy"] ?? .null]),
-        ]))
-    }
-
-    private static func checked(_ args: JSONValue) throws(Refusal) -> Request {
-        guard args["engine"]?.stringValue == "cef" else { throw .proxyRequiresCEF }
-        guard let text = args["url"]?.stringValue, let url = URL(string: text),
-              ["http", "https"].contains(url.scheme?.lowercased() ?? ""),
-              LoopbackHost(url: url)?.isLoopback == true else { throw .invalidURL }
-        guard let store = args["machineStore"], let machine = store["machine"]?.stringValue, !machine.isEmpty,
-              let proxy = store["proxy"], proxy["kind"]?.stringValue == "http", proxy["host"]?.stringValue == "127.0.0.1",
-              case .number(let number)? = proxy["port"], number.rounded() == number, number >= 1, number <= 65535,
-              let username = proxy["username"]?.stringValue, !username.isEmpty, username.utf8.count <= 128,
-              let password = proxy["password"]?.stringValue, password.utf8.count >= 32, password.utf8.count <= 256,
-              case .number(let incarnation)? = args["incarnation"], incarnation.rounded() == incarnation,
-              incarnation >= 1, incarnation <= 9_007_199_254_740_991,
-              case .number(let generation)? = args["generation"], generation.rounded() == generation,
-              generation >= 1, generation <= 9_007_199_254_740_991
-        else { throw .proxyInvalid }
-        let name = store["machineName"]?.stringValue.flatMap { $0.isEmpty ? nil : $0 } ?? machine
-        return Request(
-            url: url,
-            machine: machine,
-            machineName: name,
-            incarnation: UInt64(incarnation),
-            generation: UInt64(generation),
-            proxyPort: UInt16(number),
-            credential: ProxyCredential(username: username, password: password)
-        )
+        try Request.route(route, machineName: machineName)
     }
 }
 
@@ -120,7 +46,7 @@ extension BrowserTabOpen {
             try await open(request(args: args, page: page), services: services)
             return ["opened": .bool(true)]
         } catch {
-            throw error.pageError
+            throw PageError(code: error.code, message: error.code)
         }
     }
 
@@ -145,7 +71,8 @@ extension BrowserTabOpen {
                 generation: request.generation,
                 proxyPort: request.proxyPort,
                 proxyUsername: request.credential.username,
-                proxyPassword: request.credential.password
+                proxyPassword: request.credential.password,
+                refetch: refetch(machine: request.machine, name: request.machineName, services: services)
             )
         } catch {
             throw .proxyInvalid
@@ -164,7 +91,7 @@ extension BrowserTabOpen {
         // The daemon tab adopts the page (TabContentCache.install records
         // its tab id). The record starts blank: a page made from the record
         // before the adoption lands must not load the URL unproxied.
-        services.cache.pageRequests.proxiedTabs.expect(store)
+        services.cache.pageRequests.proxiedTabs.expect(store, machine: .init(id: request.machine, name: request.machineName))
         let open = { pane.newBrowserTab(url: nil, engine: BrowserEngineTag.cef.rawValue, adopting: page, background: background, then: then) }
         guard reportsRefusal else {
             open()
@@ -175,3 +102,48 @@ extension BrowserTabOpen {
         return opened ? nil : refusal ?? RefusalStrings.chromiumUnavailable
     }
 }
+
+extension BrowserTabOpen {
+    /// Asks the Cloud app server for `machine`'s browser route again (#25595): ``CloudBrowserRoute``
+    /// calls it when the route a tab had is gone (the link dropped or was replaced) and on the first
+    /// request of a tab restored after a relaunch. Only for a machine with a connected daemon on
+    /// the app server's link: `cloud.browser.open` connects the link, which would start a paused
+    /// machine, and a page must never do that. Each answer gets the checks of a page's request,
+    /// for this machine.
+    static func refetch(machine: String, name: String, services: AppServices) -> CloudBrowserRoute.Fetch {
+        let ops = services.cloud.appOps
+        return { [weak services] host, port in
+            guard let services, await isRunning(machine, services: services) else {
+                throw LoopbackTunnelFailure.unavailable("\(name) is not running")
+            }
+            let route = try await ops("cloud.browser.open", .object(["machine": .string(machine), "port": .number(Double(port)), "host": .string(host)]),
+                                      UUID().uuidString.lowercased(), .script)
+            let request = try await request(route: pageJSON(route), machineName: name)
+            guard request.machine == machine else { throw Refusal.proxyInvalid }
+            return CloudBrowserRoute.Answer(proxyPort: request.proxyPort, credential: request.credential)
+        }
+    }
+
+    private static func isRunning(_ machine: String, services: AppServices) -> Bool {
+        services.machines.cloud.contains { session in
+            guard session.machineID == machine, session.appLink != nil else { return false }
+            let connected = if case .connected = session.daemon.store.connectionState { true } else { false }
+            return mayRefetch(live: session.machine.status.isLive, connected: connected)
+        }
+    }
+
+    /// Only a connected daemon permits a page to ask for its route. The machine record can still
+    /// say running after idle pause, or paused after a click resumes it; neither is authoritative
+    /// for a background request that must never start the VM.
+    nonisolated static func mayRefetch(live _: Bool, connected: Bool) -> Bool { connected }
+
+    /// The store of a proxied tab restored after a relaunch (``ProxiedBrowserTabs/restore``): its
+    /// machine's route is fetched on the page's first request, once the machine is listed, linked
+    /// and running; until then the page shows the proxy's "not connected" page, which loads again
+    /// when the machine connects (``StrandedPageReload``). Never this Mac's localhost.
+    static func restoredStore(_ machine: ProxiedBrowserTabs.Machine, services: AppServices) async -> BrowserMachineStore? {
+        try? await services.remoteLocalhost.restoredCloudBrowserStore(
+            machineKey: machineKey(machine: machine.id), machineName: machine.name,
+            fetch: refetch(machine: machine.id, name: machine.name, services: services))
+    }
+}

Packages/macOS/CmuxNext/Sources/CmuxNextBridge/TerminalAttachMachine.swift

@@ -84,8 +84,8 @@ public nonisolated struct TerminalAttachMachine<Link: Hashable & Sendable>: Send
     private var hiddenOn: Link?
     /// This view's claims the stream has not echoed yet, oldest first (a live resize claims every frame).
     public private(set) var echoesDue: [CellSize] = []
-    /// The grid the stream last announced on the live link: a claim of it changes nothing, so the
-    /// stream echoes nothing and no echo is due (a stale one held the next drag's grids).
+    /// The grid the stream last announced (the live link, or a pending attach's replay, #25595): a claim of
+    /// it changes nothing, so the stream echoes nothing and no echo is due (a stale one held the next grids).
     public private(set) var announcedSize: CellSize?
     /// Last status the view was told (it starts connected: nothing shown).
     public private(set) var status: LinkStatus = .connected

Packages/macOS/CmuxNext/Sources/CmuxNextCloud/API/CloudListRetry.swift

@@ -0,0 +1,59 @@
+import CmuxNextWakeups
+
+/// Reads the machine list again after it failed (#25595). The list is read at sign-in or launch,
+/// after a mutation, on app activation and on wake; a launch while offline or while the API was down
+/// left every Cloud workspace without its machine (no terminal, no browser route) until the person
+/// switched apps. A failed read is now tried again on a capped backoff (1 s growing to 30 s), at
+/// most ``timedRetries`` times: after that only the events above read it, never a fixed period.
+@MainActor
+public final class CloudListRetry {
+    /// Timed retries before only events read the list again (about 3 minutes).
+    public static let timedRetries = 10
+
+    private let clock: any Clock<Duration>
+    private var backoff = Backoff(initial: .seconds(1), maximum: .seconds(30))
+    private var failures = 0
+    private var pending: Task<Void, Never>?
+
+    public init(clock: any Clock<Duration> = ContinuousClock()) {
+        self.clock = clock
+    }
+
+    /// Whether a retry waits now.
+    public var isScheduled: Bool { pending != nil }
+
+    /// A read failed: `read` runs again after the next backoff step, unless the budget is spent.
+    /// A retry already waiting stays the only one.
+    public func failed(read: @escaping @MainActor () async -> Void) {
+        guard pending == nil, failures < Self.timedRetries else { return }
+        failures += 1
+        let delay = backoff.next(), clock = clock
+        // task-owner: one retry at a time, cancelled by `reset` and `cancel`
+        pending = Task { [weak self] in
+            // wakeup-allow: one capped backoff step after a failed list read, a bounded number of times
+            guard (try? await clock.sleep(for: delay)) != nil, let self, !Task.isCancelled else { return }
+            pending = nil
+            await read()
+        }
+    }
+
+    /// A read landed, or the account's machines were dropped (sign-out, another account): no retry
+    /// waits and the next failure starts the backoff over.
+    public func reset() {
+        cancel()
+        failures = 0
+        backoff.reset()
+    }
+
+    /// No retry waits (signed out, the machines dropped).
+    public func cancel() {
+        pending?.cancel()
+        pending = nil
+    }
+
+    /// Whether the list loaded at `loadedAt` for `loadedFor` is the current account's and younger than `age`.
+    public nonisolated static func listIsCurrent(loadedAt: ContinuousClock.Instant?, loadedFor: [String?]?, account: [String?], now: ContinuousClock.Instant, within age: Duration) -> Bool {
+        guard let loadedAt, loadedFor == account else { return false }
+        return loadedAt.duration(to: now) < age
+    }
+}