diff --git a/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudMachineCreateFlow.swift b/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudMachineCreateFlow.swift index 5092909f5c2a..b58be29d5783 100644 --- a/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudMachineCreateFlow.swift +++ b/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudMachineCreateFlow.swift @@ -181,8 +181,8 @@ struct CloudMachineCreateFlow { if !startedByPerson, askFirst { guard await confirm(.agentRequest(named(name, sizeShown: false, askedBy: askedBy))) else { throw Failure.declined } } do { return try await create() - } catch CloudAPIError.http(let status, _, _, _, _) where status == 402 { - throw Failure.upgradeRequired(.server) + } catch CloudAPIError.http(let status, let code, _, _, _) where status == 402 { // the shared vCPU/memory pool counts only running machines + throw Failure.upgradeRequired(code == "vm_resource_pool_exceeded" ? .pool : .server) } } diff --git a/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudService.swift b/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudService.swift index f86186221940..15cd5a1dbeaa 100644 --- a/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudService.swift +++ b/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Cloud/CloudService.swift @@ -768,7 +768,7 @@ final class CloudService { let idlePause = request.persistent ? 0 : nil var machine = try await CloudMachineCap.creating(person: startedByPerson, cap: { [weak self] error in guard case CloudMachineCreateFlow.Failure.upgradeRequired(let reason) = error else { return CloudMachineCap.refusal(error) } - return self.map { CloudMachineCap.plan(freePlan: reason == .freePlan, machines: $0.machines.cloud.map(\.machine), user: $0.auth.user?.id) } + return self.map { CloudMachineCap.plan(freePlan: reason == .freePlan, pauseFrees: reason == .pool, machines: $0.machines.cloud.map(\.machine), user: $0.auth.user?.id) } }, choose: { [weak self] in await self?.chooseAtCap?($0) }, pause: { [weak self] in try await self?.pauseMachine($0) }) { again in var request = request // after a Pause, a new key: the first one's stored answer is the refusal @@ -839,7 +839,7 @@ final class CloudService { func pauseMachine(_ machineID: String) async throws { try await changes.run(.pausing, machineID: machineID, of: { machines.session(machineID) }, onRollback: { [weak self] id in self?.machines.session(machineID)?.mutationFailed(id) }) { - try await api.pauseMachine(machineID) + try await configuration.linkSource == .appServer ? CloudMachinePower.run("cloud.machine.pause", machine: machineID, ops: appOps) : api.pauseMachine(machineID) } changes.began { [weak self] in _ = await self?.refresh() }; await refresh(); logger.info("paused machine \(machineID, privacy: .public)") } @@ -847,7 +847,7 @@ final class CloudService { func resumeMachine(_ machineID: String) async throws { try await changes.run(.starting, machineID: machineID, of: { machines.session(machineID) }, onRollback: { [weak self] id in self?.machines.session(machineID)?.mutationFailed(id) }) { - try await api.resumeMachine(machineID) + try await configuration.linkSource == .appServer ? CloudMachinePower.run("cloud.machine.start", machine: machineID, ops: appOps) : api.resumeMachine(machineID) } changes.began { [weak self] in _ = await self?.refresh() }; await refresh(); logger.info("resumed machine \(machineID, privacy: .public)") } diff --git a/Packages/macOS/CmuxNext/Sources/CmuxNextCloud/API/CloudMachineCap.swift b/Packages/macOS/CmuxNext/Sources/CmuxNextCloud/API/CloudMachineCap.swift index dcf43a94cd65..65ca7f1c2284 100644 --- a/Packages/macOS/CmuxNext/Sources/CmuxNextCloud/API/CloudMachineCap.swift +++ b/Packages/macOS/CmuxNext/Sources/CmuxNextCloud/API/CloudMachineCap.swift @@ -9,7 +9,7 @@ public import Foundation /// recently used of their own idle machines, then creates, and the answer names it (`made_room`). Only when every /// machine of theirs is busy does the create come back refused (`cloud.quota.exceeded`, `details.busy`), and the app /// shows one card: Pause one (each busy machine), Upgrade, or Run on This Mac. The classic link makes no room (it -/// gets no new features before the DEV cutover): it shows the same card with the person's own running machines. A +/// gets no new features before the DEV cutover): it shows the same card, with Pause only where pausing frees room. A /// free plan has nothing to pause: Upgrade or Run on This Mac. Never another workspace's machine, never a teammate's. public nonisolated struct CloudMachineCap: Equatable, Sendable { /// One of the person's own machines that counts against the cap. @@ -72,10 +72,12 @@ public nonisolated struct CloudMachineCap: Equatable, Sendable { } } - /// The card for a refusal the app read itself (its plan gate, the classic link's 402): a free plan's offers no - /// pause; otherwise `user`'s own machines (``busy(machines:user:)``). - public static func plan(freePlan: Bool, machines: [CloudMachine], user: String?) -> CloudMachineCap { - CloudMachineCap(busy: freePlan ? [] : busy(machines: machines, user: user), freePlan: freePlan) + /// The card for a refusal the app read itself (its plan gate, the classic link's 402): `user`'s own machines + /// (``busy(machines:user:)``) only when pausing one frees what refused the create (`pauseFrees`), else no pause. + /// The classic link's machine allowance counts paused machines too (web/services/vms/repository.ts), so there only + /// its vCPU and memory pool refusal, which counts running machines alone, offers Pause (cmux#32710). + public static func plan(freePlan: Bool, pauseFrees: Bool, machines: [CloudMachine], user: String?) -> CloudMachineCap { + CloudMachineCap(busy: freePlan || !pauseFrees ? [] : busy(machines: machines, user: user), freePlan: freePlan) } /// The classic link's card: `user`'s own machines that count against the cap (running or on their way), newest diff --git a/Packages/macOS/CmuxNext/Sources/CmuxNextCloud/Bootstrap/CloudAppOp.swift b/Packages/macOS/CmuxNext/Sources/CmuxNextCloud/Bootstrap/CloudAppOp.swift index c206c42f0c3b..61d1148e9419 100644 --- a/Packages/macOS/CmuxNext/Sources/CmuxNextCloud/Bootstrap/CloudAppOp.swift +++ b/Packages/macOS/CmuxNext/Sources/CmuxNextCloud/Bootstrap/CloudAppOp.swift @@ -31,3 +31,22 @@ public nonisolated struct CloudAppOpFailure: Error, Equatable, Sendable, CustomS /// One Cloud app op: `op`, JSON `args`, an idempotency key for a mutation /// (nil for a read), and the wire origin. public typealias CloudAppOp = @Sendable (_ op: String, _ args: JSONValue, _ key: String?, _ origin: AppsRunRequest.Origin) async throws -> JSONValue + +/// A machine's pause or start on the app-server link (`cloud.machine.pause`, `cloud.machine.start`; cmux#32708), never +/// the classic `/api/vm`, which cannot reach a CloudDO machine. CloudDO answers once the provider call landed, so a +/// pause's answer means its active slot is free; a call longer than CloudDO's wait answers `mutation.indeterminate`, +/// asked again with the same key (it joins the call in flight), at most `attempts` times in all. +public nonisolated enum CloudMachinePower { + public static let indeterminate = "cmux.cloud.indeterminate" + + public static func run(_ op: String, machine: String, attempts: Int = 3, ops: CloudAppOp) async throws { + let key = UUID().uuidString + let tries = max(attempts, 1) + for attempt in 1...tries { + do { + _ = try await ops(op, .object(["machine": .string(machine)]), key, .user) + return + } catch let failure as CloudAppOpFailure where failure.code == indeterminate && attempt < tries {} + } + } +} diff --git a/Packages/macOS/CmuxNext/Sources/CmuxNextDaemonService/CloudStrings.swift b/Packages/macOS/CmuxNext/Sources/CmuxNextDaemonService/CloudStrings.swift index 1d2390599d9e..204fa6660518 100644 --- a/Packages/macOS/CmuxNext/Sources/CmuxNextDaemonService/CloudStrings.swift +++ b/Packages/macOS/CmuxNext/Sources/CmuxNextDaemonService/CloudStrings.swift @@ -80,7 +80,7 @@ public enum CloudStrings { switch reason { case .freePlan, .server: String(localized: "cloud.upgrade.detail.free", defaultValue: "Your plan has no Cloud machines. The Go is $10 a month and includes machines; Pro is $50 a month.", table: "Cloud", bundle: .module) - case .atLimit: + case .atLimit, .pool: String(localized: "cloud.upgrade.detail.limit", defaultValue: "All the running Cloud machines your plan allows are in use. Upgrade (Pro is $50 a month) or delete one.", table: "Cloud", bundle: .module) } } @@ -169,4 +169,6 @@ public enum CloudStrings { public nonisolated enum CloudUpgradeReason: Equatable, Sendable { case freePlan, server case atLimit(Int) + /// The classic link's 402 for the plan's shared vCPU and memory pool, which counts only running machines. + case pool } diff --git a/Packages/macOS/CmuxNext/Tests/CmuxNextAppTests/CloudCreateWithoutAskingTests.swift b/Packages/macOS/CmuxNext/Tests/CmuxNextAppTests/CloudCreateWithoutAskingTests.swift index 607f039b1b75..85827a6eef25 100644 --- a/Packages/macOS/CmuxNext/Tests/CmuxNextAppTests/CloudCreateWithoutAskingTests.swift +++ b/Packages/macOS/CmuxNext/Tests/CmuxNextAppTests/CloudCreateWithoutAskingTests.swift @@ -61,6 +61,11 @@ import Testing create: { () async throws -> String in throw CloudAPIError.http(status: 402, code: "vm_active_limit_exceeded", message: "full") }) } #expect(sent.prompts.isEmpty) + // cmux#32710: the pool refusal is the classic one a Pause can lift (the pool counts only running machines). + await #expect(throws: CloudMachineCreateFlow.Failure.upgradeRequired(.pool)) { + try await CloudMachineCreateFlow.legacy(name: nil, startedByPerson: true, gate: { .allowed }, confirm: { _ in false }, + create: { () async throws -> String in throw CloudAPIError.http(status: 402, code: "vm_resource_pool_exceeded", message: "full") }) + } } @Test func gateReadsThePlanLimits() { diff --git a/Packages/macOS/CmuxNext/Tests/CmuxNextCloudTests/CloudMachineCapTests.swift b/Packages/macOS/CmuxNext/Tests/CmuxNextCloudTests/CloudMachineCapTests.swift index d034574b3571..71143e5739dd 100644 --- a/Packages/macOS/CmuxNext/Tests/CmuxNextCloudTests/CloudMachineCapTests.swift +++ b/Packages/macOS/CmuxNext/Tests/CmuxNextCloudTests/CloudMachineCapTests.swift @@ -33,10 +33,56 @@ import Testing theirs.createdBy = "bob" var paused = CloudMachine(id: "vm_paused", status: .paused) paused.createdBy = "alice" - let cap = CloudMachineCap.plan(freePlan: false, machines: [mine, theirs, paused], user: "alice") + let cap = CloudMachineCap.plan(freePlan: false, pauseFrees: true, machines: [mine, theirs, paused], user: "alice") #expect(cap.busy.map(\.machine) == ["vm_mine"], "never a teammate's machine, nor one not running") #expect(CloudMachineCap.busy(machines: [mine], user: nil).isEmpty) - #expect(CloudMachineCap.plan(freePlan: true, machines: [mine], user: "alice") == CloudMachineCap(freePlan: true)) + #expect(CloudMachineCap.plan(freePlan: true, pauseFrees: true, machines: [mine], user: "alice") == CloudMachineCap(freePlan: true)) + } + + /// cmux#32710: the classic allowance counts paused machines, so a Pause there would free nothing: its card offers + /// Upgrade and Run on This Mac only. Its vCPU and memory pool counts running machines alone, so that refusal keeps Pause. + @Test func theClassicCardOffersPauseOnlyWherePausingFreesRoom() { + var mine = CloudMachine(id: "vm_mine", status: .running, displayName: "paint") + mine.createdBy = "alice" + let full = CloudMachineCap.plan(freePlan: false, pauseFrees: false, machines: [mine], user: "alice") + #expect(full == CloudMachineCap(), "a Pause that frees no slot was offered") + #expect(!full.freePlan) + #expect(CloudMachineCap.plan(freePlan: false, pauseFrees: true, machines: [mine], user: "alice").busy.map(\.machine) == ["vm_mine"]) + } + + /// cmux#32708: the app-server link pauses through `cloud.machine.pause`, never `/api/vm`; CloudDO answers once the + /// slot is free, and a pause longer than its wait (`mutation.indeterminate`) is asked again with the same key. + @Test func theAppServerPauseAsksCloudDOWithOneKeyUntilItLands() async throws { + let sent = SentOps() + let ops: CloudAppOp = { op, args, key, origin in + let count = await sent.record(op: op, machine: args["machine"]?.stringValue, key: key, origin: origin) + if count < 3 { throw CloudAppOpFailure(code: CloudMachinePower.indeterminate, message: "cut off") } + return .object(["machine": .object(["id": .string("vm_a"), "status": .string("pausing")])]) + } + try await CloudMachinePower.run("cloud.machine.pause", machine: "vm_a", ops: ops) + let calls = await sent.calls + #expect(calls.map(\.op) == Array(repeating: "cloud.machine.pause", count: 3)) + #expect(calls.allSatisfy { $0.machine == "vm_a" && $0.origin == .user }) + #expect(Set(calls.compactMap(\.key)).count == 1 && calls.allSatisfy { $0.key != nil }, "a retry made a new key") + } + + @Test func theAppServerPauseStopsAfterItsAttemptsAndOnAnyOtherError() async { + let sent = SentOps() + await #expect(throws: CloudAppOpFailure(code: CloudMachinePower.indeterminate, message: "")) { + try await CloudMachinePower.run("cloud.machine.pause", machine: "vm_a", ops: { op, args, key, origin in + _ = await sent.record(op: op, machine: args["machine"]?.stringValue, key: key, origin: origin) + throw CloudAppOpFailure(code: CloudMachinePower.indeterminate, message: "") + }) + } + #expect(await sent.calls.count == 3) + let other = SentOps() + await #expect(throws: CloudAppOpFailure(code: "cmux.cloud.not_running", message: "")) { + try await CloudMachinePower.run("cloud.machine.pause", machine: "vm_a", ops: { op, args, key, origin in + _ = await other.record(op: op, machine: args["machine"]?.stringValue, key: key, origin: origin) + throw CloudAppOpFailure(code: "cmux.cloud.not_running", message: "") + }) + } + #expect(await other.calls.count == 1) } /// Red/green for the app side: at the cap, the person's Pause pauses that machine, then the create goes again. @@ -84,3 +130,12 @@ import Testing #expect(CloudMachineCap.madeRoom(.object(["machine": .object([:])])) == nil) } } + +private actor SentOps { + struct Call { var op: String; var machine: String?; var key: String?; var origin: AppsRunRequest.Origin } + var calls: [Call] = [] + func record(op: String, machine: String?, key: String?, origin: AppsRunRequest.Origin) -> Int { + calls.append(Call(op: op, machine: machine, key: key, origin: origin)) + return calls.count + } +}