diff --git a/.github/workflows/auth-refresh-tests.yml b/.github/workflows/auth-refresh-tests.yml index be97725d6e22..8cee39a3aa13 100644 --- a/.github/workflows/auth-refresh-tests.yml +++ b/.github/workflows/auth-refresh-tests.yml @@ -18,12 +18,14 @@ concurrency: jobs: auth-refresh-tests: - runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-15' || vars.MACOS_RUNNER_26 || 'blacksmith-6vcpu-macos-26' }} + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-15' || (vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_DUAL_XCODE || 'blacksmith-6vcpu-macos-15') }} timeout-minutes: 15 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd with: persist-credentials: false + - name: Select pinned Xcode + run: ./scripts/select-ci-xcode.sh - name: Exercise shared refresh transport and deadlines without credentials run: swift test --package-path vendor/stack-auth-swift-sdk-prerelease --filter 'RefreshLifecycleTests|RefreshDeadlineTests|RefreshTransportTimingTests|RateLimitRetryTests' - name: Exercise coordinator cancellation, sessions, and recovery diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml index 98ab69e0401d..f81dbc08924f 100644 --- a/.github/workflows/ci-macos.yml +++ b/.github/workflows/ci-macos.yml @@ -1729,7 +1729,9 @@ jobs: # Separate invocations prove each selector executed tests; a healthy # suite cannot mask a miswired or accidentally empty selector. set -euo pipefail - for suite in CloudMachineDragSourceTests CloudMachineOrderingTests; do + for suite in CloudMachineDragSourceTests CloudMachineOrderingTests \ + CloudOperationRecorderTests CloudReadRequestCoordinatorTests \ + VMClientReadCoalescingTests; do test_log="$RUNNER_TEMP/cmux-cloud-ordering-${suite}.txt" bash scripts/ci/run-and-capture.sh "$test_log" \ scripts/ci/run-in-console-session.sh \ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4f1ba362dba4..187d29147131 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -877,6 +877,7 @@ jobs: uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: "20" + - name: Run standalone Claude wrapper regressions run: | set -euo pipefail diff --git a/.github/workflows/cloud-vm-guest-install.yml b/.github/workflows/cloud-vm-guest-install.yml new file mode 100644 index 000000000000..13bc5c718919 --- /dev/null +++ b/.github/workflows/cloud-vm-guest-install.yml @@ -0,0 +1,66 @@ +name: Cloud VM guest installation + +on: + pull_request: + paths: + - web/services/vms/drivers/** + - web/services/vms/guestCli*.ts + - web/services/vms/guestPrompt.ts + - web/services/vms/workflows.ts + - web/services/vms/repository.ts + - web/services/vms/routeHelpers.ts + - web/services/vms/vmErrorMessages.ts + - web/messages/** + - web/tests/vm-guest-*.test.ts + - web/tests/fixtures/freestyleGuest.ts + - web/tests/vm-workflows.test.ts + - web/package.json + - web/bun.lock + - .github/workflows/cloud-vm-guest-install.yml + workflow_dispatch: {} + +permissions: + contents: read + +concurrency: + group: cloud-vm-guest-install-${{ github.ref }} + cancel-in-progress: true + +jobs: + guest-install: + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }} + timeout-minutes: 15 + defaults: + run: + working-directory: web + services: + postgres: + image: postgres:16-alpine@sha256:3c5c8892d184f738f4fe282d14ddaa613a38f00f4189d2d94725ebe6f2909ddb + env: + POSTGRES_DB: cmux_test + POSTGRES_PASSWORD: cmux + POSTGRES_USER: cmux + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U cmux -d cmux_test" + --health-interval 1s --health-timeout 5s --health-retries 30 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + persist-credentials: false + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 + with: + bun-version: "1.3.14" + - run: bun install --frozen-lockfile + - name: SDK and guest filesystem regressions + run: bun test tests/vm-guest-install.test.ts tests/vm-guest-install-runtime.test.ts tests/vm-guest-install-deadline.test.ts tests/vm-guest-create-response.test.ts + - name: Migrate isolated database + env: + DATABASE_URL: postgres://cmux:cmux@localhost:5432/cmux_test + run: bunx drizzle-kit migrate --config drizzle.config.ts + - name: Guest provisioning and idempotency in PostgreSQL + env: + CMUX_DB_TEST: "1" + DATABASE_URL: postgres://cmux:cmux@localhost:5432/cmux_test + run: bun test --timeout=30000 --test-name-pattern 'guest bootstrap|unconfirmed guest' tests/vm-workflows.test.ts diff --git a/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Client/StackAuthClient.swift b/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Client/StackAuthClient.swift index 8f7cf5955fde..37e8adf3d1e5 100644 --- a/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Client/StackAuthClient.swift +++ b/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Client/StackAuthClient.swift @@ -58,6 +58,10 @@ public struct StackAuthClient: AuthClient { await stack.getAccessToken() } + /// Resolves access with typed cancellation/deadline failures when supported. + /// - Parameter forceRefresh: Bypasses the cached token after server rejection. + /// - Returns: A usable access token, or `nil` when unavailable. + /// - Throws: Cancellation or a classified transient refresh failure. public func resolvedAccessToken(forceRefresh: Bool) async throws -> String? { let pair = await stack.resolvedTokenPair(forceRefresh: forceRefresh) try Task.checkCancellation() diff --git a/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Coordinator/AuthCoordinator+CurrentTokens.swift b/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Coordinator/AuthCoordinator+CurrentTokens.swift index 379e2968326e..3d4e3099642d 100644 --- a/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Coordinator/AuthCoordinator+CurrentTokens.swift +++ b/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Coordinator/AuthCoordinator+CurrentTokens.swift @@ -23,7 +23,11 @@ extension AuthCoordinator { } try Task.checkCancellation() guard !deadline.hasExpired() else { throw AuthError.timedOut } - return try await runTokenTouchingPhase(.accessToken, timeout: deadline.remaining()) { + return try await runTokenTouchingPhase( + .accessToken, + timeout: deadline.remaining(), + deadline: deadline + ) { return try await self.captureCloudTokens() } } diff --git a/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Coordinator/AuthCoordinator+TokenTouchingPhaseTimeout.swift b/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Coordinator/AuthCoordinator+TokenTouchingPhaseTimeout.swift index dc0e55d1e1cc..c6442f126b14 100644 --- a/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Coordinator/AuthCoordinator+TokenTouchingPhaseTimeout.swift +++ b/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Coordinator/AuthCoordinator+TokenTouchingPhaseTimeout.swift @@ -6,6 +6,20 @@ extension AuthCoordinator { _ phase: AuthPhase, timeout: Duration, _ operation: @escaping @Sendable () async throws -> T + ) async throws -> T { + try await runTokenTouchingPhase( + phase, + timeout: timeout, + deadline: clock.authTokenDeadline(after: timeout), + operation + ) + } + + func runTokenTouchingPhase( + _ phase: AuthPhase, + timeout: Duration, + deadline: AuthTokenDeadline, + _ operation: @escaping @Sendable () async throws -> T ) async throws -> T { expireTimedOutTokenTouchingPhaseIfNeeded(phase) guard timedOutTokenTouchingPhaseStates[phase] == nil else { @@ -13,7 +27,7 @@ extension AuthCoordinator { throw AuthError.timedOut } try Task.checkCancellation() - let deadline = clock.authTokenDeadline(after: timeout) + guard !deadline.hasExpired() else { throw AuthError.timedOut } let phaseID = UUID() let generation = sessionGeneration let signOutEpoch = signOutEpoch diff --git a/Packages/Shared/CmuxAuthRuntime/Tests/CmuxAuthRuntimeTests/AuthCoordinatorTokenTouchingPhaseCancellationTests.swift b/Packages/Shared/CmuxAuthRuntime/Tests/CmuxAuthRuntimeTests/AuthCoordinatorTokenTouchingPhaseCancellationTests.swift index a376ae66e738..0798560f2d0d 100644 --- a/Packages/Shared/CmuxAuthRuntime/Tests/CmuxAuthRuntimeTests/AuthCoordinatorTokenTouchingPhaseCancellationTests.swift +++ b/Packages/Shared/CmuxAuthRuntime/Tests/CmuxAuthRuntimeTests/AuthCoordinatorTokenTouchingPhaseCancellationTests.swift @@ -48,6 +48,55 @@ import Testing await waitUntilTokenTouchingCleanupFinished(coordinator) } + @Test func inheritedDeadlineIsNotRebasedWhenTokenPhaseStartsLate() async { + let clock = ManualTestClock() + let deadline = clock.authTokenDeadline(after: .seconds(2)) + clock.advance(by: .seconds(1)) + let coordinator = makeCoordinator(client: FakeAuthClient(), clock: clock) + let entered = TestPhaseSignal() + let release = TestPhaseSignal() + let phase = Task { + try await coordinator.runTokenTouchingPhase( + .accessToken, + timeout: .seconds(2), + deadline: deadline + ) { + await entered.markStarted() + await release.waitUntilStarted() + return "captured" + } + } + + await entered.waitUntilStarted() + await clock.waitUntilSleepers() + clock.advance(by: .seconds(1)) + // Even if the operation resumes first at the boundary, the phase's + // post-operation check must apply the inherited absolute deadline. + await release.markStarted() + await #expect(throws: AuthError.timedOut) { try await phase.value } + } + + @Test func expiredInheritedDeadlineSkipsTokenOperation() async { + let clock = ManualTestClock() + let deadline = clock.authTokenDeadline(after: .seconds(2)) + clock.advance(by: .seconds(2)) + let coordinator = makeCoordinator(client: FakeAuthClient(), clock: clock) + let entered = TestPhaseSignal() + let phase = Task { + try await coordinator.runTokenTouchingPhase( + .accessToken, + timeout: .seconds(2), + deadline: deadline + ) { + await entered.markStarted() + return "unexpected" + } + } + + await #expect(throws: AuthError.timedOut) { try await phase.value } + #expect(await entered.didStart == false) + } + private func waitUntilTokenTouchingPhaseGated(_ coordinator: AuthCoordinator, phase: AuthPhase) async { for _ in 0..<100 { if coordinator.timedOutTokenTouchingPhaseStates[phase] != nil { diff --git a/Sources/AppDelegate.swift b/Sources/AppDelegate.swift index 432a186df71a..797caf4cd553 100644 --- a/Sources/AppDelegate.swift +++ b/Sources/AppDelegate.swift @@ -2474,12 +2474,12 @@ final class AppDelegate: NSObject, NSApplicationDelegate, UNUserNotificationCent let cloudTunnel = makeCloudTunnelCoordinator() cloudTunnelCoordinator = cloudTunnel CmuxTuiSurfaceProviderRegistry.shared.portAccess.coordinator = cloudTunnel - VMClient.bootstrap(auth: auth.coordinator, operations: cloudOperations) + let cloudReads = VMClient.bootstrap(auth: auth.coordinator, operations: cloudOperations) TerminalController.shared.cloudTunnel = cloudTunnel RemotesClient.bootstrap(auth: auth.coordinator) AIAccountsClient.bootstrap(auth: auth.coordinator) CoderouterClient.bootstrap(auth: auth.coordinator) - MachineUsageClient.bootstrap(auth: auth.coordinator, operations: cloudOperations) + MachineUsageClient.bootstrap(auth: auth.coordinator, operations: cloudOperations, readRequests: cloudReads) PhonePushClient.shared.configure(auth: auth.coordinator) MobileHostService.shared.configure(auth: auth.coordinator) caffeineController.onStateChange = { [weak self] enabled in @@ -14886,13 +14886,14 @@ final class AppDelegate: NSObject, NSApplicationDelegate, UNUserNotificationCent } if activeConfiguredShortcutChordPrefixForCurrentEvent == nil { - let shortcutContext = shortcutEventFocusContext(event).shortcutContext + let focusContext = shortcutEventFocusContext(event) let availableChordActions = currentConfiguredShortcutChordActions().filter { action in // Arm by the effective `when` clause (its shortcuts.when override or // the built-in context default), matching the keyDown gate, so a // `when`-broadened chord arms in its allowed context and a narrowed // one does not swallow its first stroke elsewhere (issue #5189). - KeyboardShortcutSettings.effectiveWhenClause(for: action).evaluate(shortcutContext) + KeyboardShortcutSettings.effectiveWhenClause(for: action) + .evaluate(focusContext.whenClauseContext(for: action)) } if armConfiguredShortcutChordIfNeeded(event: event, actions: availableChordActions) { return true @@ -17165,7 +17166,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate, UNUserNotificationCent /// handlers that previously ignored context (issue #5189). func shortcutWhenClauseAllows(action: KeyboardShortcutSettings.Action, event: NSEvent) -> Bool { KeyboardShortcutSettings.effectiveWhenClause(for: action) - .evaluate(shortcutEventFocusContext(event).shortcutContext) + .evaluate(shortcutEventFocusContext(event).whenClauseContext(for: action)) } /// Resolves a right-sidebar mode shortcut after applying the action's diff --git a/Sources/Cloud/CloudReadCooldownStore+Cooldown.swift b/Sources/Cloud/CloudReadCooldownStore+Cooldown.swift new file mode 100644 index 000000000000..b9fe83cbe1bc --- /dev/null +++ b/Sources/Cloud/CloudReadCooldownStore+Cooldown.swift @@ -0,0 +1,8 @@ +import Foundation + +extension CloudReadCooldownStore { + struct Cooldown: Sendable { + let until: TimeInterval + let response: CloudReadRequestCoordinator.Response + } +} diff --git a/Sources/Cloud/CloudReadCooldownStore+Session.swift b/Sources/Cloud/CloudReadCooldownStore+Session.swift new file mode 100644 index 000000000000..8df5a4ed5f1b --- /dev/null +++ b/Sources/Cloud/CloudReadCooldownStore+Session.swift @@ -0,0 +1,8 @@ +import Foundation + +extension CloudReadCooldownStore { + struct Session: Equatable, Sendable { + let accountID: String? + let generation: UInt64? + } +} diff --git a/Sources/Cloud/CloudReadCooldownStore.swift b/Sources/Cloud/CloudReadCooldownStore.swift new file mode 100644 index 000000000000..f74c9668057c --- /dev/null +++ b/Sources/Cloud/CloudReadCooldownStore.swift @@ -0,0 +1,69 @@ +import Foundation + +/// Bounds retained throttling state without turning eviction into an early retry. +struct CloudReadCooldownStore: Sendable { + private let capacity: Int + private var session: Session? + private var cooldowns: [CloudReadRequestCoordinator.Key: Cooldown] = [:] + + init(capacity: Int = 256) { + self.capacity = max(1, capacity) + } + + var retainedCount: Int { cooldowns.count } + + mutating func activateSession(for key: CloudReadRequestCoordinator.Key) { + let incoming = Session(accountID: key.accountID, generation: key.generation) + guard incoming != session else { return } + if let session { + // AuthCoordinator generations increase across account replacement. + // An old read admitted late cannot reactivate a retired session. + guard let generation = incoming.generation else { return } + if let previousGeneration = session.generation, generation <= previousGeneration { return } + } + session = incoming + cooldowns.removeAll(keepingCapacity: true) + } + + mutating func response(for key: CloudReadRequestCoordinator.Key, now: TimeInterval) -> CloudReadRequestCoordinator.Response? { + guard matchesSession(key) else { return nil } + prune(now: now) + return cooldowns[key]?.response + } + + mutating func record( + _ key: CloudReadRequestCoordinator.Key, until: TimeInterval, now: TimeInterval, + response: CloudReadRequestCoordinator.Response + ) { + guard matchesSession(key), until > now else { return } + prune(now: now) + let compactResponse = CloudReadRequestCoordinator.Response( + data: Data(#"{"error":"rate_limited"}"#.utf8), + http: HTTPURLResponse( + url: response.http.url ?? URL(fileURLWithPath: "/"), statusCode: 429, + httpVersion: nil, headerFields: nil + )! + ) + let cooldown = Cooldown(until: until, response: compactResponse) + if let existing = cooldowns[key] { + if until > existing.until { cooldowns[key] = cooldown } + } else if cooldowns.count < capacity { + cooldowns[key] = cooldown + } else if let evicted = cooldowns.min(by: { $0.value.until < $1.value.until }) { + // Evict only an exact-key minimum when the bounded store is full. + // A session-wide barrier would fabricate 429s for unrelated paths; + // an evicted key may ask the server again, where its real minimum + // is enforced without blocking another machine's read. + cooldowns.removeValue(forKey: evicted.key) + cooldowns[key] = cooldown + } + } + + private func matchesSession(_ key: CloudReadRequestCoordinator.Key) -> Bool { + session == Session(accountID: key.accountID, generation: key.generation) + } + + private mutating func prune(now: TimeInterval) { + cooldowns = cooldowns.filter { $0.value.until > now } + } +} diff --git a/Sources/Cloud/CloudReadMutation.swift b/Sources/Cloud/CloudReadMutation.swift new file mode 100644 index 000000000000..6c8dd7ad9a72 --- /dev/null +++ b/Sources/Cloud/CloudReadMutation.swift @@ -0,0 +1,52 @@ +import Foundation + +/// Identifies the read snapshots changed by one successful control-plane mutation. +struct CloudReadMutation: Sendable { + let scope: CloudReadRequestCoordinator.Key + let affectedPaths: Set + + init(method: String, scope: CloudReadRequestCoordinator.Key, responseData: Data) { + self.scope = scope + var paths: Set = [] + let parts = scope.path.split(separator: "/").map(String.init) + let createsMachine = method == "POST" && [ + "/api/vm", "/api/vm/restore", "/api/vm/base/open", "/api/vm/base/reset" + ].contains(scope.path) + var forksMachine = false + if createsMachine { + paths.insert("/api/vm") + paths.insert("/api/coderouter/vm-usage/team") + } else if parts.count >= 3, parts[0] == "api", parts[1] == "vm", + !["base", "restore", "tunnel", "publications", "domains"].contains(parts[2]) { + let machinePath = "/api/vm/\(parts[2])" + if parts.count == 3, method == "PATCH" || method == "DELETE" { + paths.insert("/api/vm") + paths.insert("/api/coderouter/vm-usage/team") + if method == "DELETE" { paths.insert("\(machinePath)/stats") } + } else if parts.count == 4, method == "POST", [ + "pause", "resume", "resize", "fork", "attach-endpoint", "ssh-endpoint", + "scp-endpoint", "sessions", "exec", "open-port" + ].contains(parts[3]) { + // Open/exec/file-transfer routes can wake a suspended machine. + // Other machines' samples remain valid while this one changes. + paths.insert("/api/vm") + paths.insert("\(machinePath)/stats") + if parts[3] == "fork" { + forksMachine = true + paths.insert("/api/coderouter/vm-usage/team") + } + } + } + if createsMachine || forksMachine { + if let object = try? JSONSerialization.jsonObject(with: responseData) as? [String: Any], + let id = object["id"] as? String, !id.isEmpty { + var allowed = CharacterSet.urlPathAllowed + allowed.remove(charactersIn: "/?#") + if let encodedID = id.addingPercentEncoding(withAllowedCharacters: allowed) { + paths.insert("/api/vm/\(encodedID)/stats") + } + } + } + affectedPaths = paths + } +} diff --git a/Sources/Cloud/CloudReadNetworkMonitor.swift b/Sources/Cloud/CloudReadNetworkMonitor.swift new file mode 100644 index 000000000000..17d8b3d048ad --- /dev/null +++ b/Sources/Cloud/CloudReadNetworkMonitor.swift @@ -0,0 +1,27 @@ +import Foundation +@preconcurrency import Network + +/// One path observer for the VM client. The actor owns the mutable +/// Network.framework monitor; only its sendable stream crosses the actor +/// boundary, and it never protects domain state or runs UI work. +actor CloudReadNetworkMonitor { + private let monitor: NWPathMonitor + nonisolated let updates: AsyncStream + + init() { + let monitor = NWPathMonitor() + self.monitor = monitor + let (updates, continuation) = AsyncStream.makeStream(bufferingPolicy: .bufferingNewest(1)) + self.updates = updates + monitor.pathUpdateHandler = { continuation.yield($0.status != .unsatisfied) } + continuation.onTermination = { _ in monitor.cancel() } + monitor.start(queue: DispatchQueue(label: "cmux.cloud.read-network")) + } + + deinit { monitor.cancel() } +} + +extension Notification.Name { + static let cmuxCloudReadNetworkChanged = Notification.Name("cmux.cloud.readNetworkChanged") + static let cmuxCloudReadNetworkRecovered = Notification.Name("cmux.cloud.readNetworkRecovered") +} diff --git a/Sources/Cloud/CloudReadRecoveryObserver.swift b/Sources/Cloud/CloudReadRecoveryObserver.swift new file mode 100644 index 000000000000..f49493f936be --- /dev/null +++ b/Sources/Cloud/CloudReadRecoveryObserver.swift @@ -0,0 +1,19 @@ +import Foundation + +/// Owns the legacy notification seam for the registry's network recovery refresh. +@MainActor +final class CloudReadRecoveryObserver { + private let notificationCenter: NotificationCenter + private var observer: NSObjectProtocol? + + init(notificationCenter: NotificationCenter, recover: @escaping @MainActor @Sendable () async -> Void) { + self.notificationCenter = notificationCenter + observer = notificationCenter.addObserver(forName: .cmuxCloudReadNetworkRecovered, object: nil, queue: .main) { _ in + Task { @MainActor in await recover() } + } + } + + deinit { + if let observer { notificationCenter.removeObserver(observer) } + } +} diff --git a/Sources/Cloud/CloudReadRequestCoordinator+Context.swift b/Sources/Cloud/CloudReadRequestCoordinator+Context.swift new file mode 100644 index 000000000000..d15662c53d03 --- /dev/null +++ b/Sources/Cloud/CloudReadRequestCoordinator+Context.swift @@ -0,0 +1,8 @@ +import Foundation + +extension CloudReadRequestCoordinator { + struct Context: Sendable { + weak var owner: CloudReadRequestCoordinator? + let key: Key + } +} diff --git a/Sources/Cloud/CloudReadRequestCoordinator+Entry.swift b/Sources/Cloud/CloudReadRequestCoordinator+Entry.swift new file mode 100644 index 000000000000..5e3b3d67677f --- /dev/null +++ b/Sources/Cloud/CloudReadRequestCoordinator+Entry.swift @@ -0,0 +1,16 @@ +import Foundation + +extension CloudReadRequestCoordinator { + struct Entry: Sendable { + let id: UUID + /// Fixed at transport admission; later callers and retries never renew it. + let transportDeadline: Duration + var waiters: [UUID: Waiter] + var work: Task? + var timer: Task? + var terminalError: URLError? + var invalidated = false + let operation: @Sendable () async throws -> Response + var pending: Pending? + } +} diff --git a/Sources/Cloud/CloudReadRequestCoordinator+Key.swift b/Sources/Cloud/CloudReadRequestCoordinator+Key.swift new file mode 100644 index 000000000000..c47bc0c4919c --- /dev/null +++ b/Sources/Cloud/CloudReadRequestCoordinator+Key.swift @@ -0,0 +1,10 @@ +import Foundation + +extension CloudReadRequestCoordinator { + struct Key: Hashable, Sendable { + let path: String + let accountID: String? + let generation: UInt64? + let teamID: String? + } +} diff --git a/Sources/Cloud/CloudReadRequestCoordinator+Pending.swift b/Sources/Cloud/CloudReadRequestCoordinator+Pending.swift new file mode 100644 index 000000000000..c437a23431db --- /dev/null +++ b/Sources/Cloud/CloudReadRequestCoordinator+Pending.swift @@ -0,0 +1,10 @@ +import Foundation + +extension CloudReadRequestCoordinator { + struct Pending: Sendable { + let id: UUID + var waiters: [UUID: Waiter] + let operation: @Sendable () async throws -> Response + var timer: Task? + } +} diff --git a/Sources/Cloud/CloudReadRequestCoordinator+Response.swift b/Sources/Cloud/CloudReadRequestCoordinator+Response.swift new file mode 100644 index 000000000000..d05563b58e15 --- /dev/null +++ b/Sources/Cloud/CloudReadRequestCoordinator+Response.swift @@ -0,0 +1,8 @@ +import Foundation + +extension CloudReadRequestCoordinator { + struct Response: Sendable { + let data: Data + let http: HTTPURLResponse + } +} diff --git a/Sources/Cloud/CloudReadRequestCoordinator+Waiter.swift b/Sources/Cloud/CloudReadRequestCoordinator+Waiter.swift new file mode 100644 index 000000000000..8d09730cef91 --- /dev/null +++ b/Sources/Cloud/CloudReadRequestCoordinator+Waiter.swift @@ -0,0 +1,8 @@ +import Foundation + +extension CloudReadRequestCoordinator { + struct Waiter: Sendable { + let deadline: Duration + let continuation: CheckedContinuation + } +} diff --git a/Sources/Cloud/CloudReadRequestCoordinator.swift b/Sources/Cloud/CloudReadRequestCoordinator.swift new file mode 100644 index 000000000000..913411f79cd4 --- /dev/null +++ b/Sources/Cloud/CloudReadRequestCoordinator.swift @@ -0,0 +1,314 @@ +import Foundation + +/// Owns overlapping read requests for one VM client. Caller cancellation releases +/// only that waiter; the last waiter cancels the transport. A cancelled request +/// keeps its slot until teardown completes, preventing replacement amplification. +/// Caller deadlines are independent upper bounds. The shared transport also has +/// a fixed lifetime cap; reaching it fails even later callers without extending it. +actor CloudReadRequestCoordinator { + @TaskLocal static var current: Context? + + private nonisolated let clock: CloudRequestClock + private nonisolated let budget: Duration + private let onNetworkChange: @Sendable (Bool) async -> Void + private(set) var entries: [Key: Entry] = [:] + private var networkTask: Task? + private var isOnline: Bool? + private var cooldowns = CloudReadCooldownStore() + private var networkSubscribers: [UUID: AsyncStream.Continuation] = [:] + + init(clock: CloudRequestClock = CloudRequestClock(ContinuousClock()), budget: Duration = .seconds(30), + onNetworkChange: @escaping @Sendable (Bool) async -> Void = { _ in }) { + self.clock = clock + self.budget = budget + self.onNetworkChange = onNetworkChange + } + + nonisolated func makeDeadline(elapsed: Duration = .zero, limit: Duration? = nil) -> Duration { + clock.now() + min(budget, limit ?? budget) - elapsed + } + + func read(_ key: Key, deadline: Duration? = nil, operation: @escaping @Sendable () async throws -> Response) async throws -> Response { + let waiter = UUID() + let response = try await withTaskCancellationHandler { + try Task.checkCancellation() + return try await withCheckedThrowingContinuation { continuation in + join(key, waiter: waiter, deadline: deadline ?? makeDeadline(), continuation: continuation, operation: operation) + } + } onCancel: { + Task { await self.cancel(key, waiter: waiter) } + } + try Task.checkCancellation() + return response + } + + private func join( + _ key: Key, waiter: UUID, deadline: Duration, continuation: CheckedContinuation, + operation: @escaping @Sendable () async throws -> Response + ) { + cooldowns.activateSession(for: key) + if clock.now() >= deadline { + continuation.resume(throwing: URLError(.timedOut)) + return + } + if isOnline == false { + continuation.resume(throwing: URLError(.notConnectedToInternet)) + return + } + if let entry = entries[key] { + if entry.terminalError != nil { + queueAfterTeardown(key, waiter: waiter, deadline: deadline, continuation: continuation, operation: operation) + } else if clock.now() >= entry.transportDeadline { + expire(key, id: entry.id) + queueAfterTeardown(key, waiter: waiter, deadline: deadline, continuation: continuation, operation: operation) + } else { + entries[key]?.waiters[waiter] = Waiter(deadline: deadline, continuation: continuation) + armTimer(key, id: entry.id) + } + return + } + if let response = cooldowns.response(for: key, now: seconds(clock.now())) { + continuation.resume(returning: response) + return + } + startEntry(key, id: UUID(), + waiters: [waiter: Waiter(deadline: deadline, continuation: continuation)], operation: operation) + } + + private func startEntry( + _ key: Key, id: UUID, waiters: [UUID: Waiter], + operation: @escaping @Sendable () async throws -> Response + ) { + let waiters = unexpired(waiters) + guard !waiters.isEmpty else { return } + let transportDeadline = makeDeadline() + if clock.now() >= transportDeadline || isOnline == false { + let error = URLError(isOnline == false ? .notConnectedToInternet : .timedOut) + for waiter in waiters.values { waiter.continuation.resume(throwing: error) } + return + } + if let response = cooldowns.response(for: key, now: seconds(clock.now())) { + for waiter in waiters.values { waiter.continuation.resume(returning: response) } + return + } + entries[key] = Entry(id: id, transportDeadline: transportDeadline, waiters: waiters, operation: operation) + startWork(key, id: id, operation: operation) + armTimer(key, id: id) + } + + private func armTimer(_ key: Key, id: UUID) { + guard let entry = entries[key], entry.terminalError == nil else { return } + entry.timer?.cancel() + let deadline = min(entry.transportDeadline, entry.waiters.values.map(\.deadline).min() ?? entry.transportDeadline) + entries[key]?.timer = Task { [weak self, clock] in + do { try await clock.sleepUntil(deadline) } catch { return } + await self?.expireDueWaiters(key, id: id) + } + } + + private func unexpired(_ waiters: [UUID: Waiter]) -> [UUID: Waiter] { + let now = clock.now() + return waiters.filter { _, waiter in + guard waiter.deadline > now else { + waiter.continuation.resume(throwing: URLError(.timedOut)) + return false + } + return true + } + } + + private func expireDueWaiters(_ key: Key, id: UUID) { + guard let entry = entries[key], entry.id == id, entry.terminalError == nil else { return } + if clock.now() >= entry.transportDeadline { expire(key, id: id); return } + entries[key]?.waiters = unexpired(entry.waiters) + if entries[key]?.waiters.isEmpty == true { expire(key, id: id) } + else { armTimer(key, id: id) } + } + + private func queueAfterTeardown( + _ key: Key, waiter: UUID, deadline: Duration, continuation: CheckedContinuation, + operation: @escaping @Sendable () async throws -> Response + ) { + if let pending = entries[key]?.pending { + entries[key]?.pending?.waiters[waiter] = Waiter(deadline: deadline, continuation: continuation) + armPendingTimer(key, id: pending.id) + return + } + let id = UUID() + entries[key]?.pending = Pending(id: id, waiters: [waiter: Waiter(deadline: deadline, continuation: continuation)], operation: operation) + armPendingTimer(key, id: id) + } + + private func armPendingTimer(_ key: Key, id: UUID) { + guard let pending = entries[key]?.pending, pending.id == id else { return } + pending.timer?.cancel() + let waiters = unexpired(pending.waiters) + entries[key]?.pending?.waiters = waiters + guard let deadline = waiters.values.map(\.deadline).min() else { + entries[key]?.pending = nil + return + } + entries[key]?.pending?.timer = Task { [weak self, clock] in + do { try await clock.sleepUntil(deadline) } catch { return } + await self?.armPendingTimer(key, id: id) + } + } + + private func expirePending(_ key: Key, id: UUID, error: URLError = URLError(.timedOut)) { + guard let pending = entries[key]?.pending, pending.id == id else { return } + entries[key]?.pending = nil + pending.timer?.cancel() + for waiter in pending.waiters.values { waiter.continuation.resume(throwing: error) } + } + + private func startWork(_ key: Key, id: UUID, operation: @escaping @Sendable () async throws -> Response) { + let context = Context(owner: self, key: key) + entries[key]?.work = Task { [weak self] in + let result: Result + do { + try Task.checkCancellation() + result = .success(try await Self.$current.withValue(context, operation: operation)) + } catch { + result = .failure(error) + } + await self?.finish(key, id: id, result: result) + } + } + + /// A completed mutation invalidates affected reads from the captured auth + /// and team scope. Its readers share a trailing pass in the original budget. + func invalidate(_ mutation: CloudReadMutation) { + for path in mutation.affectedPaths { + let key = Key(path: path, accountID: mutation.scope.accountID, + generation: mutation.scope.generation, teamID: mutation.scope.teamID) + entries[key]?.invalidated = true + } + } + + /// Retains the server's minimum retry time across cancellation and later + /// polls. Retry only while a live caller and the transport both have budget; + /// otherwise return 429 now and preserve that minimum for future reads. + func noteRetryAfter(_ key: Key, seconds: TimeInterval, response: Response) -> Bool { + // Retry-After can contain Int.max seconds. Keep that distant deadline + // as a monotonic floating-point instant rather than overflowing Duration. + let until = self.seconds(clock.now()) + seconds + cooldowns.record(key, until: until, now: self.seconds(clock.now()), response: response) + guard let entry = entries[key], entry.terminalError == nil, + let callerDeadline = entry.waiters.values.map(\.deadline).max() else { return false } + return until < self.seconds(min(entry.transportDeadline, callerDeadline)) + } + + private func seconds(_ duration: Duration) -> TimeInterval { + let parts = duration.components + return Double(parts.seconds) + Double(parts.attoseconds) / 1e18 + } + + private func cancel(_ key: Key, waiter: UUID) { + guard let removed = entries[key]?.waiters.removeValue(forKey: waiter) else { + if let removed = entries[key]?.pending?.waiters.removeValue(forKey: waiter) { + removed.continuation.resume(throwing: CancellationError()) + if entries[key]?.pending?.waiters.isEmpty == true { + entries[key]?.pending?.timer?.cancel() + entries[key]?.pending = nil + } + } + return + } + removed.continuation.resume(throwing: CancellationError()) + if entries[key]?.waiters.isEmpty == true { + entries[key]?.terminalError = URLError(.cancelled) + entries[key]?.timer?.cancel() + entries[key]?.work?.cancel() + } + } + + private func expire(_ key: Key, id: UUID, error: URLError = URLError(.timedOut)) { + guard let entry = entries[key], entry.id == id, entry.terminalError == nil else { return } + entries[key]?.terminalError = error + entries[key]?.waiters.removeAll() + entry.work?.cancel() + entry.timer?.cancel() + for waiter in entry.waiters.values { waiter.continuation.resume(throwing: error) } + } + + private func finish(_ key: Key, id: UUID, result: Result) { + guard let entry = entries[key], entry.id == id else { return } + // A response may run before the expired timer after sleep/wake. The + // monotonic deadline decides the outcome, not executor delivery order. + expireDueWaiters(key, id: id) + if entry.invalidated, entries[key]?.terminalError == nil, + case .success(let response) = result, (200...299).contains(response.http.statusCode) { + entries[key]?.invalidated = false + startWork(key, id: id, operation: entry.operation) + return + } + guard let completed = entries.removeValue(forKey: key) else { return } + completed.timer?.cancel() + for waiter in completed.waiters.values { waiter.continuation.resume(with: result) } + if let pending = completed.pending { + pending.timer?.cancel() + startEntry(key, id: pending.id, waiters: pending.waiters, operation: pending.operation) + } + } + + func observeNetwork(_ monitor: CloudReadNetworkMonitor) { + networkTask?.cancel() + networkTask = Task { [weak self, monitor] in + for await online in monitor.updates { + guard !Task.isCancelled else { return } + await self?.networkChanged(isOnline: online) + } + } + } + + /// Each consumer receives its own stream so multiple Machines panels never + /// compete for a single `AsyncStream` iterator. + func networkChanges() -> AsyncStream { + let id = UUID() + let (stream, continuation) = AsyncStream.makeStream(bufferingPolicy: .bufferingNewest(1)) + continuation.onTermination = { [weak self] _ in + Task { await self?.removeNetworkSubscriber(id) } + } + networkSubscribers[id] = continuation + if let isOnline { continuation.yield(isOnline) } + return stream + } + + private func removeNetworkSubscriber(_ id: UUID) { + networkSubscribers.removeValue(forKey: id) + } + + func networkChanged(isOnline: Bool) async { + let changed = self.isOnline != isOnline && (self.isOnline != nil || !isOnline) + self.isOnline = isOnline + if !isOnline { + for (key, entry) in entries { + expire(key, id: entry.id, error: URLError(.notConnectedToInternet)) + if let pending = entry.pending { expirePending(key, id: pending.id, error: URLError(.notConnectedToInternet)) } + } + } + if changed { + var terminated: [UUID] = [] + for (id, continuation) in networkSubscribers { + if case .terminated = continuation.yield(isOnline) { + terminated.append(id) + } + } + for id in terminated { networkSubscribers.removeValue(forKey: id) } + await onNetworkChange(isOnline) + } + } + + deinit { + networkTask?.cancel() + for entry in entries.values { + entry.work?.cancel() + entry.timer?.cancel() + for waiter in entry.waiters.values { waiter.continuation.resume(throwing: CancellationError()) } + if let pending = entry.pending { + pending.timer?.cancel() + for waiter in pending.waiters.values { waiter.continuation.resume(throwing: CancellationError()) } + } + } + } +} diff --git a/Sources/Cloud/CloudRequestClock.swift b/Sources/Cloud/CloudRequestClock.swift new file mode 100644 index 000000000000..4b062e07db75 --- /dev/null +++ b/Sources/Cloud/CloudRequestClock.swift @@ -0,0 +1,13 @@ +import Foundation + +/// An injected monotonic clock shared by a read's deadline and Retry-After gate. +struct CloudRequestClock: Sendable { + let now: @Sendable () -> Duration + let sleepUntil: @Sendable (Duration) async throws -> Void + + init>(_ clock: C) { + let origin = clock.now + now = { origin.duration(to: clock.now) } + sleepUntil = { try await clock.sleep(until: origin.advanced(by: $0), tolerance: nil) } + } +} diff --git a/Sources/Cloud/MachineUsageSnapshot.swift b/Sources/Cloud/MachineUsageSnapshot.swift new file mode 100644 index 000000000000..3b82cf799c39 --- /dev/null +++ b/Sources/Cloud/MachineUsageSnapshot.swift @@ -0,0 +1,15 @@ +import Foundation + +/// One machine's usage readout: the row shows `totals` labeled with +/// `periodDays`. `vmID` is the id `GET /api/vm` returns as the machine id, so +/// it matches ``MachineSnapshot/id`` directly. +struct MachineUsageSnapshot: Equatable, Sendable { + let vmID: String + /// The provider machine id, the `id` that `GET /api/vm` lists. Rows key + /// on it when present because `vmID` is the backend's own uuid. + let providerVmID: String? + let displayName: String? + let periodDays: Int + let asOf: Date? + let totals: MachineUsageTotals +} diff --git a/Sources/Cloud/MachineUsageTotals.swift b/Sources/Cloud/MachineUsageTotals.swift new file mode 100644 index 000000000000..f2e0058e4f3e --- /dev/null +++ b/Sources/Cloud/MachineUsageTotals.swift @@ -0,0 +1,15 @@ +import Foundation + +/// Token and spend totals for one machine over the usage window, as +/// `GET /api/coderouter/vm-usage/team` reports them. +struct MachineUsageTotals: Equatable, Sendable { + let inputTokens: Int + let cachedInputTokens: Int + let outputTokens: Int + let totalTokens: Int + /// What the same traffic would have cost at list API prices. + let apiEquivalentUsd: Double + + /// Nothing to show for a machine that has not routed a single token. + var isEmpty: Bool { totalTokens <= 0 && apiEquivalentUsd <= 0 } +} diff --git a/Sources/Cloud/MachinesPanelViewModel+Refresh.swift b/Sources/Cloud/MachinesPanelViewModel+Refresh.swift new file mode 100644 index 000000000000..3d660293fd92 --- /dev/null +++ b/Sources/Cloud/MachinesPanelViewModel+Refresh.swift @@ -0,0 +1,65 @@ +import CmuxCloudMachines +import Foundation + +/// Refresh ownership stays on the panel; only immutable requests survive suspension. +extension MachinesPanelViewModel { + func refreshStats() { + statsTask?.cancel() + statsTask = nil + statsID = nil + guard isCloudEnabled(), let client = client ?? VMClient.shared else { return } + let ids = machines.filter { $0.capabilities.stats }.map(\.id) + guard !ids.isEmpty else { return } + let requestID = UUID() + statsID = requestID + statsTask = Task { [weak self] in + await withTaskGroup(of: Void.self) { group in + for id in ids { group.addTask { _ = try? await client.stats(id: id) } } + } + guard let self, self.statsID == requestID else { return } + self.statsTask = nil + self.statsID = nil + } + } + + /// Samples machines advertising stats support. Sleeping machines report + /// `asleep` without being woken, so polling never costs the user anything. + /// Older servers omitting the flag retain the desktop-only polling policy + /// through capability decoding; explicit support overrides that fallback. + func refreshUsage() { + guard isCloudEnabled(), usageTask == nil else { return } + if let retryNotBefore = usageRetryNotBefore, retryNotBefore > Date() { return } + guard let client = MachineUsageClient.shared else { return } + let generation = refreshGeneration + usageTask = Task { [weak self] in + defer { if generation == self?.refreshGeneration { self?.usageTask = nil } } + do { + let usage = (try await client.teamUsage()).byMachineID + guard !Task.isCancelled, let self, self.isCloudEnabled() else { return } + self.usageFailureCount = 0; self.usageRetryNotBefore = nil + self.applyUsage(usage) + } catch is CancellationError { return } catch { + guard !Task.isCancelled, let self else { return } + self.usageFailureCount = min(self.usageFailureCount + 1, 4) + self.usageRetryNotBefore = Date().addingTimeInterval(Self.usageBackoffDelay(failureCount: self.usageFailureCount)) + } + } + } + + func startPolling() { + wantsPolling = true + guard isCloudEnabled() else { pausePolling(); return } + refresh() + guard pollTask == nil else { return } + pollTask = Task { [weak self, pollingClock] in + while !Task.isCancelled { + do { try await pollingClock.sleep(for: Self.pollInterval) } catch { return } + guard !Task.isCancelled, let self else { return } + self.refresh() + } + } + } + + func stopPolling() { wantsPolling = false; pausePolling() } + +} diff --git a/Sources/Cloud/MachinesPanelViewModel.swift b/Sources/Cloud/MachinesPanelViewModel.swift index 417a6aeafd45..706d1595fa78 100644 --- a/Sources/Cloud/MachinesPanelViewModel.swift +++ b/Sources/Cloud/MachinesPanelViewModel.swift @@ -79,64 +79,93 @@ final class MachinesPanelViewModel: ObservableObject { let selected = tabManager.selectedTabId return tabManager.tabs.map { CloudTreeLocalWorkspace(id: $0.id, title: $0.title, isSelected: $0.id == selected) } } - func beginOperation(_ label: String) { activeOperation = label } func endOperation() { activeOperation = nil - refresh() + if wantsPolling { refresh() } } func noteTreeFailure(_ description: String) { treeErrorDescription = description } - private var refreshTask: Task? - private var pollTask: Task? - private var statsTask: Task? + /// Projects the coordinator's typed reachability event into this panel's + /// local loading and empty-state model. The panel owns presentation state; + /// the coordinator remains the sole network-state owner. + private func applyNetworkChange(_ online: Bool) { + if online { + if wantsPolling { startPolling() } + return + } + clearUnavailableMetrics() + lastErrorDescription = URLError(.notConnectedToInternet).localizedDescription + listProblem = .unreachable + // Mark an interrupted first request as observed so the offline empty + // state renders its retry action. + hasLoadedOnce = true + // Retire the active transport and advance the generation so a late + // response cannot clear the offline state or schedule another refresh. + // `wantsPolling` remains true, allowing the online event to restart it. + pausePolling() + } + + var refreshTask: Task? + var statsID: UUID? + let client: VMClient? + let isCloudEnabled: @MainActor () -> Bool + let pollingClock: any Clock + private var networkTask: Task? + var pollTask: Task? + var statsTask: Task? private var resourceUpdatesTask: Task? - private let resourceStats: VMResourceStatsStore? - private var machineIndexByID: [String: Int] = [:] - private var usageTask: Task? - private var usageFailureCount = 0 - private var usageRetryNotBefore: Date? + let resourceStats: VMResourceStatsStore? + var machineIndexByID: [String: Int] = [:] + var usageTask: Task? + var usageFailureCount = 0 + var usageRetryNotBefore: Date? /// One-shot timer armed at the exact next free-access transition (a /// countdown day-boundary or an expiry). Expiry is client-computable from /// createdAt + window, so rows flip at the boundary itself — scheduling, /// not polling; the slow poll only covers changes made elsewhere. - private var freeAccessTransitionTask: Task? - private var freeAccessWindowDays = 0 + var freeAccessTransitionTask: Task? + var freeAccessWindowDays = 0 /// Last plan limits the list returned; the banner countdown re-derives from /// these on every local recompute without another round trip. - private var lastLimits: VMPlanLimits? + var lastLimits: VMPlanLimits? var memoryOptionsMb: [Int] { lastLimits?.memoryOptionsMb ?? [] } var lockedMemoryOptionsMb: [Int]? { lastLimits?.lockedMemoryOptionsMb } var memoryUpgradePlanId: String? { lastLimits?.memoryUpgradePlanId } var memoryUpgradePlansByMb: [String: String]? { lastLimits?.memoryUpgradePlansByMb } private var authScopeObservers: [NSObjectProtocol] = [] private var featureFlagObserver: CloudFeatureAvailabilityObserver? - private var wantsPolling = false + var wantsPolling = false private var treeChangeObserver: NSObjectProtocol? private var createChangeObserver: NSObjectProtocol? - private var treeTask: Task? - private let machineRefreshes = CloudMachineRefreshCoordinator { await SurfaceCatalog.shared.refresh(machine: $0, force: true) } - private static let statsInterval: Duration = .seconds(20) - + var treeTask: Task? + let machineRefreshes = CloudMachineRefreshCoordinator { await SurfaceCatalog.shared.refresh(machine: $0, force: true) } /// Explicit machine pins and the stable fleet order; nil keeps fleet order. let machinePinStore: CloudMachinePinStore? private let catalogProvider: @MainActor () -> SurfaceCatalogSnapshot - private var awaitingCatalogScope = false + var awaitingCatalogScope = false init( createCoordinator: MachineCreateCoordinator? = nil, machinePinStore: CloudMachinePinStore? = nil, resourceStats: VMResourceStatsStore? = nil, + client: VMClient? = nil, + pollingClock: any Clock = ContinuousClock(), + isCloudEnabled: @escaping @MainActor () -> Bool = { CloudMachinesFeature.isEnabled }, catalogProvider: @escaping @MainActor () -> SurfaceCatalogSnapshot = { SurfaceCatalog.shared.snapshot }, localWorkspacesProvider: (@MainActor () -> [CloudTreeLocalWorkspace])? = nil ) { - self.resourceStats = resourceStats ?? VMClient.shared?.resourceStats + let networkClient = client ?? VMClient.shared + self.client = networkClient + self.pollingClock = pollingClock + self.isCloudEnabled = isCloudEnabled + self.resourceStats = resourceStats ?? networkClient?.resourceStats ?? VMClient.shared?.resourceStats self.machinePinStore = machinePinStore self.catalogProvider = catalogProvider if let localWorkspacesProvider { self.localWorkspacesProvider = localWorkspacesProvider } @@ -162,7 +191,7 @@ final class MachinesPanelViewModel: ObservableObject { } } featureFlagObserver = CloudFeatureAvailabilityObserver( - isEnabled: { CloudMachinesFeature.isEnabled }, + isEnabled: isCloudEnabled, didChange: { [weak self] enabled in guard let self else { return } if enabled, self.wantsPolling { self.startPolling() } @@ -186,6 +215,16 @@ final class MachinesPanelViewModel: ObservableObject { MainActor.assumeIsolated { self?.readUnreadTerminalIDs() } } readUnreadTerminalIDs() + if let networkClient { + networkTask = Task { [weak self, networkClient] in + let changes = await networkClient.networkChanges() + for await online in changes { + guard !Task.isCancelled else { return } + guard let self else { return } + self.applyNetworkChange(online) + } + } + } if let resourceStats = self.resourceStats { let changes = resourceStats.changes() resourceUpdatesTask = Task { [weak self] in @@ -225,6 +264,13 @@ final class MachinesPanelViewModel: ObservableObject { } } deinit { + networkTask?.cancel() + refreshTask?.cancel() + pollTask?.cancel() + statsTask?.cancel() + usageTask?.cancel() + treeTask?.cancel() + freeAccessTransitionTask?.cancel() resourceUpdatesTask?.cancel() for observer in authScopeObservers { NotificationCenter.default.removeObserver(observer) @@ -255,7 +301,7 @@ final class MachinesPanelViewModel: ObservableObject { ) treeErrorDescription = String(format: format, machineID, MachineCreateOperation.headline(ofOutput: output) ?? output) } - refresh() + if wantsPolling { refresh() } } /// Publishes the catalog's current value and the local workspace list. Cheap /// (a value read), so every change notification may call it. @@ -299,53 +345,6 @@ final class MachinesPanelViewModel: ObservableObject { refreshTree(force: forceTree) } func refreshMachine(_ machine: SurfaceMachineID) { machineRefreshes.refresh(machine) } - /// Samples machines advertising stats support. Sleeping machines report - /// `asleep` without being woken, so polling never costs the user anything. - /// Older servers omitting the flag retain the desktop-only polling policy - /// through capability decoding; explicit support overrides that fallback. - func refreshStats() { - guard CloudMachinesFeature.isEnabled, let client = VMClient.shared else { return } - statsTask?.cancel() - let ids = machines.filter { $0.capabilities.stats }.map(\.id) - statsTask = Task { - await withTaskGroup(of: Void.self) { group in - for id in ids { - group.addTask { _ = try? await client.stats(id: id) } - } - } - } - } - - /// Read the shared owner's current snapshot, never a delayed poll's raw result. - private func applyResourceStats(machineIDs: Set?) { - guard CloudMachinesFeature.isEnabled, let resourceStats else { return } - for id in machineIDs ?? Set(machineIndexByID.keys) { - guard let index = machineIndexByID[id], machines.indices.contains(index), - machines[index].id == id, machines[index].capabilities.stats else { continue } - let stats = resourceStats.stats(for: id) - if machines[index].stats != stats { machines[index].stats = stats } - } - } - - func refreshUsage() { - guard CloudMachinesFeature.isEnabled, usageTask == nil else { return } - if let retryNotBefore = usageRetryNotBefore, retryNotBefore > Date() { return } - guard let client = MachineUsageClient.shared else { return } - let generation = refreshGeneration - usageTask = Task { [weak self] in - defer { if generation == self?.refreshGeneration { self?.usageTask = nil } } - do { - let usage = (try await client.teamUsage()).byMachineID - guard !Task.isCancelled, CloudMachinesFeature.isEnabled, let self else { return } - self.usageFailureCount = 0; self.usageRetryNotBefore = nil - self.applyUsage(usage) - } catch is CancellationError { return } catch { - guard !Task.isCancelled, let self else { return } - self.usageFailureCount = min(self.usageFailureCount + 1, 4) - self.usageRetryNotBefore = Date().addingTimeInterval(Self.usageBackoffDelay(failureCount: self.usageFailureCount)) - } - } - } nonisolated static func usageBackoffDelay(failureCount: Int) -> TimeInterval { [30, 30, 60, 120, 300][min(max(failureCount, 0), 4)] } @@ -354,81 +353,18 @@ final class MachinesPanelViewModel: ObservableObject { usageByMachineID = usage machines = MachineSnapshotBuilder.applyingUsage(to: machines, usage: usage) } - private static let pollInterval: Duration = .seconds(45) + static let pollInterval: Duration = .seconds(45) /// A refresh asked for while one is in flight runs again afterwards: a /// create that lands mid-poll must still replace its pending row with the /// real machine now, not on the next 45 s sweep. - private var refreshRequestedWhileLoading = false + var refreshRequestedWhileLoading = false /// Invalidates refresh completions when the Cloud gate closes. A cancelled /// URLSession task may still resume on the main actor, so cancellation /// alone is not enough to prevent stale rows or follow-up work. - private(set) var refreshGeneration: UInt64 = 0 - func refresh() { - guard CloudMachinesFeature.isEnabled else { return } - guard refreshTask == nil else { - refreshRequestedWhileLoading = true - return - } - isLoading = true - let generation = refreshGeneration - refreshTask = Task { [weak self] in - await self?.performRefresh() - guard let self else { return } - guard generation == self.refreshGeneration else { return } - self.refreshTask = nil - if self.refreshRequestedWhileLoading { - self.refreshRequestedWhileLoading = false - self.refresh() - } - } - } - func startPolling() { - wantsPolling = true - guard CloudMachinesFeature.isEnabled else { - pausePolling() - return - } - refresh() - guard pollTask == nil else { return } - pollTask = Task { [weak self] in - while !Task.isCancelled { - try? await Task.sleep(for: Self.pollInterval) - guard !Task.isCancelled else { return } - guard let self else { return } - self.refresh() - } - } - } - - func stopPolling() { - wantsPolling = false - pausePolling() - } - - private func pausePolling() { - pollTask?.cancel() - pollTask = nil - refreshTask?.cancel() - refreshTask = nil - refreshRequestedWhileLoading = false - refreshGeneration &+= 1 - isLoading = false - statsTask?.cancel() - statsTask = nil - usageTask?.cancel() - usageTask = nil - usageFailureCount = 0 - usageRetryNotBefore = nil - treeTask?.cancel() - treeTask = nil - machineRefreshes.cancelAll() - freeAccessTransitionTask?.cancel() - freeAccessTransitionTask = nil - } - + var refreshGeneration: UInt64 = 0 /// Sleeps until the earliest upcoming transition across the fleet, then /// recomputes the free-access facet locally and re-arms for the next one. - private func scheduleFreeAccessTransition(now: Date = Date()) { + func scheduleFreeAccessTransition(now: Date = Date()) { freeAccessTransitionTask?.cancel() freeAccessTransitionTask = nil guard freeAccessWindowDays > 0 else { return } @@ -514,21 +450,71 @@ final class MachinesPanelViewModel: ObservableObject { return scoped } - private func performRefresh() async { - defer { isLoading = false } - guard CloudMachinesFeature.isEnabled else { - return - } - guard let client = VMClient.shared else { - return + /// Read the current shared resource owner, retaining its resize and list fences. + func applyResourceStats(machineIDs: Set?) { + guard isCloudEnabled(), let resourceStats else { return } + for id in machineIDs ?? Set(machineIndexByID.keys) { + guard let index = machineIndexByID[id], machines.indices.contains(index), + machines[index].id == id, machines[index].capabilities.stats else { continue } + let stats = resourceStats.stats(for: id) + if machines[index].stats != stats { machines[index].stats = stats } } + } + + func refresh() { + guard isCloudEnabled(), let client = client ?? VMClient.shared else { return } + guard refreshTask == nil else { refreshRequestedWhileLoading = true; return } + isLoading = true let generation = refreshGeneration let scope = machinePinStore?.scopeIdentifier + refreshTask = Task { [weak self] in + defer { self?.isLoading = false } + let result: Result + do { result = .success(try await client.listPage()) } + catch { result = .failure(error) } + guard !Task.isCancelled, let self, generation == self.refreshGeneration else { return } + self.applyRefreshResult(result, generation: generation, scope: scope) + self.refreshTask = nil + if self.refreshRequestedWhileLoading { + self.refreshRequestedWhileLoading = false + self.refresh() + } + } + } + + func pausePolling() { + pollTask?.cancel(); pollTask = nil + refreshTask?.cancel(); refreshTask = nil + refreshRequestedWhileLoading = false + refreshGeneration &+= 1 + isLoading = false + statsTask?.cancel(); statsTask = nil; statsID = nil + usageTask?.cancel(); usageTask = nil + usageFailureCount = 0 + usageRetryNotBefore = nil + treeTask?.cancel(); treeTask = nil + machineRefreshes.cancelAll() + freeAccessTransitionTask?.cancel(); freeAccessTransitionTask = nil + } + + func clearUnavailableMetrics() { + if let resourceStats { + for id in machineIndexByID.keys { + _ = resourceStats.finishRead(resourceStats.beginRead(machineID: id), stats: nil) + } + applyResourceStats(machineIDs: nil) + } + usageByMachineID = [:] + machines = MachineSnapshotBuilder.applyingUsage(to: machines, usage: [:]) + } + + func applyRefreshResult(_ result: Result, generation: UInt64, scope: String?) { + guard generation == refreshGeneration, scope == machinePinStore?.scopeIdentifier, isCloudEnabled() else { return } do { - let page = try await client.listPage() + let page = try result.get() try Task.checkCancellation() guard generation == refreshGeneration, scope == machinePinStore?.scopeIdentifier, - CloudMachinesFeature.isEnabled else { return } + isCloudEnabled() else { return } let previous = resourceStats?.snapshot ?? [:] let freeAccessWindowDays = page.limits?.freeAccessWindowDays ?? 0 self.freeAccessWindowDays = freeAccessWindowDays diff --git a/Sources/Cloud/TeamMachineUsage.swift b/Sources/Cloud/TeamMachineUsage.swift new file mode 100644 index 000000000000..1c293941a4b8 --- /dev/null +++ b/Sources/Cloud/TeamMachineUsage.swift @@ -0,0 +1,25 @@ +import Foundation + +/// The team-wide usage payload. `kind == .unavailable` means the backend has no +/// usage store for this team (no rows are rendered, no error is surfaced). +struct TeamMachineUsage: Equatable, Sendable { + let teamID: String + let periodDays: Int + let kind: Kind + let asOf: Date? + let machines: [MachineUsageSnapshot] + + /// The lookup the machines panel keys rows by. Empty when the backend says + /// usage is unavailable; blank ids are dropped; the first entry wins when + /// the backend repeats a machine. + var byMachineID: [String: MachineUsageSnapshot] { + guard kind == .ready else { return [:] } + var result: [String: MachineUsageSnapshot] = [:] + for machine in machines { + for key in [machine.providerVmID ?? "", machine.vmID] where !key.isEmpty && result[key] == nil { + result[key] = machine + } + } + return result + } +} diff --git a/Sources/Cloud/TeamMachineUsageKind.swift b/Sources/Cloud/TeamMachineUsageKind.swift new file mode 100644 index 000000000000..3361bfa792fc --- /dev/null +++ b/Sources/Cloud/TeamMachineUsageKind.swift @@ -0,0 +1,7 @@ +extension TeamMachineUsage { + /// Whether the backend has usage records available for this team. + enum Kind: String, Sendable { + case ready + case unavailable + } +} diff --git a/Sources/Cloud/VMCapabilities.swift b/Sources/Cloud/VMCapabilities.swift new file mode 100644 index 000000000000..2956b442aaf1 --- /dev/null +++ b/Sources/Cloud/VMCapabilities.swift @@ -0,0 +1,76 @@ +import Foundation + +/// What a machine's provider can do; the app offers only verbs that can succeed +/// (Checkpoint/Fork disappear from menus when false — a verb that answers 502 +/// "not implemented" is not a verb). +struct VMCapabilities: Equatable, Sendable { + var snapshot: Bool + var restore: Bool + var fork: Bool + var exec: Bool + var stats: Bool + /// The provider can mint a browser preview URL for a machine port. + var ports: Bool + var desktop: Bool + var sizing: Bool + var persistentHome: Bool + var attachTransports: [String]? + + var ssh: Bool { attachTransports?.contains("ssh") ?? true } + var cmuxRemote: Bool { attachTransports?.contains("cmux-remote") ?? true } + + static let all = VMCapabilities( + snapshot: true, restore: true, fork: true, + exec: true, stats: true, ports: true, desktop: true, + sizing: true, persistentHome: true, attachTransports: nil) + + init( + snapshot: Bool, restore: Bool, fork: Bool, + exec: Bool = true, stats: Bool = true, ports: Bool = true, + desktop: Bool = true, sizing: Bool = true, persistentHome: Bool = true, + attachTransports: [String]? = nil + ) { + self.snapshot = snapshot + self.restore = restore + self.fork = fork + self.exec = exec + self.stats = stats + self.ports = ports + self.desktop = desktop + self.sizing = sizing + self.persistentHome = persistentHome + self.attachTransports = attachTransports + } + + /// Missing flags preserve legacy support; stats can use the historical kind fallback. + init(json: Any?, legacyStatsSupported: Bool = true) { + let dict = json as? [String: Any] + func flag(_ key: String, fallback: Bool = true) -> Bool { + if let value = dict?[key] as? Bool { return value } + if let number = dict?[key] as? NSNumber { return number.boolValue } + return fallback + } + let transports = (dict?["attachTransports"] as? [Any] ?? dict?["attach_transports"] as? [Any])? + .compactMap { $0 as? String } + self.init( + snapshot: flag("snapshot"), restore: flag("restore"), fork: flag("fork"), + exec: flag("exec"), stats: flag("stats", fallback: legacyStatsSupported), ports: flag("ports"), + desktop: flag("desktop"), sizing: flag("sizing"), + persistentHome: flag("persistentHome"), attachTransports: transports) + } + + var jsonObject: [String: Any] { + var object: [String: Any] = [ + "snapshot": snapshot, "restore": restore, "fork": fork, + "exec": exec, "stats": stats, "ports": ports, "desktop": desktop, + "sizing": sizing, "persistentHome": persistentHome, + ] + if let attachTransports { object["attach_transports"] = attachTransports } + return object + } + + init(vmResponse: [String: Any]) { + let kind = VMMachineKind.resolved(kind: vmResponse["kind"], image: vmResponse["image"]) + self.init(json: vmResponse["capabilities"], legacyStatsSupported: kind.hasDesktop) + } +} diff --git a/Sources/Cloud/VMClient.swift b/Sources/Cloud/VMClient.swift index fb8dddb85655..907caa837612 100644 --- a/Sources/Cloud/VMClient.swift +++ b/Sources/Cloud/VMClient.swift @@ -33,6 +33,7 @@ extension URLError.Code { } } + func formattedCloudVMHTTPError(status: Int, body: String) -> String { let trimmedBody = body.trimmingCharacters(in: .whitespacesAndNewlines) guard let data = trimmedBody.data(using: .utf8), @@ -351,81 +352,6 @@ struct VMExecResult: Sendable { let stderr: String } -/// What a machine's provider can do; the app offers only verbs that can succeed -/// (Checkpoint/Fork disappear from menus when false — a verb that answers 502 -/// "not implemented" is not a verb). -struct VMCapabilities: Equatable, Sendable { - var snapshot: Bool - var restore: Bool - var fork: Bool - var exec: Bool - var stats: Bool - /// The provider can mint a browser preview URL for a machine port. - var ports: Bool - var desktop: Bool - var sizing: Bool - var persistentHome: Bool - var attachTransports: [String]? - - var ssh: Bool { attachTransports?.contains("ssh") ?? true } - var cmuxRemote: Bool { attachTransports?.contains("cmux-remote") ?? true } - - static let all = VMCapabilities( - snapshot: true, restore: true, fork: true, - exec: true, stats: true, ports: true, desktop: true, - sizing: true, persistentHome: true, attachTransports: nil) - - init( - snapshot: Bool, restore: Bool, fork: Bool, - exec: Bool = true, stats: Bool = true, ports: Bool = true, - desktop: Bool = true, sizing: Bool = true, persistentHome: Bool = true, - attachTransports: [String]? = nil - ) { - self.snapshot = snapshot - self.restore = restore - self.fork = fork - self.exec = exec - self.stats = stats - self.ports = ports - self.desktop = desktop - self.sizing = sizing - self.persistentHome = persistentHome - self.attachTransports = attachTransports - } - - /// Missing flags preserve legacy support; stats can use the historical kind fallback. - init(json: Any?, legacyStatsSupported: Bool = true) { - let dict = json as? [String: Any] - func flag(_ key: String, fallback: Bool = true) -> Bool { - if let value = dict?[key] as? Bool { return value } - if let number = dict?[key] as? NSNumber { return number.boolValue } - return fallback - } - let transports = (dict?["attachTransports"] as? [Any] ?? dict?["attach_transports"] as? [Any])? - .compactMap { $0 as? String } - self.init( - snapshot: flag("snapshot"), restore: flag("restore"), fork: flag("fork"), - exec: flag("exec"), stats: flag("stats", fallback: legacyStatsSupported), ports: flag("ports"), - desktop: flag("desktop"), sizing: flag("sizing"), - persistentHome: flag("persistentHome"), attachTransports: transports) - } - - var jsonObject: [String: Any] { - var object: [String: Any] = [ - "snapshot": snapshot, "restore": restore, "fork": fork, - "exec": exec, "stats": stats, "ports": ports, "desktop": desktop, - "sizing": sizing, "persistentHome": persistentHome, - ] - if let attachTransports { object["attach_transports"] = attachTransports } - return object - } - - init(vmResponse: [String: Any]) { - let kind = VMMachineKind.resolved(kind: vmResponse["kind"], image: vmResponse["image"]) - self.init(json: vmResponse["capabilities"], legacyStatsSupported: kind.hasDesktop) - } -} - struct VMOpenPortEndpoint { let url: String let token: String @@ -555,7 +481,6 @@ struct VMPublicationDomain: Equatable, Sendable { } } - /// One row of `GET /api/vm//snapshots`: the provider snapshot id, its display name /// when one was given, and the creation time as the ISO-8601 string the server sent. struct VMSnapshotSummary: Sendable, Equatable { @@ -703,8 +628,17 @@ actor VMClient { /// Build the shared client with its injected auth dependency. Call once at /// the composition root. @MainActor - static func bootstrap(auth: AuthCoordinator, session: URLSession = .shared, operations: CloudOperationRecorder? = nil) { - shared = VMClient(session: session, auth: auth, resourceStats: VMResourceStatsStore(), checkpointRenames: SurfaceCatalog.shared.cloudRenameCoordinator, operations: operations, isCloudEnabled: { CloudMachinesFeature.offMainIsEnabled() }) + @discardableResult + static func bootstrap(auth: AuthCoordinator, session: URLSession = .shared, operations: CloudOperationRecorder? = nil) -> CloudReadRequestCoordinator { + let reads = CloudReadRequestCoordinator(onNetworkChange: { online in + await MainActor.run { + NotificationCenter.default.post(name: .cmuxCloudReadNetworkChanged, object: nil, userInfo: ["isOnline": online]) + if online { NotificationCenter.default.post(name: .cmuxCloudReadNetworkRecovered, object: nil) } + } + }) + shared = VMClient(session: session, auth: auth, resourceStats: VMResourceStatsStore(), checkpointRenames: SurfaceCatalog.shared.cloudRenameCoordinator, operations: operations, readRequests: reads, isCloudEnabled: { CloudMachinesFeature.offMainIsEnabled() }) + Task { await reads.observeNetwork(CloudReadNetworkMonitor()) } + return reads } /// Revoke endpoint credentials issued by the Cloud VM service during sign-out. @@ -748,6 +682,7 @@ actor VMClient { nonisolated let operations: CloudOperationRecorder? nonisolated let resourceStats: VMResourceStatsStore private let machineCache: CloudMachineCache + private let readRequests: CloudReadRequestCoordinator private let isCloudEnabled: @Sendable () -> Bool private let isDisabledByManagedPolicy: (@Sendable () -> Bool)? @@ -760,15 +695,17 @@ actor VMClient { operations: CloudOperationRecorder? = nil, machineCache: CloudMachineCache = CloudMachineCache(), isDisabledByManagedPolicy: (@Sendable () -> Bool)? = nil, + readRequests: CloudReadRequestCoordinator = CloudReadRequestCoordinator(), isCloudEnabled: @escaping @Sendable () -> Bool = { true } ) { self.session = session - self.resourceStats = resourceStats self.auth = auth + self.resourceStats = resourceStats self.checkpointRenames = checkpointRenames self.telemetry = telemetry self.operations = operations self.machineCache = machineCache + self.readRequests = readRequests self.isCloudEnabled = isCloudEnabled self.isDisabledByManagedPolicy = isDisabledByManagedPolicy } @@ -779,6 +716,14 @@ actor VMClient { } } + /// Typed reachability changes for scoped Cloud owners such as the machines + /// panel. The request coordinator remains the single reachability owner; + /// consumers subscribe to its stream instead of observing a process-wide + /// notification and maintaining a second state machine. + func networkChanges() async -> AsyncStream { + await readRequests.networkChanges() + } + func listPage() async throws -> VMListPage { let (retentionToken, listIdentity, listTeamID) = await MainActor.run { [auth, resourceStats] in (resourceStats.beginRetention(), auth.authenticatedSessionIdentity, auth.resolvedTeamID) @@ -2013,6 +1958,7 @@ actor VMClient { // safety net when this tail cannot reach the API. } } + private func revokeCloudAccess( deviceID: String, accessToken: String?, @@ -2057,15 +2003,60 @@ actor VMClient { allowedUnderManagedPolicy: Bool = false ) async throws -> (Data, HTTPURLResponse) { let work = { - try await self.requestMeasured(method, path: path, jsonBody: jsonBody, extraHeaders: extraHeaders, - timeoutSeconds: timeoutSeconds, retryTransientServiceUnavailable: retryTransientServiceUnavailable, - allowedUnderManagedPolicy: allowedUnderManagedPolicy) + let isSharedRead = method == "GET" + && jsonBody == nil + && extraHeaders.isEmpty + && !retryTransientServiceUnavailable + && !allowedUnderManagedPolicy + && (path == "/api/vm" || path.hasSuffix("/stats")) + if !isSharedRead { + return try await self.requestMeasured(method, path: path, jsonBody: jsonBody, extraHeaders: extraHeaders, + timeoutSeconds: timeoutSeconds, retryTransientServiceUnavailable: retryTransientServiceUnavailable, + allowedUnderManagedPolicy: allowedUnderManagedPolicy) + } + try Task.checkCancellation() + try self.checkCloudAccess(allowedUnderManagedPolicy: allowedUnderManagedPolicy) + let context = CloudOperationContext.current + let elapsed = context.map { $0.operation == .list || $0.operation == .stats ? $0.clock.duration(to: .now) : .zero } ?? .zero + let deadline = self.readRequests.makeDeadline(elapsed: elapsed) + let identity = await self.auth.authenticatedSessionIdentity + guard let identity else { + let isAuthenticated = await self.auth.isAuthenticated + let isRestoringSession = await self.auth.isRestoringSession + if isAuthenticated || isRestoringSession { + throw VMClientError.sessionRefreshFailed + } + throw VMClientError.notSignedIn + } + let teamID = await self.auth.resolvedTeamID + let key = CloudReadRequestCoordinator.Key(path: path, accountID: identity.accountID, + generation: identity.generation, teamID: teamID) + let value = try await self.readRequests.read(key, deadline: deadline) { + try await CloudOperationContext.withCurrent(context) { + let (data, http) = try await self.requestMeasured(method, path: path, timeoutSeconds: timeoutSeconds) + return CloudReadRequestCoordinator.Response(data: data, http: http) + } + } + try Task.checkCancellation() + guard await self.auth.isAuthenticatedSessionIdentityCurrent(identity), + await self.auth.resolvedTeamID == teamID else { throw CancellationError() } + try self.checkCloudAccess(allowedUnderManagedPolicy: allowedUnderManagedPolicy) + return (value.data, value.http) } if CloudOperationContext.current != nil || operations == nil { return try await work() } let kind: CloudOperationKind = path == "/api/vm" ? (method == "GET" ? .list : .create) : .resolve(path) return try await operations!.perform(kind, foreground: method != "GET", work) } + private func checkCloudAccess(allowedUnderManagedPolicy: Bool) throws { + if !allowedUnderManagedPolicy, isDisabledByManagedPolicy?() == true { + throw VMClientError.disabledByManagedPolicy + } + if !allowedUnderManagedPolicy, !isCloudEnabled() { + throw VMClientError.cloudMachinesDisabled + } + } + private func requestMeasured( _ method: String, path: String, @@ -2075,12 +2066,7 @@ actor VMClient { retryTransientServiceUnavailable: Bool = false, allowedUnderManagedPolicy: Bool = false ) async throws -> (Data, HTTPURLResponse) { - if !allowedUnderManagedPolicy, isDisabledByManagedPolicy?() == true { - throw VMClientError.disabledByManagedPolicy - } - if !allowedUnderManagedPolicy, !isCloudEnabled() { - throw VMClientError.cloudMachinesDisabled - } + try checkCloudAccess(allowedUnderManagedPolicy: allowedUnderManagedPolicy) let minted = VMRequestTraceContext.mint() let trace = CloudOperationContext.current.map { VMRequestTraceContext(traceId: $0.traceID, spanId: $0.spanID, clientRequestId: minted.clientRequestId) @@ -2181,6 +2167,7 @@ actor VMClient { } return traceId } + private func performRequest( _ method: String, path: String, @@ -2280,6 +2267,12 @@ actor VMClient { guard let http = response as? HTTPURLResponse else { throw VMClientError.malformedResponse("non-HTTP response") } + if http.statusCode == 429, let read = CloudReadRequestCoordinator.current, let owner = read.owner { + let seconds = Self.retryDelaySeconds(statusCode: 429, retryAfterHeader: http.value(forHTTPHeaderField: "Retry-After")) ?? 60 + let canRetry = await owner.noteRetryAfter(read.key, seconds: seconds, + response: .init(data: data, http: http)) + if !canRetry { return (data, http) } + } if http.statusCode == 429, retriesLeft > 0 { retriesLeft -= 1 onRetry() @@ -2327,6 +2320,12 @@ actor VMClient { throw VMClientError.notSignedIn } } + if method != "GET", (200...299).contains(http.statusCode) { + await readRequests.invalidate(CloudReadMutation(method: method, scope: .init( + path: path, accountID: sessionIdentity?.accountID, + generation: sessionIdentity?.generation, teamID: requestedTeamID + ), responseData: data)) + } return (data, http) } } @@ -2536,63 +2535,6 @@ actor VMClient { // MARK: - Per-machine coderouter usage -/// Token and spend totals for one machine over the usage window, as -/// `GET /api/coderouter/vm-usage/team` reports them. -struct MachineUsageTotals: Equatable, Sendable { - let inputTokens: Int - let cachedInputTokens: Int - let outputTokens: Int - let totalTokens: Int - /// What the same traffic would have cost at list API prices. - let apiEquivalentUsd: Double - - /// Nothing to show for a machine that has not routed a single token. - var isEmpty: Bool { totalTokens <= 0 && apiEquivalentUsd <= 0 } -} - -/// One machine's usage readout: the row shows `totals` labeled with -/// `periodDays`. `vmID` is the id `GET /api/vm` returns as the machine id, so -/// it matches ``MachineSnapshot/id`` directly. -struct MachineUsageSnapshot: Equatable, Sendable { - let vmID: String - /// The provider machine id, the `id` that `GET /api/vm` lists. Rows key - /// on it when present because `vmID` is the backend's own uuid. - let providerVmID: String? - let displayName: String? - let periodDays: Int - let asOf: Date? - let totals: MachineUsageTotals -} - -/// The team-wide usage payload. `kind == .unavailable` means the backend has no -/// usage store for this team (no rows are rendered, no error is surfaced). -struct TeamMachineUsage: Equatable, Sendable { - enum Kind: String, Sendable { - case ready - case unavailable - } - - let teamID: String - let periodDays: Int - let kind: Kind - let asOf: Date? - let machines: [MachineUsageSnapshot] - - /// The lookup the machines panel keys rows by. Empty when the backend says - /// usage is unavailable; blank ids are dropped; the first entry wins when - /// the backend repeats a machine. - var byMachineID: [String: MachineUsageSnapshot] { - guard kind == .ready else { return [:] } - var result: [String: MachineUsageSnapshot] = [:] - for machine in machines { - for key in [machine.providerVmID ?? "", machine.vmID] where !key.isEmpty && result[key] == nil { - result[key] = machine - } - } - return result - } -} - enum MachineUsageClientError: Error, CustomStringConvertible { case notSignedIn case sessionRefreshFailed @@ -2625,18 +2567,23 @@ actor MachineUsageClient { @MainActor private(set) static var shared: MachineUsageClient? @MainActor - static func bootstrap(auth: AuthCoordinator, session: URLSession = .shared, operations: CloudOperationRecorder? = nil) { - shared = MachineUsageClient(session: session, auth: auth, operations: operations) + static func bootstrap(auth: AuthCoordinator, session: URLSession = .shared, operations: CloudOperationRecorder? = nil, + readRequests: CloudReadRequestCoordinator? = nil) { + shared = MachineUsageClient(session: session, auth: auth, operations: operations, + readRequests: readRequests ?? CloudReadRequestCoordinator(budget: .seconds(15))) } private let session: URLSession private let auth: AuthCoordinator + private let readRequests: CloudReadRequestCoordinator nonisolated let operations: CloudOperationRecorder? - init(session: URLSession = .shared, auth: AuthCoordinator, operations: CloudOperationRecorder? = nil) { + init(session: URLSession = .shared, auth: AuthCoordinator, operations: CloudOperationRecorder? = nil, + readRequests: CloudReadRequestCoordinator = CloudReadRequestCoordinator(budget: .seconds(15))) { self.session = session self.auth = auth self.operations = operations + self.readRequests = readRequests } private func withOperation(_ kind: CloudOperationKind, foreground: Bool, _ work: () async throws -> T) async rethrows -> T { @@ -2646,7 +2593,36 @@ actor MachineUsageClient { func teamUsage(teamID: String? = nil) async throws -> TeamMachineUsage { return try await withOperation(.stats, foreground: false) { - let (data, _) = try await request("GET", path: "/api/coderouter/vm-usage/team", teamID: teamID) + let context = CloudOperationContext.current + let deadline = readRequests.makeDeadline(elapsed: context.map { $0.operation == .stats ? $0.clock.duration(to: .now) : .zero } ?? .zero, limit: .seconds(15)) + let identity = await auth.authenticatedSessionIdentity + guard let identity else { + let isAuthenticated = await auth.isAuthenticated + let isRestoringSession = await auth.isRestoringSession + if isAuthenticated || isRestoringSession { + throw MachineUsageClientError.sessionRefreshFailed + } + throw MachineUsageClientError.notSignedIn + } + let selectedTeam = await auth.resolvedTeamID + let explicitTeam = teamID?.trimmingCharacters(in: .whitespacesAndNewlines) + let key = CloudReadRequestCoordinator.Key(path: "/api/coderouter/vm-usage/team", accountID: identity.accountID, + generation: identity.generation, teamID: explicitTeam?.isEmpty == false ? explicitTeam : selectedTeam) + let response = try await readRequests.read(key, deadline: deadline) { + try await CloudOperationContext.withCurrent(context) { + let (data, http) = try await self.request("GET", path: key.path, teamID: teamID) + return CloudReadRequestCoordinator.Response(data: data, http: http) + } + } + guard await auth.isAuthenticatedSessionIdentityCurrent(identity) else { throw CancellationError() } + if explicitTeam?.isEmpty != false, + await auth.resolvedTeamID != selectedTeam { + throw CancellationError() + } + let (data, http) = (response.data, response.http) + guard (200...299).contains(http.statusCode) else { + throw MachineUsageClientError.httpStatus(http.statusCode, String(data: data, encoding: .utf8) ?? "") + } return try Self.decodeTeamUsage(data) } } @@ -2737,6 +2713,7 @@ actor MachineUsageClient { } catch { throw MachineUsageClientError.notSignedIn } + try Task.checkCancellation() let resolvedTeamID = await auth.resolvedTeamID guard var comps = URLComponents(url: AuthEnvironment.vmAPIBaseURL, resolvingAgainstBaseURL: false) else { @@ -2774,6 +2751,11 @@ actor MachineUsageClient { guard let http = response as? HTTPURLResponse else { throw MachineUsageClientError.malformedResponse("non-HTTP response") } + if http.statusCode == 429, let read = CloudReadRequestCoordinator.current, let owner = read.owner { + let retryAfter = CmxRetryAfterPolicy() + let seconds = TimeInterval(retryAfter.seconds(from: http) ?? retryAfter.defaultRateLimitSeconds) + _ = await owner.noteRetryAfter(read.key, seconds: seconds, response: .init(data: data, http: http)) + } guard (200...299).contains(http.statusCode) else { throw MachineUsageClientError.httpStatus(http.statusCode, String(data: data, encoding: .utf8) ?? "") } diff --git a/Sources/GhosttyTerminalView.swift b/Sources/GhosttyTerminalView.swift index fc7a739bf812..79e4c80c591c 100644 --- a/Sources/GhosttyTerminalView.swift +++ b/Sources/GhosttyTerminalView.swift @@ -2206,6 +2206,16 @@ class GhosttyApp { configurationReloadCoordinator.isReloadActive } +#if DEBUG + /// Where the app-scoped reload transaction is, so tests can assert that a + /// reload is held at the font-work barrier instead of inferring it from a + /// notification that has not arrived yet. + @MainActor + var debugConfigurationReloadPhase: TerminalConfigurationReloadPhase { + configurationReloadCoordinator.phase + } +#endif + @MainActor func terminalFontConfigurationSnapshot() -> WorkspaceTerminalFontConfigurationSnapshot { diff --git a/Sources/KeyboardShortcutActionContext.swift b/Sources/KeyboardShortcutActionContext.swift index e3bebbf99232..29df6de227f4 100644 --- a/Sources/KeyboardShortcutActionContext.swift +++ b/Sources/KeyboardShortcutActionContext.swift @@ -105,7 +105,9 @@ extension KeyboardShortcutSettings.Action { focusedBrowserPanel: context.browserPanel != nil, focusedMarkdownPanel: context.markdownPanel != nil, focusedSimulatorPanel: context.shortcutContext.bool(ShortcutContextKnownKey.simulatorFocus.rawValue), - focusedFilePreviewTextEditor: context.filePreviewTextEditorFocused, + focusedFilePreviewTextEditor: self == .filePreviewTextEditor + ? context.fileEditorFocused + : context.filePreviewTextEditorFocused, rightSidebarFocused: context.rightSidebarFocused, workspaceCanvasLayout: context.shortcutContext.bool(ShortcutContextKnownKey.workspaceCanvasLayout.rawValue) ) diff --git a/Sources/KeyboardShortcutContext.swift b/Sources/KeyboardShortcutContext.swift index 62d7297942cf..58fccab7e90e 100644 --- a/Sources/KeyboardShortcutContext.swift +++ b/Sources/KeyboardShortcutContext.swift @@ -6,7 +6,12 @@ import WebKit struct ShortcutEventFocusContext { let browserPanel: BrowserPanel? let markdownPanel: MarkdownPanel? + /// The focused text file preview's editor owns the responder. Preview zoom + /// and Canvas zoom routing use this narrow scope. let filePreviewTextEditorFocused: Bool + /// Any file editor owns the responder, including editors in the Dock and + /// Markdown text mode. File-editor actions such as word wrap use it. + let fileEditorFocused: Bool let simulatorFocused: Bool let simulatorPanel: SimulatorPanel? let simulatorTextEditorFocused: Bool @@ -18,6 +23,7 @@ struct ShortcutEventFocusContext { browserPanel: BrowserPanel?, markdownPanel: MarkdownPanel?, filePreviewTextEditorFocused: Bool, + fileEditorFocused: Bool = false, simulatorFocused: Bool, simulatorPanel: SimulatorPanel? = nil, simulatorTextEditorFocused: Bool = false, @@ -27,6 +33,7 @@ struct ShortcutEventFocusContext { self.browserPanel = browserPanel self.markdownPanel = markdownPanel self.filePreviewTextEditorFocused = filePreviewTextEditorFocused + self.fileEditorFocused = fileEditorFocused || filePreviewTextEditorFocused self.simulatorFocused = simulatorFocused self.simulatorPanel = simulatorPanel self.simulatorTextEditorFocused = simulatorTextEditorFocused @@ -49,6 +56,23 @@ struct ShortcutEventFocusContext { simulator: simulatorFocused ) } + + /// ``shortcutContext`` with any focused file editor projected onto the + /// file-editor atom, so a file-editor action's `when` clause holds in + /// editors outside a text file preview. + var fileEditorShortcutContext: ShortcutContext { + guard fileEditorFocused, !filePreviewTextEditorFocused else { return shortcutContext } + var context = shortcutContext + context.setBool(ShortcutFocusAtom.filePreviewTextEditorFocus.rawValue, true) + context.setBool(ShortcutFocusAtom.terminalFocus.rawValue, false) + return context + } + + /// The context `action`'s `when` clause evaluates against: file-editor + /// actions see every file editor, everything else the narrow preview scope. + func whenClauseContext(for action: KeyboardShortcutSettings.Action) -> ShortcutContext { + action.shortcutContext == .filePreviewTextEditor ? fileEditorShortcutContext : shortcutContext + } } func shortcutResponderAcceptsTextEditing(_ responder: NSResponder) -> Bool { @@ -110,9 +134,11 @@ extension AppDelegate { // Only treat a markdown panel as focused when no browser panel owns the // event, so a focused browser never routes markdown shortcuts. let markdownPanel = browserPanel == nil ? shortcutFocusedMarkdownPanel(in: shortcutWindow) : nil - let filePreviewTextEditorFocused = browserPanel == nil && markdownPanel == nil - ? shortcutFocusedFilePreviewTextEditor(in: shortcutWindow) + let fileEditorFocused = browserPanel == nil && markdownPanel == nil + ? shortcutFocusedSavingTextView(in: shortcutWindow) != nil : false + let filePreviewTextEditorFocused = fileEditorFocused + && shortcutFocusedFilePreviewTextEditor(in: shortcutWindow) let rightSidebarFocused = !simulatorFocused && (shortcutWindow.map { shouldRouteRightSidebarModeShortcut(in: $0) } ?? false) let focusState = ShortcutFocusState( @@ -126,6 +152,7 @@ extension AppDelegate { browserPanel: browserPanel, markdownPanel: markdownPanel, filePreviewTextEditorFocused: filePreviewTextEditorFocused, + fileEditorFocused: fileEditorFocused, simulatorFocused: simulatorFocused, simulatorPanel: simulatorPanel, simulatorTextEditorFocused: simulatorTextEditorFocused, @@ -191,9 +218,18 @@ extension AppDelegate { return tabManager?.focusedMarkdownPanel } - /// Includes text editors in the Dock and Markdown text mode by responder ownership. + /// Matches only the focused text file preview's editor, the same scope as the + /// command palette's `panelIsFilePreviewTextEditor`. Editors in the Dock and + /// Markdown text mode are file editors but not previews, so preview zoom and + /// Canvas zoom routing leave them alone. private func shortcutFocusedFilePreviewTextEditor(in window: NSWindow?) -> Bool { - shortcutFocusedSavingTextView(in: window) != nil + guard let focusedFilePreviewPanel = shortcutContextTabManager(in: window)?.focusedTextFilePreviewPanel, + let textView = shortcutFocusedSavingTextView(in: window), + let owningFilePreviewPanel = textView.panel as? FilePreviewPanel, + owningFilePreviewPanel === focusedFilePreviewPanel else { + return false + } + return true } /// Resolves the editor that owns the requested window’s keyboard responder. diff --git a/Sources/Surfaces/CmuxTuiSurfaceProviderRegistry+Routes.swift b/Sources/Surfaces/CmuxTuiSurfaceProviderRegistry+Routes.swift new file mode 100644 index 000000000000..1145d2f3f81f --- /dev/null +++ b/Sources/Surfaces/CmuxTuiSurfaceProviderRegistry+Routes.swift @@ -0,0 +1,56 @@ +import CmuxSettings +import Foundation + +/// Routes Cloud traffic through the current account-owned registry. +extension CmuxTuiSurfaceProviderRegistry { + /// The headless link's local mux socket for a machine, connecting if needed. + func linkSocketPath(machineID: String) async throws -> (socketPath: String, session: String) { + guard !isRetired, !ManagedDevicePolicy().isEnforced(.disableCloud), isCloudEnabled(), !Task.isCancelled else { + throw CloudMachineLinkManager.ManagerError.retryLater(String( + localized: "cloud.feature.disabled", + defaultValue: "Cloud Machines are temporarily unavailable." + )) + } + let epoch = accessEpoch + let connected = try await links.connected(machineID: machineID) + guard !isRetired, epoch == accessEpoch, + !ManagedDevicePolicy().isEnforced(.disableCloud), isCloudEnabled(), !Task.isCancelled else { + throw CancellationError() + } + return (connected.socketPath, connected.session) + } + + func privateRoute(machineID: String) async -> String? { + guard !isRetired, !ManagedDevicePolicy().isEnforced(.disableCloud), isCloudEnabled(), !Task.isCancelled else { + return nil + } + let epoch = accessEpoch + // The persisted device marker outlives this in-memory registry. An + // explicit open must discover its machine before using the saved-device + // shortcut, even when the first background fleet read has not run. + guard await providerRefreshingIfMissing(machineID: machineID) != nil else { return nil } + let route = await links.privateRoute(for: machineID) + guard !isRetired, epoch == accessEpoch, + !ManagedDevicePolicy().isEnforced(.disableCloud), isCloudEnabled(), !Task.isCancelled else { + return nil + } + return route + } + + func resolvedPrivateRoute(machineID: String, through hub: CloudWireGuardHub.Ready, fallbackRoute: String, addresses: [String]) async throws -> String { + guard !isRetired, !ManagedDevicePolicy().isEnforced(.disableCloud), isCloudEnabled(), !Task.isCancelled else { + throw CloudMachineLinkManager.ManagerError.retryLater(String( + localized: "cloud.feature.disabled", + defaultValue: "Cloud Machines are temporarily unavailable." + )) + } + let epoch = accessEpoch + let route = try await links.resolvedPrivateRoute(machineID: machineID, through: hub, fallbackRoute: fallbackRoute, addresses: addresses) + guard !isRetired, epoch == accessEpoch, + !ManagedDevicePolicy().isEnforced(.disableCloud), isCloudEnabled(), !Task.isCancelled else { + throw CancellationError() + } + return route + } + +} diff --git a/Sources/Surfaces/CmuxTuiSurfaceProviderRegistry.swift b/Sources/Surfaces/CmuxTuiSurfaceProviderRegistry.swift index 4b1f9392e5fe..ca7f64c18671 100644 --- a/Sources/Surfaces/CmuxTuiSurfaceProviderRegistry.swift +++ b/Sources/Surfaces/CmuxTuiSurfaceProviderRegistry.swift @@ -16,7 +16,7 @@ final class CmuxTuiSurfaceProviderRegistry { private var catalog: SurfaceCatalog? private var providers: [String: CmuxTuiSurfaceProvider] = [:] - private let links: CloudMachineLinkManager + let links: CloudMachineLinkManager /// The app's one WireGuard hub for private-network machines; nil when no cmux-tui /// client is bundled (then no link can be made at all). nonisolated let wireGuardHub: CloudWireGuardHub? @@ -29,10 +29,11 @@ final class CmuxTuiSurfaceProviderRegistry { private var accessObserver: NSObjectProtocol? private var themeObserver: NSObjectProtocol? private var activationObserver: NSObjectProtocol? + private var networkObserver: CloudReadRecoveryObserver? private var featureFlagObserver: NSObjectProtocol? private let notificationCenter: NotificationCenter /// Whether the periodic fleet read may run right now. - private let isCloudEnabled: @MainActor () -> Bool + let isCloudEnabled: @MainActor () -> Bool private let allowsBackgroundWork: @MainActor () -> Bool private let listPage: @MainActor () async -> VMListPage? private let refreshProvider: @MainActor (CmuxTuiSurfaceProvider, Bool) async -> Bool @@ -51,7 +52,7 @@ final class CmuxTuiSurfaceProviderRegistry { /// carries the epoch it was registered with and a stale one is dropped: /// without this, a `DisableCloud` teardown that lands just after the /// policy lifts would clear the freshly restarted registry. - private var accessEpoch: UInt64 = 0 + private(set) var accessEpoch: UInt64 = 0 /// Create receipts also end at team changes, which preserve the registry's observer epoch. private var creationEpoch = UUID() /// Machine IDs admitted from a successful create response remain owned by @@ -64,7 +65,8 @@ final class CmuxTuiSurfaceProviderRegistry { private var createdTrustedCarrierIDs: Set = [] /// Whether account access has ended. Retired registries reject all new Cloud work /// until ``start(catalog:)`` reactivates them for the next account. - private var isRetired = true + var isRetired = true + /// Same cadence as the Machines panel's list refresh. private let pollInterval: Duration = .seconds(45) /// In-flight forward and link teardowns for deleted machines, keyed by /// machine id; sign-out waits for them before stopping the hub. @@ -243,6 +245,10 @@ final class CmuxTuiSurfaceProviderRegistry { ) { [weak self] _ in MainActor.assumeIsolated { self?.syncPollingToActivationPolicy() } } + networkObserver = CloudReadRecoveryObserver(notificationCenter: notificationCenter) { [weak self] in + guard let self, !self.isRetired, self.allowsBackgroundWork() else { return } + _ = await self.refresh(force: false) + } syncPollingToActivationPolicy() } @@ -437,42 +443,6 @@ final class CmuxTuiSurfaceProviderRegistry { return candidates.first { $0.caseInsensitiveCompare(rawID) == .orderedSame } ?? rawID } - /// The headless link's local mux socket for a machine, connecting if needed. - func linkSocketPath(machineID: String) async throws -> (socketPath: String, session: String) { - guard isCloudEnabled() else { - throw CloudMachineLinkManager.ManagerError.retryLater(String( - localized: "cloud.feature.disabled", - defaultValue: "Cloud Machines are temporarily unavailable." - )) - } - let connected = try await links.connected(machineID: machineID) - return (connected.socketPath, connected.session) - } - - func privateRoute(machineID: String) async -> String? { - guard !isRetired, !ManagedDevicePolicy().isEnforced(.disableCloud), isCloudEnabled(), !Task.isCancelled else { - return nil - } - let epoch = accessEpoch - // The persisted device marker outlives this in-memory registry. An - // explicit open must discover its machine before using the saved-device - // shortcut, even when the first background fleet read has not run. - guard await providerRefreshingIfMissing(machineID: machineID) != nil else { return nil } - let route = await links.privateRoute(for: machineID) - guard !isRetired, epoch == accessEpoch, isCloudEnabled(), !Task.isCancelled else { return nil } - return route - } - - func resolvedPrivateRoute(machineID: String, through hub: CloudWireGuardHub.Ready, fallbackRoute: String, addresses: [String]) async throws -> String { - guard isCloudEnabled() else { - throw CloudMachineLinkManager.ManagerError.retryLater(String( - localized: "cloud.feature.disabled", - defaultValue: "Cloud Machines are temporarily unavailable." - )) - } - return try await links.resolvedPrivateRoute(machineID: machineID, through: hub, fallbackRoute: fallbackRoute, addresses: addresses) - } - /// Cancels Cloud-only transport work when the remote gate closes while /// retaining providers, catalog resources, and persisted pane identities. /// Re-enabling the gate reuses those providers on the next discovery pass. @@ -556,6 +526,7 @@ final class CmuxTuiSurfaceProviderRegistry { /// Synchronous publication fence shared by team switching and full teardown. private func invalidateAccess() { + networkObserver = nil isRetired = true accessEpoch &+= 1 creationEpoch = UUID() diff --git a/cmux.xcodeproj/project.pbxproj b/cmux.xcodeproj/project.pbxproj index de7396ca4c5d..d2e75c2dc1cc 100644 --- a/cmux.xcodeproj/project.pbxproj +++ b/cmux.xcodeproj/project.pbxproj @@ -866,10 +866,31 @@ 0C45DC8D9BE54DDEA2397892 /* CloudPrivateRouteSelectionTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1E6077A4B6CC441EB3668651 /* CloudPrivateRouteSelectionTests.swift */; }; D1AC3F6282A94EAEB79D160B /* CloudProviderRefreshCoordinator.swift in Sources */ = {isa = PBXBuildFile; fileRef = A09B853868B54438A5B14F26 /* CloudProviderRefreshCoordinator.swift */; }; AC38407F09E94DA38B94F168 /* CloudProviderRefreshCoordinatorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5C2A7143303749BE82E51F69 /* CloudProviderRefreshCoordinatorTests.swift */; }; + C13151C00000000000000014 /* CloudReadCooldownStore+Cooldown.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C00000000000000013 /* CloudReadCooldownStore+Cooldown.swift */; }; + C13151C00000000000000012 /* CloudReadCooldownStore+Session.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C00000000000000011 /* CloudReadCooldownStore+Session.swift */; }; + C13151C00000000000000002 /* CloudReadCooldownStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C00000000000000001 /* CloudReadCooldownStore.swift */; }; + C13151C00000000000000031 /* CloudReadIdentityAuthClient.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C00000000000000030 /* CloudReadIdentityAuthClient.swift */; }; + C12625B00000000000000006 /* CloudReadManualClock.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12625B00000000000000005 /* CloudReadManualClock.swift */; }; + C13151C00000000000000004 /* CloudReadMutation.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C00000000000000003 /* CloudReadMutation.swift */; }; + C12625B00000000000000002 /* CloudReadNetworkMonitor.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12625B00000000000000001 /* CloudReadNetworkMonitor.swift */; }; + C12636A00000000000000004 /* CloudReadRecoveryObserver.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12636A00000000000000003 /* CloudReadRecoveryObserver.swift */; }; + C13151C0000000000000000A /* CloudReadRequestCoordinator+Context.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C00000000000000009 /* CloudReadRequestCoordinator+Context.swift */; }; + C13151C0000000000000000E /* CloudReadRequestCoordinator+Entry.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C0000000000000000D /* CloudReadRequestCoordinator+Entry.swift */; }; + C13151C00000000000000006 /* CloudReadRequestCoordinator+Key.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C00000000000000005 /* CloudReadRequestCoordinator+Key.swift */; }; + C13151C00000000000000010 /* CloudReadRequestCoordinator+Pending.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C0000000000000000F /* CloudReadRequestCoordinator+Pending.swift */; }; + C13151C00000000000000008 /* CloudReadRequestCoordinator+Response.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C00000000000000007 /* CloudReadRequestCoordinator+Response.swift */; }; + C13151C0000000000000000C /* CloudReadRequestCoordinator+Waiter.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C0000000000000000B /* CloudReadRequestCoordinator+Waiter.swift */; }; + C12625A00000000000000004 /* CloudReadRequestCoordinator.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12625A00000000000000003 /* CloudReadRequestCoordinator.swift */; }; + C12625B00000000000000004 /* CloudReadRequestCoordinatorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12625B00000000000000003 /* CloudReadRequestCoordinatorTests.swift */; }; + 9F046FC7A14BBDB098F471B3 /* CloudReadResponseGate.swift in Sources */ = {isa = PBXBuildFile; fileRef = E9A79D080D5B1E7996CC8586 /* CloudReadResponseGate.swift */; }; + C13151C00000000000000023 /* CloudRefreshAuthClient.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C00000000000000022 /* CloudRefreshAuthClient.swift */; }; + C13151C00000000000000021 /* CloudRefreshFixture.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C00000000000000020 /* CloudRefreshFixture.swift */; }; + C12625000000000000000004 /* CloudRefreshURLProtocol.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12625000000000000000003 /* CloudRefreshURLProtocol.swift */; }; C12449010000000000000001 /* CloudRemoteColorOwnershipTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12449020000000000000001 /* CloudRemoteColorOwnershipTests.swift */; }; 595EC249E2E015ACE304F0EE /* CloudRemoteOperationStep.swift in Sources */ = {isa = PBXBuildFile; fileRef = 64BF8B9B8328108560B40B7F /* CloudRemoteOperationStep.swift */; }; B12438020000000000000001 /* CloudRenameCoordinator.swift in Sources */ = {isa = PBXBuildFile; fileRef = B12438020000000000000002 /* CloudRenameCoordinator.swift */; }; C12575000000000000000050 /* CloudRenameOptimismTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12575000000000000000051 /* CloudRenameOptimismTests.swift */; }; + C12625A00000000000000002 /* CloudRequestClock.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12625A00000000000000001 /* CloudRequestClock.swift */; }; 46383D21CF46427F36EBEB7B /* CloudSidebarAcceptanceTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8150EA26C949866059F3B6EA /* CloudSidebarAcceptanceTests.swift */; }; 5E3484F75CE048CC95018CF1 /* CloudSidebarAttentionLayoutTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5DB4F197C29A4C03B61418A1 /* CloudSidebarAttentionLayoutTests.swift */; }; D075FB64A68C4CA0A4042528 /* CloudSidebarConsistencyTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = B27A4938294E4A109BC6D2B3 /* CloudSidebarConsistencyTests.swift */; }; @@ -1562,6 +1583,7 @@ 646ACD2F9B5044B8A2AB6C3A /* CmuxTuiSurfaceProvider+TerminalPrimitives.swift in Sources */ = {isa = PBXBuildFile; fileRef = 122E7DF3DF3044E68322B11C /* CmuxTuiSurfaceProvider+TerminalPrimitives.swift */; }; D12439050000000000000001 /* CmuxTuiSurfaceProvider+WorkspaceLifecycle.swift in Sources */ = {isa = PBXBuildFile; fileRef = D12439050000000000000002 /* CmuxTuiSurfaceProvider+WorkspaceLifecycle.swift */; }; 175CFE773740410786E08A2B /* CmuxTuiSurfaceProviderRegistry+Production.swift in Sources */ = {isa = PBXBuildFile; fileRef = 95278207545546F08C9A325F /* CmuxTuiSurfaceProviderRegistry+Production.swift */; }; + DAE9491D73F526601DFC97A2 /* CmuxTuiSurfaceProviderRegistry+Routes.swift in Sources */ = {isa = PBXBuildFile; fileRef = A6040C2AA5B187C05EEC1A67 /* CmuxTuiSurfaceProviderRegistry+Routes.swift */; }; 7D46CAA4C3C848ACAD2E46A7 /* CmuxTuiSurfaceProviderRegistry+Teardown.swift in Sources */ = {isa = PBXBuildFile; fileRef = 69821B8A79404A3CA4D07699 /* CmuxTuiSurfaceProviderRegistry+Teardown.swift */; }; A1B2C3D4E5F6071827364961 /* CmuxTuiSurfaceProviderRegistry.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1B2C3D4E5F6071827364960 /* CmuxTuiSurfaceProviderRegistry.swift */; }; D01100800000000000000008 /* CmuxTuiSurfaceProviderRegistryDiscoveryTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D01100800000000000000007 /* CmuxTuiSurfaceProviderRegistryDiscoveryTests.swift */; }; @@ -2267,8 +2289,11 @@ 98E6802001D61AC24F2ABB09 /* MachinesPanelUncappedPlanTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9A986C3063FCE5F28AA10ACA /* MachinesPanelUncappedPlanTests.swift */; }; E879C292A2D38E3DDBCB004D /* MachinesPanelView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 08610D114C8CF23877176D7E /* MachinesPanelView.swift */; }; C1257500000000000000001A /* MachinesPanelViewModel+MachinePins.swift in Sources */ = {isa = PBXBuildFile; fileRef = C1257500000000000000001B /* MachinesPanelViewModel+MachinePins.swift */; }; + F9ADD9F062DEA2BC5924BC38 /* MachinesPanelViewModel+Refresh.swift in Sources */ = {isa = PBXBuildFile; fileRef = E8DCA0BDC648E6CDCFD3E228 /* MachinesPanelViewModel+Refresh.swift */; }; D7E6477A2FC01DCCF353840D /* MachinesPanelViewModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 68672B3C63D4E348AFA4DEF7 /* MachinesPanelViewModel.swift */; }; 7A0CE1000000000000000724 /* MachinesTunnelBanner.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7A0CE1000000000000000723 /* MachinesTunnelBanner.swift */; }; + C12625D00000000000000004 /* MachineUsageSnapshot.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12625D00000000000000003 /* MachineUsageSnapshot.swift */; }; + C12625D00000000000000002 /* MachineUsageTotals.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12625D00000000000000001 /* MachineUsageTotals.swift */; }; D1F0A00600000000000000C1 /* MacPairedMacBackupBody.swift in Sources */ = {isa = PBXBuildFile; fileRef = D1F0A00600000000000000C2 /* MacPairedMacBackupBody.swift */; }; D1F0A00700000000000000C1 /* MacPairedMacBackupOpWire.swift in Sources */ = {isa = PBXBuildFile; fileRef = D1F0A00700000000000000C2 /* MacPairedMacBackupOpWire.swift */; }; D1F0A00100000000000000C1 /* MacPairedMacBackupPublisher.swift in Sources */ = {isa = PBXBuildFile; fileRef = D1F0A00100000000000000C2 /* MacPairedMacBackupPublisher.swift */; }; @@ -2663,6 +2688,7 @@ C0DEA771000000000000000D /* PricingPlansScreen.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0DEA771000000000000000E /* PricingPlansScreen.swift */; }; C0DEA771000000000000000B /* ProBadgeStyle.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0DEA771000000000000000C /* ProBadgeStyle.swift */; }; F0ACC0DE000000000000000D /* ProcessDetectedResumeIndexes.swift in Sources */ = {isa = PBXBuildFile; fileRef = F0ACC0DE000000000000000E /* ProcessDetectedResumeIndexes.swift */; }; + AC0FCEFB6665CF134634D60B /* ProcessExitWait.swift in Sources */ = {isa = PBXBuildFile; fileRef = E06E7588C05AC6A8659CFA51 /* ProcessExitWait.swift */; }; C47110010000000000000001 /* ProcessPipeReadCrashRegressionTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C47110010000000000000002 /* ProcessPipeReadCrashRegressionTests.swift */; }; 0A327C88E531C7E407CEF58C /* ProcessSnapshotMeasurement.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4587EDAC6B63EF4846FB24A6 /* ProcessSnapshotMeasurement.swift */; }; D75280010000000000000001 /* ProfilingSignposts.swift in Sources */ = {isa = PBXBuildFile; fileRef = D75280010000000000000002 /* ProfilingSignposts.swift */; }; @@ -3481,6 +3507,8 @@ C7A502000000000000000002 /* TaskManagerWindowController.swift in Sources */ = {isa = PBXBuildFile; fileRef = C7A502000000000000000001 /* TaskManagerWindowController.swift */; }; 7490C00F7490C00F7490C00F /* TaskVMInfoMemoryPressureFootprintSampler.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7490D00F7490D00F7490D00F /* TaskVMInfoMemoryPressureFootprintSampler.swift */; }; B82090010000000000000001 /* TCCUsageDescriptionUITests.swift in Sources */ = {isa = PBXBuildFile; fileRef = B82090010000000000000002 /* TCCUsageDescriptionUITests.swift */; }; + C12625C00000000000000002 /* TeamMachineUsage.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12625C00000000000000001 /* TeamMachineUsage.swift */; }; + C13151D00000000000000002 /* TeamMachineUsageKind.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151D00000000000000001 /* TeamMachineUsageKind.swift */; }; 0044CBDD567D9CD29E6A6F12 /* TelemetrySettings.swift in Sources */ = {isa = PBXBuildFile; fileRef = 257DFD20F0961FE0BDAF70F2 /* TelemetrySettings.swift */; }; A10199010000000000000001 /* TerminalAdaptiveDefaultThemeSettings.swift in Sources */ = {isa = PBXBuildFile; fileRef = A10199010000000000000002 /* TerminalAdaptiveDefaultThemeSettings.swift */; }; C10190010000000000000002 /* TerminalAgentPanelInitialTitleTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C10190010000000000000001 /* TerminalAgentPanelInitialTitleTests.swift */; }; @@ -3911,9 +3939,12 @@ 66141EC09936A3AD5EC83307 /* View+CloudErrorCopy.swift in Sources */ = {isa = PBXBuildFile; fileRef = B51E0FBEDF07F4E6A6A792F7 /* View+CloudErrorCopy.swift */; }; A28B087F0000000000000005 /* ViewerNavigationKeyRouter.swift in Sources */ = {isa = PBXBuildFile; fileRef = A28B087F0000000000000006 /* ViewerNavigationKeyRouter.swift */; }; A3340002A3340002A3340002 /* ViewerNavigationTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = A3340001A3340001A3340001 /* ViewerNavigationTests.swift */; }; + C12625A00000000000000006 /* VMCapabilities.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12625A00000000000000005 /* VMCapabilities.swift */; }; 60AA592C22BFA579A064782E /* VMClient+ResourceStats.swift in Sources */ = {isa = PBXBuildFile; fileRef = 65A5357644AB29D74F4AC368 /* VMClient+ResourceStats.swift */; }; ACE4FE1A8F1CC6C0B6A673AB /* VMClient.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9CEC6EF35B71AE59FA45BA69 /* VMClient.swift */; }; EF56B870FD0D4AC490D99CD5 /* VMClientError.swift in Sources */ = {isa = PBXBuildFile; fileRef = 98F890512F8E425FAD71FE7C /* VMClientError.swift */; }; + C13151C00000000000000033 /* VMClientReadCoalescingTests+Identity.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C00000000000000032 /* VMClientReadCoalescingTests+Identity.swift */; }; + C12625000000000000000002 /* VMClientReadCoalescingTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12625000000000000000001 /* VMClientReadCoalescingTests.swift */; }; 7A0CE1000000000000000044 /* VMClientSocketCommands+Tunnel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7A0CE1000000000000000018 /* VMClientSocketCommands+Tunnel.swift */; }; ACE4FE2A8F1CC6C0B6A673AB /* VMClientSocketCommands.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9CEC6F0035B71AE59FA45BA7 /* VMClientSocketCommands.swift */; }; 93AF7E8EC956FE883E252322 /* VMClientTelemetry.swift in Sources */ = {isa = PBXBuildFile; fileRef = 831145877779546B628FCAFC /* VMClientTelemetry.swift */; }; @@ -5171,10 +5202,31 @@ 1E6077A4B6CC441EB3668651 /* CloudPrivateRouteSelectionTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudPrivateRouteSelectionTests.swift; sourceTree = ""; }; A09B853868B54438A5B14F26 /* CloudProviderRefreshCoordinator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudProviderRefreshCoordinator.swift"; sourceTree = ""; }; 5C2A7143303749BE82E51F69 /* CloudProviderRefreshCoordinatorTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudProviderRefreshCoordinatorTests.swift; sourceTree = ""; }; + C13151C00000000000000013 /* CloudReadCooldownStore+Cooldown.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudReadCooldownStore+Cooldown.swift"; sourceTree = ""; }; + C13151C00000000000000011 /* CloudReadCooldownStore+Session.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudReadCooldownStore+Session.swift"; sourceTree = ""; }; + C13151C00000000000000001 /* CloudReadCooldownStore.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudReadCooldownStore.swift; sourceTree = ""; }; + C13151C00000000000000030 /* CloudReadIdentityAuthClient.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudReadIdentityAuthClient.swift"; sourceTree = ""; }; + C12625B00000000000000005 /* CloudReadManualClock.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudReadManualClock.swift; sourceTree = ""; }; + C13151C00000000000000003 /* CloudReadMutation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudReadMutation.swift; sourceTree = ""; }; + C12625B00000000000000001 /* CloudReadNetworkMonitor.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudReadNetworkMonitor.swift; sourceTree = ""; }; + C12636A00000000000000003 /* CloudReadRecoveryObserver.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudReadRecoveryObserver.swift; sourceTree = ""; }; + C13151C00000000000000009 /* CloudReadRequestCoordinator+Context.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudReadRequestCoordinator+Context.swift"; sourceTree = ""; }; + C13151C0000000000000000D /* CloudReadRequestCoordinator+Entry.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudReadRequestCoordinator+Entry.swift"; sourceTree = ""; }; + C13151C00000000000000005 /* CloudReadRequestCoordinator+Key.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudReadRequestCoordinator+Key.swift"; sourceTree = ""; }; + C13151C0000000000000000F /* CloudReadRequestCoordinator+Pending.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudReadRequestCoordinator+Pending.swift"; sourceTree = ""; }; + C13151C00000000000000007 /* CloudReadRequestCoordinator+Response.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudReadRequestCoordinator+Response.swift"; sourceTree = ""; }; + C13151C0000000000000000B /* CloudReadRequestCoordinator+Waiter.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudReadRequestCoordinator+Waiter.swift"; sourceTree = ""; }; + C12625A00000000000000003 /* CloudReadRequestCoordinator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudReadRequestCoordinator.swift; sourceTree = ""; }; + C12625B00000000000000003 /* CloudReadRequestCoordinatorTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudReadRequestCoordinatorTests.swift; sourceTree = ""; }; + E9A79D080D5B1E7996CC8586 /* CloudReadResponseGate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudReadResponseGate.swift"; sourceTree = ""; }; + C13151C00000000000000022 /* CloudRefreshAuthClient.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudRefreshAuthClient.swift; sourceTree = ""; }; + C13151C00000000000000020 /* CloudRefreshFixture.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudRefreshFixture.swift; sourceTree = ""; }; + C12625000000000000000003 /* CloudRefreshURLProtocol.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudRefreshURLProtocol.swift; sourceTree = ""; }; C12449020000000000000001 /* CloudRemoteColorOwnershipTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudRemoteColorOwnershipTests.swift; sourceTree = ""; }; 64BF8B9B8328108560B40B7F /* CloudRemoteOperationStep.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudRemoteOperationStep.swift"; sourceTree = ""; }; B12438020000000000000002 /* CloudRenameCoordinator.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = CloudRenameCoordinator.swift; sourceTree = ""; }; C12575000000000000000051 /* CloudRenameOptimismTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudRenameOptimismTests.swift; sourceTree = ""; }; + C12625A00000000000000001 /* CloudRequestClock.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudRequestClock.swift; sourceTree = ""; }; 8150EA26C949866059F3B6EA /* CloudSidebarAcceptanceTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudSidebarAcceptanceTests.swift"; sourceTree = ""; }; 5DB4F197C29A4C03B61418A1 /* CloudSidebarAttentionLayoutTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudSidebarAttentionLayoutTests.swift"; sourceTree = ""; }; B27A4938294E4A109BC6D2B3 /* CloudSidebarConsistencyTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudSidebarConsistencyTests.swift; sourceTree = ""; }; @@ -5762,6 +5814,7 @@ 122E7DF3DF3044E68322B11C /* CmuxTuiSurfaceProvider+TerminalPrimitives.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = "CmuxTuiSurfaceProvider+TerminalPrimitives.swift"; sourceTree = ""; }; D12439050000000000000002 /* CmuxTuiSurfaceProvider+WorkspaceLifecycle.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = "CmuxTuiSurfaceProvider+WorkspaceLifecycle.swift"; sourceTree = ""; }; 95278207545546F08C9A325F /* CmuxTuiSurfaceProviderRegistry+Production.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = "CmuxTuiSurfaceProviderRegistry+Production.swift"; sourceTree = ""; }; + A6040C2AA5B187C05EEC1A67 /* CmuxTuiSurfaceProviderRegistry+Routes.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = "CmuxTuiSurfaceProviderRegistry+Routes.swift"; sourceTree = ""; }; 69821B8A79404A3CA4D07699 /* CmuxTuiSurfaceProviderRegistry+Teardown.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CmuxTuiSurfaceProviderRegistry+Teardown.swift"; sourceTree = ""; }; A1B2C3D4E5F6071827364960 /* CmuxTuiSurfaceProviderRegistry.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = CmuxTuiSurfaceProviderRegistry.swift; sourceTree = ""; }; D01100800000000000000007 /* CmuxTuiSurfaceProviderRegistryDiscoveryTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CmuxTuiSurfaceProviderRegistryDiscoveryTests.swift; sourceTree = ""; }; @@ -6455,8 +6508,11 @@ 9A986C3063FCE5F28AA10ACA /* MachinesPanelUncappedPlanTests.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = MachinesPanelUncappedPlanTests.swift; sourceTree = ""; }; 08610D114C8CF23877176D7E /* MachinesPanelView.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = MachinesPanelView.swift; sourceTree = ""; }; C1257500000000000000001B /* MachinesPanelViewModel+MachinePins.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "MachinesPanelViewModel+MachinePins.swift"; sourceTree = ""; }; + E8DCA0BDC648E6CDCFD3E228 /* MachinesPanelViewModel+Refresh.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "MachinesPanelViewModel+Refresh.swift"; sourceTree = ""; }; 68672B3C63D4E348AFA4DEF7 /* MachinesPanelViewModel.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = MachinesPanelViewModel.swift; sourceTree = ""; }; 7A0CE1000000000000000723 /* MachinesTunnelBanner.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = MachinesTunnelBanner.swift; sourceTree = ""; }; + C12625D00000000000000003 /* MachineUsageSnapshot.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MachineUsageSnapshot.swift; sourceTree = ""; }; + C12625D00000000000000001 /* MachineUsageTotals.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MachineUsageTotals.swift; sourceTree = ""; }; D1F0A00600000000000000C2 /* MacPairedMacBackupBody.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = MacPairedMacBackupBody.swift; sourceTree = ""; }; D1F0A00700000000000000C2 /* MacPairedMacBackupOpWire.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = MacPairedMacBackupOpWire.swift; sourceTree = ""; }; D1F0A00100000000000000C2 /* MacPairedMacBackupPublisher.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = MacPairedMacBackupPublisher.swift; sourceTree = ""; }; @@ -6849,6 +6905,7 @@ C0DEA771000000000000000E /* PricingPlansScreen.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PricingPlansScreen.swift; sourceTree = ""; }; C0DEA771000000000000000C /* ProBadgeStyle.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProBadgeStyle.swift; sourceTree = ""; }; F0ACC0DE000000000000000E /* ProcessDetectedResumeIndexes.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProcessDetectedResumeIndexes.swift; sourceTree = ""; }; + E06E7588C05AC6A8659CFA51 /* ProcessExitWait.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProcessExitWait.swift; sourceTree = ""; }; C47110010000000000000002 /* ProcessPipeReadCrashRegressionTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProcessPipeReadCrashRegressionTests.swift; sourceTree = ""; }; 4587EDAC6B63EF4846FB24A6 /* ProcessSnapshotMeasurement.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "ProcessSnapshotMeasurement.swift"; sourceTree = ""; }; D75280010000000000000002 /* ProfilingSignposts.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProfilingSignposts.swift; sourceTree = ""; }; @@ -7654,6 +7711,8 @@ C7A502000000000000000001 /* TaskManagerWindowController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TaskManagerWindowController.swift; sourceTree = ""; }; 7490D00F7490D00F7490D00F /* TaskVMInfoMemoryPressureFootprintSampler.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = App/TaskVMInfoMemoryPressureFootprintSampler.swift; sourceTree = ""; }; B82090010000000000000002 /* TCCUsageDescriptionUITests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TCCUsageDescriptionUITests.swift; sourceTree = ""; }; + C12625C00000000000000001 /* TeamMachineUsage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TeamMachineUsage.swift; sourceTree = ""; }; + C13151D00000000000000001 /* TeamMachineUsageKind.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TeamMachineUsageKind.swift; sourceTree = ""; }; 257DFD20F0961FE0BDAF70F2 /* TelemetrySettings.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "TelemetrySettings.swift"; sourceTree = ""; }; A10199010000000000000002 /* TerminalAdaptiveDefaultThemeSettings.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TerminalAdaptiveDefaultThemeSettings.swift; sourceTree = ""; }; C10190010000000000000001 /* TerminalAgentPanelInitialTitleTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TerminalAgentPanelInitialTitleTests.swift; sourceTree = ""; }; @@ -8084,9 +8143,12 @@ B51E0FBEDF07F4E6A6A792F7 /* View+CloudErrorCopy.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "View+CloudErrorCopy.swift"; sourceTree = ""; }; A28B087F0000000000000006 /* ViewerNavigationKeyRouter.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Panels/ViewerNavigationKeyRouter.swift; sourceTree = ""; }; A3340001A3340001A3340001 /* ViewerNavigationTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ViewerNavigationTests.swift; sourceTree = ""; }; + C12625A00000000000000005 /* VMCapabilities.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VMCapabilities.swift; sourceTree = ""; }; 65A5357644AB29D74F4AC368 /* VMClient+ResourceStats.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "VMClient+ResourceStats.swift"; sourceTree = ""; }; 9CEC6EF35B71AE59FA45BA69 /* VMClient.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = VMClient.swift; sourceTree = ""; }; 98F890512F8E425FAD71FE7C /* VMClientError.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = VMClientError.swift; sourceTree = ""; }; + C13151C00000000000000032 /* VMClientReadCoalescingTests+Identity.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "VMClientReadCoalescingTests+Identity.swift"; sourceTree = ""; }; + C12625000000000000000001 /* VMClientReadCoalescingTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VMClientReadCoalescingTests.swift; sourceTree = ""; }; 7A0CE1000000000000000018 /* VMClientSocketCommands+Tunnel.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = "VMClientSocketCommands+Tunnel.swift"; sourceTree = ""; }; 9CEC6F0035B71AE59FA45BA7 /* VMClientSocketCommands.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = VMClientSocketCommands.swift; sourceTree = ""; }; 831145877779546B628FCAFC /* VMClientTelemetry.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = VMClientTelemetry.swift; sourceTree = ""; }; @@ -8937,6 +8999,7 @@ 1EDD243C7C87A2AB78B12BAD /* LocalSurfaceProvider.swift */, A1B2C3D4E5F6071827364960 /* CmuxTuiSurfaceProviderRegistry.swift */, 95278207545546F08C9A325F /* CmuxTuiSurfaceProviderRegistry+Production.swift */, + A6040C2AA5B187C05EEC1A67 /* CmuxTuiSurfaceProviderRegistry+Routes.swift */, 63F5BDFB110A8A96C9CA0A75 /* CmuxTuiSurfaceProviders.swift */, D9F800000000000000000002 /* CmuxTuiRemoteRouting.swift */, 1FE7223A65E72C46927A4F02 /* SurfacePaneFactory.swift */, @@ -8982,6 +9045,7 @@ 590E4F5F342E5B27010ECC8D /* Cloud */ = { isa = PBXGroup; children = ( + C12636A00000000000000003 /* CloudReadRecoveryObserver.swift */, C11325A20000000000000001 /* CloudTuiTerminalProjectionTarget.swift */, C11324120000000000000001 /* CloudTuiManualIOCommand.swift */, C11324320000000000000001 /* CloudTuiManualIODescriptorLease.swift */, @@ -9038,6 +9102,25 @@ B12B8221BA9A4271E0919C72 /* MachinePlanSnapshot.swift */, BC1FC55E8DA750AD0DABABF4 /* VMResourceStatsStore.swift */, 65A5357644AB29D74F4AC368 /* VMClient+ResourceStats.swift */, + C12625A00000000000000005 /* VMCapabilities.swift */, + C12625C00000000000000001 /* TeamMachineUsage.swift */, + C13151D00000000000000001 /* TeamMachineUsageKind.swift */, + C12625D00000000000000001 /* MachineUsageTotals.swift */, + C12625D00000000000000003 /* MachineUsageSnapshot.swift */, + C12625A00000000000000001 /* CloudRequestClock.swift */, + C13151C00000000000000001 /* CloudReadCooldownStore.swift */, + C13151C00000000000000003 /* CloudReadMutation.swift */, + C13151C00000000000000005 /* CloudReadRequestCoordinator+Key.swift */, + C13151C00000000000000007 /* CloudReadRequestCoordinator+Response.swift */, + C13151C00000000000000009 /* CloudReadRequestCoordinator+Context.swift */, + C13151C0000000000000000B /* CloudReadRequestCoordinator+Waiter.swift */, + C13151C0000000000000000D /* CloudReadRequestCoordinator+Entry.swift */, + C13151C0000000000000000F /* CloudReadRequestCoordinator+Pending.swift */, + C13151C00000000000000011 /* CloudReadCooldownStore+Session.swift */, + C13151C00000000000000013 /* CloudReadCooldownStore+Cooldown.swift */, + C12625A00000000000000003 /* CloudReadRequestCoordinator.swift */, + C12625B00000000000000001 /* CloudReadNetworkMonitor.swift */, + E8DCA0BDC648E6CDCFD3E228 /* MachinesPanelViewModel+Refresh.swift */, FA1253800000000000000004 /* VMStats+State.swift */, FA1253800000000000000006 /* CloudTreeRowIcon.swift */, 98F890512F8E425FAD71FE7C /* VMClientError.swift */, @@ -11671,6 +11754,14 @@ B8E9500A0000000000000002 /* PostHogAnalyticsPropertiesTests.swift */, 8F8A0358C8DD0A7D998A86E9 /* VMClientTelemetryTests.swift */, 7B7B816D0E6DDD18F2D66C77 /* CloudOperationRecorderTests.swift */, + C13151C00000000000000030 /* CloudReadIdentityAuthClient.swift */, + C13151C00000000000000032 /* VMClientReadCoalescingTests+Identity.swift */, + C12625000000000000000001 /* VMClientReadCoalescingTests.swift */, + C13151C00000000000000020 /* CloudRefreshFixture.swift */, + C13151C00000000000000022 /* CloudRefreshAuthClient.swift */, + C12625000000000000000003 /* CloudRefreshURLProtocol.swift */, + C12625B00000000000000003 /* CloudReadRequestCoordinatorTests.swift */, + C12625B00000000000000005 /* CloudReadManualClock.swift */, 604100010000000000000002 /* GhosttyDrawableSizeRetryTests.swift */, BCBC0A0E0000000000000D02 /* BrowserChromeMetricsTests.swift */, B5A5E55B0000000000000002 /* BrowserEphemeralRestorationTests.swift */, @@ -11859,6 +11950,7 @@ F6724603A1B2C3D4E5F67246 /* RemoteTmuxHostRemoteCommandOverrideTests.swift */, F6724601A1B2C3D4E5F67246 /* SSHConfiguredRemoteCommandHostTests.swift */, 4575F4BB7F52CF88AB6D0022 /* SSHStartupCommandTestSupport.swift */, + E06E7588C05AC6A8659CFA51 /* ProcessExitWait.swift */, 37B04BC7D56945E59781D11F /* SSHStartupManualReconnectTests+Fixtures.swift */, EDF9EEAA233347858BCF3320 /* SSHConfiguredRemoteCommandHostTests+Harness.swift */, A82715B8BDE347A18031067C /* CLINotifyProcessIntegrationRegressionTests+JSON.swift */, @@ -12661,6 +12753,7 @@ 52476C8E96AAA6928BA35B95 /* CloudDesktopOpenActionTests.swift */, AC9CBF6AD1163240A7A1E161 /* CloudDesktopOpenFixture.swift */, C09C35915CA32C6A8B13F0A0 /* CloudDesktopOpenTestProvider.swift */, + E9A79D080D5B1E7996CC8586 /* CloudReadResponseGate.swift */, 0AB37AFFC13C8507E428527F /* SSHTuiMigrationTests.swift */, 1ABECB16C9B3F79933C712B5 /* CLILegacyCodexRestoreLeaseTests.swift */, D96D0A50A9C2220332A4B28A /* DeferredAdmissionTestOwner.swift */, @@ -13926,8 +14019,22 @@ 7A0CE1000000000000000504 /* CloudPrivateNetworkPurpose.swift in Sources */, 7A0CE1000000000000000304 /* CloudPrivateNetworkUse.swift in Sources */, D1AC3F6282A94EAEB79D160B /* CloudProviderRefreshCoordinator.swift in Sources */, + C13151C00000000000000014 /* CloudReadCooldownStore+Cooldown.swift in Sources */, + C13151C00000000000000012 /* CloudReadCooldownStore+Session.swift in Sources */, + C13151C00000000000000002 /* CloudReadCooldownStore.swift in Sources */, + C13151C00000000000000004 /* CloudReadMutation.swift in Sources */, + C12625B00000000000000002 /* CloudReadNetworkMonitor.swift in Sources */, + C12636A00000000000000004 /* CloudReadRecoveryObserver.swift in Sources */, + C13151C0000000000000000A /* CloudReadRequestCoordinator+Context.swift in Sources */, + C13151C0000000000000000E /* CloudReadRequestCoordinator+Entry.swift in Sources */, + C13151C00000000000000006 /* CloudReadRequestCoordinator+Key.swift in Sources */, + C13151C00000000000000010 /* CloudReadRequestCoordinator+Pending.swift in Sources */, + C13151C00000000000000008 /* CloudReadRequestCoordinator+Response.swift in Sources */, + C13151C0000000000000000C /* CloudReadRequestCoordinator+Waiter.swift in Sources */, + C12625A00000000000000004 /* CloudReadRequestCoordinator.swift in Sources */, 595EC249E2E015ACE304F0EE /* CloudRemoteOperationStep.swift in Sources */, B12438020000000000000001 /* CloudRenameCoordinator.swift in Sources */, + C12625A00000000000000002 /* CloudRequestClock.swift in Sources */, B12357000000000000000004 /* CloudSidebarDebugLabWindow.swift in Sources */, 51B4FF9048D7FC13A294FEC2 /* CloudSidebarDebugMetrics.swift in Sources */, 6EAF84C73F1793C18F24DEE8 /* CloudSidebarDebugResetButton.swift in Sources */, @@ -14231,6 +14338,7 @@ 646ACD2F9B5044B8A2AB6C3A /* CmuxTuiSurfaceProvider+TerminalPrimitives.swift in Sources */, D12439050000000000000001 /* CmuxTuiSurfaceProvider+WorkspaceLifecycle.swift in Sources */, 175CFE773740410786E08A2B /* CmuxTuiSurfaceProviderRegistry+Production.swift in Sources */, + DAE9491D73F526601DFC97A2 /* CmuxTuiSurfaceProviderRegistry+Routes.swift in Sources */, 7D46CAA4C3C848ACAD2E46A7 /* CmuxTuiSurfaceProviderRegistry+Teardown.swift in Sources */, A1B2C3D4E5F6071827364961 /* CmuxTuiSurfaceProviderRegistry.swift in Sources */, 036359ABDB1F58B489D378A4 /* CmuxTuiSurfaceProviders.swift in Sources */, @@ -14686,8 +14794,11 @@ 4A6A6A54919FEE7C66D41678 /* MachinesPanelBanners.swift in Sources */, E879C292A2D38E3DDBCB004D /* MachinesPanelView.swift in Sources */, C1257500000000000000001A /* MachinesPanelViewModel+MachinePins.swift in Sources */, + F9ADD9F062DEA2BC5924BC38 /* MachinesPanelViewModel+Refresh.swift in Sources */, D7E6477A2FC01DCCF353840D /* MachinesPanelViewModel.swift in Sources */, 7A0CE1000000000000000724 /* MachinesTunnelBanner.swift in Sources */, + C12625D00000000000000004 /* MachineUsageSnapshot.swift in Sources */, + C12625D00000000000000002 /* MachineUsageTotals.swift in Sources */, D1F0A00600000000000000C1 /* MacPairedMacBackupBody.swift in Sources */, D1F0A00700000000000000C1 /* MacPairedMacBackupOpWire.swift in Sources */, D1F0A00100000000000000C1 /* MacPairedMacBackupPublisher.swift in Sources */, @@ -15526,6 +15637,8 @@ C7A506000000000000000002 /* TaskManagerView.swift in Sources */, C7A502000000000000000002 /* TaskManagerWindowController.swift in Sources */, 7490C00F7490C00F7490C00F /* TaskVMInfoMemoryPressureFootprintSampler.swift in Sources */, + C12625C00000000000000002 /* TeamMachineUsage.swift in Sources */, + C13151D00000000000000002 /* TeamMachineUsageKind.swift in Sources */, 0044CBDD567D9CD29E6A6F12 /* TelemetrySettings.swift in Sources */, A10199010000000000000001 /* TerminalAdaptiveDefaultThemeSettings.swift in Sources */, 8362A0030000000000000003 /* TerminalCallerTTYBinding.swift in Sources */, @@ -15866,6 +15979,7 @@ C9A57209C9A57209C9A57209 /* VerticalTabsSidebar+WorkspaceGroups.swift in Sources */, 66141EC09936A3AD5EC83307 /* View+CloudErrorCopy.swift in Sources */, A28B087F0000000000000005 /* ViewerNavigationKeyRouter.swift in Sources */, + C12625A00000000000000006 /* VMCapabilities.swift in Sources */, 60AA592C22BFA579A064782E /* VMClient+ResourceStats.swift in Sources */, ACE4FE1A8F1CC6C0B6A673AB /* VMClient.swift in Sources */, EF56B870FD0D4AC490D99CD5 /* VMClientError.swift in Sources */, @@ -16697,6 +16811,13 @@ DBF72463931EE07F5759C449 /* CloudPortsVPNAffordanceTests.swift in Sources */, 0C45DC8D9BE54DDEA2397892 /* CloudPrivateRouteSelectionTests.swift in Sources */, AC38407F09E94DA38B94F168 /* CloudProviderRefreshCoordinatorTests.swift in Sources */, + C13151C00000000000000031 /* CloudReadIdentityAuthClient.swift in Sources */, + C12625B00000000000000006 /* CloudReadManualClock.swift in Sources */, + C12625B00000000000000004 /* CloudReadRequestCoordinatorTests.swift in Sources */, + 9F046FC7A14BBDB098F471B3 /* CloudReadResponseGate.swift in Sources */, + C13151C00000000000000023 /* CloudRefreshAuthClient.swift in Sources */, + C13151C00000000000000021 /* CloudRefreshFixture.swift in Sources */, + C12625000000000000000004 /* CloudRefreshURLProtocol.swift in Sources */, C12449010000000000000001 /* CloudRemoteColorOwnershipTests.swift in Sources */, C12575000000000000000050 /* CloudRenameOptimismTests.swift in Sources */, 46383D21CF46427F36EBEB7B /* CloudSidebarAcceptanceTests.swift in Sources */, @@ -17111,6 +17232,7 @@ F83620010000000000000001 /* PortScannerTTYFreshnessTests.swift in Sources */, B8E9500A0000000000000001 /* PostHogAnalyticsPropertiesTests.swift in Sources */, DE71CE00000000000000000A /* PresenceHeartbeatClientTests.swift in Sources */, + AC0FCEFB6665CF134634D60B /* ProcessExitWait.swift in Sources */, C47110010000000000000001 /* ProcessPipeReadCrashRegressionTests.swift in Sources */, 0A327C88E531C7E407CEF58C /* ProcessSnapshotMeasurement.swift in Sources */, A64610020000000000000001 /* QuitConfirmationAlertPresenterTests.swift in Sources */, @@ -17415,6 +17537,8 @@ 62F6C9E77A6A40B982D91040 /* VaultSessionSearchQueryTests.swift in Sources */, C5BEA2B0EA914759A8BFD577 /* VaultSocketPayloadTests.swift in Sources */, A3340002A3340002A3340002 /* ViewerNavigationTests.swift in Sources */, + C13151C00000000000000033 /* VMClientReadCoalescingTests+Identity.swift in Sources */, + C12625000000000000000002 /* VMClientReadCoalescingTests.swift in Sources */, 6EE115A36CAD9082FBC4593E /* VMClientTelemetryTests.swift in Sources */, C37800000000000000000003 /* VMDefaultCloudCommandTests.swift in Sources */, B81607322100000000000003 /* VMMachineTerminalResolution.swift in Sources */, diff --git a/cmuxTests/AgentHookDeliveryQueueTests.swift b/cmuxTests/AgentHookDeliveryQueueTests.swift index d56930f05de7..5f9d44e2289d 100644 --- a/cmuxTests/AgentHookDeliveryQueueTests.swift +++ b/cmuxTests/AgentHookDeliveryQueueTests.swift @@ -321,16 +321,29 @@ struct AgentHookDeliveryQueueTests { await probe.deliver(event) } - for index in 1...3 { + // Tool telemetry has a single ingress slot, so a second tool event is + // admitted only after the drain task has moved the first one into a + // lane. Waiting for each delivery to start makes that hand-off + // observable instead of racing the drain task. Only two tool + // deliveries may run, so the third stays resident behind them. + for index in 1...2 { #expect(queue.enqueue(try makeEvent( agent: "cursor", subcommand: "shell-exec", payload: "tool-\(index)", surfaceID: "surface-\(index)" ))) + try await probe.waitUntilStarted(count: index) } - try await probe.waitUntilStarted(count: 2) + #expect(queue.enqueue(try makeEvent( + agent: "cursor", + subcommand: "shell-exec", + payload: "tool-3", + surfaceID: "surface-3" + ))) + // Three outstanding tool events exhaust the best-effort reservation + // whether or not tool-3 has left ingress yet. #expect(!queue.enqueue(try makeEvent( agent: "cursor", subcommand: "shell-exec", diff --git a/cmuxTests/AppDelegateEqualizeSplitsShortcutTests.swift b/cmuxTests/AppDelegateEqualizeSplitsShortcutTests.swift index 0aa94215199d..30ca5afc7cdb 100644 --- a/cmuxTests/AppDelegateEqualizeSplitsShortcutTests.swift +++ b/cmuxTests/AppDelegateEqualizeSplitsShortcutTests.swift @@ -367,6 +367,30 @@ private func waitWhileSuspended( } } +/// Finishes any configuration reload still fanning out to surfaces. +/// +/// A reload commits synchronously but completes after its bounded surface +/// fanout, which `TerminalConfigurationApplyScheduler` drives through +/// main-actor tasks. A nested `RunLoop.main.run` inside a synchronous +/// main-actor test cannot execute those tasks, so reload cases must suspend +/// instead. A reload issued here merges with any queued one, and its +/// completion runs only after the shared coordinator has finished them. +@MainActor +private func settleConfigurationReload( + _ app: GhosttyApp +) async { + let settled = expectation( + description: "earlier configuration reload settled" + ) + app.reloadConfiguration( + source: "test.settleConfigurationReload", + reloadSettingsFromFile: false + ) { + settled.fulfill() + } + await waitWhileSuspended(for: [settled], timeout: 5) +} + @MainActor private extension TabManager { @discardableResult @@ -5499,11 +5523,20 @@ final class AppDelegateEqualizeSplitsShortcutTests { @Test func testFullConfigurationReloadStagesAppearanceUntilConfigurationCommit() - throws { + async throws { #if DEBUG let app = GhosttyApp.shared + await settleConfigurationReload(app) let originalProfile = GhosttyStartupAppearancePreviewState.profile + // An idle reload commits synchronously. Hold one transaction in its + // surface fanout so the staged reload below is queued behind it. It + // reloads the unchanged configuration, so the appearance captured + // next is still the one to restore. + app.reloadConfiguration( + source: "test.stageAppearance.active", + reloadSettingsFromFile: false + ) let originalBackgroundHex = app.defaultBackgroundColor.hexString() let targetProfile: GhosttyStartupAppearancePreviewProfile = @@ -5528,33 +5561,6 @@ final class AppDelegateEqualizeSplitsShortcutTests { ) { _ in reloadCompleted.fulfill() } - defer { - NotificationCenter.default.removeObserver(observer) - GhosttyStartupAppearancePreviewState.profile = - originalProfile - GhosttyConfig.invalidateLoadCache() - - let restoreCompleted = expectation( - description: "original appearance restored" - ) - let restoreObserver = - NotificationCenter.default.addObserver( - forName: .ghosttyConfigDidReload, - object: nil, - queue: .main - ) { _ in - restoreCompleted.fulfill() - } - app.reloadConfiguration( - source: "test.restoreStagedAppearance", - reloadSettingsFromFile: false - ) - wait(for: [restoreCompleted], timeout: 5) - NotificationCenter.default.removeObserver( - restoreObserver - ) - withExtendedLifetime(retainedPanels) {} - } GhosttyStartupAppearancePreviewState.profile = targetProfile @@ -5568,14 +5574,36 @@ final class AppDelegateEqualizeSplitsShortcutTests { XCTAssertEqual( app.defaultBackgroundColor.hexString(), originalBackgroundHex, - "A pending full reload must not publish its new background before the matching Ghostty config commits" + "A full reload queued behind an active transaction must not publish its new background before its own Ghostty config commits" + ) + await waitWhileSuspended( + for: [reloadCompleted], + timeout: 5 ) - wait(for: [reloadCompleted], timeout: 5) XCTAssertNotEqual( app.defaultBackgroundColor.hexString(), originalBackgroundHex, "The staged appearance must publish when the full configuration commits" ) + + NotificationCenter.default.removeObserver(observer) + GhosttyStartupAppearancePreviewState.profile = + originalProfile + GhosttyConfig.invalidateLoadCache() + let restoreCompleted = expectation( + description: "original appearance restored" + ) + app.reloadConfiguration( + source: "test.restoreStagedAppearance", + reloadSettingsFromFile: false + ) { + restoreCompleted.fulfill() + } + await waitWhileSuspended( + for: [restoreCompleted], + timeout: 5 + ) + withExtendedLifetime(retainedPanels) {} #else throw XCTSkip("Startup appearance previews require DEBUG") #endif @@ -5583,9 +5611,10 @@ final class AppDelegateEqualizeSplitsShortcutTests { @Test func testConfigurationReloadRemainsActiveUntilAsyncReconciliationCompletes() - throws { + async throws { #if DEBUG let app = GhosttyApp.shared + await settleConfigurationReload(app) let retainedPanels = (0..<16).map { _ in TerminalPanel( workspaceId: UUID(), @@ -5607,7 +5636,10 @@ final class AppDelegateEqualizeSplitsShortcutTests { app.isConfigurationReloadActive, "Appearance synchronization must stay deferred while incremental reconciliation is pending" ) - wait(for: [reloadCompleted], timeout: 5) + await waitWhileSuspended( + for: [reloadCompleted], + timeout: 5 + ) XCTAssertFalse(app.isConfigurationReloadActive) withExtendedLifetime(retainedPanels) {} #else @@ -5701,9 +5733,10 @@ final class AppDelegateEqualizeSplitsShortcutTests { @Test func testConfigurationReloadQueuesRequestDuringAsyncReconciliation() - throws { + async throws { #if DEBUG let app = GhosttyApp.shared + await settleConfigurationReload(app) let retainedPanels = (0..<16).map { _ in TerminalPanel( workspaceId: UUID(), @@ -5742,7 +5775,7 @@ final class AppDelegateEqualizeSplitsShortcutTests { secondCompletionGeneration, "A request queued during reconciliation must not report success against the active transaction" ) - wait( + await waitWhileSuspended( for: [ firstReloadCompleted, secondReloadCompleted @@ -6227,11 +6260,12 @@ final class AppDelegateEqualizeSplitsShortcutTests { } @Test - func testGhosttyAppConfigUpdateWaitsForFontBarrier() { + func testGhosttyAppConfigUpdateWaitsForFontBarrier() async { guard let appDelegate = AppDelegate.shared else { XCTFail("Expected AppDelegate.shared") return } + await settleConfigurationReload(GhosttyApp.shared) let manager = TabManager() guard let workspace = manager.selectedWorkspace, let panelId = workspace.focusedPanelId, @@ -6280,21 +6314,47 @@ final class AppDelegateEqualizeSplitsShortcutTests { XCTAssertEqual(applyAttemptCount, 2) var didUpdateGhosttyAppConfig = false + let configUpdated = expectation( + description: "ghostty app config update published" + ) let observer = NotificationCenter.default.addObserver( forName: .ghosttyConfigDidReload, object: nil, queue: .main ) { _ in didUpdateGhosttyAppConfig = true + configUpdated.fulfill() } defer { NotificationCenter.default.removeObserver(observer) } + var didCommitGhosttyAppConfig = false GhosttyApp.shared.reloadConfiguration( soft: true, source: "test.fontBarrier", - reloadSettingsFromFile: false + reloadSettingsFromFile: false, + commitCompletion: { _ in didCommitGhosttyAppConfig = true } + ) + // Held, not merely not yet run: the transaction is parked at the + // font-work barrier, and giving the main actor turns does not move it. + // Only releasing the barrier below may. +#if DEBUG + XCTAssertEqual( + GhosttyApp.shared.debugConfigurationReloadPhase, + .waitingForFontWork, + "The app config update must wait at the font-work barrier" + ) + for _ in 0..<5 { await Task.yield() } + XCTAssertEqual( + GhosttyApp.shared.debugConfigurationReloadPhase, + .waitingForFontWork, + "Main-actor turns must not release the font-work barrier" + ) +#endif + XCTAssertFalse( + didCommitGhosttyAppConfig, + "The app config must not commit before font work finishes" ) XCTAssertFalse( didUpdateGhosttyAppConfig, @@ -6305,6 +6365,10 @@ final class AppDelegateEqualizeSplitsShortcutTests { scheduler.fire(at: 2) } XCTAssertEqual(applyAttemptCount, 3) + // Releasing the barrier commits the app configuration; the reload + // notification is published after its bounded surface fanout, which + // runs on later main-actor turns. + await waitWhileSuspended(for: [configUpdated], timeout: 5) XCTAssertTrue(didUpdateGhosttyAppConfig) } @@ -7567,7 +7631,16 @@ final class AppDelegateEqualizeSplitsShortcutTests { } @Test - func testConfiguredWorkspaceTerminalFontSizeResetRestoresEverySplit() { + func testConfiguredWorkspaceTerminalFontSizeResetRestoresEverySplit() async { + // A reload an earlier case left fanning out holds the app-wide + // font-size barrier, and the arbiter queues a reset issued behind it + // until the barrier lifts, which is after these synchronous checks. + // Settle it while suspended so the reset below applies immediately. + await settleConfigurationReload(GhosttyApp.shared) + XCTAssertFalse( + GhosttyApp.shared.isConfigurationReloadActive, + "A configuration reload in flight would queue the reset past these checks" + ) withTemporaryShortcut(action: .resetWorkspaceTerminalFontSize) { guard let appDelegate = AppDelegate.shared else { XCTFail("Expected AppDelegate.shared") diff --git a/cmuxTests/AppDelegateShortcutRoutingTests.swift b/cmuxTests/AppDelegateShortcutRoutingTests.swift index fcfb3c5f7375..30ab6a12ed11 100644 --- a/cmuxTests/AppDelegateShortcutRoutingTests.swift +++ b/cmuxTests/AppDelegateShortcutRoutingTests.swift @@ -1146,17 +1146,18 @@ final class AppDelegateShortcutRoutingTests: XCTestCase { originatingWindow.orderFront(nil) previouslyFocusedWindow.makeKeyAndOrderFront(nil) - RunLoop.main.run(until: Date(timeIntervalSinceNow: 0.05)) - - XCTAssertTrue(appDelegate.shortcutRoutingKeyWindow === previouslyFocusedWindow) + // A key-window change lands on a later run-loop turn, and a loaded + // runner can take several; wait for it instead of a fixed 50 ms. + XCTAssertTrue( + waitForCondition(timeout: 5) { appDelegate.shortcutRoutingKeyWindow === previouslyFocusedWindow } + ) XCTAssertTrue( appDelegate.toggleSidebarInActiveMainWindow(preferredWindow: originatingWindow) ) - RunLoop.main.run(until: Date(timeIntervalSinceNow: 0.05)) XCTAssertTrue( - appDelegate.shortcutRoutingKeyWindow === originatingWindow, + waitForCondition(timeout: 5) { appDelegate.shortcutRoutingKeyWindow === originatingWindow }, "An in-window action must request key status for its originating window before mutating window state" ) XCTAssertEqual( diff --git a/cmuxTests/CLINotifyProcessIntegrationRegressionTests.swift b/cmuxTests/CLINotifyProcessIntegrationRegressionTests.swift index e455fdbb68ee..aa9b66894fad 100644 --- a/cmuxTests/CLINotifyProcessIntegrationRegressionTests.swift +++ b/cmuxTests/CLINotifyProcessIntegrationRegressionTests.swift @@ -5032,12 +5032,7 @@ final class CLINotifyProcessIntegrationRegressionTests: XCTestCase { allowBridgeResponse.signal() XCTAssertEqual(handshakeReceived.wait(timeout: .now() + 5), .success) - let exited = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .userInitiated).async { - process.waitUntilExit() - exited.signal() - } - XCTAssertEqual(exited.wait(timeout: .now() + 5), .success) + XCTAssertEqual(waitForProcessExit(process, timeout: 5), .success) wait(for: [socketHandled, bridgeHandled], timeout: 5) let stderr = String(data: stderrPipe.fileHandleForReading.readDataToEndOfFile(), encoding: .utf8) ?? "" @@ -5244,12 +5239,7 @@ final class CLINotifyProcessIntegrationRegressionTests: XCTestCase { wait(for: [bridgeHandled], timeout: 5) allowResizeResponse.signal() - let exited = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .userInitiated).async { - process.waitUntilExit() - exited.signal() - } - XCTAssertEqual(exited.wait(timeout: .now() + 5), .success) + XCTAssertEqual(waitForProcessExit(process, timeout: 5), .success) wait(for: [socketHandled, unexpectedReadinessAfterAcknowledgement], timeout: 0.5) let stdout = String(data: stdoutPipe.fileHandleForReading.readDataToEndOfFile(), encoding: .utf8) ?? "" @@ -5436,12 +5426,7 @@ final class CLINotifyProcessIntegrationRegressionTests: XCTestCase { closeBridge.signal() wait(for: [bridgeHandled], timeout: 5) - let exited = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .userInitiated).async { - process.waitUntilExit() - exited.signal() - } - XCTAssertEqual(exited.wait(timeout: .now() + 5), .success) + XCTAssertEqual(waitForProcessExit(process, timeout: 5), .success) wait(for: [socketHandled], timeout: 5) let stdout = String( diff --git a/cmuxTests/CLIRemoteShellStartupPerformanceTests.swift b/cmuxTests/CLIRemoteShellStartupPerformanceTests.swift index 513211cda6f3..4efc72afeb71 100644 --- a/cmuxTests/CLIRemoteShellStartupPerformanceTests.swift +++ b/cmuxTests/CLIRemoteShellStartupPerformanceTests.swift @@ -395,15 +395,10 @@ struct CLIRemoteShellStartupPerformanceTests { } private func waitForProcess(_ running: RunningProcess, timeout: TimeInterval) -> ProcessRunResult { - let done = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .userInitiated).async { - running.process.waitUntilExit() - done.signal() - } - let timedOut = done.wait(timeout: .now() + timeout) == .timedOut + let timedOut = waitForProcessExit(running.process, timeout: timeout) == .timedOut if timedOut { running.process.terminate() - _ = done.wait(timeout: .now() + 1) + _ = waitForProcessExit(running.process, timeout: 1) } let stderr = String(data: running.stderrPipe.fileHandleForReading.readDataToEndOfFile(), encoding: .utf8) ?? "" return ProcessRunResult( diff --git a/cmuxTests/CLISSHPTYResizeInputTests.swift b/cmuxTests/CLISSHPTYResizeInputTests.swift index 3cd6701099cb..17e4ca1b39c6 100644 --- a/cmuxTests/CLISSHPTYResizeInputTests.swift +++ b/cmuxTests/CLISSHPTYResizeInputTests.swift @@ -149,12 +149,7 @@ struct CLISSHPTYResizeInputTests { closeBridge.signal() #expect(bridgeCloseObserved.wait(timeout: .now() + 5) == .success) - let exited = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .userInitiated).async { - process.waitUntilExit() - exited.signal() - } - let didExit = exited.wait(timeout: .now() + 5) == .success + let didExit = waitForProcessExit(process, timeout: 5) == .success #expect(didExit, "Expected ssh-pty-attach to exit") guard didExit else { if process.isRunning { diff --git a/cmuxTests/CLISSHSessionAttachAnchorTests.swift b/cmuxTests/CLISSHSessionAttachAnchorTests.swift index ace925b7876b..890a4024a953 100644 --- a/cmuxTests/CLISSHSessionAttachAnchorTests.swift +++ b/cmuxTests/CLISSHSessionAttachAnchorTests.swift @@ -516,18 +516,13 @@ struct CLISSHSessionAttachAnchorTests { return ProcessRunResult(status: -1, stdout: "", stderr: String(describing: error), timedOut: false) } - let exitSignal = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .userInitiated).async { - process.waitUntilExit() - exitSignal.signal() - } - let timedOut = exitSignal.wait(timeout: .now() + timeout) == .timedOut + let timedOut = waitForProcessExit(process, timeout: timeout) == .timedOut if timedOut { process.terminate() - if exitSignal.wait(timeout: .now() + 1) == .timedOut { + if waitForProcessExit(process, timeout: 1) == .timedOut { kill(process.processIdentifier, SIGKILL) - _ = exitSignal.wait(timeout: .now() + 1) + _ = waitForProcessExit(process, timeout: 1) } } diff --git a/cmuxTests/CLISendQueuedOutputTests.swift b/cmuxTests/CLISendQueuedOutputTests.swift index 1b9b34443e3f..2dc93e4a38e1 100644 --- a/cmuxTests/CLISendQueuedOutputTests.swift +++ b/cmuxTests/CLISendQueuedOutputTests.swift @@ -201,18 +201,13 @@ struct CLISendQueuedOutputTests { return ProcessRunResult(status: -1, stdout: "", stderr: String(describing: error), timedOut: false) } - let exitSignal = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .userInitiated).async { - process.waitUntilExit() - exitSignal.signal() - } - let timedOut = exitSignal.wait(timeout: .now() + timeout) == .timedOut + let timedOut = waitForProcessExit(process, timeout: timeout) == .timedOut if timedOut { process.terminate() - if exitSignal.wait(timeout: .now() + 1) == .timedOut { + if waitForProcessExit(process, timeout: 1) == .timedOut { kill(process.processIdentifier, SIGKILL) - _ = exitSignal.wait(timeout: .now() + 1) + _ = waitForProcessExit(process, timeout: 1) } } diff --git a/cmuxTests/CLIStdioSIGPIPERegressionTests.swift b/cmuxTests/CLIStdioSIGPIPERegressionTests.swift index 166becfce49a..fd86c5e261c9 100644 --- a/cmuxTests/CLIStdioSIGPIPERegressionTests.swift +++ b/cmuxTests/CLIStdioSIGPIPERegressionTests.swift @@ -56,12 +56,7 @@ final class CLIStdioSIGPIPERegressionTests: XCTestCase { description: String, timeout: TimeInterval = 5 ) -> XCTWaiter.Result { - let exited = expectation(description: description) - DispatchQueue.global(qos: .userInitiated).async { - process.waitUntilExit() - exited.fulfill() - } - return XCTWaiter().wait(for: [exited], timeout: timeout) + waitForProcessExit(process, timeout: timeout) == .success ? .completed : .timedOut } private func runProcess( @@ -91,16 +86,11 @@ final class CLIStdioSIGPIPERegressionTests: XCTestCase { ) } - let exitSignal = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .userInitiated).async { - process.waitUntilExit() - exitSignal.signal() - } - let timedOut = exitSignal.wait(timeout: .now() + timeout) == .timedOut + let timedOut = waitForProcessExit(process, timeout: timeout) == .timedOut if timedOut { process.terminate() - _ = exitSignal.wait(timeout: .now() + 1) + _ = waitForProcessExit(process, timeout: 1) } let stdout = String(data: stdoutPipe.fileHandleForReading.readDataToEndOfFile(), encoding: .utf8) ?? "" diff --git a/cmuxTests/CLITmuxCompatStoreConcurrencyTests.swift b/cmuxTests/CLITmuxCompatStoreConcurrencyTests.swift index 65f2f4c97724..219295f4ff6a 100644 --- a/cmuxTests/CLITmuxCompatStoreConcurrencyTests.swift +++ b/cmuxTests/CLITmuxCompatStoreConcurrencyTests.swift @@ -206,17 +206,12 @@ struct CLITmuxCompatStoreConcurrencyTests { return ProcessRunResult(status: -1, stderr: String(describing: error), timedOut: false) } - let exited = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .userInitiated).async { - process.waitUntilExit() - exited.signal() - } - let timedOut = exited.wait(timeout: .now() + timeout) == .timedOut + let timedOut = waitForProcessExit(process, timeout: timeout) == .timedOut if timedOut { process.terminate() - if exited.wait(timeout: .now() + 1) == .timedOut { + if waitForProcessExit(process, timeout: 1) == .timedOut { kill(process.processIdentifier, SIGKILL) - _ = exited.wait(timeout: .now() + 1) + _ = waitForProcessExit(process, timeout: 1) } } let stderr = String( diff --git a/cmuxTests/CLIVMLayoutEnvTests.swift b/cmuxTests/CLIVMLayoutEnvTests.swift index de291847a0cc..5a7a3889e011 100644 --- a/cmuxTests/CLIVMLayoutEnvTests.swift +++ b/cmuxTests/CLIVMLayoutEnvTests.swift @@ -168,17 +168,12 @@ extension CLINotifyProcessIntegrationRegressionTests { } stdinPipe.fileHandleForWriting.write(Data(standardInput.utf8)) try? stdinPipe.fileHandleForWriting.close() - let exitSignal = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .userInitiated).async { - process.waitUntilExit() - exitSignal.signal() - } - let timedOut = exitSignal.wait(timeout: .now() + timeout) == .timedOut + let timedOut = waitForProcessExit(process, timeout: timeout) == .timedOut if timedOut { process.terminate() - if exitSignal.wait(timeout: .now() + 1) == .timedOut { + if waitForProcessExit(process, timeout: 1) == .timedOut { kill(process.processIdentifier, SIGKILL) - _ = exitSignal.wait(timeout: .now() + 1) + _ = waitForProcessExit(process, timeout: 1) } } let stdout = String(data: stdoutPipe.fileHandleForReading.readDataToEndOfFile(), encoding: .utf8) ?? "" diff --git a/cmuxTests/CMUXOpenCommandTests.swift b/cmuxTests/CMUXOpenCommandTests.swift index ab0e3cb91bf0..3c340490099c 100644 --- a/cmuxTests/CMUXOpenCommandTests.swift +++ b/cmuxTests/CMUXOpenCommandTests.swift @@ -2930,22 +2930,17 @@ final class CMUXOpenCommandTests: XCTestCase { return ProcessRunResult(status: -1, stdout: "", stderr: String(describing: error), timedOut: false) } - let exitSignal = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .userInitiated).async { - process.waitUntilExit() - exitSignal.signal() - } if let stdinText, let stdinPipe { stdinPipe.fileHandleForWriting.write(Data(stdinText.utf8)) stdinPipe.fileHandleForWriting.closeFile() } - let timedOut = exitSignal.wait(timeout: .now() + timeout) == .timedOut + let timedOut = waitForProcessExit(process, timeout: timeout) == .timedOut if timedOut { process.terminate() - if exitSignal.wait(timeout: .now() + 1) == .timedOut, process.isRunning { + if waitForProcessExit(process, timeout: 1) == .timedOut, process.isRunning { kill(process.processIdentifier, SIGKILL) - _ = exitSignal.wait(timeout: .now() + 1) + _ = waitForProcessExit(process, timeout: 1) } } @@ -3011,14 +3006,9 @@ final class CMUXOpenCommandTests: XCTestCase { private func terminateProcess(_ process: Process) { guard process.isRunning else { return } process.terminate() - let finished = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .utility).async { - process.waitUntilExit() - finished.signal() - } - if finished.wait(timeout: .now() + 1) == .timedOut, process.isRunning { + if waitForProcessExit(process, timeout: 1) == .timedOut, process.isRunning { kill(process.processIdentifier, SIGKILL) - _ = finished.wait(timeout: .now() + 1) + _ = waitForProcessExit(process, timeout: 1) } } diff --git a/cmuxTests/CampfireHookNotificationTests.swift b/cmuxTests/CampfireHookNotificationTests.swift index 6ccd2e577b2a..9daae401d437 100644 --- a/cmuxTests/CampfireHookNotificationTests.swift +++ b/cmuxTests/CampfireHookNotificationTests.swift @@ -307,18 +307,13 @@ struct CampfireHookNotificationTests { stdinPipe.fileHandleForWriting.write(Data(standardInput.utf8)) try? stdinPipe.fileHandleForWriting.close() - let exitSignal = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .userInitiated).async { - process.waitUntilExit() - exitSignal.signal() - } - let timedOut = exitSignal.wait(timeout: .now() + timeout) == .timedOut + let timedOut = waitForProcessExit(process, timeout: timeout) == .timedOut if timedOut { process.terminate() - if exitSignal.wait(timeout: .now() + 1) == .timedOut { + if waitForProcessExit(process, timeout: 1) == .timedOut { kill(process.processIdentifier, SIGKILL) - _ = exitSignal.wait(timeout: .now() + 1) + _ = waitForProcessExit(process, timeout: 1) } } diff --git a/cmuxTests/CanvasShortcutContextTests.swift b/cmuxTests/CanvasShortcutContextTests.swift index 8a22e37bacd7..471ffb849e3e 100644 --- a/cmuxTests/CanvasShortcutContextTests.swift +++ b/cmuxTests/CanvasShortcutContextTests.swift @@ -14,13 +14,14 @@ import Testing private final class CanvasRoutingViewportSpy: CanvasViewportControlling { var revealedPanelIds: [UUID] = [] var resetZoomCount = 0 + var zoomFactors: [CGFloat] = [] var currentMagnification: CGFloat = 1 var currentCenterInCanvas: CGPoint = .zero func revealPane(_ panelId: UUID, animated: Bool) { revealedPanelIds.append(panelId) } func resetZoom() { resetZoomCount += 1 } func toggleOverview() {} - func zoom(by factor: CGFloat) {} + func zoom(by factor: CGFloat) { zoomFactors.append(factor) } func setViewport(center: CGPoint, magnification: CGFloat?) {} func modelDidChangeExternally(animated: Bool) {} } @@ -53,6 +54,45 @@ struct CanvasShortcutContextTests { } } + @MainActor + @Test + func fileEditorFocusWidensOnlyFileEditorActions() { + var shortcutContext = ShortcutFocusState(browser: false, markdown: false, sidebar: false).context + shortcutContext.setBool(ShortcutContextKnownKey.workspaceCanvasLayout.rawValue, true) + // A Markdown source or Dock editor: a file editor, but not a text file preview. + let focus = ShortcutEventFocusContext( + browserPanel: nil, + markdownPanel: nil, + filePreviewTextEditorFocused: false, + fileEditorFocused: true, + simulatorFocused: false, + rightSidebarFocused: false, + shortcutContext: shortcutContext + ) + + let wordWrap = KeyboardShortcutSettings.Action.toggleFileEditorWordWrap + #expect(KeyboardShortcutSettings.effectiveWhenClause(for: wordWrap).evaluate(focus.whenClauseContext(for: wordWrap))) + #expect(wordWrap.shortcutContext.isAvailable(focus)) + + let canvasZoomActions: [KeyboardShortcutSettings.Action] = [.canvasZoomIn, .canvasZoomOut, .canvasZoomReset] + for action in canvasZoomActions { + #expect( + KeyboardShortcutSettings.effectiveWhenClause(for: action).evaluate(focus.whenClauseContext(for: action)), + "\(action.rawValue) must reach the canvas while a non-preview file editor is focused" + ) + #expect(action.shortcutContext.isAvailable(focus)) + } + + let previewZoomActions: [KeyboardShortcutSettings.Action] = [.browserZoomIn, .browserZoomOut, .browserZoomReset] + for action in previewZoomActions { + #expect( + !KeyboardShortcutSettings.effectiveWhenClause(for: action).evaluate(focus.whenClauseContext(for: action)), + "\(action.rawValue) is scoped to text file previews, not every file editor" + ) + #expect(!action.shortcutContext.isAvailable(focus)) + } + } + @Test func canvasLayoutContextOverlapsNormalTerminalFocusShortcuts() { let canvas = KeyboardShortcutSettings.Action.canvasOverview.shortcutContext @@ -331,6 +371,71 @@ struct CanvasShortcutRoutingFeedbackTests { } } + /// Canvas zoom in/out must reach the canvas, not the Markdown source editor: + /// the editor is a file editor for word wrap, but not a text file preview. + @Test func canvasZoomShortcutsZoomCanvasWhenMarkdownSourceEditorIsFocused() throws { + try withIsolatedShortcutSettings { + let appDelegate = try #require(AppDelegate.shared) + let windowId = appDelegate.createMainWindow() + defer { closeWindow(withId: windowId) } + + let window = try #require(mainWindow(for: windowId)) + let manager = try #require(appDelegate.tabManagerFor(windowId: windowId)) + let workspace = try #require(manager.selectedWorkspace) + let zoomIn = try #require(makeKeyDownEvent( + key: "=", + modifiers: [.command, .option], + keyCode: UInt16(kVK_ANSI_Equal), + windowNumber: window.windowNumber + )) + let zoomOut = try #require(makeKeyDownEvent( + key: "-", + modifiers: [.command, .option], + keyCode: UInt16(kVK_ANSI_Minus), + windowNumber: window.windowNumber + )) + + window.makeKeyAndOrderFront(nil) + workspace.setLayoutMode(.canvas) + let viewport = CanvasRoutingViewportSpy() + workspace.canvasModel.viewport = viewport + + let firstPane = try #require(workspace.bonsplitController.allPaneIds.first) + let fileURL = try temporaryMarkdownFile(contents: "# Preview\n") + defer { try? FileManager.default.removeItem(at: fileURL) } + let panel = try #require(workspace.newMarkdownSurface( + inPane: firstPane, + filePath: fileURL.path, + focus: true + )) + panel.setDisplayMode(.text) + + let textView = SavingTextView.makeFilePreviewTextView() + textView.frame = NSRect(x: 0, y: 0, width: 200, height: 120) + textView.string = panel.textContent + textView.panel = panel + panel.attachTextView(textView) + window.contentView?.addSubview(textView) + defer { textView.removeFromSuperview() } + #expect(window.makeFirstResponder(textView)) + #expect(workspace.focusedPanelId == panel.id) + #expect(manager.focusedTextFilePreviewPanel == nil) + let initialPointSize = try #require(textView.font?.pointSize) + +#if DEBUG + #expect(appDelegate.debugHandleCustomShortcut(event: zoomIn)) + #expect(appDelegate.debugHandleCustomShortcut(event: zoomOut)) +#else + Issue.record("debugHandleCustomShortcut is only available in DEBUG") +#endif + + #expect(viewport.zoomFactors.count == 2) + #expect((viewport.zoomFactors.first ?? 1) > 1) + #expect((viewport.zoomFactors.last ?? 1) < 1) + #expect(abs((textView.font?.pointSize ?? 0) - initialPointSize) < 0.01) + } + } + @Test func chordedViewZoomShortcutZoomsFocusedTextPreviewThroughAppRouter() throws { try withIsolatedShortcutSettings { KeyboardShortcutSettings.setShortcut( diff --git a/cmuxTests/CloudOperationRecorderTests.swift b/cmuxTests/CloudOperationRecorderTests.swift index 72a01a7cdf88..5fdbb54f1596 100644 --- a/cmuxTests/CloudOperationRecorderTests.swift +++ b/cmuxTests/CloudOperationRecorderTests.swift @@ -8,6 +8,49 @@ import Testing @MainActor struct CloudOperationRecorderTests { + @Test("Placement receipts distinguish acceptance from permanent rejection", arguments: [202, 400]) + func placementUploaderReceipt(status: Int) async throws { + let fixture = try await CloudRefreshFixture.make() + defer { fixture.session.invalidateAndCancel() } + let directory = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + defer { try? FileManager.default.removeItem(at: directory) } + let queueURL = directory.appendingPathComponent("queue.json") + let configuration = URLSessionConfiguration.ephemeral + configuration.protocolClasses = [PlacementReceiptURLProtocol.self] + let session = URLSession(configuration: configuration) + defer { session.invalidateAndCancel() } + let uploader = CloudTelemetryUploader( + auth: fixture.auth, + baseURL: try #require(URL(string: "https://receipt-\(status).test")), + client: CloudTelemetryClient.current(info: [:], flavor: .dev), + session: session, + queueURL: queueURL + ) + let identity = try #require(fixture.auth.authenticatedSessionIdentity) + let recorder = CloudOperationRecorder(uploader: uploader, identity: { identity }) + let operation = recorder.begin(.open) + await PlacementReceiptURLProtocol.resetCapture(status: status) + await recorder.finish(operation, error: CmuxTuiSurfaceProvider.ProviderError.remotePlacementUnavailable("fixture-machine")) + let deadline = ContinuousClock.now.advanced(by: .seconds(10)) + var acknowledged = false + while ContinuousClock.now < deadline { + let data = try Data(contentsOf: queueURL) + let entries = try #require(JSONSerialization.jsonObject(with: data) as? [[String: Any]]) + if entries.isEmpty, await PlacementReceiptURLProtocol.captured(status: status) != nil { + acknowledged = true + break + } + try await Task.sleep(for: .milliseconds(10)) + } + #expect(acknowledged) + let captured = try #require(await PlacementReceiptURLProtocol.captured(status: status)) + #expect(captured.failure == "placement") + #expect(captured.traceID == operation.traceID) + #expect(captured.operationID == operation.operationID.uuidString.lowercased()) + #expect(await uploader.droppedCount == (status == 400 ? 1 : 0)) + await uploader.clearForSignOut() + } + @Test func authenticationFailuresKeepTheirTelemetryClassification() async throws { let identity = AuthenticatedSessionIdentity(generation: 1, accountID: "synthetic") let sink = CapturedCloudDiagnostics() @@ -228,6 +271,7 @@ struct CloudOperationRecorderTests { @Test func metadataSeparatesNightlyFromItsBackend() { let info: [String: Any] = ["CFBundleShortVersionString": "1.2.3", "CFBundleVersion": "45", "CMUXCommit": "abcdef123"] #expect(CloudTelemetryClient.current(info: info, flavor: .nightly).channel == "nightly") + #expect(CloudTelemetryClient.current(info: info, flavor: .rc).channel == "rc") #expect(CloudTelemetryClient.current(info: info, flavor: .stable).channel == "production") #expect(CloudTelemetryClient.current(info: info, flavor: .dev).channel == "dev") #expect(CloudTelemetryClient.current(info: info, flavor: .nightly).revision == "abcdef123") @@ -248,4 +292,82 @@ private actor CapturedCloudDiagnostics: CloudTelemetrySending { func enqueue(_ span: CloudTelemetrySpan, identity: AuthenticatedSessionIdentity) { spans.append(span) } func clearForSignOut() { spans.removeAll() } } + +/// The parsed immutable span crosses to the capture actor before any receipt is delivered. +/// URLProtocol callbacks can arrive off-actor; mutable capture state stays actor-isolated. +private final class PlacementReceiptURLProtocol: URLProtocol, @unchecked Sendable { + fileprivate struct UploadedSpan: Sendable { + let failure: String + let traceID: String + let operationID: String + } + + private actor Capture { + private var values: [Int: UploadedSpan] = [:] + + func reset(status: Int) { + values.removeValue(forKey: status) + } + + func record(status: Int, span: UploadedSpan) { + values[status] = span + } + + func value(status: Int) -> UploadedSpan? { + return values[status] + } + } + + private static let capture = Capture() + fileprivate static func resetCapture(status: Int) async { await capture.reset(status: status) } + fileprivate static func captured(status: Int) async -> UploadedSpan? { await capture.value(status: status) } + + override class func canInit(with request: URLRequest) -> Bool { true } + override class func canonicalRequest(for request: URLRequest) -> URLRequest { request } + override func stopLoading() {} + override func startLoading() { + do { + var body = request.httpBody ?? Data() + if body.isEmpty, let stream = request.httpBodyStream { + stream.open() + defer { stream.close() } + var buffer = [UInt8](repeating: 0, count: 4096) + while true { + let count = stream.read(&buffer, maxLength: buffer.count) + if count < 0 { throw URLError(.cannotDecodeContentData) } + if count == 0 { break } + body.append(contentsOf: buffer.prefix(count)) + } + } + guard let batch = try JSONSerialization.jsonObject(with: body) as? [String: Any], + let spans = batch["spans"] as? [[String: Any]], spans.count == 1, + let span = spans.first, let eventID = span["eventId"] as? String, + span["failure"] as? String == "placement", + let operationID = span["operationId"] as? String, !operationID.isEmpty, + (span["traceId"] as? String)?.count == 32, + (span["spanId"] as? String)?.count == 16 else { + throw URLError(.cannotDecodeContentData) + } + let status = request.url?.host == "receipt-202.test" ? 202 : 400 + let uploaded = UploadedSpan( + failure: span["failure"] as? String ?? "", + traceID: span["traceId"] as? String ?? "", + operationID: operationID + ) + guard let url = request.url, + let response = HTTPURLResponse(url: url, statusCode: status, httpVersion: nil, headerFields: nil) else { + throw URLError(.badURL) + } + let receipt = try JSONSerialization.data(withJSONObject: ["eventIds": [eventID]]) + Task { + await Self.capture.record(status: status, span: uploaded) + client?.urlProtocol(self, didReceive: response, cacheStoragePolicy: .notAllowed) + client?.urlProtocol(self, didLoad: receipt) + client?.urlProtocolDidFinishLoading(self) + } + } catch { + client?.urlProtocol(self, didFailWithError: error) + } + } +} #endif diff --git a/cmuxTests/CloudReadIdentityAuthClient.swift b/cmuxTests/CloudReadIdentityAuthClient.swift new file mode 100644 index 000000000000..456b6dec0732 --- /dev/null +++ b/cmuxTests/CloudReadIdentityAuthClient.swift @@ -0,0 +1,55 @@ +import CMUXAuthCore +import CmuxAuthRuntime +import Foundation + +/// Holds only the revalidation token probe; coherent token reads remain available. +actor CloudReadIdentityAuthClient: AuthClient { + private var accountID: String? = "fixture" + private var holdsProbe = false + private var probe: CheckedContinuation? + private var probeWaiters: [CheckedContinuation] = [] + + func holdValidationProbe() { holdsProbe = true } + func waitUntilValidationProbe() async { + if probe != nil { return } + await withCheckedContinuation { probeWaiters.append($0) } + } + func releaseValidationProbe() { + holdsProbe = false + probe?.resume() + probe = nil + } + func accessToken() async -> String? { + if holdsProbe { + await withCheckedContinuation { continuation in + probe = continuation + let waiters = probeWaiters + probeWaiters.removeAll() + for waiter in waiters { waiter.resume() } + } + } + return accountID.map { "fixture-access-\($0)" } + } + func resolvedAccessToken(forceRefresh: Bool) async throws -> String? { accountID.map { "fixture-access-\($0)" } } + func refreshToken() async -> String? { accountID.map { "fixture-refresh-\($0)" } } + func forceRefreshAccessToken() async -> String? { accountID.map { "fixture-access-\($0)" } } + func currentUser(throwOnMissing: Bool) async throws -> CMUXAuthUser? { + accountID.map { CMUXAuthUser(id: $0, primaryEmail: "\($0)@example.test", displayName: "Fixture") } + } + func listTeams() async throws -> [CMUXAuthTeam] { + [CMUXAuthTeam(id: "selected", displayName: "Selected")] + } + func setSelectedTeam(id: String?) async throws {} + func sendMagicLinkEmail(email: String, callbackURL: String) async throws -> String { "fixture" } + func signInWithMagicLink(code: String) async throws {} + func signInWithCredential(email: String, password: String) async throws { accountID = "replacement" } + func signInWithOAuth(provider: String, anchor: any AuthPresentationAnchoring) async throws {} + func storedAccessToken() async -> String? { accountID.map { "fixture-access-\($0)" } } + func clearLocalSession() async { accountID = nil } + func clearLocalSession(ifRefreshTokenMatches refreshToken: String) async { + guard let accountID, refreshToken == "fixture-refresh-\(accountID)" else { return } + self.accountID = nil + } + func revokeSession(accessToken: String?, refreshToken: String?) async throws {} + func freshAccessToken(accessToken: String?, refreshToken: String) async -> String? { accessToken } +} diff --git a/cmuxTests/CloudReadManualClock.swift b/cmuxTests/CloudReadManualClock.swift new file mode 100644 index 000000000000..76316a9f8844 --- /dev/null +++ b/cmuxTests/CloudReadManualClock.swift @@ -0,0 +1,48 @@ +import Foundation + +/// Clock.now is synchronous. A lock protects this test clock's instant and +/// continuation registration; no production state uses this lock. +final class CloudReadManualClock: Clock, @unchecked Sendable { + struct Instant: InstantProtocol { + let offset: Duration + func advanced(by duration: Duration) -> Self { Self(offset: offset + duration) } + func duration(to other: Self) -> Duration { other.offset - offset } + static func < (lhs: Self, rhs: Self) -> Bool { lhs.offset < rhs.offset } + } + private struct Waiter { + let deadline: Instant + let continuation: CheckedContinuation + } + private let lock = NSLock() + private var instant = Instant(offset: .zero) + private var sleepers: [UUID: Waiter] = [:] + var now: Instant { lock.withLock { instant } } + var minimumResolution: Duration { .zero } + var pendingSleeperCount: Int { lock.withLock { sleepers.count } } + + func sleep(until deadline: Instant, tolerance: Duration?) async throws { + let id = UUID() + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + lock.withLock { + if Task.isCancelled { continuation.resume(throwing: CancellationError()) } + else if deadline <= instant { continuation.resume() } + else { sleepers[id] = Waiter(deadline: deadline, continuation: continuation) } + } + } + } onCancel: { + self.lock.withLock { self.sleepers.removeValue(forKey: id) }?.continuation.resume(throwing: CancellationError()) + } + } + + func advance(by duration: Duration, deliverTimers: Bool = true) { + let ready: [Waiter] = lock.withLock { + instant = instant.advanced(by: duration) + guard deliverTimers else { return [] } + let ready = sleepers.filter { $0.value.deadline <= instant } + for id in ready.keys { sleepers.removeValue(forKey: id) } + return Array(ready.values) + } + for waiter in ready { waiter.continuation.resume() } + } +} diff --git a/cmuxTests/CloudReadRequestCoordinatorTests.swift b/cmuxTests/CloudReadRequestCoordinatorTests.swift new file mode 100644 index 000000000000..45dc89a7d5ad --- /dev/null +++ b/cmuxTests/CloudReadRequestCoordinatorTests.swift @@ -0,0 +1,489 @@ +import Foundation +import Testing +#if canImport(cmux_DEV) +@testable import cmux_DEV +#elseif canImport(cmux) +@testable import cmux +#else +@testable import CloudReadFixture +#endif + +@Suite("Cloud read deadline and cancellation", .timeLimit(.minutes(1))) +struct CloudReadRequestCoordinatorTests { + private typealias Owner = CloudReadRequestCoordinator + private func key(_ id: String = "vm", account: String = "fixture", generation: UInt64 = 1, team: String = "team") -> Owner.Key { + .init(path: id, accountID: account, generation: generation, teamID: team) + } + private func response(_ status: Int = 200) -> Owner.Response { + .init(data: Data(), http: HTTPURLResponse(url: URL(string: "https://fixture.invalid")!, statusCode: status, httpVersion: nil, headerFields: nil)!) + } + + @Test("Four owners issue one request per machine", arguments: [1, 10, 100, 1000]) + func scale(machines: Int) async throws { + let gate = CloudReadResponseGate() + let owner = Owner() + let tasks = (0..<(machines * 4)).map { index in + Task { try await owner.read(key("vm-\(index % machines)")) { await gate.read(response()) } } + } + try await eventually { await owner.entries.values.reduce(0) { $0 + $1.waiters.count } == machines * 4 } + await gate.release() + for task in tasks { _ = try await task.value } + #expect(await gate.requests == machines) + #expect(await owner.entries.isEmpty) + print("cloud-read-scale machines=\(machines) owners=4 requests=\(await gate.requests) requests_per_machine=1") + } + + @Test("Reachability changes fan out to every panel subscriber") + func networkChangesBroadcast() async { + let owner = Owner() + let first = await owner.networkChanges() + let second = await owner.networkChanges() + let firstValue = Task { + var iterator = first.makeAsyncIterator() + return await iterator.next() + } + let secondValue = Task { + var iterator = second.makeAsyncIterator() + return await iterator.next() + } + await owner.networkChanged(isOnline: false) + #expect(await firstValue.value == .some(false)) + #expect(await secondValue.value == .some(false)) + } + + @Test("A late reachability subscriber receives the current offline state") + func networkChangesReplayCurrentState() async { + let owner = Owner() + await owner.networkChanged(isOnline: false) + let stream = await owner.networkChanges() + var iterator = stream.makeAsyncIterator() + #expect(await iterator.next() == .some(false)) + } + + @Test("The final waiter cancels and holds the draining slot after another caller leaves", arguments: [false, true]) + func independentCancellation(expireFirst: Bool) async throws { + let gate = CloudReadResponseGate() + let clock = CloudReadManualClock() + let owner = Owner(clock: CloudRequestClock(clock)) + let first = Task { try await owner.read(key(), deadline: .seconds(5)) { await gate.read(response()) } } + try await eventually { await gate.requests == 1 } + let second = Task { try await owner.read(key()) { await gate.read(response()) } } + try await eventually { await owner.entries.values.first?.waiters.count == 2 } + if expireFirst { clock.advance(by: .seconds(6)) } else { first.cancel() } + do { _ = try await first.value; Issue.record("retired waiter returned a value") } + catch is CancellationError { #expect(!expireFirst) } + catch { #expect(expireFirst && (error as? URLError)?.code == .timedOut) } + #expect(await owner.entries.values.first?.waiters.count == 1) + second.cancel() + do { _ = try await second.value; Issue.record("final cancelled waiter returned a value") } + catch is CancellationError {} catch { Issue.record("\(error)") } + let replacement = Task { try await owner.read(key()) { await gate.read(response()) } } + try await eventually { await owner.entries.values.first?.pending?.waiters.count == 1 } + #expect(await gate.requests == 1) + await gate.release() + #expect(try await replacement.value.http.statusCode == 200) + #expect(await gate.requests == 2) + try await eventually { await owner.entries.isEmpty } + } + + @Test("Deadline returns before an uncooperative loader, retaining only its draining slot") + func deadlineDoesNotJoinStuckChild() async throws { + let clock = CloudReadManualClock() + let owner = Owner(clock: CloudRequestClock(clock), budget: .seconds(30)) + let gate = CloudReadResponseGate() + let task = Task { try await owner.read(key()) { await gate.read(response()) } } + try await eventually { await gate.requests == 1 } + clock.advance(by: .seconds(31)) + do { _ = try await task.value; Issue.record("expired request succeeded") } + catch { #expect((error as? URLError)?.code == .timedOut) } + #expect(await owner.entries.count == 1) + #expect(await owner.entries.values.first?.waiters.isEmpty == true) + await gate.release() + try await eventually { await owner.entries.isEmpty } + } + + @Test("A queued replacement has its own deadline while old cleanup stays held") + func queuedReplacementDeadline() async throws { + let clock = CloudReadManualClock() + let owner = Owner(clock: CloudRequestClock(clock), budget: .seconds(30)) + let gate = CloudReadResponseGate() + let first = Task { try await owner.read(key()) { await gate.read(response()) } } + try await eventually { await gate.requests == 1 } + first.cancel() + _ = await first.result + let replacement = Task { try await owner.read(key()) { await gate.read(response()) } } + try await eventually { await owner.entries.values.first?.pending?.waiters.count == 1 } + clock.advance(by: .seconds(31)) + do { _ = try await replacement.value; Issue.record("queued deadline missed") } + catch { #expect((error as? URLError)?.code == .timedOut) } + #expect(await gate.requests == 1) + await gate.release() + try await eventually { await owner.entries.isEmpty } + } + + @Test("Time spent acquiring read scope consumes the original budget") + func admissionUsesOriginalDeadline() async throws { + let clock = CloudReadManualClock() + let owner = Owner(clock: CloudRequestClock(clock)) + let deadline = owner.makeDeadline() + clock.advance(by: .seconds(600), deliverTimers: false) + do { + _ = try await owner.read(key(), deadline: deadline) { Issue.record("started after admission expired"); return response() } + } catch { #expect((error as? URLError)?.code == .timedOut) } + #expect(await owner.entries.isEmpty) + } + + @Test("A short waiter cannot shorten another caller's deadline", arguments: [false, true], [false, true]) + func independentDeadlines(queued: Bool, deliverTimers: Bool) async throws { + let clock = CloudReadManualClock() + let owner = Owner(clock: CloudRequestClock(clock)) + let gate = CloudReadResponseGate() + if queued { + let retired = Task { try await owner.read(key()) { await gate.read(response()) } } + try await eventually { await gate.requests == 1 } + retired.cancel() + _ = await retired.result + } + let long = Task { try await owner.read(key(), deadline: .seconds(20)) { await gate.read(response()) } } + try await eventually { + let entry = await owner.entries.values.first + return queued ? entry?.pending?.waiters.count == 1 : entry?.waiters.count == 1 + } + let short = Task { try await owner.read(key(), deadline: .seconds(1)) { await gate.read(response()) } } + try await eventually { + let entry = await owner.entries.values.first + return queued ? entry?.pending?.waiters.count == 2 : entry?.waiters.count == 2 + } + clock.advance(by: .seconds(2), deliverTimers: deliverTimers) + if deliverTimers { + _ = await short.result + let entry = await owner.entries.values.first + #expect(queued ? entry?.pending?.waiters.count == 1 : entry?.waiters.count == 1) + } + await gate.release() + do { _ = try await short.value; Issue.record("short waiter exceeded its deadline") } + catch { #expect((error as? URLError)?.code == .timedOut) } + #expect(try await long.value.http.statusCode == 200) + #expect(await gate.requests == (queued ? 2 : 1)) + #expect(await owner.entries.isEmpty) + } + + @Test("An elapsed first caller cannot shorten a later caller's budget", arguments: [false, true], [false, true]) + func shortFirstDeadline(queued: Bool, deliverTimers: Bool) async throws { + let clock = CloudReadManualClock() + let owner = Owner(clock: CloudRequestClock(clock)) + let gate = CloudReadResponseGate() + if queued { + let retired = Task { try await owner.read(key()) { await gate.read(response()) } } + try await eventually { await gate.requests == 1 } + retired.cancel() + _ = await retired.result + } + let shortDeadline = owner.makeDeadline(elapsed: .seconds(25)) + let short = Task { try await owner.read(key(), deadline: shortDeadline) { await gate.read(response()) } } + try await eventually { + let entry = await owner.entries.values.first + return queued ? entry?.pending?.waiters.count == 1 : entry?.waiters.count == 1 + } + clock.advance(by: .seconds(1)) + let longDeadline = owner.makeDeadline() + let long = Task { try await owner.read(key(), deadline: longDeadline) { await gate.read(response()) } } + try await eventually { + let entry = await owner.entries.values.first + return queued ? entry?.pending?.waiters.count == 2 : entry?.waiters.count == 2 + } + clock.advance(by: .seconds(5), deliverTimers: deliverTimers) + if deliverTimers { _ = await short.result } + await gate.release() + do { _ = try await short.value; Issue.record("elapsed caller exceeded its deadline") } + catch { #expect((error as? URLError)?.code == .timedOut) } + #expect(try await long.value.http.statusCode == 200) + #expect(await gate.requests == (queued ? 2 : 1)) + #expect(await owner.entries.isEmpty) + } + + @Test("Later callers cannot rejuvenate the transport beyond its fixed budget") + func transportLifetimeCap() async throws { + let clock = CloudReadManualClock() + let owner = Owner(clock: CloudRequestClock(clock)) + let gate = CloudReadResponseGate() + let first = Task { try await owner.read(key()) { await gate.read(response()) } } + try await eventually { await gate.requests == 1 } + clock.advance(by: .seconds(29)) + let late = Task { try await owner.read(key()) { await gate.read(response()) } } + try await eventually { await owner.entries.values.first?.waiters.count == 2 } + #expect(await owner.noteRetryAfter(key(), seconds: 2, response: response(429)) == false) + clock.advance(by: .seconds(2)) + for request in [first, late] { + do { _ = try await request.value; Issue.record("shared transport exceeded its cap") } + catch { #expect((error as? URLError)?.code == .timedOut) } + } + #expect(await owner.entries.values.first?.waiters.isEmpty == true) + let replacement = Task { try await owner.read(key()) { await gate.read(response()) } } + try await eventually { await owner.entries.values.first?.pending?.waiters.count == 1 } + #expect(await gate.requests == 1) + await gate.release() + #expect(try await replacement.value.http.statusCode == 200) + #expect(await gate.requests == 2) + } + + @Test("Retry admission uses the latest live caller within the transport cap") + func retryUsesRemainingCallerBudgets() async throws { + let clock = CloudReadManualClock() + let owner = Owner(clock: CloudRequestClock(clock)) + let gate = CloudReadResponseGate() + let short = Task { try await owner.read(key(), deadline: .seconds(5)) { await gate.read(response(429)) } } + try await eventually { await gate.requests == 1 } + clock.advance(by: .seconds(1)) + let long = Task { try await owner.read(key(), deadline: .seconds(20)) { await gate.read(response(429)) } } + try await eventually { await owner.entries.values.first?.waiters.count == 2 } + #expect(await owner.noteRetryAfter(key(), seconds: 8, response: response(429))) + long.cancel() + _ = await long.result + #expect(await owner.noteRetryAfter(key(), seconds: 8, response: response(429)) == false) + await gate.release() + #expect(try await short.value.http.statusCode == 429) + #expect(try await owner.read(key()) { Issue.record("retried before the server minimum"); return response() }.http.statusCode == 429) + clock.advance(by: .seconds(8)) + #expect(try await owner.read(key()) { response() }.http.statusCode == 200) + } + + @Test("Response-first delivery after a simulated wake still expires") + func responseAfterDeadline() async throws { + let clock = CloudReadManualClock() + let owner = Owner(clock: CloudRequestClock(clock)) + let gate = CloudReadResponseGate() + let task = Task { try await owner.read(key()) { await gate.read(response()) } } + try await eventually { await gate.requests == 1 } + try await eventually { clock.pendingSleeperCount == 1 } + clock.advance(by: .seconds(600), deliverTimers: false) + await gate.release() + do { _ = try await task.value; Issue.record("late response succeeded") } + catch { #expect((error as? URLError)?.code == .timedOut) } + } + + @Test("A reader arriving after wake waits for a fresh pass instead of inheriting the old timeout") + func freshReaderAfterWake() async throws { + let clock = CloudReadManualClock() + let owner = Owner(clock: CloudRequestClock(clock)) + let gate = CloudReadResponseGate() + let old = Task { try await owner.read(key()) { await gate.read(response()) } } + try await eventually { await gate.requests == 1 && clock.pendingSleeperCount == 1 } + clock.advance(by: .seconds(600), deliverTimers: false) + let fresh = Task { try await owner.read(key()) { await gate.read(response()) } } + try await eventually { await owner.entries.values.first?.pending?.waiters.count == 1 } + await gate.release() + do { _ = try await old.value; Issue.record("old request survived its deadline") } + catch { #expect((error as? URLError)?.code == .timedOut) } + #expect(try await fresh.value.http.statusCode == 200) + #expect(await gate.requests == 2) + } + + @Test("Retry-After survives operation completion and offline recovery") + func serverCooldown() async throws { + let clock = CloudReadManualClock() + let owner = Owner(clock: CloudRequestClock(clock)) + let throttled = response(429) + let first = try await owner.read(key()) { + #expect(await owner.noteRetryAfter(key(), seconds: 60, response: throttled) == false) + return throttled + } + #expect(first.http.statusCode == 429) + await owner.networkChanged(isOnline: false) + await owner.networkChanged(isOnline: true) + let cached = try await owner.read(key()) { Issue.record("retried before the server allowed it"); return response() } + #expect(cached.http.statusCode == 429) + clock.advance(by: .seconds(60)) + let recovered = try await owner.read(key()) { response() } + #expect(recovered.http.statusCode == 200) + } + + @Test("An arbitrarily long Retry-After does not overflow or become an early retry") + func oversizedRetryAfter() async throws { + let owner = Owner() + let throttled = response(429) + _ = try await owner.read(key()) { + #expect(await owner.noteRetryAfter(key(), seconds: TimeInterval(Int.max), response: throttled) == false) + return throttled + } + #expect(try await owner.read(key()) { Issue.record("ignored long server cooldown"); return response() }.http.statusCode == 429) + } + + @Test("Offline cancels readers, refuses more work, and reconnect recovers") + func offlineRecovery() async throws { + let owner = Owner() + let gate = CloudReadResponseGate() + let task = Task { try await owner.read(key()) { await gate.read(response()) } } + try await eventually { await gate.requests == 1 } + await owner.networkChanged(isOnline: false) + do { _ = try await task.value; Issue.record("offline returned live data") } + catch { #expect((error as? URLError)?.code == .notConnectedToInternet) } + await gate.release() + try await eventually { await owner.entries.isEmpty } + do { _ = try await owner.read(key()) { Issue.record("started offline"); return response() } } + catch { #expect((error as? URLError)?.code == .notConnectedToInternet) } + await owner.networkChanged(isOnline: true) + #expect(try await owner.read(key()) { response() }.http.statusCode == 200) + } + + @Test("Account and session generations never share responses") + func identityIsolation() async throws { + let owner = Owner() + let gate = CloudReadResponseGate() + let first = Task { try await owner.read(key()) { await gate.read(response()) } } + try await eventually { await gate.requests == 1 } + let replacement = try await owner.read(key(generation: 2)) { response(201) } + #expect(replacement.http.statusCode == 201) + await gate.release() + #expect(try await first.value.http.statusCode == 200) + } + + @Test("A mutation landing during a read shares one fresh trailing pass") + func mutationInvalidatesRunningRead() async throws { + let owner = Owner() + let gate = CloudReadResponseGate() + let list = key("/api/vm") + let operation: @Sendable () async -> Owner.Response = { + await gate.read(response(await gate.requests == 0 ? 200 : 201)) + } + let first = Task { try await owner.read(list, operation: operation) } + try await eventually { await gate.requests == 1 } + await owner.invalidate(CloudReadMutation(method: "POST", scope: list, responseData: Data())) + let second = Task { try await owner.read(list, operation: operation) } + try await eventually { await owner.entries.values.first?.waiters.count == 2 } + await gate.release() + #expect(try await first.value.http.statusCode == 201) + #expect(try await second.value.http.statusCode == 201) + #expect(await gate.requests == 2) + } + + @Test("Resize refreshes only the captured team's list and target machine") + func mutationScopeIsolation() async throws { + let owner = Owner() + let scenarios: [(Owner.Key, Bool)] = [ + (key("/api/vm"), true), (key("/api/vm/target/stats"), true), + (key("/api/vm/other/stats"), false), + (key("/api/coderouter/vm-usage/team"), false), + (key("/api/vm", account: "other"), false), + (key("/api/vm", generation: 2), false), + (key("/api/vm", team: "other"), false) + ] + let gates = scenarios.map { _ in CloudReadResponseGate() } + let requests = zip(scenarios, gates).map { scenario, gate in + Task { try await owner.read(scenario.0) { + await gate.read(response(await gate.requests == 0 ? 200 : 201)) + } } + } + try await eventually { await owner.entries.count == scenarios.count } + for gate in gates { try await eventually { await gate.requests == 1 } } + await owner.invalidate(CloudReadMutation(method: "POST", scope: key("/api/vm/target/resize"), responseData: Data())) + for gate in gates { await gate.release() } + for (index, request) in requests.enumerated() { + #expect(try await request.value.http.statusCode == (scenarios[index].1 ? 201 : 200)) + #expect(await gates[index].requests == (scenarios[index].1 ? 2 : 1)) + } + } + + @Test("Unrelated mutations do not restart machine reads", arguments: [ + "/api/vm/tunnel", "/api/vm/publications", "/api/billing/checkout", + "/api/vm/target/snapshot", "/api/vm/target/files" + ]) + func unrelatedMutation(path: String) async throws { + let owner = Owner() + let gate = CloudReadResponseGate() + let request = Task { try await owner.read(key("/api/vm")) { await gate.read(response()) } } + try await eventually { await gate.requests == 1 } + await owner.invalidate(CloudReadMutation(method: "POST", scope: key(path), responseData: Data())) + await gate.release() + #expect(try await request.value.http.statusCode == 200) + #expect(await gate.requests == 1) + } + + @Test("Cooldown capacity retains bounded compact errors and every server minimum") + func boundedCooldowns() throws { + var store = CloudReadCooldownStore(capacity: 8) + store.activateSession(for: key()) + let largeResponse = Owner.Response(data: Data(repeating: 65, count: 1_000_000), http: response(429).http) + for index in 0..<1000 { + store.record(key("vm-\(index)"), until: 60, now: 0, response: largeResponse) + } + #expect(store.retainedCount == 8) + let cachedResponses = (0..<1000).compactMap { store.response(for: key("vm-\($0)"), now: 59) } + #expect(cachedResponses.count == 8) + for cached in cachedResponses { + #expect(cached.http.statusCode == 429) + #expect(cached.data == Data(#"{"error":"rate_limited"}"#.utf8)) + #expect(cached.http.allHeaderFields.isEmpty) + } + // An evicted key and an unrelated path must not inherit another + // machine's Retry-After minimum. + #expect(store.response(for: key("unseen"), now: 59) == nil) + let expiredResponse = store.response(for: key("vm-999"), now: 60) + #expect(expiredResponse == nil) + #expect(store.retainedCount == 0) + } + + @Test("A key keeps its longest cooldown without leaking to other paths") + func cooldownKeepsLongestMinimumPerKey() { + var store = CloudReadCooldownStore(capacity: 1) + store.activateSession(for: key()) + store.record(key("first"), until: 10, now: 0, response: response(429)) + store.record(key("first"), until: 100, now: 0, response: response(429)) + store.record(key("first"), until: 20, now: 0, response: response(429)) + let throttledResponse = store.response(for: key("first"), now: 99) + #expect(throttledResponse?.http.statusCode == 429) + #expect(store.retainedCount == 1) + #expect(store.response(for: key("other"), now: 99) == nil) + let expiredResponse = store.response(for: key("first"), now: 100) + #expect(expiredResponse == nil) + } + + @Test("Session replacement discards old cooldowns and late completions cannot revive them") + func retiredSessionCooldowns() { + var store = CloudReadCooldownStore(capacity: 8) + let old = key() + for generation in 1...1000 { + let current = key(account: "account-\(generation)", generation: UInt64(generation)) + store.activateSession(for: current) + store.record(current, until: .infinity, now: 0, response: response(429)) + #expect(store.retainedCount == 1) + } + let current = key(account: "account-1000", generation: 1000) + store.activateSession(for: old) + store.record(old, until: .infinity, now: 0, response: response(429)) + let oldResponse = store.response(for: old, now: 0) + #expect(oldResponse == nil) + let currentResponse = store.response(for: current, now: 0) + #expect(currentResponse?.http.statusCode == 429) + #expect(store.retainedCount == 1) + let replacement = key(account: "replacement", generation: 1001) + store.activateSession(for: replacement) + store.record(current, until: .infinity, now: 0, response: response(429)) + let replacementResponse = store.response(for: replacement, now: 0) + #expect(replacementResponse == nil) + #expect(store.retainedCount == 0) + } + + private func eventually(_ condition: @escaping () async -> Bool) async throws { + let deadline = ContinuousClock.now.advanced(by: .seconds(10)) + while !(await condition()), ContinuousClock.now < deadline { await Task.yield() } + try #require(await condition(), "Fixture did not reach its expected actor state") + } +} + +@Suite("Cloud machines offline empty state") +struct CloudMachinesOfflineStateTests { + @Test("Offline before the first list load exposes retry state") + @MainActor + func offlineBeforeFirstLoadIsVisible() async throws { + let fixture = try await CloudRefreshFixture.make() + defer { fixture.session.invalidateAndCancel() } + let model = MachinesPanelViewModel(client: fixture.client, isCloudEnabled: { true }) + await fixture.readRequests.networkChanged(isOnline: false) + for _ in 0..<10 { await Task.yield() } + #expect(model.hasLoadedOnce) + #expect(model.listProblem == .unreachable) + #expect(model.lastErrorDescription != nil) + model.stopPolling() + } +} diff --git a/cmuxTests/CloudReadResponseGate.swift b/cmuxTests/CloudReadResponseGate.swift new file mode 100644 index 000000000000..a91840783703 --- /dev/null +++ b/cmuxTests/CloudReadResponseGate.swift @@ -0,0 +1,28 @@ +import Foundation + +#if canImport(cmux_DEV) +@testable import cmux_DEV +#elseif canImport(cmux) +@testable import cmux +#else +@testable import CloudReadFixture +#endif + +actor CloudReadResponseGate { + private(set) var requests = 0 + private var released = false + private var waiting: [(CloudReadRequestCoordinator.Response, CheckedContinuation)] = [] + + func read(_ response: CloudReadRequestCoordinator.Response) async -> CloudReadRequestCoordinator.Response { + requests += 1 + if released { return response } + return await withCheckedContinuation { waiting.append((response, $0)) } + } + + func release() { + released = true + let pending = waiting + waiting.removeAll() + for (response, continuation) in pending { continuation.resume(returning: response) } + } +} diff --git a/cmuxTests/CloudRefreshAuthClient.swift b/cmuxTests/CloudRefreshAuthClient.swift new file mode 100644 index 000000000000..e1f74a1b306c --- /dev/null +++ b/cmuxTests/CloudRefreshAuthClient.swift @@ -0,0 +1,22 @@ +import CMUXAuthCore +import CmuxAuthRuntime +import Foundation + +actor CloudRefreshAuthClient: AuthClient { + func accessToken() async -> String? { "fixture-access" } + func refreshToken() async -> String? { "fixture-refresh" } + func forceRefreshAccessToken() async -> String? { "fixture-access" } + func currentUser(throwOnMissing: Bool) async throws -> CMUXAuthUser? { + CMUXAuthUser(id: "fixture", primaryEmail: "fixture@example.test", displayName: "Fixture") + } + func listTeams() async throws -> [CMUXAuthTeam] { [] } + func sendMagicLinkEmail(email: String, callbackURL: String) async throws -> String { "fixture" } + func signInWithMagicLink(code: String) async throws {} + func signInWithCredential(email: String, password: String) async throws {} + func signInWithOAuth(provider: String, anchor: any AuthPresentationAnchoring) async throws {} + func storedAccessToken() async -> String? { "fixture-access" } + func clearLocalSession() async {} + func clearLocalSession(ifRefreshTokenMatches refreshToken: String) async {} + func revokeSession(accessToken: String?, refreshToken: String?) async throws {} + func freshAccessToken(accessToken: String?, refreshToken: String) async -> String? { accessToken } +} diff --git a/cmuxTests/CloudRefreshFixture.swift b/cmuxTests/CloudRefreshFixture.swift new file mode 100644 index 000000000000..1f549e559fda --- /dev/null +++ b/cmuxTests/CloudRefreshFixture.swift @@ -0,0 +1,55 @@ +import CMUXAuthCore +import CmuxAuthRuntime +import Foundation +import Testing +import os + +#if canImport(cmux_DEV) +@testable import cmux_DEV +#elseif canImport(cmux) +@testable import cmux +#endif + +@MainActor +struct CloudRefreshFixture { + let client: VMClient + let auth: AuthCoordinator + let session: URLSession + let readRequests: CloudReadRequestCoordinator + + static func make( + readRequests: CloudReadRequestCoordinator = CloudReadRequestCoordinator(), + authClient: (any AuthClient)? = nil, + isDisabledByManagedPolicy: (@Sendable () -> Bool)? = nil, + isCloudEnabled: @escaping @Sendable () -> Bool = { true } + ) async throws -> Self { + let defaults = try #require(UserDefaults(suiteName: "CloudRefreshFixture.\(UUID())")) + let auth = AuthCoordinator( + client: authClient ?? CloudRefreshAuthClient(), + sessionCache: CMUXAuthSessionCache(keyValueStore: defaults, key: "session"), + userCache: CMUXAuthIdentityStore(keyValueStore: defaults, key: "user"), + teamSelection: CMUXAuthTeamSelectionStore(keyValueStore: defaults, key: "team"), + anchor: AuthPresentationContextProvider(), + config: AuthConfig( + stack: CMUXAuthConfig(projectId: "fixture", publishableClientKey: "fixture"), + magicLinkCallbackURL: "http://127.0.0.1:1/callback", apiBaseURL: "http://127.0.0.1:1" + ), + launch: AuthLaunchOptions( + clearAuthRequested: false, mockDataEnabled: false, + environment: ["CMUX_UITEST_AUTH_FIXTURE": "1", "CMUX_UITEST_AUTH_USER_ID": "fixture"], + includesDevAuth: true + ) + ) + auth.start() + await auth.awaitBootstrapped() + try #require(auth.isAuthenticated) + let configuration = URLSessionConfiguration.ephemeral + configuration.protocolClasses = [CloudRefreshURLProtocol.self] + let session = URLSession(configuration: configuration) + return Self(client: VMClient( + session: session, auth: auth, resourceStats: VMResourceStatsStore(), checkpointRenames: CloudRenameCoordinator(), + machineCache: CloudMachineCache(defaults: defaults), isDisabledByManagedPolicy: isDisabledByManagedPolicy, + readRequests: readRequests, isCloudEnabled: isCloudEnabled + ), auth: auth, session: session, readRequests: readRequests) + } +} diff --git a/cmuxTests/CloudRefreshURLProtocol.swift b/cmuxTests/CloudRefreshURLProtocol.swift new file mode 100644 index 000000000000..5ec179330817 --- /dev/null +++ b/cmuxTests/CloudRefreshURLProtocol.swift @@ -0,0 +1,97 @@ +import Foundation + +/// URLProtocol's synchronous callbacks hand off to one actor; only that actor +/// reads the fixture state or calls the client, including after stopLoading. +final class CloudRefreshURLProtocol: URLProtocol, @unchecked Sendable { + enum Behavior: Sendable { case normal, statsUnavailable, listUnavailable, throttled } + private static let responses = Responses() + static func holdResponses() async { await responses.hold() } + static func releaseResponses() async { await responses.release() } + static func configure(_ behavior: Behavior) async { await responses.configure(behavior) } + static func waitUntilStarted(_ count: Int = 1) async { await responses.waitUntilStarted(count) } + static func currentStopCount() async -> Int { await responses.stopCount } + static func waitUntilStopped(after baseline: Int) async { await responses.waitUntilStopped(after: baseline) } + static func reset() async { await responses.reset() } + static func requestCounts() async -> [String: Int] { await responses.counts } + override class func canInit(with request: URLRequest) -> Bool { true } + override class func canonicalRequest(for request: URLRequest) -> URLRequest { request } + override func startLoading() { Task { await Self.responses.start(self) } } + override func stopLoading() { Task { await Self.responses.stop(self) } } + + private actor Responses { + private(set) var counts: [String: Int] = [:] + private var tasks: [ObjectIdentifier: Task] = [:] + private var behavior = Behavior.normal + private var held = false + private var responseWaiters: [ObjectIdentifier: CheckedContinuation] = [:] + func hold() { held = true } + func release() { + held = false + let pending = responseWaiters + responseWaiters.removeAll() + for waiter in pending.values { waiter.resume() } + } + private var stoppedRequests: Set = [] + private(set) var stopCount = 0 + private var startWaiters: [(Int, CheckedContinuation)] = [] + private var stopWaiters: [(after: Int, CheckedContinuation)] = [] + func configure(_ value: Behavior) { behavior = value } + func waitUntilStarted(_ count: Int) async { + guard counts.values.reduce(0, +) < count else { return } + await withCheckedContinuation { startWaiters.append((count, $0)) } + } + func waitUntilStopped(after baseline: Int) async { + guard stopCount <= baseline else { return } + await withCheckedContinuation { stopWaiters.append((baseline, $0)) } + } + func reset() { + release() + for task in tasks.values { task.cancel() } + tasks.removeAll() + counts.removeAll() + behavior = .normal + stoppedRequests.removeAll() + stopCount = 0 + } + func start(_ source: CloudRefreshURLProtocol) { + let key = ObjectIdentifier(source) + guard !stoppedRequests.contains(key) else { return } + let path = source.request.url!.path + counts[path, default: 0] += 1 + let count = counts.values.reduce(0, +) + let ready = startWaiters.filter { $0.0 <= count } + startWaiters.removeAll { $0.0 <= count } + for (_, waiter) in ready { waiter.resume() } + let behavior = self.behavior + tasks[key] = Task { + guard !Task.isCancelled else { return } + if self.held { await withCheckedContinuation { self.responseWaiters[key] = $0 } } + guard self.tasks.removeValue(forKey: key) != nil else { return } + let unavailable = path.hasSuffix("/stats") ? behavior == .statsUnavailable : behavior == .listUnavailable + let response = HTTPURLResponse(url: source.request.url!, statusCode: behavior == .throttled ? 429 : unavailable ? 503 : 200, httpVersion: nil, + headerFields: behavior == .throttled ? ["Retry-After": "60"] : nil)! + source.client?.urlProtocol(source, didReceive: response, cacheStoragePolicy: .notAllowed) + let body: String + if path == "/api/coderouter/vm-usage/team" { + body = #"{"teamId":"fixture-team","kind":"ready","periodDays":30,"machines":[]}"# + } else if path.hasSuffix("/stats") { + body = #"{"state":"awake","cpus":2}"# + } else { + body = #"{"vms":[{"id":"fixture-0","provider":"fixture","image":"desktop-vnc","status":"running","createdAt":0,"capabilities":{"stats":true}}]}"# + } + source.client?.urlProtocol(source, didLoad: Data(body.utf8)) + source.client?.urlProtocolDidFinishLoading(source) + } + } + func stop(_ source: CloudRefreshURLProtocol) { + let key = ObjectIdentifier(source) + guard stoppedRequests.insert(key).inserted else { return } + tasks.removeValue(forKey: key)?.cancel() + responseWaiters.removeValue(forKey: key)?.resume() + stopCount += 1 + let waiters = stopWaiters.filter { $0.after < stopCount } + stopWaiters.removeAll { $0.after < stopCount } + for (_, waiter) in waiters { waiter.resume() } + } + } +} diff --git a/cmuxTests/CodexTerminalErrorNotificationTests.swift b/cmuxTests/CodexTerminalErrorNotificationTests.swift index 32c2466333a0..ffef2620d534 100644 --- a/cmuxTests/CodexTerminalErrorNotificationTests.swift +++ b/cmuxTests/CodexTerminalErrorNotificationTests.swift @@ -264,17 +264,12 @@ private struct CodexTerminalErrorProcess { stdin.fileHandleForWriting.write(Data(standardInput.utf8)) try? stdin.fileHandleForWriting.close() - let finished = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .userInitiated).async { - process.waitUntilExit() - finished.signal() - } - let timedOut = finished.wait(timeout: .now() + timeout) == .timedOut + let timedOut = waitForProcessExit(process, timeout: timeout) == .timedOut if timedOut { process.terminate() - if finished.wait(timeout: .now() + 1) == .timedOut { + if waitForProcessExit(process, timeout: 1) == .timedOut { Darwin.kill(process.processIdentifier, SIGKILL) - guard finished.wait(timeout: .now() + 1) == .success else { + guard waitForProcessExit(process, timeout: 1) == .success else { return Result( status: -1, stdout: "", diff --git a/cmuxTests/FishShellIntegrationTests.swift b/cmuxTests/FishShellIntegrationTests.swift index c4a1e8374e90..8b159d8b3db3 100644 --- a/cmuxTests/FishShellIntegrationTests.swift +++ b/cmuxTests/FishShellIntegrationTests.swift @@ -491,16 +491,11 @@ struct FishShellIntegrationTests { return ProcessRunResult(status: -1, stdout: "", stderr: String(describing: error), timedOut: false) } - let exitSignal = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .userInitiated).async { - process.waitUntilExit() - exitSignal.signal() - } - let timedOut = exitSignal.wait(timeout: .now() + timeout) == .timedOut + let timedOut = waitForProcessExit(process, timeout: timeout) == .timedOut if timedOut { process.terminate() - _ = exitSignal.wait(timeout: .now() + 1) + _ = waitForProcessExit(process, timeout: 1) } return ProcessRunResult( diff --git a/cmuxTests/GhosttyConfigTests.swift b/cmuxTests/GhosttyConfigTests.swift index 2616e243f331..9eeea7f47deb 100644 --- a/cmuxTests/GhosttyConfigTests.swift +++ b/cmuxTests/GhosttyConfigTests.swift @@ -1397,16 +1397,11 @@ final class GhosttyConfigTests: XCTestCase { ) } - let exitSignal = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .userInitiated).async { - process.waitUntilExit() - exitSignal.signal() - } - let timedOut = exitSignal.wait(timeout: .now() + timeout) == .timedOut + let timedOut = waitForProcessExit(process, timeout: timeout) == .timedOut if timedOut { process.terminate() - _ = exitSignal.wait(timeout: .now() + 1) + _ = waitForProcessExit(process, timeout: 1) } let stdout = String( diff --git a/cmuxTests/OpenCodeHookRegressionTests.swift b/cmuxTests/OpenCodeHookRegressionTests.swift index 621650e4ad1d..3a59cb205eba 100644 --- a/cmuxTests/OpenCodeHookRegressionTests.swift +++ b/cmuxTests/OpenCodeHookRegressionTests.swift @@ -227,15 +227,10 @@ const fs = require("node:fs"); } catch { return ProcessRunResult(status: -1, stdout: "", stderr: String(describing: error), timedOut: false) } - let exitSignal = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .userInitiated).async { - process.waitUntilExit() - exitSignal.signal() - } - let timedOut = exitSignal.wait(timeout: .now() + timeout) == .timedOut + let timedOut = waitForProcessExit(process, timeout: timeout) == .timedOut if timedOut { process.terminate() - _ = exitSignal.wait(timeout: .now() + 1) + _ = waitForProcessExit(process, timeout: 1) } return ProcessRunResult( status: process.terminationStatus, diff --git a/cmuxTests/ProcessExitWait.swift b/cmuxTests/ProcessExitWait.swift new file mode 100644 index 000000000000..c7d3f8bac32e --- /dev/null +++ b/cmuxTests/ProcessExitWait.swift @@ -0,0 +1,21 @@ +import Foundation + +/// Waits for `process` to exit on the calling thread, polling `isRunning`. +/// +/// Returns `.success` once the process has exited, `.timedOut` if it is still +/// running at the deadline. Use this instead of queueing a blocking +/// `waitUntilExit()` on `DispatchQueue.global()` and waiting on a semaphore: +/// every queued waiter pins a pool thread until its child exits, waiters for +/// children that never exit accumulate across a test batch, and once the pool +/// is exhausted a new waiter never starts, so a child that exited normally is +/// reported as a timeout. +func waitForProcessExit(_ process: Process, timeout: TimeInterval) -> DispatchTimeoutResult { + let deadline = Date().addingTimeInterval(timeout) + while process.isRunning { + if Date() >= deadline { + return .timedOut + } + usleep(5_000) + } + return .success +} diff --git a/cmuxTests/SSHDeepSleepReattachTests.swift b/cmuxTests/SSHDeepSleepReattachTests.swift index 8a311d2f467c..f323911b601d 100644 --- a/cmuxTests/SSHDeepSleepReattachTests.swift +++ b/cmuxTests/SSHDeepSleepReattachTests.swift @@ -361,7 +361,12 @@ struct SSHDeepSleepReattachTests { sessionID: "ssh-test-session", foregroundAuth: Self.foregroundAuth() ).replacingOccurrences(of: "/usr/bin/ssh", with: fakeSSH.path), - environment: environment + environment: environment, + // Every attach attempt spawns uuidgen, the fake ssh, cmux and + // sleep, so the full budget costs ~4.5 s on an idle runner and + // more under shard load. The deadline only bounds a hang; it + // scales with the attempts the case expects to run. + timeout: max(5, Double(expectedSleepCount + 1) * 2) ) #expect(!result.timedOut, Comment(rawValue: result.stderr)) @@ -527,7 +532,11 @@ struct SSHDeepSleepReattachTests { try (lines.joined(separator: "\n") + "\n").write(to: url, atomically: true, encoding: .utf8) } - private static func runProcess(command: String, environment: [String: String]) -> ProcessRunResult { + private static func runProcess( + command: String, + environment: [String: String], + timeout: TimeInterval = 5 + ) -> ProcessRunResult { let process = Process() let stderrPipe = Pipe() process.executableURL = URL(fileURLWithPath: "/bin/sh") @@ -541,15 +550,10 @@ struct SSHDeepSleepReattachTests { } catch { return ProcessRunResult(status: -1, stderr: String(describing: error), timedOut: false) } - let exitSignal = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .userInitiated).async { - process.waitUntilExit() - exitSignal.signal() - } - let timedOut = exitSignal.wait(timeout: .now() + 5) == .timedOut + let timedOut = waitForProcessExit(process, timeout: timeout) == .timedOut if timedOut { process.terminate() - _ = exitSignal.wait(timeout: .now() + 1) + _ = waitForProcessExit(process, timeout: 1) } let stderr = String(data: stderrPipe.fileHandleForReading.readDataToEndOfFile(), encoding: .utf8) ?? "" return ProcessRunResult(status: process.terminationStatus, stderr: stderr, timedOut: timedOut) diff --git a/cmuxTests/SSHRemoteCWDRegressionTests.swift b/cmuxTests/SSHRemoteCWDRegressionTests.swift index 4a4a8c7b68db..59b646e40102 100644 --- a/cmuxTests/SSHRemoteCWDRegressionTests.swift +++ b/cmuxTests/SSHRemoteCWDRegressionTests.swift @@ -45,15 +45,10 @@ struct SSHRemoteCWDRegressionTests { return ProcessRunResult(status: -1, stdout: "", stderr: String(describing: error), timedOut: false) } - let exitSignal = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .userInitiated).async { - process.waitUntilExit() - exitSignal.signal() - } - let timedOut = exitSignal.wait(timeout: .now() + timeout) == .timedOut + let timedOut = waitForProcessExit(process, timeout: timeout) == .timedOut if timedOut { process.terminate() - _ = exitSignal.wait(timeout: .now() + 1) + _ = waitForProcessExit(process, timeout: 1) } return ProcessRunResult( status: process.terminationStatus, diff --git a/cmuxTests/SurfacePaneFactoryFocusTests.swift b/cmuxTests/SurfacePaneFactoryFocusTests.swift index 3c2cef923e6b..7b4992002a1f 100644 --- a/cmuxTests/SurfacePaneFactoryFocusTests.swift +++ b/cmuxTests/SurfacePaneFactoryFocusTests.swift @@ -262,7 +262,18 @@ import SwiftUI overlay.layoutSubtreeIfNeeded() await Task.yield() } - #expect(overlay.frame.width > 100 && overlay.frame.height > 50) + #expect( + overlay.frame.width > 100 && overlay.frame.height > 50, + """ + card=\(overlay.frame) inContainer=\(overlay.superview === target.container) \ + source=\(target.container.convert(source.hostedView.bounds, from: source.hostedView)) \ + sourceVisible=\(source.hostedView.visibleRect) sourceHidden=\(source.hostedView.isHiddenOrHasHiddenAncestor) \ + sourceWindowMatches=\(source.hostedView.window === window) \ + reference=\(target.container.convert(target.reference.bounds, from: target.reference)) \ + window=\(window.frame) visible=\(window.isVisible) key=\(window.isKeyWindow) \ + mainWindows=\(NSApp.windows.filter { $0.identifier?.rawValue.hasPrefix("cmux.main.") == true }.map { "\($0.frame)" }) + """ + ) let terminalFrame = target.container.convert(source.hostedView.bounds, from: source.hostedView) #expect(abs(overlay.frame.midX - terminalFrame.midX) < 2) #expect(abs(overlay.frame.midY - terminalFrame.midY) < 2) diff --git a/cmuxTests/VMClientReadCoalescingTests+Identity.swift b/cmuxTests/VMClientReadCoalescingTests+Identity.swift new file mode 100644 index 000000000000..37fb7cbc27b5 --- /dev/null +++ b/cmuxTests/VMClientReadCoalescingTests+Identity.swift @@ -0,0 +1,116 @@ +import CmuxAuthRuntime +import Foundation +import Testing + +#if canImport(cmux_DEV) +@testable import cmux_DEV +#elseif canImport(cmux) +@testable import cmux +#endif + +@MainActor +extension VMClientReadCoalescingTests { + @Test("Reads without a transition identity fail before HTTP and recover for the next account", arguments: ["list", "stats", "usage"]) + func nilIdentityAdmission(operation: String) async throws { + let authClient = CloudReadIdentityAuthClient() + let fixture = try await CloudRefreshFixture.make(authClient: authClient) + defer { fixture.session.invalidateAndCancel() } + await CloudRefreshURLProtocol.reset() + await authClient.holdValidationProbe() + let validation = Task { await fixture.auth.revalidateSession() } + await authClient.waitUntilValidationProbe() + #expect(fixture.auth.isAuthenticated) + #expect(fixture.auth.authenticatedSessionIdentity == nil) + do { + try await identityRead(operation, fixture: fixture) + Issue.record("A read with no authenticated identity reached the server") + } catch VMClientError.sessionRefreshFailed { + } catch MachineUsageClientError.sessionRefreshFailed { + } catch { Issue.record("Unexpected transition error: \(error)") } + #expect(await fixture.readRequests.entries.isEmpty) + #expect(await CloudRefreshURLProtocol.requestCounts().isEmpty) + await authClient.releaseValidationProbe() + await validation.value + try await fixture.auth.signInWithPassword(email: "replacement@example.test", password: "fixture") + #expect(fixture.auth.authenticatedSessionIdentity?.accountID == "replacement") + try await identityRead(operation, fixture: fixture) + #expect(await CloudRefreshURLProtocol.requestCounts().values.reduce(0, +) == 1) + } + + @Test("An account change rejects a response bound to the prior session", arguments: ["list", "stats", "usage"]) + func accountChangeRejectsRead(operation: String) async throws { + let fixture = try await CloudRefreshFixture.make(authClient: CloudReadIdentityAuthClient()) + defer { fixture.session.invalidateAndCancel() } + await CloudRefreshURLProtocol.reset() + await CloudRefreshURLProtocol.holdResponses() + let original = try #require(fixture.auth.authenticatedSessionIdentity) + let reader = Task { try await identityRead(operation, fixture: fixture) } + await CloudRefreshURLProtocol.waitUntilStarted() + try await fixture.auth.signInWithPassword(email: "replacement@example.test", password: "fixture") + #expect(fixture.auth.authenticatedSessionIdentity != original) + await CloudRefreshURLProtocol.releaseResponses() + do { try await reader.value; Issue.record("The previous account's response was published") } + catch is CancellationError {} catch VMClientError.notSignedIn {} catch { Issue.record("\(error)") } + try await identityRead(operation, fixture: fixture) + #expect(await CloudRefreshURLProtocol.requestCounts().values.reduce(0, +) == 2) + } + + @Test("A known identity read survives same-account revalidation", arguments: ["list", "stats", "usage"]) + func knownIdentityDuringRevalidation(operation: String) async throws { + let authClient = CloudReadIdentityAuthClient() + let fixture = try await CloudRefreshFixture.make(authClient: authClient) + defer { fixture.session.invalidateAndCancel() } + await CloudRefreshURLProtocol.reset() + await CloudRefreshURLProtocol.holdResponses() + let original = try #require(fixture.auth.authenticatedSessionIdentity) + let reader = Task { try await identityRead(operation, fixture: fixture) } + await CloudRefreshURLProtocol.waitUntilStarted() + await authClient.holdValidationProbe() + let validation = Task { await fixture.auth.revalidateSession() } + await authClient.waitUntilValidationProbe() + #expect(fixture.auth.authenticatedSessionIdentity == nil) + #expect(fixture.auth.isAuthenticatedSessionIdentityCurrent(original)) + await CloudRefreshURLProtocol.releaseResponses() + let result = await reader.result + await authClient.releaseValidationProbe() + await validation.value + try result.get() + #expect(await CloudRefreshURLProtocol.requestCounts().values.reduce(0, +) == 1) + } + + @Test("Signed-out shared reads fail before the network", arguments: ["list", "stats", "usage"]) + func signedOutReadAdmission(operation: String) async throws { + let fixture = try await CloudRefreshFixture.make(authClient: CloudReadIdentityAuthClient()) + defer { fixture.session.invalidateAndCancel() } + await CloudRefreshURLProtocol.reset() + await fixture.auth.signOut() + do { try await identityRead(operation, fixture: fixture); Issue.record("Signed-out read succeeded") } + catch VMClientError.notSignedIn {} catch MachineUsageClientError.notSignedIn {} catch { Issue.record("\(error)") } + #expect(await CloudRefreshURLProtocol.requestCounts().isEmpty) + #expect(await fixture.readRequests.entries.isEmpty) + } + + @Test("Explicit team usage does not require the selected team") + @MainActor + func explicitTeamUsageAllowsCrossTeamQuery() async throws { + let fixture = try await CloudRefreshFixture.make(authClient: CloudReadIdentityAuthClient()) + defer { fixture.session.invalidateAndCancel() } + await CloudRefreshURLProtocol.reset() + let usage = MachineUsageClient(session: fixture.session, auth: fixture.auth, readRequests: fixture.readRequests) + + let result = try await usage.teamUsage(teamID: "explicit-team") + + #expect(result.kind == .ready) + #expect(await CloudRefreshURLProtocol.requestCounts().values.reduce(0, +) == 1) + } + + private func identityRead(_ operation: String, fixture: CloudRefreshFixture) async throws { + switch operation { + case "list": _ = try await fixture.client.listPage() + case "stats": _ = try await fixture.client.stats(id: "fixture-0") + default: + let usage = MachineUsageClient(session: fixture.session, auth: fixture.auth, readRequests: fixture.readRequests) + _ = try await usage.teamUsage() + } + } +} diff --git a/cmuxTests/VMClientReadCoalescingTests.swift b/cmuxTests/VMClientReadCoalescingTests.swift new file mode 100644 index 000000000000..e89f526a660a --- /dev/null +++ b/cmuxTests/VMClientReadCoalescingTests.swift @@ -0,0 +1,421 @@ +import CMUXAuthCore +import CmuxAuthRuntime +import Foundation +import Testing +import os + +#if canImport(cmux_DEV) +@testable import cmux_DEV +#elseif canImport(cmux) +@testable import cmux +#endif + +@MainActor +@Suite("Cloud read request ownership", .serialized) +struct VMClientReadCoalescingTests { + @Test("Overlapping machine stats callers share one HTTP request") + func statsReadersShareTransport() async throws { + let fixture = try await CloudRefreshFixture.make() + defer { fixture.session.invalidateAndCancel() } + await CloudRefreshURLProtocol.reset() + await CloudRefreshURLProtocol.holdResponses() + let requests = Task { + await withTaskGroup(of: Void.self) { group in + for _ in 0..<4 { + for machine in 0..<10 { + group.addTask { _ = try? await fixture.client.stats(id: "fixture-\(machine)") } + } + } + } + } + try await eventually { await fixture.readRequests.entries.values.reduce(0) { $0 + $1.waiters.count } == 40 } + try await eventually { await CloudRefreshURLProtocol.requestCounts().count == 10 } + await CloudRefreshURLProtocol.releaseResponses() + await requests.value + let counts = await CloudRefreshURLProtocol.requestCounts() + #expect(counts.count == 10) + #expect(counts.values.allSatisfy { $0 == 1 }, "Four owners must share one read per machine: \(counts.values.sorted())") + } + + @Test("Overlapping machine list callers share one HTTP request") + func listReadersShareTransport() async throws { + let fixture = try await CloudRefreshFixture.make() + defer { fixture.session.invalidateAndCancel() } + await CloudRefreshURLProtocol.reset() + await CloudRefreshURLProtocol.holdResponses() + let requests = Task { + await withTaskGroup(of: Void.self) { group in + for _ in 0..<4 { group.addTask { _ = try? await fixture.client.listPage() } } + } + } + try await eventually { await fixture.readRequests.entries.values.first?.waiters.count == 4 } + await CloudRefreshURLProtocol.releaseResponses() + await requests.value + let counts = await CloudRefreshURLProtocol.requestCounts() + #expect(counts.values.reduce(0, +) == 1) + } + @Test("A hidden panel cancels its list and cannot start stats from a late result") + func hiddenPanelCancelsFollowupWork() async throws { + let fixture = try await CloudRefreshFixture.make() + defer { fixture.session.invalidateAndCancel() } + await CloudRefreshURLProtocol.reset() + await CloudRefreshURLProtocol.holdResponses() + let model = MachinesPanelViewModel(client: fixture.client, isCloudEnabled: { true }) + model.startPolling() + await CloudRefreshURLProtocol.waitUntilStarted() + model.beginOperation("fixture operation") + let stopBaseline = await CloudRefreshURLProtocol.currentStopCount() + model.stopPolling() + model.endOperation() + await CloudRefreshURLProtocol.waitUntilStopped(after: stopBaseline) + #expect(!model.isLoading) + #expect(model.machines.isEmpty) + await fixture.readRequests.networkChanged(isOnline: true) + for _ in 0..<10 { await Task.yield() } + #expect(await CloudRefreshURLProtocol.requestCounts().values.reduce(0, +) == 1) + } + + @Test("Dropping a view model releases it and cancels its pending list") + func viewModelTeardown() async throws { + let fixture = try await CloudRefreshFixture.make() + defer { fixture.session.invalidateAndCancel() } + await CloudRefreshURLProtocol.reset() + await CloudRefreshURLProtocol.holdResponses() + var model: MachinesPanelViewModel? = MachinesPanelViewModel(client: fixture.client, isCloudEnabled: { true }) + weak var weakModel: MachinesPanelViewModel? + weakModel = model + model?.refresh() + await CloudRefreshURLProtocol.waitUntilStarted() + let stopBaseline = await CloudRefreshURLProtocol.currentStopCount() + model = nil + #expect(weakModel == nil) + await CloudRefreshURLProtocol.waitUntilStopped(after: stopBaseline) + } + + @Test("A failed stats sample clears the last live reading") + func failedStatsAreUnavailable() async throws { + let fixture = try await CloudRefreshFixture.make() + defer { fixture.session.invalidateAndCancel() } + await CloudRefreshURLProtocol.reset() + let model = MachinesPanelViewModel(client: fixture.client, isCloudEnabled: { true }) + defer { model.stopPolling() } + model.refresh() + try await eventually { model.machines.first?.stats?.state == .awake } + await CloudRefreshURLProtocol.configure(.statsUnavailable) + model.refresh() + try await eventually { !model.isLoading && model.machines.first?.stats?.state == .unknown } + #expect(model.machines.count == 1) + #expect(model.listProblem == nil) + #expect(model.machines.first?.stats?.cpus == 2) + #expect(model.machines.first?.stats?.cpuPercent == nil) + #expect(model.machines.first?.stats?.memoryUsedMb == nil) + #expect(model.machines.first?.stats?.diskUsedMb == nil) + } + + @Test("Known offline state clears live samples without waiting for the next poll") + func offlinePresentation() async throws { + let fixture = try await CloudRefreshFixture.make() + defer { fixture.session.invalidateAndCancel() } + await CloudRefreshURLProtocol.reset() + let model = MachinesPanelViewModel(client: fixture.client, isCloudEnabled: { true }) + defer { model.stopPolling() } + model.refresh() + try await eventually { model.machines.first?.stats?.state == .awake } + await fixture.readRequests.networkChanged(isOnline: false) + for _ in 0..<10 { await Task.yield() } + #expect(model.machines.first?.stats?.state == .unknown) + #expect(model.machines.first?.stats?.cpus == 2) + #expect(model.machines.first?.stats?.cpuPercent == nil) + #expect(model.listProblem == .unreachable) + #expect(model.lastErrorDescription == URLError(.notConnectedToInternet).localizedDescription) + } + + @Test("The VM operation budget cancels a slow transport") + func totalRequestBudget() async throws { + let clock = CloudReadManualClock() + let reads = CloudReadRequestCoordinator(clock: CloudRequestClock(clock), budget: .milliseconds(100)) + let fixture = try await CloudRefreshFixture.make(readRequests: reads) + defer { fixture.session.invalidateAndCancel() } + await CloudRefreshURLProtocol.reset() + await CloudRefreshURLProtocol.holdResponses() + let request = Task { try await fixture.client.stats(id: "fixture-0") } + await CloudRefreshURLProtocol.waitUntilStarted() + let stopBaseline = await CloudRefreshURLProtocol.currentStopCount() + clock.advance(by: .milliseconds(101)) + do { _ = try await request.value; Issue.record("request exceeded its total budget") } + catch { #expect((error as? URLError)?.code == .timedOut) } + await CloudRefreshURLProtocol.waitUntilStopped(after: stopBaseline) + await CloudRefreshURLProtocol.releaseResponses() + } + + @Test("HTTP Retry-After exceeds the budget without an early automatic retry") + func retryAfterAcrossCalls() async throws { + let clock = CloudReadManualClock() + let reads = CloudReadRequestCoordinator(clock: CloudRequestClock(clock)) + let fixture = try await CloudRefreshFixture.make(readRequests: reads) + defer { fixture.session.invalidateAndCancel() } + await CloudRefreshURLProtocol.reset() + await CloudRefreshURLProtocol.configure(.throttled) + for _ in 0..<2 { + do { _ = try await fixture.client.stats(id: "fixture-0"); Issue.record("throttle succeeded") } + catch VMClientError.httpStatus(429, _) {} catch { Issue.record("\(error)") } + } + #expect(await CloudRefreshURLProtocol.requestCounts().values.reduce(0, +) == 1) + await CloudRefreshURLProtocol.configure(.normal) + clock.advance(by: .seconds(60)) + #expect(try await fixture.client.stats(id: "fixture-0").state == .awake) + #expect(await CloudRefreshURLProtocol.requestCounts().values.reduce(0, +) == 2) + } + + @Test("Disabled Cloud rejects cached throttles before reuse", arguments: ["/api/vm", "/api/vm/fixture-0/stats"], [false, true]) + func gateClosesDuringCooldown(path: String, managedPolicy: Bool) async throws { + // Injected flags are synchronous across actors; this lock protects only test state. + let blocked = OSAllocatedUnfairLock(initialState: false) + let fixture = try await CloudRefreshFixture.make( + isDisabledByManagedPolicy: { managedPolicy && blocked.withLock { $0 } }, + isCloudEnabled: { managedPolicy || !blocked.withLock { $0 } } + ) + defer { fixture.session.invalidateAndCancel() } + await CloudRefreshURLProtocol.reset() + await CloudRefreshURLProtocol.configure(.throttled) + let first = try await fixture.client.request("GET", path: path) + #expect(first.1.statusCode == 429) + blocked.withLock { $0 = true } + do { + _ = try await fixture.client.request("GET", path: path) + Issue.record("Disabled Cloud reused a cached response") + } catch VMClientError.disabledByManagedPolicy where managedPolicy { + } catch VMClientError.cloudMachinesDisabled where !managedPolicy { + } catch { Issue.record("Unexpected gate error: \(error)") } + #expect(await CloudRefreshURLProtocol.requestCounts().values.reduce(0, +) == 1) + } + + @Test("A joined read cannot publish after its Cloud gate closes", arguments: ["/api/vm", "/api/vm/fixture-0/stats"], [false, true]) + func gateClosesDuringJoinedRead(path: String, managedPolicy: Bool) async throws { + // Injected flags are synchronous across actors; this lock protects only test state. + let blocked = OSAllocatedUnfairLock(initialState: false) + let fixture = try await CloudRefreshFixture.make( + isDisabledByManagedPolicy: { managedPolicy && blocked.withLock { $0 } }, + isCloudEnabled: { managedPolicy || !blocked.withLock { $0 } } + ) + defer { fixture.session.invalidateAndCancel() } + await CloudRefreshURLProtocol.reset() + let identity = try #require(fixture.auth.authenticatedSessionIdentity) + let key = CloudReadRequestCoordinator.Key(path: path, accountID: identity.accountID, + generation: identity.generation, teamID: fixture.auth.resolvedTeamID) + let gate = CloudReadResponseGate() + // Hold an already-admitted response at the shared owner boundary, after + // its transport's gate check, so the joining caller must enforce its own gate. + let existing = Task { try await fixture.readRequests.read(key) { + await gate.read(.init(data: Data(), http: HTTPURLResponse( + url: URL(string: "https://fixture.invalid")!, statusCode: 200, httpVersion: nil, headerFields: nil + )!)) + } } + try await eventually { await gate.requests == 1 } + let joined = Task { try await fixture.client.request("GET", path: path) } + try await eventually { await fixture.readRequests.entries[key]?.waiters.count == 2 } + blocked.withLock { $0 = true } + await gate.release() + _ = try await existing.value + do { + _ = try await joined.value + Issue.record("Disabled Cloud published an in-flight response") + } catch VMClientError.disabledByManagedPolicy where managedPolicy { + } catch VMClientError.cloudMachinesDisabled where !managedPolicy { + } catch { Issue.record("Unexpected gate error: \(error)") } + #expect(await CloudRefreshURLProtocol.requestCounts().isEmpty) + } + + @Test("Cloud access revocation remains available with both gates closed") + func revocationBypassesClosedGates() async throws { + let fixture = try await CloudRefreshFixture.make(isDisabledByManagedPolicy: { true }, isCloudEnabled: { false }) + defer { fixture.session.invalidateAndCancel() } + await CloudRefreshURLProtocol.reset() + try await fixture.client.revokeCloudAccess(deviceID: "fixture-device") + #expect(await CloudRefreshURLProtocol.requestCounts() == ["/api/vm/tunnel": 1]) + } + + @Test("A list delayed beyond a polling interval stays owned and stops when hidden") + func delayedListAcrossPoll() async throws { + let clock = CloudReadManualClock() + let reads = CloudReadRequestCoordinator(clock: CloudRequestClock(clock), budget: .seconds(90)) + let fixture = try await CloudRefreshFixture.make(readRequests: reads) + defer { fixture.session.invalidateAndCancel() } + await CloudRefreshURLProtocol.reset() + await CloudRefreshURLProtocol.holdResponses() + let model = MachinesPanelViewModel(client: fixture.client, pollingClock: clock, isCloudEnabled: { true }) + model.startPolling() + await CloudRefreshURLProtocol.waitUntilStarted() + try await eventually { clock.pendingSleeperCount == 2 } + clock.advance(by: .seconds(45)) + try await eventually { clock.pendingSleeperCount == 2 } + #expect(await CloudRefreshURLProtocol.requestCounts().values.reduce(0, +) == 1) + let stopBaseline = await CloudRefreshURLProtocol.currentStopCount() + model.stopPolling() + await CloudRefreshURLProtocol.waitUntilStopped(after: stopBaseline) + await CloudRefreshURLProtocol.releaseResponses() + #expect(!model.isLoading) + #expect(model.machines.isEmpty) + } + + @Test("Visible and hidden panels share the same machines without hidden follow-up work") + func visibleAndHiddenOwners() async throws { + let fixture = try await CloudRefreshFixture.make() + defer { fixture.session.invalidateAndCancel() } + await CloudRefreshURLProtocol.reset() + await CloudRefreshURLProtocol.holdResponses() + let models = (0..<4).map { _ in MachinesPanelViewModel(client: fixture.client, isCloudEnabled: { true }) } + defer { for model in models { model.stopPolling() } } + for model in models { model.startPolling() } + models[2].stopPolling() + models[3].stopPolling() + try await eventually { await fixture.readRequests.entries.values.first?.waiters.count == 2 } + await CloudRefreshURLProtocol.releaseResponses() + try await eventually { models[0].machines.first?.stats != nil && models[1].machines.first?.stats != nil } + let counts = await CloudRefreshURLProtocol.requestCounts() + #expect(counts["/api/vm"] == 1) + #expect(counts["/api/vm/fixture-0/stats"] == 1) + #expect(models[2].machines.isEmpty && models[3].machines.isEmpty) + } + + @Test("An authoritative fleet replaces the old stats batch and preserves other readers") + func changedFleetReplacesStatsBatch() async throws { + let fixture = try await CloudRefreshFixture.make() + defer { fixture.session.invalidateAndCancel() } + await CloudRefreshURLProtocol.reset() + await CloudRefreshURLProtocol.holdResponses() + let model = MachinesPanelViewModel(client: fixture.client, isCloudEnabled: { true }) + defer { model.stopPolling() } + func acceptFleet(_ ids: [String]) { + let page = VMListPage(vms: ids.map { + VMSummary(id: $0, provider: "fixture", status: "running", image: "desktop-vnc", createdAt: 0, base: nil) + }, limits: nil) + model.applyRefreshResult(.success(page), generation: model.refreshGeneration, scope: nil) + } + let otherReader = Task { try await fixture.client.stats(id: "shared") } + defer { otherReader.cancel() } + await CloudRefreshURLProtocol.waitUntilStarted() + acceptFleet(["removed", "shared"]) + try await eventually { + let entries = await fixture.readRequests.entries + return entries.values.reduce(0) { $0 + $1.waiters.count } == 3 + } + await CloudRefreshURLProtocol.waitUntilStarted(2) + let oldID = try #require(model.statsID) + let oldTask = try #require(model.statsTask) + + acceptFleet(["added", "shared"]) + try #require(model.statsID != oldID, "The changed fleet must replace the running batch") + let replacementID = try #require(model.statsID) + let replacementTask = try #require(model.statsTask) + await oldTask.value + #expect(model.statsID == replacementID, "Old completion must not clear the replacement owner") + try await eventually { + let entries = await fixture.readRequests.entries + return entries.first { $0.key.path == "/api/vm/removed/stats" } == nil + && entries.first { $0.key.path == "/api/vm/shared/stats" }?.value.waiters.count == 2 + && entries.first { $0.key.path == "/api/vm/added/stats" }?.value.waiters.count == 1 + } + await CloudRefreshURLProtocol.waitUntilStarted(3) + #expect(await CloudRefreshURLProtocol.requestCounts() == [ + "/api/vm/removed/stats": 1, "/api/vm/shared/stats": 1, "/api/vm/added/stats": 1 + ]) + + acceptFleet([]) + try #require(model.statsID == nil && model.statsTask == nil) + await replacementTask.value + #expect(model.statsID == nil && model.statsTask == nil) + try await eventually { + let entries = await fixture.readRequests.entries + return entries.count == 1 && entries.first?.key.path == "/api/vm/shared/stats" + && entries.first?.value.waiters.count == 1 + } + await CloudRefreshURLProtocol.releaseResponses() + #expect(try await otherReader.value.state == .awake) + #expect(model.machines.isEmpty) + } + + @Test("Team usage shares offline state with list and stats while keeping its shorter budget") + func teamUsageNetworkState() async throws { + let clock = CloudReadManualClock() + let reads = CloudReadRequestCoordinator(clock: CloudRequestClock(clock)) + let fixture = try await CloudRefreshFixture.make(readRequests: reads) + defer { fixture.session.invalidateAndCancel() } + await CloudRefreshURLProtocol.reset() + await CloudRefreshURLProtocol.holdResponses() + let usage = MachineUsageClient(session: fixture.session, auth: fixture.auth, readRequests: reads) + let request = Task { try await usage.teamUsage() } + await CloudRefreshURLProtocol.waitUntilStarted() + let stopBaseline = await CloudRefreshURLProtocol.currentStopCount() + await reads.networkChanged(isOnline: false) + do { _ = try await request.value; Issue.record("usage survived offline") } + catch { #expect((error as? URLError)?.code == .notConnectedToInternet) } + await CloudRefreshURLProtocol.waitUntilStopped(after: stopBaseline) + do { _ = try await usage.teamUsage(); Issue.record("usage started offline") } + catch { #expect((error as? URLError)?.code == .notConnectedToInternet) } + #expect(await CloudRefreshURLProtocol.requestCounts().values.reduce(0, +) == 1) + await reads.networkChanged(isOnline: true) + let recovery = Task { try await usage.teamUsage() } + await CloudRefreshURLProtocol.waitUntilStarted(2) + clock.advance(by: .seconds(16)) + do { _ = try await recovery.value; Issue.record("usage exceeded its 15 second budget") } + catch { #expect((error as? URLError)?.code == .timedOut) } + await CloudRefreshURLProtocol.releaseResponses() + } + + @Test("Throttled team usage preserves Retry-After and failed request telemetry") + func teamUsageCooldown() async throws { + let clock = CloudReadManualClock() + let reads = CloudReadRequestCoordinator(clock: CloudRequestClock(clock)) + let fixture = try await CloudRefreshFixture.make(readRequests: reads) + defer { fixture.session.invalidateAndCancel() } + await CloudRefreshURLProtocol.reset() + await CloudRefreshURLProtocol.configure(.throttled) + let recorder = CloudOperationRecorder() + let usage = MachineUsageClient(session: fixture.session, auth: fixture.auth, operations: recorder, readRequests: reads) + for call in 0..<4 { + if call == 2 { clock.advance(by: .seconds(59)) } + if call == 3 { clock.advance(by: .seconds(1)) } + do { _ = try await usage.teamUsage(); Issue.record("throttled usage succeeded") } + catch MachineUsageClientError.httpStatus(429, _) {} catch { Issue.record("\(error)") } + #expect(await CloudRefreshURLProtocol.requestCounts() == ["/api/coderouter/vm-usage/team": call == 3 ? 2 : 1]) + } + let requests = recorder.operations.flatMap(\.steps).filter { $0.phase == .request } + #expect(requests.count == 2) + #expect(requests.allSatisfy { $0.outcome == .failure && $0.failure == .rateLimit }) + } + + @Test("Successful resize invalidates only its authenticated list and machine stats") + func mutationInvalidationUsesRequestScope() async throws { + let fixture = try await CloudRefreshFixture.make() + defer { fixture.session.invalidateAndCancel() } + await CloudRefreshURLProtocol.reset() + let identity = try #require(fixture.auth.authenticatedSessionIdentity) + let paths = ["/api/vm", "/api/vm/fixture-0/stats", "/api/vm/fixture-1/stats"] + let gates = paths.map { _ in CloudReadResponseGate() } + let requests = zip(paths, gates).map { path, gate in + let key = CloudReadRequestCoordinator.Key(path: path, accountID: identity.accountID, + generation: identity.generation, teamID: fixture.auth.resolvedTeamID) + return Task { try await fixture.readRequests.read(key) { + let status = await gate.requests == 0 ? 200 : 201 + return await gate.read(.init(data: Data(), http: HTTPURLResponse( + url: URL(string: "https://fixture.invalid")!, statusCode: status, httpVersion: nil, headerFields: nil + )!)) + } } + } + for gate in gates { try await eventually { await gate.requests == 1 } } + _ = try await fixture.client.request("POST", path: "/api/vm/fixture-0/resize", jsonBody: ["cpu": 4]) + for gate in gates { await gate.release() } + for (index, request) in requests.enumerated() { + #expect(try await request.value.http.statusCode == (index < 2 ? 201 : 200)) + #expect(await gates[index].requests == (index < 2 ? 2 : 1)) + } + } + + private func eventually(_ condition: () async -> Bool) async throws { + let deadline = ContinuousClock.now.advanced(by: .seconds(10)) + while !(await condition()), ContinuousClock.now < deadline { await Task.yield() } + try #require(await condition()) + } + +} diff --git a/cmuxTests/WorkspaceRemoteConnectionTests.swift b/cmuxTests/WorkspaceRemoteConnectionTests.swift index 890cff7d7318..646be5bdcddd 100644 --- a/cmuxTests/WorkspaceRemoteConnectionTests.swift +++ b/cmuxTests/WorkspaceRemoteConnectionTests.swift @@ -159,16 +159,11 @@ final class WorkspaceRemoteConnectionTests: XCTestCase { ) } - let exitSignal = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .userInitiated).async { - process.waitUntilExit() - exitSignal.signal() - } - let timedOut = exitSignal.wait(timeout: .now() + timeout) == .timedOut + let timedOut = waitForProcessExit(process, timeout: timeout) == .timedOut if timedOut { process.terminate() - _ = exitSignal.wait(timeout: .now() + 1) + _ = waitForProcessExit(process, timeout: 1) } let stdout = String(data: stdoutPipe.fileHandleForReading.readDataToEndOfFile(), encoding: .utf8) ?? "" @@ -5065,15 +5060,10 @@ final class CLINotifyProcessIntegrationTests: XCTestCase { process.standardError = stderrPipe try process.run() - let exitSignal = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .userInitiated).async { - process.waitUntilExit() - exitSignal.signal() - } defer { if process.isRunning { process.terminate() - _ = exitSignal.wait(timeout: .now() + 1) + _ = waitForProcessExit(process, timeout: 1) } _ = stdoutPipe.fileHandleForReading.readDataToEndOfFile() _ = stderrPipe.fileHandleForReading.readDataToEndOfFile() @@ -5165,15 +5155,10 @@ final class CLINotifyProcessIntegrationTests: XCTestCase { process.standardError = stderrPipe try process.run() - let exitSignal = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .userInitiated).async { - process.waitUntilExit() - exitSignal.signal() - } defer { if process.isRunning { process.terminate() - _ = exitSignal.wait(timeout: .now() + 1) + _ = waitForProcessExit(process, timeout: 1) } _ = stdoutPipe.fileHandleForReading.readDataToEndOfFile() _ = stderrPipe.fileHandleForReading.readDataToEndOfFile() @@ -5352,11 +5337,6 @@ final class CLINotifyProcessIntegrationTests: XCTestCase { } } - let exitSignal = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .userInitiated).async { - process.waitUntilExit() - exitSignal.signal() - } XCTAssertTrue( waitForProcess(process, toHoldOpenFile: transcriptURL.path, timeout: 2), @@ -5374,10 +5354,10 @@ final class CLINotifyProcessIntegrationTests: XCTestCase { try appendHandle.close() let serverTimedOut = serverHandled.wait(timeout: .now() + 5) == .timedOut - let timedOut = exitSignal.wait(timeout: .now() + 5) == .timedOut + let timedOut = waitForProcessExit(process, timeout: 5) == .timedOut if timedOut { process.terminate() - _ = exitSignal.wait(timeout: .now() + 1) + _ = waitForProcessExit(process, timeout: 1) } let stdout = String(data: stdoutPipe.fileHandleForReading.readDataToEndOfFile(), encoding: .utf8) ?? "" let stderr = String(data: stderrPipe.fileHandleForReading.readDataToEndOfFile(), encoding: .utf8) ?? "" diff --git a/scripts/ci/cmux_unit_test_shard.py b/scripts/ci/cmux_unit_test_shard.py index a4b2cbbebb77..45776c3f3979 100755 --- a/scripts/ci/cmux_unit_test_shard.py +++ b/scripts/ci/cmux_unit_test_shard.py @@ -45,6 +45,8 @@ "cmuxTests/ClaudeBackgroundWorkNotifyTests", "cmuxTests/CloudMachineDragSourceTests", "cmuxTests/CloudMachineOrderingTests", + "cmuxTests/CloudOperationRecorderTests", + "cmuxTests/CloudReadRequestCoordinatorTests", "cmuxTests/DeviceDirectoryLifecycleTests", "cmuxTests/DeviceDirectoryMergeTests", "cmuxTests/DeviceLinkReconnectPolicyTests", @@ -72,6 +74,7 @@ "cmuxTests/SidebarIssue8373StressTests", "cmuxTests/SidebarWorkspaceSwitchLayoutFaultTests", "cmuxTests/SocketACLReloadRegressionTests", + "cmuxTests/VMClientReadCoalescingTests", "cmuxTests/SurfaceMachineIDDeviceEncodingTests", } diff --git a/web/messages/ar.json b/web/messages/ar.json index 238d099960f8..58e02af0bee0 100644 --- a/web/messages/ar.json +++ b/web/messages/ar.json @@ -1,5 +1,16 @@ { "vmErrors": { + "guestInstall": { + "title": "لم يكتمل إعداد Cloud VM", + "reason": "لم يكتمل إعداد الضيف لـ Cloud VM.", + "message": "تعذر على cmux إكمال إعداد Cloud VM.", + "action": "أعد المحاولة. إذا استمرت المشكلة، تواصل مع الدعم." + }, + "createCleanupPending": { + "title": "تنظيف آلة Cloud الافتراضية معلق", + "message": "فشل إعداد آلة Cloud الافتراضية وما زال التنظيف معلقًا.", + "action": "تواصل مع الدعم وأرفق معرّف التتبع قبل إنشاء آلة أخرى." + }, "unsupported": { "title": "عملية Cloud VM غير متاحة", "reason": "هذا النوع من الأجهزة لا يوفر هذه العملية.", diff --git a/web/messages/bs.json b/web/messages/bs.json index 1667a99c4736..dcf64811f0c9 100644 --- a/web/messages/bs.json +++ b/web/messages/bs.json @@ -1,5 +1,16 @@ { "vmErrors": { + "guestInstall": { + "title": "Postavljanje Cloud VM-a nije završeno", + "reason": "Postavljanje gosta za Cloud VM nije završeno.", + "message": "cmux nije mogao završiti postavljanje Cloud VM-a.", + "action": "Pokušajte ponovo. Ako se problem nastavi, obratite se podršci." + }, + "createCleanupPending": { + "title": "Čišćenje Cloud VM-a na čekanju", + "message": "Postavljanje Cloud VM-a nije uspjelo i čišćenje je još na čekanju.", + "action": "Obratite se podršci s ID-jem praćenja prije kreiranja druge mašine." + }, "unsupported": { "title": "Operacija Cloud VM-a nije dostupna", "reason": "Ova vrsta mašine ne pruža ovu operaciju.", diff --git a/web/messages/da.json b/web/messages/da.json index 970737acc742..29815e2b1c50 100644 --- a/web/messages/da.json +++ b/web/messages/da.json @@ -1,5 +1,16 @@ { "vmErrors": { + "guestInstall": { + "title": "Opsætning af Cloud VM blev ikke fuldført", + "reason": "Gæsteopsætningen af Cloud VM blev ikke fuldført.", + "message": "cmux kunne ikke fuldføre opsætningen af Cloud VM.", + "action": "Prøv igen. Kontakt support, hvis problemet fortsætter." + }, + "createCleanupPending": { + "title": "Oprydning af Cloud VM afventer", + "message": "Opsætningen af Cloud VM mislykkedes, og oprydningen afventer stadig.", + "action": "Kontakt support med sporings-ID’et, før du opretter en ny maskine." + }, "unsupported": { "title": "Cloud VM-handling er ikke tilgængelig", "reason": "Denne maskintype tilbyder ikke denne handling.", diff --git a/web/messages/de.json b/web/messages/de.json index 54ae8ee2929a..7e51a03b1ff1 100644 --- a/web/messages/de.json +++ b/web/messages/de.json @@ -1,5 +1,16 @@ { "vmErrors": { + "guestInstall": { + "title": "Cloud-VM-Einrichtung nicht abgeschlossen", + "reason": "Die Gast-Einrichtung der Cloud-VM wurde nicht abgeschlossen.", + "message": "cmux konnte die Cloud-VM-Einrichtung nicht abschließen.", + "action": "Wiederholen Sie den Vorgang. Wenn der Fehler bestehen bleibt, wenden Sie sich an den Support." + }, + "createCleanupPending": { + "title": "Cloud-VM-Bereinigung ausstehend", + "message": "Die Einrichtung der Cloud-VM ist fehlgeschlagen. Die Bereinigung steht noch aus.", + "action": "Wenden Sie sich mit der Trace-ID an den Support, bevor Sie eine weitere Maschine erstellen." + }, "unsupported": { "title": "Cloud-VM-Vorgang nicht verfügbar", "reason": "Dieser Maschinentyp stellt diesen Vorgang nicht bereit.", diff --git a/web/messages/en.json b/web/messages/en.json index 31174665f263..70123fff875e 100644 --- a/web/messages/en.json +++ b/web/messages/en.json @@ -136,6 +136,17 @@ "expiredBody": "{machine} and everything on it are fine — only this link's session ended. Reopen the desktop from cmux (right-click the machine, then Open Desktop) to get a fresh one." }, "vmErrors": { + "guestInstall": { + "title": "Cloud VM setup did not finish.", + "reason": "The Cloud VM guest setup did not complete.", + "message": "cmux could not finish setting up the Cloud VM.", + "action": "Retry the operation. If it keeps failing, contact support." + }, + "createCleanupPending": { + "title": "Cloud VM cleanup pending", + "message": "Cloud VM setup failed and cleanup is still pending.", + "action": "Contact support with the trace ID before creating another machine." + }, "artifactUnavailable": { "title": "Cloud VM setup unavailable", "message": "Cloud VM setup is temporarily unavailable.", diff --git a/web/messages/es.json b/web/messages/es.json index e9db66686241..f33598714fdf 100644 --- a/web/messages/es.json +++ b/web/messages/es.json @@ -1,5 +1,16 @@ { "vmErrors": { + "guestInstall": { + "title": "La configuración de la Cloud VM no terminó", + "reason": "La configuración del invitado de la Cloud VM no se completó.", + "message": "cmux no pudo terminar de configurar la Cloud VM.", + "action": "Vuelve a intentarlo. Si el problema continúa, contacta con soporte." + }, + "createCleanupPending": { + "title": "Limpieza de la VM Cloud pendiente", + "message": "La configuración de la VM Cloud falló y la limpieza sigue pendiente.", + "action": "Contacta con soporte e indica el ID de seguimiento antes de crear otra máquina." + }, "unsupported": { "title": "Operación de Cloud VM no disponible", "reason": "Este tipo de máquina no ofrece esta operación.", diff --git a/web/messages/fr.json b/web/messages/fr.json index 1d4eac64962d..a015d1196fee 100644 --- a/web/messages/fr.json +++ b/web/messages/fr.json @@ -1,5 +1,16 @@ { "vmErrors": { + "guestInstall": { + "title": "Configuration de la Cloud VM incomplète", + "reason": "La configuration invitée de la Cloud VM n'est pas terminée.", + "message": "cmux n'a pas pu terminer la configuration de la Cloud VM.", + "action": "Réessayez. Si le problème persiste, contactez le support." + }, + "createCleanupPending": { + "title": "Nettoyage de la VM Cloud en attente", + "message": "La configuration de la VM Cloud a échoué et le nettoyage est encore en attente.", + "action": "Contactez le support avec l’identifiant de trace avant de créer une autre machine." + }, "unsupported": { "title": "Opération de Cloud VM indisponible", "reason": "Ce type de machine ne propose pas cette opération.", diff --git a/web/messages/it.json b/web/messages/it.json index bd930b601f7c..bd2e4718b5d1 100644 --- a/web/messages/it.json +++ b/web/messages/it.json @@ -1,5 +1,16 @@ { "vmErrors": { + "guestInstall": { + "title": "Configurazione della Cloud VM incompleta", + "reason": "La configurazione guest della Cloud VM non è stata completata.", + "message": "cmux non ha potuto completare la configurazione della Cloud VM.", + "action": "Riprova. Se il problema continua, contatta l'assistenza." + }, + "createCleanupPending": { + "title": "Pulizia della VM Cloud in sospeso", + "message": "La configurazione della VM Cloud non è riuscita e la pulizia è ancora in sospeso.", + "action": "Contatta il supporto con l’ID della traccia prima di creare un’altra macchina." + }, "unsupported": { "title": "Operazione Cloud VM non disponibile", "reason": "Questo tipo di macchina non offre questa operazione.", diff --git a/web/messages/ja.json b/web/messages/ja.json index 8224ba5493ff..af04bf5a1e46 100644 --- a/web/messages/ja.json +++ b/web/messages/ja.json @@ -136,6 +136,17 @@ "expiredBody": "{machine} とその中身は無事です。このリンクのセッションだけが終了しました。cmux でマシンを右クリックし、「デスクトップを開く」から新しく開き直してください。" }, "vmErrors": { + "guestInstall": { + "title": "Cloud VM のセットアップを完了できませんでした", + "reason": "Cloud VM のゲストセットアップが完了しませんでした。", + "message": "cmux は Cloud VM のセットアップを完了できませんでした。", + "action": "操作を再試行してください。問題が続く場合はサポートにお問い合わせください。" + }, + "createCleanupPending": { + "title": "Cloud VM のクリーンアップが保留中です", + "message": "Cloud VM のセットアップに失敗し、クリーンアップがまだ完了していません。", + "action": "別のマシンを作成する前に、トレース ID を添えてサポートにお問い合わせください。" + }, "artifactUnavailable": { "title": "Cloud VM のセットアップを利用できません", "message": "Cloud VM のセットアップを一時的に利用できません。", diff --git a/web/messages/km.json b/web/messages/km.json index f2e894ef39a1..20d2d2e2d508 100644 --- a/web/messages/km.json +++ b/web/messages/km.json @@ -1,5 +1,16 @@ { "vmErrors": { + "guestInstall": { + "title": "ការរៀបចំ Cloud VM មិនបានបញ្ចប់ទេ", + "reason": "ការរៀបចំបរិស្ថានភ្ញៀវរបស់ Cloud VM មិនបានបញ្ចប់ទេ។", + "message": "cmux មិនអាចបញ្ចប់ការរៀបចំ Cloud VM បានទេ។", + "action": "សូមព្យាយាមប្រតិបត្តិការម្តងទៀត។ ប្រសិនបើបញ្ហានៅតែបន្ត សូមទាក់ទងជំនួយ។" + }, + "createCleanupPending": { + "title": "ការសម្អាត Cloud VM កំពុងរង់ចាំ", + "message": "ការដំឡើង Cloud VM បានបរាជ័យ ហើយការសម្អាតមិនទាន់បញ្ចប់ទេ។", + "action": "សូមទាក់ទងផ្នែកគាំទ្រដោយផ្ដល់លេខសម្គាល់តាមដាន មុនពេលបង្កើតម៉ាស៊ីនផ្សេងទៀត។" + }, "unsupported": { "title": "ប្រតិបត្តិការ Cloud VM មិនអាចប្រើបាន", "reason": "ប្រភេទម៉ាស៊ីននេះមិនផ្តល់ប្រតិបត្តិការនេះទេ។", diff --git a/web/messages/ko.json b/web/messages/ko.json index 970540e942f4..2fdde0fcfa2d 100644 --- a/web/messages/ko.json +++ b/web/messages/ko.json @@ -1,5 +1,16 @@ { "vmErrors": { + "guestInstall": { + "title": "Cloud VM 설정이 완료되지 않았습니다", + "reason": "Cloud VM 게스트 설정이 완료되지 않았습니다.", + "message": "cmux가 Cloud VM 설정을 완료하지 못했습니다.", + "action": "작업을 다시 시도하세요. 문제가 계속되면 지원팀에 문의하세요." + }, + "createCleanupPending": { + "title": "Cloud VM 정리 대기 중", + "message": "Cloud VM 설정에 실패했으며 정리가 아직 완료되지 않았습니다.", + "action": "다른 머신을 만들기 전에 추적 ID와 함께 지원팀에 문의하세요." + }, "unsupported": { "title": "Cloud VM 작업을 사용할 수 없음", "reason": "이 머신 종류는 이 작업을 제공하지 않습니다.", diff --git a/web/messages/no.json b/web/messages/no.json index c21a346c8c08..642ec06fdbff 100644 --- a/web/messages/no.json +++ b/web/messages/no.json @@ -1,5 +1,16 @@ { "vmErrors": { + "guestInstall": { + "title": "Oppsettet av Cloud VM ble ikke fullført", + "reason": "Gjesteoppsettet for Cloud VM ble ikke fullført.", + "message": "cmux kunne ikke fullføre oppsettet av Cloud VM.", + "action": "Prøv operasjonen på nytt. Kontakt kundestøtte hvis problemet fortsetter." + }, + "createCleanupPending": { + "title": "Opprydding av Cloud VM venter", + "message": "Oppsettet av Cloud VM mislyktes, og oppryddingen venter fortsatt.", + "action": "Kontakt kundestøtte med sporings-ID-en før du oppretter en ny maskin." + }, "unsupported": { "title": "Cloud VM-operasjon er ikke tilgjengelig", "reason": "Denne maskintypen tilbyr ikke denne operasjonen.", diff --git a/web/messages/pl.json b/web/messages/pl.json index 5ed5c5e3e1fc..12f820c1cd9e 100644 --- a/web/messages/pl.json +++ b/web/messages/pl.json @@ -1,5 +1,16 @@ { "vmErrors": { + "guestInstall": { + "title": "Konfiguracja Cloud VM nie została ukończona", + "reason": "Konfiguracja gościa Cloud VM nie została ukończona.", + "message": "cmux nie może ukończyć konfiguracji Cloud VM.", + "action": "Spróbuj ponownie. Jeśli problem będzie się powtarzał, skontaktuj się z pomocą techniczną." + }, + "createCleanupPending": { + "title": "Oczekiwanie na usunięcie pozostałości maszyny Cloud VM", + "message": "Konfiguracja maszyny Cloud VM nie powiodła się, a usuwanie pozostałości nadal oczekuje.", + "action": "Przed utworzeniem kolejnej maszyny skontaktuj się z pomocą techniczną, podając identyfikator śledzenia." + }, "unsupported": { "title": "Operacja Cloud VM jest niedostępna", "reason": "Ten typ maszyny nie udostępnia tej operacji.", diff --git a/web/messages/pt-BR.json b/web/messages/pt-BR.json index a703329c5883..b60af8314d9e 100644 --- a/web/messages/pt-BR.json +++ b/web/messages/pt-BR.json @@ -1,5 +1,16 @@ { "vmErrors": { + "guestInstall": { + "title": "A configuração da Cloud VM não foi concluída", + "reason": "A configuração convidada da Cloud VM não foi concluída.", + "message": "O cmux não conseguiu concluir a configuração da Cloud VM.", + "action": "Tente novamente. Se o problema persistir, entre em contato com o suporte." + }, + "createCleanupPending": { + "title": "Limpeza da VM Cloud pendente", + "message": "A configuração da VM Cloud falhou e a limpeza ainda está pendente.", + "action": "Entre em contato com o suporte e informe o ID de rastreamento antes de criar outra máquina." + }, "unsupported": { "title": "Operação de Cloud VM indisponível", "reason": "Este tipo de máquina não oferece esta operação.", diff --git a/web/messages/ru.json b/web/messages/ru.json index e0c50ae14071..dd6186a92180 100644 --- a/web/messages/ru.json +++ b/web/messages/ru.json @@ -1,5 +1,16 @@ { "vmErrors": { + "guestInstall": { + "title": "Настройка Cloud VM не завершена", + "reason": "Настройка гостевой среды Cloud VM не завершена.", + "message": "cmux не удалось завершить настройку Cloud VM.", + "action": "Повторите операцию. Если проблема сохраняется, обратитесь в поддержку." + }, + "createCleanupPending": { + "title": "Очистка Cloud VM ожидается", + "message": "Не удалось настроить Cloud VM. Очистка ещё не завершена.", + "action": "Перед созданием другой машины обратитесь в поддержку и укажите идентификатор трассировки." + }, "unsupported": { "title": "Операция Cloud VM недоступна", "reason": "Этот тип машины не поддерживает эту операцию.", diff --git a/web/messages/th.json b/web/messages/th.json index be271bb9bc29..b9134bf253a2 100644 --- a/web/messages/th.json +++ b/web/messages/th.json @@ -1,5 +1,16 @@ { "vmErrors": { + "guestInstall": { + "title": "การตั้งค่า Cloud VM ไม่เสร็จสมบูรณ์", + "reason": "การตั้งค่าเกสต์ของ Cloud VM ไม่เสร็จสมบูรณ์", + "message": "cmux ไม่สามารถตั้งค่า Cloud VM ให้เสร็จสมบูรณ์ได้", + "action": "ลองดำเนินการอีกครั้ง หากปัญหายังคงอยู่ โปรดติดต่อฝ่ายสนับสนุน" + }, + "createCleanupPending": { + "title": "รอการล้างข้อมูล Cloud VM", + "message": "ตั้งค่า Cloud VM ไม่สำเร็จและยังล้างข้อมูลไม่เสร็จ", + "action": "ติดต่อฝ่ายสนับสนุนพร้อมรหัสการติดตามก่อนสร้างเครื่องอื่น" + }, "unsupported": { "title": "ไม่สามารถใช้การดำเนินการ Cloud VM ได้", "reason": "ประเภทเครื่องนี้ไม่มีการดำเนินการนี้", diff --git a/web/messages/tr.json b/web/messages/tr.json index c61616c5fb2a..ccd7e21162bc 100644 --- a/web/messages/tr.json +++ b/web/messages/tr.json @@ -1,5 +1,16 @@ { "vmErrors": { + "guestInstall": { + "title": "Cloud VM kurulumu tamamlanamadı", + "reason": "Cloud VM konuk kurulumu tamamlanmadı.", + "message": "cmux, Cloud VM kurulumunu tamamlayamadı.", + "action": "İşlemi yeniden deneyin. Sorun devam ederse destek ekibiyle iletişime geçin." + }, + "createCleanupPending": { + "title": "Cloud VM temizliği bekleniyor", + "message": "Cloud VM kurulumu başarısız oldu ve temizlik henüz tamamlanmadı.", + "action": "Başka bir makine oluşturmadan önce izleme kimliğiyle destek ekibine başvurun." + }, "unsupported": { "title": "Cloud VM işlemi kullanılamıyor", "reason": "Bu makine türü bu işlemi sağlamıyor.", diff --git a/web/messages/uk.json b/web/messages/uk.json index bb46460e97ec..30b67e5f6d91 100644 --- a/web/messages/uk.json +++ b/web/messages/uk.json @@ -1,5 +1,16 @@ { "vmErrors": { + "guestInstall": { + "title": "Налаштування Cloud VM не завершено", + "reason": "Налаштування гостьового середовища Cloud VM не завершено.", + "message": "cmux не вдалося завершити налаштування Cloud VM.", + "action": "Повторіть операцію. Якщо проблема не зникає, зверніться до служби підтримки." + }, + "createCleanupPending": { + "title": "Очищення Cloud VM очікується", + "message": "Не вдалося налаштувати Cloud VM. Очищення ще не завершено.", + "action": "Перш ніж створювати іншу машину, зверніться до підтримки та вкажіть ідентифікатор трасування." + }, "unsupported": { "title": "Операція Cloud VM недоступна", "reason": "Цей тип машини не підтримує цю операцію.", diff --git a/web/messages/zh-CN.json b/web/messages/zh-CN.json index e1fd8fe3b57d..a54b855929f0 100644 --- a/web/messages/zh-CN.json +++ b/web/messages/zh-CN.json @@ -1,5 +1,16 @@ { "vmErrors": { + "guestInstall": { + "title": "云端虚拟机设置未完成", + "reason": "云端虚拟机的来宾设置未完成。", + "message": "cmux 无法完成云端虚拟机的设置。", + "action": "请重试此操作。如果问题持续,请联系支持。" + }, + "createCleanupPending": { + "title": "Cloud VM 清理待完成", + "message": "Cloud VM 设置失败,清理仍未完成。", + "action": "创建另一台机器之前,请提供跟踪 ID 联系支持团队。" + }, "unsupported": { "title": "云端虚拟机操作不可用", "reason": "此机器类型不提供此操作。", diff --git a/web/messages/zh-TW.json b/web/messages/zh-TW.json index d176daa5e8d2..8d22b4705288 100644 --- a/web/messages/zh-TW.json +++ b/web/messages/zh-TW.json @@ -1,5 +1,16 @@ { "vmErrors": { + "guestInstall": { + "title": "Cloud VM 設定未完成", + "reason": "Cloud VM 的來賓環境設定未完成。", + "message": "cmux 無法完成 Cloud VM 的設定。", + "action": "請重試此操作。如果問題持續,請聯絡支援。" + }, + "createCleanupPending": { + "title": "Cloud VM 清理待完成", + "message": "Cloud VM 設定失敗,清理仍未完成。", + "action": "建立另一台機器之前,請提供追蹤 ID 聯絡支援團隊。" + }, "unsupported": { "title": "雲端虛擬機操作無法使用", "reason": "此機器類型不提供這項操作。", diff --git a/web/services/observability/cloudServerError.ts b/web/services/observability/cloudServerError.ts index 67bb3c67ac7d..df38a8f30a19 100644 --- a/web/services/observability/cloudServerError.ts +++ b/web/services/observability/cloudServerError.ts @@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto"; import { after } from "next/server"; import type { VmRequestContext } from "../vms/requestContext"; import type { VmErrorResponseInput } from "../vms/routeHelpers"; -import { cloudOperations, type CloudTelemetryClient, type CloudTelemetrySpan } from "./cloudTelemetryContract"; +import { cloudChannels, cloudOperations, type CloudTelemetryClient, type CloudTelemetrySpan } from "./cloudTelemetryContract"; import { acceptCloudTelemetry } from "./cloudTelemetryRepository"; import { drainCloudDiagnostics } from "./cloudTelemetryDelivery"; @@ -11,7 +11,7 @@ export function retainCloudServerError(input: VmErrorResponseInput, context: VmR if (!context?.userId || !context.traceId || !context.spanId) return; const now = Date.now(); const rawChannel = context.client.channel; - const channel = rawChannel === "nightly" ? "nightly" : rawChannel === "stable" || rawChannel === "production" ? "production" : rawChannel === "dev" ? "dev" : "unknown"; + const channel = rawChannel === "stable" ? "production" : cloudChannels.find((value) => value === rawChannel) ?? "unknown"; const client: CloudTelemetryClient = { channel, version: safeVersion(context.client.version), build: context.client.build?.match(/^[0-9]{1,20}$/)?.[0] ?? "0", diff --git a/web/services/observability/cloudTelemetryContract.ts b/web/services/observability/cloudTelemetryContract.ts index c5b292aebe25..f954b650cda8 100644 --- a/web/services/observability/cloudTelemetryContract.ts +++ b/web/services/observability/cloudTelemetryContract.ts @@ -17,8 +17,9 @@ export const cloudPhases = [ export const cloudFailures = [ "authentication", "session_refresh", "permission", "plan", "rate_limit", "conflict", "network", "timeout", "server", "response", "unsupported", "process", "protocol", - "not_found", "resource_limit", "storage", "cancelled", "unknown", + "not_found", "placement", "resource_limit", "storage", "cancelled", "unknown", ] as const; +export const cloudChannels = ["dev", "nightly", "production", "rc", "unknown"] as const; export type CloudTelemetrySpan = { readonly eventId: string; @@ -40,7 +41,7 @@ export type CloudTelemetrySpan = { readonly sourceLine?: number; }; export type CloudTelemetryClient = { - readonly channel: "dev" | "nightly" | "production" | "unknown"; + readonly channel: typeof cloudChannels[number]; readonly tag?: string; readonly version: string; readonly build: string; @@ -75,7 +76,7 @@ export function parseCloudTelemetryBatch(value: unknown, now = Date.now()): Clou function validClient(value: unknown): value is CloudTelemetryClient { if (!record(value) || !onlyKeys(value, clientKeys)) return false; - return member(value.channel, ["dev", "nightly", "production", "unknown"]) + return member(value.channel, cloudChannels) && (value.tag === undefined || textMatches(value.tag, /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/)) && textMatches(value.version, /^[0-9][0-9A-Za-z.+-]{0,39}$/) && textMatches(value.build, /^[0-9]{1,20}$/) diff --git a/web/services/vms/drivers/freestyleGuestCli.ts b/web/services/vms/drivers/freestyleGuestCli.ts new file mode 100644 index 000000000000..b1da16b2568d --- /dev/null +++ b/web/services/vms/drivers/freestyleGuestCli.ts @@ -0,0 +1,132 @@ +import { randomBytes } from "node:crypto"; +import { Clock, Data, Effect, Exit } from "effect"; +import type { Freestyle } from "freestyle"; +import { GUEST_CMUX_SHIM, GUEST_CMUX_SHIM_PATH } from "../guestCli"; +import type { GuestPromptIdentity } from "../guestPrompt"; +import type { GuestCliDistribution } from "../guestCliDistribution"; +import { guestCliInstallCommand } from "./guestCliInstallCommand"; + +export type FreestyleClientFactory = (timeoutMs?: number, signal?: AbortSignal) => Freestyle; +export const GUEST_INSTALL_DEADLINE_MS = 45_000; +const CLEANUP_DEADLINE_MS = 10_000; +type InstallStage = "upload" | "install" | "validate" | "verify" | "browser" | "prompt" | "publish"; +type InstallOutcome = "missing_status" | "invalid_status" | "provider_timeout" | "guest_exit" + | "cancelled" | "transport_timeout" | "transport" | "deadline"; + +export class GuestCliInstallError extends Data.TaggedError("GuestCliInstallError")<{ + readonly stage: InstallStage; + readonly outcome: InstallOutcome; + readonly exitCode?: number | null; + readonly diagnostic?: string; + readonly rollbackError?: string; + readonly elapsedMs?: number; + readonly cause?: unknown; + readonly cleanupCause?: unknown; +}> { + get message(): string { + return `guest cmux shim ${this.stage}: ${this.outcome}` + + (typeof this.exitCode === "number" ? ` (exit ${this.exitCode})` : "") + + (this.diagnostic ? `: ${this.diagnostic}` : ""); + } +} + +function rollbackErrorName(value: unknown): string | undefined { + return typeof value === "string" && /^[A-Za-z]{1,40}$/.test(value) ? value : undefined; +} + +function guestFailure(result: unknown): GuestCliInstallError | undefined { + const response = result && typeof result === "object" ? result as Record : {}; + const status = response.statusCode; + if (status === 0) return; + const outcome: InstallOutcome = status === undefined ? "missing_status" + : status === null ? "provider_timeout" + : typeof status === "number" && Number.isInteger(status) ? "guest_exit" : "invalid_status"; + const failure: { stage: InstallStage; diagnostic?: string; rollbackError?: string } = { stage: "install" }; + // Only our own bounded diagnostic record enters telemetry. Arbitrary guest + // stdout/stderr may contain private data and is deliberately not logged. + const marker = typeof response.stderr === "string" + ? response.stderr.split("\n").find((line) => line.startsWith("CMUX_GUEST_INSTALL_FAILURE=")) : undefined; + if (marker && marker.length < 512) { + try { + const detail = JSON.parse(marker.slice("CMUX_GUEST_INSTALL_FAILURE=".length)); + if (["validate", "verify", "browser", "prompt", "publish"].includes(detail.stage)) failure.stage = detail.stage; + if (typeof detail.error === "string" && /^[A-Za-z]{1,40}$/.test(detail.error)) { + failure.diagnostic = detail.error + + (Number.isInteger(detail.errno) ? ` errno=${detail.errno}` : "") + + (Number.isInteger(detail.exitCode) ? ` exit=${detail.exitCode}` : ""); + } + failure.rollbackError = rollbackErrorName(detail.rollbackError); + } catch { /* Unknown provider output is not evidence of an install stage. */ } + } + return new GuestCliInstallError({ + ...failure, outcome, + exitCode: status === null || typeof status === "number" ? status : undefined, + }); +} + +function transportFailure(stage: InstallStage, cause: unknown): GuestCliInstallError { + if (cause instanceof GuestCliInstallError) return cause; + const name = cause instanceof Error ? cause.name : undefined; + return new GuestCliInstallError({ + stage, cause, + outcome: name === "AbortError" ? "cancelled" : name === "TimeoutError" ? "transport_timeout" : "transport", + }); +} + +/** One overall deadline includes upload, exec and SDK background polling. */ +export function installFreestyleGuestCli( + client: FreestyleClientFactory, + vmId: string, + identity?: GuestPromptIdentity, + distribution?: GuestCliDistribution, +) { + return Effect.gen(function* () { + const started = yield* Clock.currentTimeMillis; + const temporaryPath = `${GUEST_CMUX_SHIM_PATH}.tmp-${randomBytes(12).toString("hex")}`; + let stage: InstallStage = "upload"; + let cleanupCause: unknown; + const install = Effect.tryPromise({ + try: async (signal) => { + const vm = client(GUEST_INSTALL_DEADLINE_MS, signal).vms.ref(vmId); + await vm.fs.writeTextFile(temporaryPath, GUEST_CMUX_SHIM, { mode: 0o755 }); + signal.throwIfAborted(); + stage = "install"; + const result = await vm.exec({ + command: guestCliInstallCommand(temporaryPath, identity, distribution), timeoutMs: 30_000, linuxUser: "root", + }); + const failure = guestFailure(result); + if (failure) throw failure; + }, + catch: (cause) => transportFailure(stage, cause), + }).pipe(Effect.timeoutFail({ + duration: GUEST_INSTALL_DEADLINE_MS, + onTimeout: () => new GuestCliInstallError({ stage, outcome: "deadline" }), + })); + const cleanup = Effect.tryPromise({ + // A fresh request owns cleanup; an aborted install signal cannot cancel it. + try: (signal) => client(CLEANUP_DEADLINE_MS, signal).vms.ref(vmId).fs.remove(temporaryPath), + catch: (cause) => cause, + }).pipe( + Effect.interruptible, + Effect.timeoutFail({ duration: CLEANUP_DEADLINE_MS, onTimeout: () => new Error("guest upload cleanup deadline") }), + Effect.catchAll((cause) => Effect.sync(() => { cleanupCause = cause; })), + ); + yield* Effect.acquireUseRelease( + Effect.void, + () => install, + (_, exit) => Exit.isFailure(exit) ? cleanup : Effect.void, + ).pipe(Effect.catchAll((error) => Clock.currentTimeMillis.pipe(Effect.flatMap((finished) => + Effect.fail(new GuestCliInstallError({ + stage: error.stage, + outcome: error.outcome, + elapsedMs: finished - started, + exitCode: error.exitCode, + diagnostic: error.diagnostic, + rollbackError: error.rollbackError, + cause: error.cause, + cleanupCause, + })) + )))); + return { elapsedMs: (yield* Clock.currentTimeMillis) - started }; + }); +} diff --git a/web/services/vms/drivers/freestyleRequestTiming.ts b/web/services/vms/drivers/freestyleRequestTiming.ts index 07d06c91a838..b19d34830a21 100644 --- a/web/services/vms/drivers/freestyleRequestTiming.ts +++ b/web/services/vms/drivers/freestyleRequestTiming.ts @@ -12,6 +12,7 @@ export type FreestyleRequestTiming = { type Options = { readonly timeoutMs: number; + readonly signal?: AbortSignal; readonly fetch?: typeof fetch; readonly record?: (event: FreestyleRequestTiming) => void; readonly now?: () => number; @@ -22,9 +23,11 @@ export function freestyleRequestFetch(options: Options): typeof fetch { const fetchImpl = options.fetch ?? fetch; const timeoutSignal = () => AbortSignal.timeout(options.timeoutMs); const request = ((input, init) => fetchImpl(input, { - ...(init ?? {}), signal: init?.signal - ? AbortSignal.any([init.signal, timeoutSignal()]) - : timeoutSignal(), + ...(init ?? {}), signal: AbortSignal.any([ + timeoutSignal(), + ...[options.signal, init?.signal ?? (input instanceof Request ? input.signal : undefined)] + .filter((signal): signal is AbortSignal => !!signal), + ]), })) as typeof fetch; if (!options.record) return request; diff --git a/web/services/vms/drivers/guestCliInstallCommand.ts b/web/services/vms/drivers/guestCliInstallCommand.ts new file mode 100644 index 000000000000..532f76548517 --- /dev/null +++ b/web/services/vms/drivers/guestCliInstallCommand.ts @@ -0,0 +1,324 @@ +import { createHash } from "node:crypto"; +import { GUEST_CMUX_SHIM, GUEST_CMUX_SHIM_PATH } from "../guestCli"; +import { GUEST_BROWSER_FILES, guestBrowserInstallCommand } from "../guestBrowser"; +import { + defaultGuestCliDistribution, + guestCliDistributionCommand, + guestCliDistributionInstallPaths, + guestCliDistributionPruneCommand, + type GuestCliDistribution, +} from "../guestCliDistribution"; +import { guestPromptInstallFiles, type GuestPromptIdentity } from "../guestPrompt"; +import { shellQuote } from "./cmuxTuiDaemon"; + +const digest = createHash("sha256").update(GUEST_CMUX_SHIM).digest("hex"); +function distributionPaths(manifest: GuestCliDistribution = defaultGuestCliDistribution): string[] { + return guestCliDistributionInstallPaths(manifest); +} + +const installPaths = [ + GUEST_CMUX_SHIM_PATH, + ...GUEST_BROWSER_FILES.map(({ path }) => path), + "/etc/cmux/browser-opener-version", + "/etc/cmux/prompt.bash", + "/etc/cmux/bashrc", + "/etc/cmux/.prompt-identity", + "/etc/cmux/vm-name", + "/etc/bash.bashrc", + "/etc/zsh/zshenv", + ...distributionPaths(), +]; +const promptName = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/; + +// Validate, stage, and publish the shim and its companion files as one guest +// transaction. The prompt lock is held across the browser and prompt writes, +// so a rename cannot observe or race a partially-installed generation. +const install = String.raw` +import fcntl, hashlib, json, os, shutil, stat, subprocess, sys, tempfile +source, target, digest, browser, distribution, prompt_json, paths_json, transaction_token, cleanup = sys.argv[1:] +stage = "validate" +lock = None +backup_root = None +backups = {} +paths = list(dict.fromkeys(json.loads(paths_json))) +lock_path = "/etc/cmux/.prompt-lock" + +def mime_paths(): + result = [] + for username in ("root", "cmux", "ubuntu"): + try: + lookup = subprocess.run(["getent", "passwd", username], check=False, + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True) + except OSError: + continue + fields = lookup.stdout.strip().split(":") if lookup.returncode == 0 else [] + home = fields[5] if len(fields) > 5 else "" + if not home.startswith("/"): + continue + result.extend([ + os.path.join(home, ".config/mimeapps.list"), + os.path.join(home, ".local/share/applications/mimeapps.list"), + ]) + return result + +paths = list(dict.fromkeys(paths + mime_paths())) + +def parent_dirs(path): + result = [] + current = os.path.dirname(path) + while current and current != "/": + result.append(current) + current = os.path.dirname(current) + return result + +directory_paths = list(dict.fromkeys( + os.path.dirname(path) for path in paths + [lock_path] if os.path.dirname(path) +)) + +def unsafe_symlink_target(path): + return path.endswith("/bash.bashrc") or path.endswith("/zshenv") or path.endswith("/.config/mimeapps.list") or path.endswith("/.local/share/applications/mimeapps.list") + +def validate_target(path): + if os.path.lexists(path): + if os.path.isdir(path) and not os.path.islink(path): + raise IsADirectoryError(path) + if os.path.islink(path) and unsafe_symlink_target(path): + raise ValueError("refusing to mutate a symlink target") + if not os.path.islink(path) and not stat.S_ISREG(os.lstat(path).st_mode): + raise ValueError("target is not a regular file") + parent = os.path.dirname(path) + if os.path.islink(parent): + raise ValueError("refusing a symlink parent directory") + if os.path.lexists(parent) and not os.path.isdir(parent): + raise NotADirectoryError(parent) + +def remove_path(path): + if not os.path.lexists(path): + return + if os.path.islink(path) or os.path.isfile(path): + os.unlink(path) + return + if os.path.isdir(path): + # Never recursively delete a guest directory while rolling back a + # file install. An empty directory created by a failed shell command + # is safe to remove; user data makes the rollback explicitly fail. + os.rmdir(path) + return + raise ValueError("refusing to remove a non-file install target") + +def snapshot_path(path): + if not os.path.lexists(path): + backups[path] = None + return + if os.path.isdir(path) and not os.path.islink(path): + raise IsADirectoryError(path) + if os.path.islink(path): + backups[path] = ("link", os.readlink(path)) + return + if not stat.S_ISREG(os.lstat(path).st_mode): + raise ValueError("install target is not a regular file") + backup = os.path.join(backup_root, str(len(backups))) + metadata = os.stat(path, follow_symlinks=False) + shutil.copyfile(path, backup) + backups[path] = ("file", backup, metadata.st_mode, metadata.st_uid, metadata.st_gid, metadata.st_atime_ns, metadata.st_mtime_ns) + +def restore_paths(): + failures = [] + for path, backup in reversed(list(backups.items())): + try: + remove_path(path) + if backup is None: + continue + kind, value, *metadata = backup + os.makedirs(os.path.dirname(path), exist_ok=True) + if kind == "link": + os.symlink(value, path) + else: + shutil.copyfile(value, path) + mode, uid, gid, atime_ns, mtime_ns = metadata + os.chmod(path, stat.S_IMODE(mode), follow_symlinks=False) + os.chown(path, uid, gid, follow_symlinks=False) + os.utime(path, ns=(atime_ns, mtime_ns), follow_symlinks=False) + except Exception as error: + failures.append(error) + if failures: + raise RuntimeError("rollback failed") + +def cleanup_generated(): + failures = [] + prefixes = [".prompt-" + transaction_token + "-", ".cmux-cli-", ".cmux-link-"] + prefixes.extend(os.path.basename(path) + "." + transaction_token + "." for path in paths) + for directory in directory_paths: + if not os.path.isdir(directory) or os.path.islink(directory): + continue + try: + for name in os.listdir(directory): + if not any(name.startswith(prefix) for prefix in prefixes): + continue + candidate = os.path.join(directory, name) + if os.path.isfile(candidate) or os.path.islink(candidate): + os.unlink(candidate) + except Exception as error: + failures.append(error) + if failures: + raise RuntimeError("temporary cleanup failed") + +def cleanup_created_directories(before): + for directory in sorted(directory_paths, key=len, reverse=True): + if before.get(directory, False) or not os.path.isdir(directory) or os.path.islink(directory): + continue + try: + os.rmdir(directory) + except OSError: + # Non-empty directories are never recursively removed. They may + # contain an existing guest file created outside this transaction. + continue + +def replace_file(directory, name, content): + target_path = os.path.join(directory, name) + if os.path.isfile(target_path) and not os.path.islink(target_path): + with open(target_path, "r") as stream: + if stream.read() == content: + return + fd, temporary = tempfile.mkstemp(prefix=".prompt-" + transaction_token + "-", dir=directory) + try: + with os.fdopen(fd, "w") as stream: + stream.write(content) + os.fchmod(stream.fileno(), 0o644) + os.replace(temporary, target_path) + finally: + if os.path.exists(temporary): + os.unlink(temporary) + +def install_prompt(payload): + if not payload: + return + directory = "/etc/cmux" + os.makedirs(directory, exist_ok=True) + incoming = payload["identity"] + try: + with open(os.path.join(directory, ".prompt-identity")) as stream: + current = json.load(stream) + except (FileNotFoundError, ValueError, OSError): + current = {} + if not isinstance(current, dict) or not isinstance(current.get("revision", -1), int): + current = {} + if current.get("machineId") != incoming["machineId"] or current.get("revision", -1) <= incoming["revision"]: + replace_file(directory, "vm-name", incoming["name"] + "\n") + replace_file(directory, ".prompt-identity", json.dumps(incoming)) + for name, content in payload["files"].items(): + replace_file(directory, name, content) + +try: + os.makedirs(os.path.dirname(lock_path), exist_ok=True) + lock = open(lock_path, "a+") + fcntl.flock(lock.fileno(), fcntl.LOCK_EX) + before_directories = { + path: os.path.isdir(path) for path in directory_paths + } + for path in paths: + validate_target(path) + fd = os.open(source, os.O_RDONLY | os.O_NOFOLLOW) + with os.fdopen(fd, "rb") as stream: + if not stat.S_ISREG(os.fstat(stream.fileno()).st_mode): + raise ValueError("upload is not a regular file") + if hashlib.sha256(stream.read()).hexdigest() != digest: + raise ValueError("upload checksum mismatch") + os.fchmod(stream.fileno(), 0o755) + backup_root = tempfile.mkdtemp(prefix=".cmux-install-", dir=os.path.dirname(lock_path)) + for path in paths: + snapshot_path(path) + stage = "verify" + subprocess.run([source, "--help"], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) + stage = "browser" + subprocess.run(["/bin/sh", "-c", browser], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) + stage = "publish" + subprocess.run(["/bin/sh", "-c", distribution], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) + stage = "prompt" + install_prompt(json.loads(prompt_json) if prompt_json else None) + stage = "publish" + os.replace(source, target) + cleanup_generated() + shutil.rmtree(backup_root) + backup_root = None + # Distribution releases and aliases were snapshotted with the rest of the + # install. Prune only after cleanup and backup disposal have committed the + # transaction; a stale-cache failure cannot trigger rollback with the old + # release already deleted. + if cleanup: + subprocess.run(["/bin/sh", "-c", cleanup], check=False, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) +except Exception as error: + rollback_error = None + restore_failed = False + try: + cleanup_generated() + except Exception as cleanup_error: + rollback_error = cleanup_error + try: + if backups: + restore_paths() + except Exception as restore_error: + rollback_error = restore_error + restore_failed = True + try: + cleanup_created_directories(before_directories if "before_directories" in globals() else {}) + except Exception as directory_error: + rollback_error = directory_error + if backup_root and not restore_failed: + try: + shutil.rmtree(backup_root) + except Exception as backup_error: + rollback_error = backup_error + # A distribution release created by this failed transaction is immutable + # cache state, not a previous generation. Remove only empty unreferenced + # release directories after restoring all snapshotted files; never recurse + # into a non-empty directory or touch an alias target. + try: + for directory in sorted(directory_paths, key=len, reverse=True): + if not os.path.basename(directory).startswith("cmux-cloud-"): + continue + if before_directories.get(directory, False) or not os.path.isdir(directory) or os.path.islink(directory): + continue + os.rmdir(directory) + except OSError: + pass + detail = { + "stage": stage, + "error": type(error).__name__, + "rollbackError": type(rollback_error).__name__ if rollback_error else None, + "errno": getattr(error, "errno", None), + "exitCode": getattr(error, "returncode", None), + } + print("CMUX_GUEST_INSTALL_FAILURE=" + json.dumps(detail), file=sys.stderr) + sys.exit(1) +finally: + if lock is not None: + fcntl.flock(lock.fileno(), fcntl.LOCK_UN) + lock.close() +`; + +export function guestCliInstallCommand( + temporaryPath: string, + identity?: GuestPromptIdentity, + manifest: GuestCliDistribution = defaultGuestCliDistribution, +): string { + if (identity && (!promptName.test(identity.name) || !Number.isSafeInteger(identity.revision))) { + throw new Error("Invalid Cloud prompt identity"); + } + const prompt = identity ? JSON.stringify({ + identity, + files: guestPromptInstallFiles, + }) : ""; + const transactionToken = temporaryPath.replace(/[^A-Za-z0-9_-]/g, "_"); + const browser = guestBrowserInstallCommand().replaceAll("XXXXXX", `${transactionToken}.XXXXXX`); + const distribution = guestCliDistributionCommand(false, manifest, undefined, undefined, false); + const cleanup = guestCliDistributionPruneCommand(manifest); + const paths = manifest === defaultGuestCliDistribution + ? installPaths + : [ + ...installPaths.filter((path) => !path.startsWith("/usr/local/libexec/cmux-cloud-")), + ...distributionPaths(manifest), + ]; + return `python3 -c ${shellQuote(install)} ${shellQuote(temporaryPath)} ${shellQuote(GUEST_CMUX_SHIM_PATH)} ${shellQuote(digest)} ${shellQuote(browser)} ${shellQuote(distribution)} ${shellQuote(prompt)} ${shellQuote(JSON.stringify(paths))} ${shellQuote(transactionToken)} ${shellQuote(cleanup)}`; +} diff --git a/web/services/vms/drivers/providerCreateCleanup.ts b/web/services/vms/drivers/providerCreateCleanup.ts new file mode 100644 index 000000000000..aa010f4f8c77 --- /dev/null +++ b/web/services/vms/drivers/providerCreateCleanup.ts @@ -0,0 +1,34 @@ +import { Effect } from "effect"; +import { FreestyleApiError } from "freestyle"; +import { ProviderError } from "./types"; +import type { FreestyleClientFactory } from "./freestyleGuestCli"; + +export function isProviderCreateCleanupError(value: unknown): value is ProviderCreateCleanupError { + return value instanceof ProviderCreateCleanupError; +} + +/** Retains the allocation until deletion is confirmed, including both failures. */ +export class ProviderCreateCleanupError extends ProviderError { + readonly name = "ProviderCreateCleanupError"; + constructor( + readonly providerVmId: string, + cause: unknown, + readonly cleanupCause: unknown, + ) { + super("freestyle", "create rollback is unconfirmed", cause); + } +} + +export function rollbackFreestyleCreate(client: FreestyleClientFactory, vmId: string, cause: unknown) { + return Effect.tryPromise({ + try: (signal) => client(15_000, signal).vms.ref(vmId).delete(), + catch: (cleanupCause) => new ProviderCreateCleanupError(vmId, cause, cleanupCause), + }).pipe(Effect.catchAll((error) => + error.cleanupCause instanceof FreestyleApiError + && error.cleanupCause.status === 404 && error.cleanupCause.code === "NOT_FOUND" + ? Effect.void : Effect.fail(error) + ), Effect.timeoutFail({ + duration: 15_000, + onTimeout: () => new ProviderCreateCleanupError(vmId, cause, new Error("create rollback deadline")), + })); +} diff --git a/web/services/vms/guestCliDistribution.ts b/web/services/vms/guestCliDistribution.ts index 0dd0f422d0fa..312889551491 100644 --- a/web/services/vms/guestCliDistribution.ts +++ b/web/services/vms/guestCliDistribution.ts @@ -2,6 +2,8 @@ // daemon. The archive contains the Rust facade and official CodeRouter core. import distribution from "./guestCliDistribution.json"; +export const defaultGuestCliDistribution: GuestCliDistribution = distribution; + export const GUEST_CMUX_ADAPTER_PATH = "/usr/local/libexec/cmux-cloud-adapter"; const LIBEXEC = "/usr/local/libexec"; const BIN = "/usr/local/bin"; @@ -12,18 +14,71 @@ export type GuestCliDistribution = { binaries: Record; }; +/** Every mutable path published by the distribution installer. */ +export function guestCliDistributionInstallPaths( + manifest: GuestCliDistribution = defaultGuestCliDistribution, + libexec = LIBEXEC, + bin = BIN, +): string[] { + const release = `${libexec}/cmux-cloud-${manifest.archiveSha256}`; + return [ + `${release}/cmux-cloud-cli`, + `${release}/coderouter`, + `${libexec}/cmux-coderouter`, + `${bin}/cmux`, + `${bin}/coderouter`, + `${bin}/cr`, + ]; +} + +/** Prune stale immutable releases without touching the active release. */ +export function guestCliDistributionPruneCommand( + manifest: GuestCliDistribution = defaultGuestCliDistribution, + libexec = LIBEXEC, + bin = BIN, +): string { + const settings = Buffer.from(JSON.stringify({ manifest })).toString("base64"); + const script = `import base64, json, pathlib, shutil, sys +config = json.loads(base64.b64decode('${settings}')) +root = pathlib.Path(sys.argv[1]) +release = root / ('cmux-cloud-' + config['manifest']['archiveSha256']) +if not root.is_dir(): + raise SystemExit(0) +active = set() +for path in (root / 'cmux-coderouter', pathlib.Path(sys.argv[2]) / 'cmux', pathlib.Path(sys.argv[2]) / 'coderouter', pathlib.Path(sys.argv[2]) / 'cr'): + try: + if path.is_symlink(): + active.add(path.resolve().parent) + except OSError: + pass +releases = [] +for candidate in root.iterdir(): + suffix = candidate.name.removeprefix('cmux-cloud-') + if candidate == release or candidate.is_symlink() or not candidate.is_dir() or len(suffix) != 64 or any(char not in '0123456789abcdef' for char in suffix): + continue + releases.append(candidate) +releases.sort(key=lambda path: path.stat().st_mtime_ns, reverse=True) +for obsolete in releases[2:]: + if obsolete.resolve() not in active: + shutil.rmtree(obsolete, ignore_errors=True) +`; + const quote = (value: string) => `'${value.replace(/'/g, `'\\''`)}'`; + return `python3 -c '${script.replace(/'/g, `'\\''`)}' ${quote(libexec)} ${quote(bin)}`; +} + /** Generate the same installer for create, attach healing, and local fixtures. */ export function guestCliDistributionCommand( verify = false, - manifest: GuestCliDistribution = distribution, + manifest: GuestCliDistribution = defaultGuestCliDistribution, libexec = LIBEXEC, bin = BIN, + prune = true, ): string { - const settings = Buffer.from(JSON.stringify({ verify, manifest, libexec, bin })).toString("base64"); - const script = `import base64, hashlib, io, json, os, pathlib, shutil, tarfile, tempfile, urllib.request + const settings = Buffer.from(JSON.stringify({ verify, manifest, prune })).toString("base64"); + const script = `import base64, hashlib, io, json, os, pathlib, shutil, sys, tarfile, tempfile, urllib.request config = json.loads(base64.b64decode('${settings}')) manifest = config['manifest'] -libexec, bindir = pathlib.Path(config['libexec']), pathlib.Path(config['bin']) +libexec, bindir = pathlib.Path(sys.argv[1]), pathlib.Path(sys.argv[2]) release = libexec / ('cmux-cloud-' + manifest['archiveSha256']) names = {'cmux-cloud-cli', 'coderouter'} if set(manifest['binaries']) != names: @@ -74,15 +129,25 @@ for path, target in links.items(): os.replace(temporary, path) finally: if os.path.lexists(temporary): os.unlink(temporary) -release_dirs = [] -for candidate in libexec.iterdir(): - suffix = candidate.name.removeprefix('cmux-cloud-') - if candidate == release or candidate.is_symlink() or not candidate.is_dir() or len(suffix) != 64 or any(char not in '0123456789abcdef' for char in suffix): - continue - release_dirs.append(candidate) -release_dirs.sort(key=lambda path: path.stat().st_mtime_ns, reverse=True) -for obsolete in release_dirs[2:]: - shutil.rmtree(obsolete, ignore_errors=True) +if config['prune']: + release_dirs = [] + for candidate in libexec.iterdir(): + suffix = candidate.name.removeprefix('cmux-cloud-') + if candidate == release or candidate.is_symlink() or not candidate.is_dir() or len(suffix) != 64 or any(char not in '0123456789abcdef' for char in suffix): + continue + release_dirs.append(candidate) + release_dirs.sort(key=lambda path: path.stat().st_mtime_ns, reverse=True) + active = set() + for path in links: + try: + if path.is_symlink(): + active.add(path.resolve().parent) + except OSError: + pass + for obsolete in release_dirs[2:]: + if obsolete.resolve() not in active: + shutil.rmtree(obsolete, ignore_errors=True) `; - return `python3 -c '${script.replace(/'/g, `'\\''`)}'`; + const quote = (value: string) => `'${value.replace(/'/g, `'\\''`)}'`; + return `python3 -c '${script.replace(/'/g, `'\\''`)}' ${quote(libexec)} ${quote(bin)}`; } diff --git a/web/services/vms/guestPrompt.ts b/web/services/vms/guestPrompt.ts index c231885a7d44..b6bb82439ddd 100644 --- a/web/services/vms/guestPrompt.ts +++ b/web/services/vms/guestPrompt.ts @@ -32,6 +32,10 @@ const assetPath = (relativePath: string) => fileURLToPath(new URL(relativePath, import.meta.url).toString()); const bashrc = readFileSync(assetPath("./images/devbox/cmux-bashrc"), "utf8"); const prompt = readFileSync(assetPath("./images/devbox/cmux-prompt.bash"), "utf8"); +export const guestPromptInstallFiles = { + "prompt.bash": prompt, + bashrc, +} as const; // Runs on lifecycle operations, never during shell startup or prompt drawing. // A lock serializes competing attaches/renames. Atomic replacement gives every diff --git a/web/services/vms/providerErrors.ts b/web/services/vms/providerErrors.ts index 8583a66f51eb..82a52401f9e2 100644 --- a/web/services/vms/providerErrors.ts +++ b/web/services/vms/providerErrors.ts @@ -108,6 +108,25 @@ export function isProviderNotFoundError(err: unknown): boolean { return legacyNotFound; } +/** + * A destructive delete is confirmed only by the provider's explicit + * not-found contract. A bare 404 with an unknown code may be an endpoint or + * gateway error and must retain the allocation for a later retry. + */ +export function isProviderDeletionConfirmed(err: unknown): boolean { + const seen = new Set(); + let current = err; + while (current && typeof current === "object" && !seen.has(current)) { + seen.add(current); + const candidate = current as ProviderFailure; + const status = httpStatus(candidate); + const code = String(candidate.code ?? "").toUpperCase(); + if (status !== undefined) return status === 404 && code === "NOT_FOUND"; + current = candidate.cause; + } + return false; +} + export function isProviderIdentityNotFoundError(err: unknown): boolean { if (!err || typeof err !== "object") return false; const candidate = err as ProviderFailure; diff --git a/web/services/vms/repository.ts b/web/services/vms/repository.ts index 6b390a0ef773..458b2f76c292 100644 --- a/web/services/vms/repository.ts +++ b/web/services/vms/repository.ts @@ -1,5 +1,5 @@ import { randomUUID } from "node:crypto"; -import { and, asc, count, desc, eq, gt, gte, inArray, isNotNull, isNull, lt, lte, ne, or, sql } from "drizzle-orm"; +import { and, asc, count, desc, eq, gt, gte, inArray, isNotNull, isNull, lt, lte, ne, or, sql, type SQL } from "drizzle-orm"; import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; @@ -333,6 +333,7 @@ export type VmRepositoryShape = { readonly userId: string; readonly code: string; readonly message: string; + readonly cleanupProviderVmId?: string; }) => Effect.Effect; readonly activeLimitCandidates: (input: { readonly userId: string; @@ -416,6 +417,32 @@ export type VmRepositoryShape = { readonly reconciliationCandidates: (input: { readonly limit: number; }) => Effect.Effect; + /** Provider allocations retained after an unconfirmed create rollback. */ + readonly pendingCreateCleanupCandidates?: (input: { + readonly limit: number; + }) => Effect.Effect; + /** Claim one retained allocation before provider I/O. The claim is fenced by a lease. */ + readonly claimCreateCleanup?: (input: { + readonly id: string; + readonly providerVmId: string; + readonly leaseId: string; + readonly now: Date; + readonly leaseExpiresAt: Date; + }) => Effect.Effect<{ readonly attempt: number } | null, VmDatabaseError>; + /** Release a failed cleanup attempt and persist its next retry time. */ + readonly deferCreateCleanup?: (input: { + readonly id: string; + readonly providerVmId: string; + readonly leaseId: string; + readonly nextAttemptAt: Date; + readonly now: Date; + }) => Effect.Effect; + /** Mark a retained create allocation failed after provider deletion is confirmed. */ + readonly resolveCreateCleanup?: (input: { + readonly id: string; + readonly providerVmId: string; + readonly leaseId: string; + }) => Effect.Effect; /** Live VM rows that currently claim a persistent home volume. */ readonly listLiveHomeVolumeNames?: (input: { readonly provider: ProviderId; @@ -452,6 +479,8 @@ export type VmRepositoryShape = { readonly id: string; readonly code: string; readonly message: string; + /** Keeps the allocation reserved and unready until an operator confirms cleanup. */ + readonly cleanupProviderVmId?: string; }) => Effect.Effect; /** Durable deletion intents not yet finalized, scoped to their source machine. */ readonly pendingSnapshotDeletions: (input: { @@ -659,6 +688,24 @@ export const FAILED_CREATE_RETRY_WINDOW_MS = 15 * 60 * 1000; * provider failure). */ export const PROVIDER_CREATE_UNAVAILABLE_FAILURE_CODE = "provider_create_unavailable"; +/** Provider allocation remains owned by the failed row until cleanup is confirmed. */ +export const PROVIDER_CREATE_CLEANUP_PENDING_FAILURE_CODE = "provider_create_cleanup_pending"; +/** Durable metadata keys for a provider allocation retained after create failure. */ +export const CREATE_CLEANUP_PROVIDER_VM_ID_KEY = "createCleanupProviderVmId"; +export const CREATE_CLEANUP_ATTEMPT_KEY = "createCleanupAttempt"; +export const CREATE_CLEANUP_NEXT_ATTEMPT_AT_KEY = "createCleanupNextAttemptAtMs"; +export const CREATE_CLEANUP_LEASE_ID_KEY = "createCleanupLeaseId"; +export const CREATE_CLEANUP_LEASE_EXPIRES_AT_KEY = "createCleanupLeaseExpiresAtMs"; + +/** Treat malformed/oversized cleanup timestamps as immediately eligible without an unsafe cast. */ +function cleanupTimestampDueSql(raw: SQL, nowMs: number) { + return sql`case + when ${raw} ~ '^[0-9]{1,16}$' + and (length(${raw}) < 16 or ${raw} <= '9007199254740991') + then (${raw})::numeric <= ${nowMs} + else true + end`; +} const RETRYABLE_FAILED_CREATE_CODES = new Set([ "billing_credits_insufficient", @@ -672,6 +719,98 @@ const RETRYABLE_FAILED_CREATE_CODES = new Set([ LEGACY_MODEL_PLANE_ENTITLEMENT_FAILURE_CODE, ]); +/** + * Finish the transactional half of a Base create failure. Cleanup-pending + * creates call this only after the provider confirms the allocation is gone; + * ordinary failures call it immediately. Keeping the same transaction for + * both paths prevents a late cleanup worker from promoting the wrong retained + * generation. + */ +async function restoreBaseAfterCreateFailure( + tx: CloudDbTransaction, + input: { + readonly baseId: string; + readonly generation: number; + readonly vmId: string; + readonly userId: string; + readonly code: string; + readonly message: string; + }, +): Promise { + const now = new Date(); + await tx + .update(cloudVmBaseGenerations) + .set({ state: "failed", updatedAt: now }) + .where(and( + eq(cloudVmBaseGenerations.baseId, input.baseId), + eq(cloudVmBaseGenerations.generation, input.generation), + eq(cloudVmBaseGenerations.vmId, input.vmId), + )); + const [retained] = await tx + .select({ + generation: cloudVmBaseGenerations, + vm: cloudVms, + }) + .from(cloudVmBaseGenerations) + .innerJoin(cloudVms, eq(cloudVms.id, cloudVmBaseGenerations.vmId)) + .where(and( + eq(cloudVmBaseGenerations.baseId, input.baseId), + sql`${cloudVmBaseGenerations.generation} < ${input.generation}`, + eq(cloudVmBaseGenerations.state, "retained"), + ne(cloudVms.status, "failed"), + ne(cloudVms.status, "destroyed"), + )) + .orderBy(desc(cloudVmBaseGenerations.generation)) + .limit(1); + if (retained?.generation && retained.vm) { + // Fence promotion on the failed generation still being the active Base + // row. A concurrent reset/open may have moved the Base on while provider + // cleanup was in flight; in that case leave the newer generation alone. + const activeBase = await tx + .update(cloudVmBases) + .set({ + activeGeneration: retained.generation.generation, + activeVmId: retained.vm.id, + activeProvider: retained.vm.provider, + activeProviderVmId: retained.vm.providerVmId, + state: retained.vm.providerVmId ? "ready" : "creating", + updatedAt: now, + }) + .where(and( + eq(cloudVmBases.id, input.baseId), + eq(cloudVmBases.activeGeneration, input.generation), + eq(cloudVmBases.activeVmId, input.vmId), + )) + .returning({ id: cloudVmBases.id }); + if (activeBase.length > 0) { + await tx + .update(cloudVmBaseGenerations) + .set({ state: "active", updatedAt: now }) + .where(eq(cloudVmBaseGenerations.id, retained.generation.id)); + } + } else { + await tx + .update(cloudVmBases) + .set({ state: "failed", updatedAt: now }) + .where(and( + eq(cloudVmBases.id, input.baseId), + eq(cloudVmBases.activeGeneration, input.generation), + eq(cloudVmBases.activeVmId, input.vmId), + )); + } + await tx.insert(cloudVmBaseEvents).values({ + baseId: input.baseId, + userId: input.userId, + eventType: "base.create_failed", + oldGeneration: retained?.generation.generation ?? null, + newGeneration: input.generation, + oldVmId: retained?.vm.id ?? null, + newVmId: input.vmId, + oldProviderVmId: retained?.vm.providerVmId ?? null, + metadata: { code: input.code, message: input.message }, + }); +} + function isRetryableFailedCreate(vm: CloudVmRow, now: Date): boolean { if (vm.status === "destroyed") return true; if (vm.status !== "failed") return false; @@ -752,7 +891,12 @@ function providerMetadataPatchForPersistence( key !== VM_RESOURCE_FORK_PENDING_METADATA_KEY && key !== VM_RESOURCE_RECONCILE_RETRY_METADATA_KEY && key !== VM_RESOURCE_RESIZE_PENDING_METADATA_KEY && - key !== VM_RESOURCE_RESIZE_UNCONFIRMED_METADATA_KEY, + key !== VM_RESOURCE_RESIZE_UNCONFIRMED_METADATA_KEY && + key !== CREATE_CLEANUP_PROVIDER_VM_ID_KEY && + key !== CREATE_CLEANUP_ATTEMPT_KEY && + key !== CREATE_CLEANUP_NEXT_ATTEMPT_AT_KEY && + key !== CREATE_CLEANUP_LEASE_ID_KEY && + key !== CREATE_CLEANUP_LEASE_EXPIRES_AT_KEY, ), ); } @@ -1964,80 +2108,34 @@ export const vmRepositoryLiveShape: VmRepositoryShape = { const db = cloudDb(); await db.transaction(async (tx) => { const now = new Date(); + const cleanupProviderVmId = input.code === PROVIDER_CREATE_CLEANUP_PENDING_FAILURE_CODE + ? input.cleanupProviderVmId?.trim() || null + : null; await tx .update(cloudVms) .set({ - status: "failed", + status: cleanupProviderVmId ? "provisioning" : "failed", failureCode: input.code, failureMessage: input.message, + ...(cleanupProviderVmId ? { + providerMetadata: sql`( + coalesce(${cloudVms.providerMetadata}, '{}'::jsonb) + || jsonb_build_object( + '${sql.raw(CREATE_CLEANUP_PROVIDER_VM_ID_KEY)}', ${cleanupProviderVmId}::text, + '${sql.raw(CREATE_CLEANUP_ATTEMPT_KEY)}', 0, + '${sql.raw(CREATE_CLEANUP_NEXT_ATTEMPT_AT_KEY)}', 0 + ) + ) + #- '{${sql.raw(CREATE_CLEANUP_LEASE_ID_KEY)}}' + #- '{${sql.raw(CREATE_CLEANUP_LEASE_EXPIRES_AT_KEY)}}'`, + } : {}), updatedAt: now, }) .where(eq(cloudVms.id, input.vmId)); - await tx - .update(cloudVmBaseGenerations) - .set({ state: "failed", updatedAt: now }) - .where(and( - eq(cloudVmBaseGenerations.baseId, input.baseId), - eq(cloudVmBaseGenerations.generation, input.generation), - eq(cloudVmBaseGenerations.vmId, input.vmId), - )); - const [retained] = await tx - .select({ - generation: cloudVmBaseGenerations, - vm: cloudVms, - }) - .from(cloudVmBaseGenerations) - .innerJoin(cloudVms, eq(cloudVms.id, cloudVmBaseGenerations.vmId)) - .where(and( - eq(cloudVmBaseGenerations.baseId, input.baseId), - sql`${cloudVmBaseGenerations.generation} < ${input.generation}`, - eq(cloudVmBaseGenerations.state, "retained"), - ne(cloudVms.status, "failed"), - ne(cloudVms.status, "destroyed"), - )) - .orderBy(desc(cloudVmBaseGenerations.generation)) - .limit(1); - if (retained?.generation && retained.vm) { - await tx - .update(cloudVmBaseGenerations) - .set({ state: "active", updatedAt: now }) - .where(eq(cloudVmBaseGenerations.id, retained.generation.id)); - await tx - .update(cloudVmBases) - .set({ - activeGeneration: retained.generation.generation, - activeVmId: retained.vm.id, - activeProvider: retained.vm.provider, - activeProviderVmId: retained.vm.providerVmId, - state: retained.vm.providerVmId ? "ready" : "creating", - updatedAt: now, - }) - .where(and( - eq(cloudVmBases.id, input.baseId), - eq(cloudVmBases.activeGeneration, input.generation), - eq(cloudVmBases.activeVmId, input.vmId), - )); - } else { - await tx - .update(cloudVmBases) - .set({ state: "failed", updatedAt: now }) - .where(and( - eq(cloudVmBases.id, input.baseId), - eq(cloudVmBases.activeGeneration, input.generation), - eq(cloudVmBases.activeVmId, input.vmId), - )); - } - await tx.insert(cloudVmBaseEvents).values({ - baseId: input.baseId, - userId: input.userId, - eventType: "base.create_failed", - oldGeneration: retained?.generation.generation ?? null, - newGeneration: input.generation, - oldVmId: retained?.vm.id ?? null, - newVmId: input.vmId, - oldProviderVmId: retained?.vm.providerVmId ?? null, - metadata: { code: input.code, message: input.message }, - }); + // Leave the Base generation reserved too; reset/open must not allocate + // a replacement while its failed guest still exists at the provider. + if (cleanupProviderVmId) return; + await restoreBaseAfterCreateFailure(tx, input); }); }), @@ -2554,6 +2652,172 @@ export const vmRepositoryLiveShape: VmRepositoryShape = { .limit(input.limit); }), + pendingCreateCleanupCandidates: (input) => + dbEffect("pendingCreateCleanupCandidates", async () => { + const db = cloudDb(); + const cleanupProviderVmId = sql`${cloudVms.providerMetadata}->>'${sql.raw(CREATE_CLEANUP_PROVIDER_VM_ID_KEY)}'`; + const nextAttemptAtMs = sql`${cloudVms.providerMetadata}->>'${sql.raw(CREATE_CLEANUP_NEXT_ATTEMPT_AT_KEY)}'`; + const leaseExpiresAtMs = sql`${cloudVms.providerMetadata}->>'${sql.raw(CREATE_CLEANUP_LEASE_EXPIRES_AT_KEY)}'`; + const nowMs = Date.now(); + const limit = Number.isFinite(input.limit) + ? Math.max(0, Math.min(200, Math.trunc(input.limit))) + : 200; + return await db + .select() + .from(cloudVms) + .where(and( + eq(cloudVms.status, "provisioning"), + eq(cloudVms.failureCode, PROVIDER_CREATE_CLEANUP_PENDING_FAILURE_CODE), + // The current registry has one provider. Keep retired enum values + // out of the worker batch so they cannot starve live cleanup rows. + eq(cloudVms.provider, "freestyle"), + sql`${cleanupProviderVmId} is not null and ${cleanupProviderVmId} <> ''`, + cleanupTimestampDueSql(nextAttemptAtMs, nowMs), + cleanupTimestampDueSql(leaseExpiresAtMs, nowMs), + )) + .orderBy(asc(cloudVms.updatedAt), asc(cloudVms.id)) + .limit(limit); + }), + + claimCreateCleanup: (input) => + dbEffect("claimCreateCleanup", async () => { + const providerVmId = input.providerVmId.trim(); + const leaseId = input.leaseId.trim(); + if (!providerVmId || !leaseId) return null; + const db = cloudDb(); + const cleanupProviderVmId = sql`${cloudVms.providerMetadata}->>'${sql.raw(CREATE_CLEANUP_PROVIDER_VM_ID_KEY)}'`; + const leaseIdValue = sql`${cloudVms.providerMetadata}->>'${sql.raw(CREATE_CLEANUP_LEASE_ID_KEY)}'`; + const leaseExpiresAtMs = sql`${cloudVms.providerMetadata}->>'${sql.raw(CREATE_CLEANUP_LEASE_EXPIRES_AT_KEY)}'`; + const nextAttemptAtMs = sql`${cloudVms.providerMetadata}->>'${sql.raw(CREATE_CLEANUP_NEXT_ATTEMPT_AT_KEY)}'`; + const attempt = sql`${cloudVms.providerMetadata}->>'${sql.raw(CREATE_CLEANUP_ATTEMPT_KEY)}'`; + const [row] = await db + .update(cloudVms) + .set({ + providerMetadata: sql`( + coalesce(${cloudVms.providerMetadata}, '{}'::jsonb) + || jsonb_build_object( + '${sql.raw(CREATE_CLEANUP_ATTEMPT_KEY)}', + case + when ${attempt} ~ '^[0-9]{1,9}$' then least((${attempt})::numeric + 1, 1000000)::integer + else 1 + end, + -- Keep the row ineligible for the entire claim lease. A second + -- worker may already have selected the same candidate before + -- this update commits, so the retry fence must be future-dated + -- independently of the lease-id predicate. + '${sql.raw(CREATE_CLEANUP_NEXT_ATTEMPT_AT_KEY)}', ${input.leaseExpiresAt.getTime()}::bigint, + '${sql.raw(CREATE_CLEANUP_LEASE_ID_KEY)}', ${leaseId}::text, + '${sql.raw(CREATE_CLEANUP_LEASE_EXPIRES_AT_KEY)}', ${input.leaseExpiresAt.getTime()}::bigint + ) + )`, + updatedAt: input.now, + }) + .where(and( + eq(cloudVms.id, input.id), + eq(cloudVms.status, "provisioning"), + eq(cloudVms.failureCode, PROVIDER_CREATE_CLEANUP_PENDING_FAILURE_CODE), + eq(cleanupProviderVmId, providerVmId), + cleanupTimestampDueSql(nextAttemptAtMs, input.now.getTime()), + sql`( + ${leaseIdValue} is null + or ${leaseIdValue} = '' + or ${cleanupTimestampDueSql(leaseExpiresAtMs, input.now.getTime())} + )`, + )) + .returning({ providerMetadata: cloudVms.providerMetadata }); + if (!row) return null; + const rawAttempt = row.providerMetadata[CREATE_CLEANUP_ATTEMPT_KEY]; + const parsedAttempt = typeof rawAttempt === "number" && Number.isSafeInteger(rawAttempt) + ? rawAttempt + : Number.parseInt(String(rawAttempt ?? "1"), 10); + return { attempt: Number.isSafeInteger(parsedAttempt) && parsedAttempt > 0 ? parsedAttempt : 1 }; + }), + + deferCreateCleanup: (input) => + dbEffect("deferCreateCleanup", async () => { + const providerVmId = input.providerVmId.trim(); + const leaseId = input.leaseId.trim(); + if (!providerVmId || !leaseId) return false; + const db = cloudDb(); + const cleanupProviderVmId = sql`${cloudVms.providerMetadata}->>'${sql.raw(CREATE_CLEANUP_PROVIDER_VM_ID_KEY)}'`; + const cleanupLeaseId = sql`${cloudVms.providerMetadata}->>'${sql.raw(CREATE_CLEANUP_LEASE_ID_KEY)}'`; + const rows = await db + .update(cloudVms) + .set({ + providerMetadata: sql`( + coalesce(${cloudVms.providerMetadata}, '{}'::jsonb) + || jsonb_build_object('${sql.raw(CREATE_CLEANUP_NEXT_ATTEMPT_AT_KEY)}', ${input.nextAttemptAt.getTime()}::bigint) + ) + #- '{${sql.raw(CREATE_CLEANUP_LEASE_ID_KEY)}}' + #- '{${sql.raw(CREATE_CLEANUP_LEASE_EXPIRES_AT_KEY)}}'`, + updatedAt: input.now, + }) + .where(and( + eq(cloudVms.id, input.id), + eq(cloudVms.status, "provisioning"), + eq(cloudVms.failureCode, PROVIDER_CREATE_CLEANUP_PENDING_FAILURE_CODE), + eq(cleanupProviderVmId, providerVmId), + eq(cleanupLeaseId, leaseId), + )) + .returning({ id: cloudVms.id }); + return rows.length > 0; + }), + + resolveCreateCleanup: (input) => + dbEffect("resolveCreateCleanup", async () => { + const db = cloudDb(); + return await db.transaction(async (tx) => { + const cleanupProviderVmId = sql`${cloudVms.providerMetadata}->>'${sql.raw(CREATE_CLEANUP_PROVIDER_VM_ID_KEY)}'`; + const cleanupLeaseId = sql`${cloudVms.providerMetadata}->>'${sql.raw(CREATE_CLEANUP_LEASE_ID_KEY)}'`; + const now = new Date(); + const rows = await tx + .update(cloudVms) + .set({ + status: "failed", + failureCode: PROVIDER_CREATE_UNAVAILABLE_FAILURE_CODE, + failureMessage: "Provider rollback was confirmed by background cleanup.", + providerMetadata: sql`( + coalesce(${cloudVms.providerMetadata}, '{}'::jsonb) + #- '{${sql.raw(CREATE_CLEANUP_PROVIDER_VM_ID_KEY)}}' + #- '{${sql.raw(CREATE_CLEANUP_ATTEMPT_KEY)}}' + #- '{${sql.raw(CREATE_CLEANUP_NEXT_ATTEMPT_AT_KEY)}}' + #- '{${sql.raw(CREATE_CLEANUP_LEASE_ID_KEY)}}' + #- '{${sql.raw(CREATE_CLEANUP_LEASE_EXPIRES_AT_KEY)}}' + )`, + updatedAt: now, + }) + .where(and( + eq(cloudVms.id, input.id), + eq(cloudVms.status, "provisioning"), + eq(cloudVms.failureCode, PROVIDER_CREATE_CLEANUP_PENDING_FAILURE_CODE), + eq(cleanupProviderVmId, input.providerVmId), + eq(cleanupLeaseId, input.leaseId), + )) + .returning({ id: cloudVms.id, userId: cloudVms.userId }); + const vm = rows[0]; + if (!vm) return false; + + const [generation] = await tx + .select() + .from(cloudVmBaseGenerations) + .where(and( + eq(cloudVmBaseGenerations.vmId, vm.id), + eq(cloudVmBaseGenerations.state, "creating"), + )) + .limit(1); + if (!generation) return true; + await restoreBaseAfterCreateFailure(tx, { + baseId: generation.baseId, + generation: generation.generation, + vmId: vm.id, + userId: vm.userId, + code: PROVIDER_CREATE_UNAVAILABLE_FAILURE_CODE, + message: "Provider rollback was confirmed by background cleanup.", + }); + return true; + }); + }), + listLiveHomeVolumeNames: (input) => dbEffect("listLiveHomeVolumeNames", async () => { const volumeNames = boundedReaperVolumeNames(input.volumeNames); @@ -2686,12 +2950,27 @@ export const vmRepositoryLiveShape: VmRepositoryShape = { markCreateFailed: (input) => dbEffect("markCreateFailed", async () => { const db = cloudDb(); + const cleanupProviderVmId = input.code === PROVIDER_CREATE_CLEANUP_PENDING_FAILURE_CODE + ? input.cleanupProviderVmId?.trim() || null + : null; await db .update(cloudVms) .set({ - status: "failed", + status: cleanupProviderVmId ? "provisioning" : "failed", failureCode: input.code, failureMessage: input.message, + ...(cleanupProviderVmId ? { + providerMetadata: sql`( + coalesce(${cloudVms.providerMetadata}, '{}'::jsonb) + || jsonb_build_object( + '${sql.raw(CREATE_CLEANUP_PROVIDER_VM_ID_KEY)}', ${cleanupProviderVmId}::text, + '${sql.raw(CREATE_CLEANUP_ATTEMPT_KEY)}', 0, + '${sql.raw(CREATE_CLEANUP_NEXT_ATTEMPT_AT_KEY)}', 0 + ) + ) + #- '{${sql.raw(CREATE_CLEANUP_LEASE_ID_KEY)}}' + #- '{${sql.raw(CREATE_CLEANUP_LEASE_EXPIRES_AT_KEY)}}'`, + } : {}), updatedAt: new Date(), }) .where(eq(cloudVms.id, input.id)); diff --git a/web/services/vms/routeHelpers.ts b/web/services/vms/routeHelpers.ts index 98b89933d502..0c78fbb6d58c 100644 --- a/web/services/vms/routeHelpers.ts +++ b/web/services/vms/routeHelpers.ts @@ -56,6 +56,8 @@ import { import { vmArtifactUnavailableCopy, vmDisplayNameCopy, + vmCreateCleanupPendingCopy, + vmGuestInstallCopy, vmRequestLocale, vmRequiresProCopy, vmMemoryErrorCopy, @@ -65,6 +67,8 @@ import { } from "./vmErrorMessages"; import { DISPLAY_NAME_MAX_LENGTH } from "./displayName"; import { ProviderArtifactUnavailableError } from "./drivers/types"; +import { isProviderCreateCleanupError } from "./drivers/providerCreateCleanup"; +import { PROVIDER_CREATE_CLEANUP_PENDING_FAILURE_CODE } from "./repository"; import type { Locale } from "../../i18n/routing"; /** Bearer + refresh token pair the mac app stashes in keychain. */ @@ -753,6 +757,13 @@ export const vmWorkflowErrorResponders = { if (providerArtifactUnavailable(error.cause)) { return vmArtifactUnavailableResponse(error, context.locale); } + if (isProviderCreateCleanupError(error.cause)) { + return vmCreateCleanupPendingResponse(context.locale); + } + const guestInstall = guestCliInstallFailure(error.cause); + if (guestInstall) { + return vmGuestInstallFailureResponse(error, context.locale, guestInstall); + } return vmProviderOperationErrorResponse(error); }, VmAccountDeletionInProgressError: (error) => @@ -955,6 +966,9 @@ export async function respondVmWorkflowError( context: VmWorkflowErrorResponderContext, overrides?: VmWorkflowErrorOverrides, ): Promise { + if (error._tag === "VmCreateFailedError" && error.code === PROVIDER_CREATE_CLEANUP_PENDING_FAILURE_CODE) { + return vmCreateCleanupPendingResponse(context.locale); + } const responders: VmWorkflowErrorResponders = overrides ? { ...vmWorkflowErrorResponders, ...overrides } : vmWorkflowErrorResponders; @@ -987,6 +1001,71 @@ function providerArtifactUnavailable(cause: unknown): boolean { return false; } +type GuestCliInstallFailure = { + readonly stage?: string; + readonly outcome?: string; + readonly cleanupFailed: boolean; +}; +const guestInstallStages = new Set(["upload", "install", "validate", "verify", "browser", "prompt", "publish"]); +const guestInstallOutcomes = new Set(["missing_status", "invalid_status", "provider_timeout", "guest_exit", "cancelled", "transport_timeout", "transport", "deadline"]); + +/** Match the typed guest installer failure without exposing its English diagnostics. */ +function guestCliInstallFailure(cause: unknown): GuestCliInstallFailure | null { + let current = cause; + for (let depth = 0; depth < 8 && current; depth += 1) { + if (typeof current === "object") { + const record = current as { + _tag?: unknown; + stage?: unknown; + outcome?: unknown; + cleanupCause?: unknown; + cause?: unknown; + }; + if (record._tag === "GuestCliInstallError") { + return { + ...(typeof record.stage === "string" && guestInstallStages.has(record.stage) ? { stage: record.stage } : {}), + ...(typeof record.outcome === "string" && guestInstallOutcomes.has(record.outcome) ? { outcome: record.outcome } : {}), + cleanupFailed: record.cleanupCause !== undefined, + }; + } + current = record.cause; + } else { + current = undefined; + } + } + return null; +} + +/** Keep stage/outcome/cleanup state in operator telemetry while returning only safe copy. */ +async function vmGuestInstallFailureResponse( + error: VmProviderOperationError, + locale: Locale, + failure: GuestCliInstallFailure, +): Promise { + const copy = await vmGuestInstallCopy(locale); + const phase = vmPhaseForOperation(error.operation); + const retryAfterSeconds = retryAfterForOperation(error.operation); + return vmErrorResponse({ + error: "vm_guest_install_failed", + status: 502, + message: copy.message, + reason: copy.reason, + action: copy.action, + phase, + retryable: true, + retryAfterSeconds, + displayTitle: copy.title, + displayMessage: copy.message, + details: { operation: error.operation, retryable: true }, + diagnostics: { + provider: error.provider, + ...(failure.stage ? { guestInstallStage: failure.stage } : {}), + ...(failure.outcome ? { guestInstallOutcome: failure.outcome } : {}), + guestInstallCleanupFailed: failure.cleanupFailed, + }, + }); +} + /** Keep manifest diagnostics in server error traces and return only localized setup guidance. */ async function vmArtifactUnavailableResponse(error: VmProviderOperationError, locale: Locale): Promise { const copy = await vmArtifactUnavailableCopy(locale); @@ -1003,6 +1082,20 @@ async function vmArtifactUnavailableResponse(error: VmProviderOperationError, lo }); } +async function vmCreateCleanupPendingResponse(locale: Locale): Promise { + const copy = await vmCreateCleanupPendingCopy(locale); + return vmErrorResponse({ + error: "vm_cloud_create_cleanup_pending", + status: 503, + message: copy.message, + action: copy.action, + phase: "create", + retryable: false, + displayTitle: copy.title, + details: { operation: "create", cleanupPending: true, retryable: false }, + }); +} + function vmProviderOperationErrorResponse(error: VmProviderOperationError): Response { const providerCause = providerCauseSummary(error.cause); const phase = vmPhaseForOperation(error.operation); @@ -1174,7 +1267,7 @@ function normalizedRetryAfterSeconds(value: number | undefined): number | undefi } function vmPhaseForOperation(operation: string): VmLifecyclePhase { - if (operation.includes("openAttach")) return "attach"; + if (operation.includes("openAttach") || operation.includes("openCmuxRemote")) return "attach"; if (operation.includes("openSSH")) return "ssh"; // Before the "create" check: createTunnel/createNetwork are network setup, // not machine creation, and a client that read them as "create" would show diff --git a/web/services/vms/vmErrorMessages.ts b/web/services/vms/vmErrorMessages.ts index 56e81adaa281..596a97d9d906 100644 --- a/web/services/vms/vmErrorMessages.ts +++ b/web/services/vms/vmErrorMessages.ts @@ -102,6 +102,31 @@ export async function vmArtifactUnavailableCopy(locale: Locale): Promise { + const translator = createTranslator({ + locale, + messages: await loadMessages(locale), + namespace: "vmErrors.guestInstall", + }) as unknown as (key: string) => string; + return { + title: translator("title"), + reason: translator("reason"), + message: translator("message"), + action: translator("action"), + }; +} + +export async function vmCreateCleanupPendingCopy(locale: Locale): Promise { + const translator = createTranslator({ + locale, + messages: await loadMessages(locale), + namespace: "vmErrors.createCleanupPending", + }) as unknown as (key: string) => string; + return { title: translator("title"), message: translator("message"), action: translator("action") }; +} + /** Load and translate the `vm_requires_pro` response copy for the request locale. */ export async function vmRequiresProCopy( locale: Locale, diff --git a/web/services/vms/workflows.ts b/web/services/vms/workflows.ts index ae530ae15197..efcdd0c598c7 100644 --- a/web/services/vms/workflows.ts +++ b/web/services/vms/workflows.ts @@ -95,10 +95,13 @@ import { import { getGoVmUsage, GO_INCLUDED_VM_HOURS } from "./goUsage"; import { GO_PAUSE_INTENT_KEY, pauseGoVm } from "./goPause"; import { networkSlugForUser, privateNetworkUnavailableReason, resolveOwnerNetwork } from "./privateNetwork"; -import { isProviderIdentityNotFoundError, isProviderNotFoundError } from "./providerErrors"; +import { isProviderDeletionConfirmed, isProviderIdentityNotFoundError, isProviderNotFoundError } from "./providerErrors"; import { VmProviderGateway, VmProviderGatewayLive, type VmProviderGatewayShape } from "./providerGateway"; +import { isProviderCreateCleanupError } from "./drivers/providerCreateCleanup"; import { withVmProductAnalytics } from "./productAnalytics"; import { + CREATE_CLEANUP_PROVIDER_VM_ID_KEY, + PROVIDER_CREATE_CLEANUP_PENDING_FAILURE_CODE, PROVIDER_CREATE_UNAVAILABLE_FAILURE_CODE, VmRepository, vmRepositoryLiveShape, @@ -218,6 +221,12 @@ const IDENTITY_REVOKE_PROVIDER_TIMEOUT = "5 seconds"; const ACTIVE_IDENTITY_REVOKE_HOT_PATH_LIMIT = 8; const ACCOUNT_DELETION_IDENTITY_REVOKE_BATCH = 8; const VM_STATUS_RECONCILE_BATCH_LIMIT = 200; +const CREATE_CLEANUP_CONCURRENCY = 4; +const CREATE_CLEANUP_PROVIDER_TIMEOUT = "15 seconds"; +const CREATE_CLEANUP_LEASE_MS = 60 * 1000; +const CREATE_CLEANUP_BACKOFF_BASE_MS = 5 * 1000; +const CREATE_CLEANUP_BACKOFF_MAX_MS = 15 * 60 * 1000; +const CREATE_CLEANUP_BATCH_LIMIT = 20; const LEGACY_RESOURCE_RECONCILE_BATCH_LIMIT = 50; const LEGACY_RESOURCE_RECONCILE_CONCURRENCY = 5; const LEGACY_RESOURCE_RECONCILE_RETRY_AFTER_MS = 5 * 60 * 1000; @@ -392,6 +401,11 @@ export function reconcileVmProviderStatuses(input: { yield* reconcileLegacyResourceReservations(repo, providers, { limit: LEGACY_RESOURCE_RECONCILE_BATCH_LIMIT, }); + yield* reconcilePendingCreateCleanups(repo, providers, { + // At four concurrent 15-second provider calls, twenty rows fit inside + // the five-minute cron budget while leaving time for status probes. + limit: Math.min(boundedVmStatusReconcileLimit(input.limit), CREATE_CLEANUP_BATCH_LIMIT), + }); const getStatus = providers.getStatus; if (!getStatus) { return { @@ -449,6 +463,84 @@ export function reconcileVmProviderStatuses(input: { }); } +/** + * A provider allocation retained after a failed create is not a normal VM row: + * its public provider id is intentionally absent until deletion is confirmed. + * Reconcile those ids before ordinary status probing so a failed provider + * cleanup cannot remain reserved forever or block the next Base generation. + */ +function reconcilePendingCreateCleanups( + repo: VmRepositoryShape, + providers: VmProviderGatewayShape, + input: { readonly limit: number }, +): Effect.Effect { + const listCandidates = repo.pendingCreateCleanupCandidates; + const claimCleanup = repo.claimCreateCleanup; + const deferCleanup = repo.deferCreateCleanup; + const resolveCleanup = repo.resolveCreateCleanup; + if (!listCandidates || !claimCleanup || !deferCleanup || !resolveCleanup) return Effect.void; + return Effect.gen(function* () { + const candidates = yield* listCandidates({ limit: input.limit }).pipe( + Effect.catchAll(() => Effect.succeed([] as CloudVmRow[])), + ); + yield* Effect.forEach( + candidates, + (vm) => { + // The row can outlive its provider driver. Never pass a retired + // provider or a coderouter/model-plane failure to the cleanup worker. + if (isRetiredProviderRow(vm) || vm.failureCode !== PROVIDER_CREATE_CLEANUP_PENDING_FAILURE_CODE) { + return Effect.void; + } + const rawProviderVmId = vm.providerMetadata?.[CREATE_CLEANUP_PROVIDER_VM_ID_KEY]; + if (typeof rawProviderVmId !== "string" || rawProviderVmId.trim().length === 0) return Effect.void; + const providerVmId = rawProviderVmId.trim(); + const leaseId = randomUUID(); + const now = new Date(); + const leaseExpiresAt = new Date(now.getTime() + CREATE_CLEANUP_LEASE_MS); + return claimCleanup({ + id: vm.id, + providerVmId, + leaseId, + now, + leaseExpiresAt, + }).pipe( + Effect.flatMap((claim) => { + if (!claim) return Effect.void; + const destroy = providers.destroy(vm.provider, providerVmId).pipe( + Effect.timeoutFail({ + duration: CREATE_CLEANUP_PROVIDER_TIMEOUT, + onTimeout: () => new Error("provider cleanup deadline"), + }), + Effect.catchAll((error) => isProviderDeletionConfirmed(error) + ? Effect.succeed("confirmed" as const) + : Effect.fail(error)), + ); + return destroy.pipe( + Effect.flatMap(() => resolveCleanup({ id: vm.id, providerVmId, leaseId })), + Effect.asVoid, + Effect.catchAll(() => { + const backoff = Math.min( + CREATE_CLEANUP_BACKOFF_MAX_MS, + CREATE_CLEANUP_BACKOFF_BASE_MS * 2 ** Math.min(20, Math.max(0, claim.attempt - 1)), + ); + return deferCleanup({ + id: vm.id, + providerVmId, + leaseId, + nextAttemptAt: new Date(Date.now() + backoff), + now: new Date(), + }).pipe(Effect.asVoid, Effect.catchAll(() => Effect.void)); + }), + ); + }), + Effect.catchAll(() => Effect.void), + ); + }, + { concurrency: CREATE_CLEANUP_CONCURRENCY, discard: true }, + ); + }); +} + /** * The home volume a destroyed machine owns exclusively, or null when there is * nothing safe to delete. Per-machine volumes are marked at create @@ -506,6 +598,10 @@ function rollbackProviderCreate( }); } +function isFailedVmCreate(vm: Pick): boolean { + return vm.status === "failed" || vm.failureCode === PROVIDER_CREATE_CLEANUP_PENDING_FAILURE_CODE; +} + /** Check the copied or requested shape before provisioning side effects. */ function requireMemoryPlan(planId: string, memoryMb: number | null) { const maxMemoryMb = maxMemoryMbForPlan(planId); @@ -653,7 +749,7 @@ export function createVm(input: CreateVmInput): Effect.Effect { }); describe("Cloud diagnostic durable storage", () => { + dbTest("accepted placement diagnostics retain their category through the durable outbox", async () => { + const { owner, batch } = fixture(); + const submitted = { ...batch, spans: [{ ...batch.spans[0]!, failure: "placement" }] }; + const parsed = parseCloudTelemetryBatch(submitted); + expect(parsed).not.toBeNull(); + expect(await acceptCloudTelemetry(owner, parsed!)).toBe(1); + const claimed = await claimCloudDiagnostics(100, owner); + expect(claimed.rows).toHaveLength(1); + expect(claimed.rows[0]!.payload.span).toEqual(submitted.spans[0]); + await finishCloudDiagnostics(claimed.leaseId, true); + expect((await claimCloudDiagnostics(100, owner)).rows).toEqual([]); + }); + dbTest("concurrent retry receipts store one event and charge once", async () => { const { owner, batch } = fixture(); expect(await Promise.all(Array.from({ length: 6 }, () => acceptCloudTelemetry(owner, batch)))).toEqual([1, 1, 1, 1, 1, 1]); diff --git a/web/tests/cloud-telemetry.test.ts b/web/tests/cloud-telemetry.test.ts index 5ebb2eaa635c..87ce4da24d92 100644 --- a/web/tests/cloud-telemetry.test.ts +++ b/web/tests/cloud-telemetry.test.ts @@ -21,7 +21,7 @@ function span(overrides: Record = {}) { ...overrides, }; } -function batch(spans = [span()]) { return { version: 1, client: { channel: "nightly", version: "0.1.0", build: "123", revision: "abcdef1234567", osVersion: "26.0", architecture: "arm64" }, spans }; } +function batch(spans = [span()], channel = "nightly") { return { version: 1, client: { channel, version: "0.1.0", build: "123", revision: "abcdef1234567", osVersion: "26.0", architecture: "arm64" }, spans }; } function request(body: unknown = batch(), headers: Record = {}) { return new Request("https://cmux.test/api/observability/cloud", { method: "POST", headers: { "content-type": "application/json", ...headers }, @@ -30,6 +30,94 @@ function request(body: unknown = batch(), headers: Record = {}) } describe("Cloud diagnostic boundary", () => { + test("acknowledges and exports placement failures without dropping neighboring spans", async () => { + const { exportCloudDiagnostics } = await import("../services/observability/cloudTelemetryExport"); + const payload = batch([ + span({ operation: "terminal", phase: "materialize", failure: "placement" }), + span({ eventId: "8cc333de-a1bc-4eb1-8d69-1decd01e17a9" }), + ]); + const accepted: unknown[] = []; + const sent: { url: string; body: any }[] = []; + const handler = makeCloudTelemetryHandler({ + authenticate: async () => ({ id: "synthetic-owner" }), + checkIngress: async () => true, + accept: async (userId, value) => { + accepted.push(value); + await exportCloudDiagnostics(value.spans.map((span) => ({ + userId, eventId: span.eventId, attempts: 1, payload: { client: value.client, span }, + })), { + origin: "https://us-east-1.aws.edge.axiom.co", token: "test-only", + identityKey: "test-only-identity-key".repeat(2), tracesDataset: "test-traces", + errorsDataset: "test-errors", environment: "preview", revision: "abcdef123", + }, (async (url, init) => { + sent.push({ url: String(url), body: JSON.parse(String(init?.body)) }); + return new Response("{}"); + }) as typeof fetch); + return value.spans.length; + }, + scheduleDrain: () => {}, now: () => now, + }); + const response = await handler(request(payload)); + expect(response.status).toBe(202); + expect(await response.json()).toEqual({ + accepted: 2, eventIds: payload.spans.map((span) => span.eventId), + }); + expect(accepted).toEqual([payload]); + const errors = sent.find((item) => item.url.endsWith("/v1/ingest/test-errors"))!.body; + expect(errors.map((item: any) => item.failure)).toEqual(["placement", "network"]); + expect(errors[0]).toMatchObject({ + event_id: payload.spans[0]!.eventId, operation_id: payload.spans[0]!.operationId, + trace_id: payload.spans[0]!.traceId, span_id: payload.spans[0]!.spanId, + operation: "terminal", phase: "materialize", outcome: "failure", + }); + const spans = sent.find((item) => item.url.endsWith("/v1/traces"))!.body.resourceSpans; + expect(spans[0].scopeSpans[0].spans[0].attributes).toContainEqual({ + key: "error.type", value: { stringValue: "placement" }, + }); + expect(JSON.stringify(sent)).not.toContain("synthetic-owner"); + }); + + test("accepts authenticated RC diagnostics and exports rc separately from production", async () => { + const { exportCloudDiagnostics } = await import("../services/observability/cloudTelemetryExport"); + const sent: { url: string; body: any }[] = []; + const configuration = { + origin: "https://us-east-1.aws.edge.axiom.co", token: "test-only", + identityKey: "test-only-identity-key".repeat(2), tracesDataset: "test-traces", + errorsDataset: "test-errors", environment: "production", revision: "abcdef123", + } as const; + const handler = makeCloudTelemetryHandler({ + authenticate: async () => ({ id: "synthetic-owner" }), + checkIngress: async () => true, + accept: async (userId, value) => { + await exportCloudDiagnostics(value.spans.map((span) => ({ + userId, eventId: span.eventId, attempts: 1, payload: { client: value.client, span }, + })), configuration, (async (url, init) => { + sent.push({ url: String(url), body: JSON.parse(String(init?.body)) }); + return new Response("{}"); + }) as typeof fetch); + return value.spans.length; + }, + scheduleDrain: () => {}, now: () => now, + }); + const rcPayload = batch([span({ eventId: "e51c27bc-b0ad-4149-9d92-0fc79c5d2292" })], "rc"); + const productionPayload = batch([span({ eventId: "8cc333de-a1bc-4eb1-8d69-1decd01e17a9" })], "production"); + + const rcResponse = await handler(request(rcPayload)); + const productionResponse = await handler(request(productionPayload)); + expect(rcResponse.status).toBe(202); + expect(productionResponse.status).toBe(202); + expect((await rcResponse.json()).accepted).toBe(1); + expect((await productionResponse.json()).accepted).toBe(1); + + const errorChannels = sent.filter((item) => item.url.endsWith("/v1/ingest/test-errors")) + .flatMap((item) => item.body.map((row: any) => row.client_channel)).sort(); + expect(errorChannels).toEqual(["production", "rc"]); + const traceChannels = sent.filter((item) => item.url.endsWith("/v1/traces")) + .map((item) => item.body.resourceSpans[0].resource.attributes + .find((attribute: any) => attribute.key === "cmux.client.channel").value.stringValue).sort(); + expect(traceChannels).toEqual(["production", "rc"]); + }); + test("accepts a timed operation with a real parent span", () => { expect(parseCloudTelemetryBatch(batch(), now)?.spans[0]).toEqual(span()); }); diff --git a/web/tests/fixtures/freestyleGuest.ts b/web/tests/fixtures/freestyleGuest.ts new file mode 100644 index 000000000000..fd66c9d0fb6b --- /dev/null +++ b/web/tests/fixtures/freestyleGuest.ts @@ -0,0 +1,140 @@ +import { createHash } from "node:crypto"; +import { spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { Effect } from "effect"; +import { Freestyle } from "freestyle"; +import { FreestyleProvider } from "../../services/vms/drivers/freestyle"; +import { installFreestyleGuestCli } from "../../services/vms/drivers/freestyleGuestCli"; +import { rollbackFreestyleCreate } from "../../services/vms/drivers/providerCreateCleanup"; +import { ProviderError, type CreateOptions } from "../../services/vms/drivers/types"; +import type { GuestCliDistribution } from "../../services/vms/guestCliDistribution"; + +export type GuestExecRequest = { + command: string; + timeoutMs: number; + linuxUser: string; +}; + +const testDistribution = (() => { + const root = mkdtempSync(join(tmpdir(), "cmux-guest-cli-fixture-")); + const source = join(root, "source"); + mkdirSync(source); + const facade = "#!/bin/sh\nprintf 'synthetic facade\\n'\n"; + const core = "#!/bin/sh\nprintf 'synthetic core\\n'\n"; + writeFileSync(join(source, "cmux-cloud-cli"), facade, { mode: 0o755 }); + writeFileSync(join(source, "coderouter"), core, { mode: 0o755 }); + const archive = join(root, "cli.tar.gz"); + const tar = spawnSync("tar", ["-czf", archive, "-C", source, "cmux-cloud-cli", "coderouter"], { + env: { ...process.env, COPYFILE_DISABLE: "1" }, + }); + if (tar.status !== 0) throw new Error("could not create the synthetic guest CLI archive"); + const digest = (value: string | Buffer) => createHash("sha256").update(value).digest("hex"); + return { + url: pathToFileURL(archive).href, + archiveSha256: digest(readFileSync(archive)), + binaries: { "cmux-cloud-cli": digest(facade), coderouter: digest(core) }, + } satisfies GuestCliDistribution; +})(); + +/** Real pinned SDK, synthetic HTTP only. No provider credentials or network. */ +export function freestyleGuestFixture(options: { + exec?: (request: GuestExecRequest, signal?: AbortSignal | null) => Promise; + write?: (path: string, bytes: Uint8Array, signal?: AbortSignal | null) => void | Promise; + remove?: (path: string, signal?: AbortSignal | null) => void | Promise; + deleteFailure?: boolean; + idPrefix?: string; + guestCliDistribution?: GuestCliDistribution; +} = {}) { + const requests: Array<{ method: string; path: string }> = []; + const writes: string[] = []; + const removals: string[] = []; + const liveVms = new Set(); + let allocations = 0; + let installPending = false; + const client = (_timeoutMs?: number, signal?: AbortSignal) => new Freestyle({ + apiKey: "synthetic-test-key", + baseUrl: "https://provider.invalid", + fetch: (async (input: RequestInfo | URL, init?: RequestInit) => { + const url = new URL(String(input)); + const method = init?.method ?? "GET"; + requests.push({ method, path: url.pathname }); + if (method === "POST" && url.pathname === "/v5/vms") { + const id = `${options.idPrefix ?? "vm-fixture"}-${++allocations}`; + liveVms.add(id); + return Response.json({ id, state: "running", resources: { cpu: 2, memory: 8192, storage: 32768 }, vpcs: [{ ipv4: "192.0.2.10" }] }); + } + if (url.pathname.endsWith("/fs/write")) { + const path = url.searchParams.get("path")!; + writes.push(path); + await options.write?.(path, new Uint8Array(await new Response(init?.body).arrayBuffer()), signal ?? init?.signal); + installPending = true; + return Response.json({}); + } + if (url.pathname.endsWith("/fs/remove")) { + const path = url.searchParams.get("path")!; + removals.push(path); + await options.remove?.(path, signal ?? init?.signal); + return Response.json({}); + } + if (url.pathname.endsWith("/exec-await")) { + const request = JSON.parse(String(init?.body)) as GuestExecRequest; + if (installPending) { + installPending = false; + return options.exec?.(request, signal ?? init?.signal) ?? Response.json({ statusCode: 0 }); + } + return Response.json({ statusCode: 0, stdout: "", stderr: "" }); + } + if (url.pathname.endsWith("/resize")) { + return Response.json({ id: url.pathname.split("/").at(-2), state: "running", resources: JSON.parse(String(init?.body)) }); + } + if (method === "DELETE" && /^\/v5\/vms\/[a-z0-9-]+-\d+$/.test(url.pathname)) { + if (options.deleteFailure) return Response.json({ code: "UNAVAILABLE", message: "synthetic delete failure" }, { status: 503 }); + liveVms.delete(url.pathname.split("/").at(-1)!); + return new Response(null, { status: 204 }); + } + throw new Error(`Unexpected synthetic provider request: ${method} ${url.pathname}`); + }) as typeof fetch, + }); + const provider = new FreestyleProvider({ + client, + }); + /** + * Exercise guest bootstrap as a separate workflow. The production provider's + * create path follows snapshot-v2 and intentionally performs no guest setup. + */ + const createWithGuestInstall = async (createOptions: CreateOptions) => { + const handle = await provider.create(createOptions); + const install = await Effect.runPromise(Effect.either(installFreestyleGuestCli( + client, + handle.providerVmId, + createOptions.promptIdentity, + options.guestCliDistribution ?? testDistribution, + ))); + if (install._tag === "Right") { + return handle; + } + const rollback = await Effect.runPromise(Effect.either( + rollbackFreestyleCreate(client, handle.providerVmId, install.left), + )); + if (rollback._tag === "Left") throw rollback.left; + throw new ProviderError("freestyle", "guest bootstrap failed", install.left); + }; + return { + provider, + client, + createWithGuestInstall, + requests, + writes, + removals, + liveVms, + allocations: () => allocations, + }; +} + +export const guestCreateOptions = { + image: "sh-synthetic", + imageSize: { name: "md", cpu: 2, memoryMb: 8192, storageMb: 32768 }, +} as const; diff --git a/web/tests/freestyle-network-announcement.test.ts b/web/tests/freestyle-network-announcement.test.ts index 34824975410a..4b81eb68abb0 100644 --- a/web/tests/freestyle-network-announcement.test.ts +++ b/web/tests/freestyle-network-announcement.test.ts @@ -60,7 +60,7 @@ describe("Freestyle private network readiness", () => { exec: async ({ command }: { command: string }) => { // Adapter, reporter, and hook preparation can add probes. This test // guards publication/rollback ordering, not the number of setup execs. - if (command.startsWith("python3 -c ")) events.push("guest-network"); + if (command.includes("Private network addresses are not ready")) events.push("guest-network"); return { statusCode: 0, stdout: "", stderr: "" }; }, fs: { @@ -72,6 +72,10 @@ describe("Freestyle private network readiness", () => { const client = { vms: { create: async () => { events.push("allocated"); return { vm, vmId: data.id, data }; }, get: async () => data, + ref: (id: string) => { + if (id !== data.id) throw new Error(`unexpected VM ref: ${id}`); + return vm; + }, } } as unknown as Freestyle; const provider = new FreestyleProvider({ client: () => client, @@ -99,7 +103,7 @@ describe("Freestyle private network readiness", () => { }; const vm = { exec: async ({ command }: { command: string }) => { - if (command.startsWith("python3 -c ")) { + if (command.includes("Private network addresses are not ready")) { events.push("guest-network"); return { statusCode: 124, stdout: "", stderr: "network state probe timed out" }; } @@ -110,6 +114,10 @@ describe("Freestyle private network readiness", () => { }; const client = { vms: { create: async () => { events.push("allocated"); return { vm, vmId: data.id, data }; }, + ref: (id: string) => { + if (id !== data.id) throw new Error(`unexpected VM ref: ${id}`); + return vm; + }, } } as unknown as Freestyle; const provider = new FreestyleProvider({ client: () => client, diff --git a/web/tests/freestyle-request-timing.test.ts b/web/tests/freestyle-request-timing.test.ts index 41e50dc05659..f9e8259e8181 100644 --- a/web/tests/freestyle-request-timing.test.ts +++ b/web/tests/freestyle-request-timing.test.ts @@ -2,6 +2,23 @@ import { describe, expect, test } from "bun:test"; import { freestyleRequestFetch, type FreestyleRequestTiming } from "../services/vms/drivers/freestyleRequestTiming"; describe("Freestyle enrollment request timings", () => { + test.each(["workflow", "init", "request"])("forwards %s cancellation alongside the provider timeout", async (source) => { + const controller = new AbortController(); + const reason = new DOMException("synthetic cancellation", "AbortError"); + const traced = freestyleRequestFetch({ + timeoutMs: 60_000, signal: source === "workflow" ? controller.signal : undefined, + fetch: (async (_input, init) => { + controller.abort(reason); + init!.signal!.throwIfAborted(); + return new Response("must not complete"); + }) as typeof fetch, + }); + const request = new Request("https://example.test/provider", { + signal: source === "request" ? controller.signal : undefined, + }); + await expect(traced(request, source === "init" ? { signal: controller.signal } : undefined)).rejects.toBe(reason); + }); + test("distinguishes the initial request from background completion without logging access material", async () => { const events: FreestyleRequestTiming[] = []; const forwarded: Array<{ input: RequestInfo | URL; init?: RequestInit }> = []; diff --git a/web/tests/vm-guest-create-response.test.ts b/web/tests/vm-guest-create-response.test.ts new file mode 100644 index 000000000000..a3eeaf2569ff --- /dev/null +++ b/web/tests/vm-guest-create-response.test.ts @@ -0,0 +1,57 @@ +import { expect, test } from "bun:test"; +import { VmCreateFailedError, VmProviderOperationError } from "../services/vms/errors"; +import { ProviderCreateCleanupError } from "../services/vms/drivers/providerCreateCleanup"; +import { GuestCliInstallError } from "../services/vms/drivers/freestyleGuestCli"; +import { ProviderError } from "../services/vms/drivers/types"; +import { vmWorkflowErrorResponse } from "../services/vms/routeHelpers"; +import { locales } from "../i18n/routing"; + +test.each(locales)("cleanup-pending response is localized and never suggests retry (%s)", async (locale) => { + const original = new VmProviderOperationError({ provider: "freestyle", operation: "create", cause: + new ProviderCreateCleanupError("private-provider-id", new Error("private guest output"), new Error("private delete output")), + }); + const retry = new VmCreateFailedError({ idempotencyKey: "synthetic", code: "provider_create_cleanup_pending", message: "private diagnostic" }); + const first = await vmWorkflowErrorResponse(original, { locale }); + const repeated = await vmWorkflowErrorResponse(retry, { locale }); + expect(first?.status).toBe(503); + expect(first?.headers.has("retry-after")).toBe(false); + const body = await first!.json(); + expect(body.error).toBe("vm_cloud_create_cleanup_pending"); + expect(body.retryable).toBe(false); + expect(body.ui.retryable).toBe(false); + expect(JSON.stringify(body)).not.toContain("private"); + expect(await repeated!.json()).toEqual(body); + if (locale !== "en") expect(body.message).not.toBe("Cloud VM setup failed and cleanup is still pending."); +}); + +test.each(locales)("confirmed guest install failures use localized safe guidance for %s", async (locale) => { + const install = new GuestCliInstallError({ stage: "install", outcome: "guest_exit", exitCode: 7 }); + for (const operation of ["create", "openCmuxRemote"]) { + const provider = new VmProviderOperationError({ + provider: "freestyle", operation, + cause: new ProviderError("freestyle", `${operation}(private-machine) failed`, install), + }); + const response = await vmWorkflowErrorResponse(provider, { locale }); + expect(response?.status).toBe(502); + const body = await response!.json(); + expect(body.error).toBe("vm_guest_install_failed"); + expect(body.retryable).toBe(true); + expect(body.phase).toBe(operation === "create" ? "create" : "attach"); + expect(JSON.stringify(body)).not.toContain("private-machine"); + expect(JSON.stringify(body)).not.toContain("guest cmux shim"); + expect(JSON.stringify(body)).not.toContain("guest_exit"); + } +}); + +test("pending cleanup keeps its non-retryable response when guest install also failed", async () => { + const install = new GuestCliInstallError({ stage: "publish", outcome: "guest_exit", exitCode: 1 }); + const response = await vmWorkflowErrorResponse(new VmProviderOperationError({ + provider: "freestyle", operation: "create", + cause: new ProviderCreateCleanupError("private-provider-id", install, new Error("private delete output")), + }), { locale: "en" }); + expect(response?.status).toBe(503); + const body = await response!.json(); + expect(body.error).toBe("vm_cloud_create_cleanup_pending"); + expect(body.retryable).toBe(false); + expect(JSON.stringify(body)).not.toContain("guest_exit"); +}); diff --git a/web/tests/vm-guest-install-deadline.test.ts b/web/tests/vm-guest-install-deadline.test.ts new file mode 100644 index 000000000000..78538fa689c5 --- /dev/null +++ b/web/tests/vm-guest-install-deadline.test.ts @@ -0,0 +1,126 @@ +import { expect, test } from "bun:test"; +import { Deferred, Effect, Fiber, TestClock, TestContext } from "effect"; +import { GUEST_INSTALL_DEADLINE_MS, installFreestyleGuestCli } from "../services/vms/drivers/freestyleGuestCli"; +import { freestyleGuestFixture } from "./fixtures/freestyleGuest"; +import { Freestyle, FreestyleApiError } from "freestyle"; +import { rollbackFreestyleCreate } from "../services/vms/drivers/providerCreateCleanup"; + +test("an active stalled exec is aborted at the overall deadline, then cleaned with a fresh signal", async () => { + let active = 0; + let aborted = 0; + await Effect.runPromise(Effect.gen(function* () { + const entered = yield* Deferred.make(); + const fixture = freestyleGuestFixture({ + exec: async (_, signal) => { + active++; + Effect.runSync(Deferred.succeed(entered, undefined)); + return new Promise((_, reject) => signal!.addEventListener("abort", () => { + active--; aborted++; reject(signal!.reason); + }, { once: true })); + }, + remove: (_, signal) => { expect(signal?.aborted).toBe(false); expect(active).toBe(0); }, + }); + const work = yield* Effect.fork(Effect.either(installFreestyleGuestCli(fixture.client, "vm-fixture-1"))); + yield* Deferred.await(entered); + expect(active).toBe(1); + yield* TestClock.adjust(GUEST_INSTALL_DEADLINE_MS - 1); + expect((yield* Fiber.poll(work))._tag).toBe("None"); + expect(active).toBe(1); + yield* TestClock.adjust(1); + const outcome = yield* Fiber.join(work); + expect(outcome._tag).toBe("Left"); + if (outcome._tag === "Left") expect(outcome.left).toMatchObject({ outcome: "deadline", stage: "install" }); + expect(active).toBe(0); + expect(aborted).toBe(1); + expect(fixture.removals).toEqual(fixture.writes); + }).pipe(Effect.provide(TestContext.TestContext))); +}); + +test("cancelling during upload never starts exec and still cleans the upload", async () => { + let aborted = false; + await Effect.runPromise(Effect.gen(function* () { + const entered = yield* Deferred.make(); + const fixture = freestyleGuestFixture({ write: async (_, __, signal) => { + Effect.runSync(Deferred.succeed(entered, undefined)); + await new Promise((_, reject) => signal!.addEventListener("abort", () => { + aborted = true; reject(signal!.reason); + }, { once: true })); + } }); + const work = yield* Effect.fork(installFreestyleGuestCli(fixture.client, "vm-fixture-1")); + yield* Deferred.await(entered); + yield* Fiber.interrupt(work); + expect(aborted).toBe(true); + expect(fixture.requests.some(({ path }) => path.endsWith("exec-await"))).toBe(false); + expect(fixture.removals).toEqual(fixture.writes); + })); +}); + +test("a stalled cleanup finalizer is bounded and retains the original install failure", async () => { + let aborted = false; + await Effect.runPromise(Effect.gen(function* () { + const entered = yield* Deferred.make(); + const fixture = freestyleGuestFixture({ + exec: async () => Response.json({ statusCode: 124 }), + remove: async (_, signal) => { + Effect.runSync(Deferred.succeed(entered, undefined)); + await new Promise((_, reject) => signal!.addEventListener("abort", () => { + aborted = true; reject(signal!.reason); + }, { once: true })); + }, + }); + const work = yield* Effect.fork(Effect.either(installFreestyleGuestCli(fixture.client, "vm-fixture-1"))); + yield* Deferred.await(entered); + yield* TestClock.adjust(10_000); + const outcome = yield* Fiber.join(work); + expect(outcome._tag).toBe("Left"); + if (outcome._tag === "Left") { + expect(outcome.left).toMatchObject({ outcome: "guest_exit", exitCode: 124 }); + expect(outcome.left.cleanupCause).toBeInstanceOf(Error); + } + expect(aborted).toBe(true); + }).pipe(Effect.provide(TestContext.TestContext))); +}); + +test("rollback accepts typed absence, while missing endpoints and provider outages stay unconfirmed", async () => { + for (const [status, code] of [[404, "NOT_FOUND"], [404, "UNKNOWN"], [503, "UNAVAILABLE"]] as const) { + const cause = new Error("bootstrap failed"); + const client = () => new Freestyle({ apiKey: "synthetic", baseUrl: "https://provider.invalid", fetch: + (async () => Response.json({ code, message: "synthetic deletion response" }, { status })) as typeof fetch, + }); + const result = await Effect.runPromise(Effect.either(rollbackFreestyleCreate(client, "vm-synthetic", cause))); + if (code === "NOT_FOUND") expect(result._tag).toBe("Right"); + else { + expect(result._tag).toBe("Left"); + if (result._tag === "Left") { + expect(result.left.cause).toBe(cause); + expect(result.left.cleanupCause).toBeInstanceOf(FreestyleApiError); + } + } + } +}); + +test("a stalled VM rollback is cancelled and retains the allocated id at its deadline", async () => { + let aborted = false; + const cause = new Error("bootstrap failed"); + await Effect.runPromise(Effect.gen(function* () { + const entered = yield* Deferred.make(); + const client = (_?: number, signal?: AbortSignal) => new Freestyle({ + apiKey: "synthetic", baseUrl: "https://provider.invalid", fetch: (async () => { + Effect.runSync(Deferred.succeed(entered, undefined)); + return new Promise((_, reject) => signal!.addEventListener("abort", () => { + aborted = true; reject(signal!.reason); + }, { once: true })); + }) as typeof fetch, + }); + const work = yield* Effect.fork(Effect.either(rollbackFreestyleCreate(client, "vm-synthetic", cause))); + yield* Deferred.await(entered); + yield* TestClock.adjust(15_000); + const result = yield* Fiber.join(work); + expect(result._tag).toBe("Left"); + if (result._tag === "Left") { + expect(result.left.providerVmId).toBe("vm-synthetic"); + expect(result.left.cause).toBe(cause); + } + expect(aborted).toBe(true); + }).pipe(Effect.provide(TestContext.TestContext))); +}); diff --git a/web/tests/vm-guest-install-runtime.test.ts b/web/tests/vm-guest-install-runtime.test.ts new file mode 100644 index 000000000000..f3b59de6e72f --- /dev/null +++ b/web/tests/vm-guest-install-runtime.test.ts @@ -0,0 +1,184 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { spawnSync } from "node:child_process"; +import { existsSync, mkdtempSync, mkdirSync, readFileSync, readdirSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { GUEST_CMUX_SHIM } from "../services/vms/guestCli"; +import { GUEST_BROWSER_FILES } from "../services/vms/guestBrowser"; +import { freestyleGuestFixture, guestCreateOptions } from "./fixtures/freestyleGuest"; + +const roots: string[] = []; +afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); + +/** Isolated guest filesystem; run the actual uploaded bytes and install command. */ +function guest(options: { corruptUpload?: boolean; promptFails?: boolean; publishFails?: boolean } = {}) { + const root = mkdtempSync(join(tmpdir(), "cmux-guest-install-")); + roots.push(root); + mkdirSync(join(root, "bin")); + mkdirSync(join(root, "libexec")); + mkdirSync(join(root, "fixture-bin")); + const trigger = options.promptFails || options.publishFails; + writeFileSync(join(root, "fixture-bin/getent"), trigger + ? "#!/bin/sh\n[ \"$1\" = passwd ] || exit 1\ncase \"$2\" in root|cmux|ubuntu) printf '%s:x:0:0::%s:/bin/sh\\n' \"$2\" \"$CMUX_GUEST_FIXTURE_ROOT/home/$2\"; exit 0;; esac\nexit 1\n" + : "#!/bin/sh\nexit 1\n", { mode: 0o755 }); + writeFileSync(join(root, "fixture-bin/runuser"), trigger + ? "#!/bin/sh\n[ \"$1\" = -u ] || exit 1\nuser=\"$2\"; shift 2; [ \"$1\" = -- ] || exit 1; shift\nHOME=\"$CMUX_GUEST_FIXTURE_ROOT/home/$user\"; export HOME\nexec \"$@\"\n" + : "#!/bin/sh\nexit 1\n", { mode: 0o755 }); + writeFileSync(join(root, "fixture-bin/xdg-mime"), trigger + ? "#!/bin/sh\nif [ \"$CMUX_GUEST_FAILURE_STAGE\" = prompt ]; then rm -f \"$CMUX_GUEST_FIXTURE_ROOT/etc/prompt.bash\"; mkdir -p \"$CMUX_GUEST_FIXTURE_ROOT/etc/prompt.bash\"; fi\nif [ \"$CMUX_GUEST_FAILURE_STAGE\" = publish ]; then rm -f \"$CMUX_GUEST_FIXTURE_ROOT/bin/cmux\"; mkdir -p \"$CMUX_GUEST_FIXTURE_ROOT/bin/cmux\"; fi\nexit 0\n" + : "#!/bin/sh\nexit 0\n", { mode: 0o755 }); + const prefixes: Record = { + "/usr/local/bin": join(root, "bin"), "/usr/local/libexec": join(root, "libexec"), + "/usr/local/share": join(root, "share"), + "/etc/cmux": join(root, "etc"), "/etc": join(root, "system-etc"), + }; + const rebase = (value: string) => value.replace(/\/usr\/local\/bin|\/usr\/local\/libexec|\/usr\/local\/share|\/etc\/cmux|\/etc(?=\/)/g, + (prefix) => prefixes[prefix]!); + const target = join(root, "libexec/cmux-cloud-adapter"); + const fixture = freestyleGuestFixture({ + write: (path, bytes) => writeFileSync(rebase(path), options.corruptUpload ? "#!/bin/sh\nexit 0\n" : bytes), + remove: (path) => rmSync(rebase(path), { force: true }), + exec: async (request) => { + const result = spawnSync("/bin/sh", ["-c", rebase(request.command)], { + encoding: "utf8", timeout: 5_000, + env: { + ...process.env, HOME: root, PATH: `${join(root, "fixture-bin")}:${process.env.PATH}`, + CMUX_GUEST_FIXTURE_ROOT: root, + ...(options.promptFails ? { CMUX_GUEST_FAILURE_STAGE: "prompt" } : {}), + ...(options.publishFails ? { CMUX_GUEST_FAILURE_STAGE: "publish" } : {}), + }, + }); + return Response.json({ statusCode: result.status, stdout: result.stdout, stderr: result.stderr }); + }, + }); + return { root, target, fixture }; +} + +describe("guest CLI publication in an isolated filesystem", () => { + test("successful create installs the complete executable CLI and answers help", async () => { + const { fixture, target, root } = guest(); + const handle = await fixture.createWithGuestInstall(guestCreateOptions); + expect(handle.status).toBe("running"); + expect(readFileSync(target, "utf8")).toBe(GUEST_CMUX_SHIM); + for (const file of GUEST_BROWSER_FILES.filter((file) => file.path.startsWith("/usr/local/bin/"))) { + expect(readFileSync(join(root, "bin", file.path.split("/").at(-1)!), "utf8")).toBe(file.content); + } + expect(statSync(target).mode & 0o777).toBe(0o755); + const result = spawnSync(target, ["--help"], { encoding: "utf8", timeout: 5_000 }); + expect(result.status).toBe(0); + expect(result.stdout).toContain("cmux"); + const daemon = join(root, "cmux-tui"); + writeFileSync(daemon, '#!/bin/sh\nprintf \'%s\\n\' "$@" > "$HOME/daemon-args"\nprintf \'%s\\n\' \'{"session":"cloud","workspaces":[]}\'\n', { mode: 0o755 }); + const tree = spawnSync(target, ["tree", "--json"], { + encoding: "utf8", timeout: 5_000, + env: { ...process.env, HOME: root, CMUX_TUI_BIN: daemon }, + }); + expect(tree.status).toBe(0); + expect(JSON.parse(tree.stdout)).toEqual({ session: "cloud", workspaces: [] }); + expect(readFileSync(join(root, "daemon-args"), "utf8").trim().split("\n")) + .toEqual(["--session", "cloud", "--json", "session", "current", "snapshot"]); + expect(readdirSync(join(root, "bin")).sort()).toEqual([ + "cmux", "cmux-open-url", "coderouter", "cr", "sensible-browser", "x-www-browser", "xdg-open", + ]); + }); + + test("a fresh prompt install keeps its identity when an older revision attaches", async () => { + const { fixture, root } = guest(); + await fixture.createWithGuestInstall({ + ...guestCreateOptions, + promptIdentity: { machineId: "synthetic", name: "fresh-machine", revision: 10 }, + }); + await fixture.createWithGuestInstall({ + ...guestCreateOptions, + promptIdentity: { machineId: "synthetic", name: "stale-machine", revision: 9 }, + }); + expect(readFileSync(join(root, "etc/vm-name"), "utf8")).toBe("fresh-machine\n"); + expect(JSON.parse(readFileSync(join(root, "etc/.prompt-identity"), "utf8"))).toEqual({ + machineId: "synthetic", name: "fresh-machine", revision: 10, + }); + }); + + test("replaces a target symlink without modifying its referent", async () => { + const { fixture, root, target } = guest(); + const unrelated = join(root, "unrelated"); + writeFileSync(unrelated, "preserve me"); + symlinkSync(unrelated, target); + await fixture.createWithGuestInstall(guestCreateOptions); + expect(readFileSync(unrelated, "utf8")).toBe("preserve me"); + expect(readFileSync(target, "utf8")).toBe(GUEST_CMUX_SHIM); + }); + + test("a directory destination fails instead of moving the shim inside and reporting ready", async () => { + const { fixture, target } = guest(); + mkdirSync(target); + const result = await fixture.createWithGuestInstall(guestCreateOptions).then(() => "ready", () => "failed"); + expect(result).toBe("failed"); + expect(fixture.liveVms.size).toBe(0); + }); + + test("rejects a corrupted upload before replacing the previous generation", async () => { + const { fixture, target } = guest({ corruptUpload: true }); + writeFileSync(target, "previous generation"); + const result = await fixture.createWithGuestInstall(guestCreateOptions).then(() => "ready", () => "failed"); + expect(result).toBe("failed"); + expect(readFileSync(target, "utf8")).toBe("previous generation"); + expect(fixture.liveVms.size).toBe(0); + }); + + test("prompt failure does not publish a new shim generation", async () => { + const { fixture, target, root } = guest({ promptFails: true }); + writeFileSync(target, "previous generation"); + const browserOpener = join(root, "bin/cmux-open-url"); + writeFileSync(browserOpener, "previous browser generation"); + mkdirSync(join(root, "etc")); + writeFileSync(join(root, "etc/prompt.bash"), "previous prompt generation"); + writeFileSync(join(root, "etc/bashrc"), "previous bashrc generation"); + writeFileSync(join(root, "etc/.prompt-identity"), "previous identity"); + writeFileSync(join(root, "etc/vm-name"), "previous name\n"); + const failure = await fixture.createWithGuestInstall({ + ...guestCreateOptions, + promptIdentity: { machineId: "synthetic", name: "synthetic", revision: 1 }, + }).then(() => undefined, (error) => error); + expect(failure?.cause?.stage).toBe("prompt"); + expect(readFileSync(target, "utf8")).toBe("previous generation"); + expect(readFileSync(browserOpener, "utf8")).toBe("previous browser generation"); + expect(readFileSync(join(root, "etc/prompt.bash"), "utf8")).toBe("previous prompt generation"); + expect(readFileSync(join(root, "etc/bashrc"), "utf8")).toBe("previous bashrc generation"); + expect(readFileSync(join(root, "etc/.prompt-identity"), "utf8")).toBe("previous identity"); + expect(readFileSync(join(root, "etc/vm-name"), "utf8")).toBe("previous name\n"); + const promptArtifacts = readdirSync(join(root, "etc")); + expect(promptArtifacts.some((name) => name.startsWith(".cmux-install-") || (name.startsWith(".prompt-") && ![".prompt-lock", ".prompt-identity"].includes(name)))).toBe(false); + // A failed transaction may retain an unreferenced immutable release cache; + // the safety property is that no alias points at that new generation. + expect(existsSync(join(root, "libexec", "cmux-coderouter"))).toBe(false); + expect(existsSync(join(root, "bin", "coderouter"))).toBe(false); + expect(fixture.liveVms.size).toBe(0); + }); + + test("publish failure restores browser, prompt, and shim generations without temp artifacts", async () => { + const { fixture, target, root } = guest({ publishFails: true }); + writeFileSync(target, "previous generation"); + writeFileSync(join(root, "bin/cmux-open-url"), "previous browser generation"); + mkdirSync(join(root, "etc")); + writeFileSync(join(root, "etc/prompt.bash"), "previous prompt generation"); + writeFileSync(join(root, "etc/bashrc"), "previous bashrc generation"); + writeFileSync(join(root, "etc/.prompt-identity"), "previous identity"); + writeFileSync(join(root, "etc/vm-name"), "previous name\n"); + const failure = await fixture.createWithGuestInstall({ + ...guestCreateOptions, + promptIdentity: { machineId: "synthetic", name: "synthetic", revision: 1 }, + }).then(() => undefined, (error) => error); + expect(failure?.cause?.stage).toBe("publish"); + expect(readFileSync(target, "utf8")).toBe("previous generation"); + expect(readFileSync(join(root, "bin/cmux-open-url"), "utf8")).toBe("previous browser generation"); + expect(readFileSync(join(root, "etc/prompt.bash"), "utf8")).toBe("previous prompt generation"); + expect(readFileSync(join(root, "etc/bashrc"), "utf8")).toBe("previous bashrc generation"); + expect(readFileSync(join(root, "etc/.prompt-identity"), "utf8")).toBe("previous identity"); + expect(readFileSync(join(root, "etc/vm-name"), "utf8")).toBe("previous name\n"); + const publishArtifacts = readdirSync(join(root, "etc")); + expect(publishArtifacts.some((name) => name.startsWith(".cmux-install-") || (name.startsWith(".prompt-") && ![".prompt-lock", ".prompt-identity"].includes(name)))).toBe(false); + expect(existsSync(join(root, "libexec", "cmux-coderouter"))).toBe(false); + expect(existsSync(join(root, "bin", "coderouter"))).toBe(false); + expect(fixture.liveVms.size).toBe(0); + }); +}); diff --git a/web/tests/vm-guest-install.test.ts b/web/tests/vm-guest-install.test.ts new file mode 100644 index 000000000000..549a7a5b2eeb --- /dev/null +++ b/web/tests/vm-guest-install.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, test } from "bun:test"; +import { freestyleGuestFixture, guestCreateOptions } from "./fixtures/freestyleGuest"; + +describe("Freestyle guest install result contract (SDK 0.2.10)", () => { + test("a reported zero exit publishes a ready machine", async () => { + const fixture = freestyleGuestFixture(); + const handle = await fixture.createWithGuestInstall(guestCreateOptions); + expect(handle.status).toBe("running"); + expect(fixture.liveVms.size).toBe(1); + expect(fixture.allocations()).toBe(1); + }); + + // The incident preserved only the old 'exited 124' message. These are + // synthetic candidate responses, not invented copies of the lost body. + test.each([ + [{ stdout: "", stderr: "" }, "missing_status", undefined], + [{ exitCode: 0 }, "missing_status", undefined], + [{ statusCode: null, stdout: "", stderr: "" }, "provider_timeout", null], + [{ statusCode: 124, stderr: "synthetic guest exit" }, "guest_exit", 124], + [{ statusCode: 1, stderr: "synthetic rename failure" }, "guest_exit", 1], + [{ statusCode: "0" }, "invalid_status", undefined], + ] as Array<[Record, string, number | null | undefined]>) + ("preserves outcome %s without a false ready result", async (body, outcome, statusCode) => { + const fixture = freestyleGuestFixture({ exec: async () => Response.json(body) }); + const error = await fixture.createWithGuestInstall(guestCreateOptions).catch((error: unknown) => error); + expect(error).toBeInstanceOf(Error); + const installError = (error as { cause: { name: string; outcome: string; exitCode?: number | null } }).cause; + expect(installError.name).toBe("GuestCliInstallError"); + expect(installError.outcome).toBe(outcome); + expect(installError.exitCode).toBe(statusCode); + expect(fixture.allocations()).toBe(1); + expect(fixture.liveVms.size).toBe(0); + expect(fixture.removals).toEqual(fixture.writes); + }); + + test.each([ + ["AbortError", "cancelled"], + ["TimeoutError", "transport_timeout"], + ])("preserves transport %s separately from a guest exit", async (name, outcome) => { + const cause = new DOMException("synthetic transport interruption", name); + const fixture = freestyleGuestFixture({ exec: async () => { throw cause; } }); + const error = await fixture.createWithGuestInstall(guestCreateOptions).catch((error: unknown) => error); + expect((error as { cause?: unknown }).cause).toMatchObject({ name: "GuestCliInstallError", outcome, cause }); + expect(fixture.liveVms.size).toBe(0); + expect(fixture.removals).toEqual(fixture.writes); + }); + + test("failed rollback retains the allocated machine and both failures", async () => { + const fixture = freestyleGuestFixture({ + exec: async () => Response.json({ statusCode: 1, stderr: "synthetic install failure" }), + deleteFailure: true, + }); + const error = await fixture.createWithGuestInstall(guestCreateOptions).catch((error: unknown) => error); + expect(error).toMatchObject({ + name: "ProviderCreateCleanupError", + providerVmId: "vm-fixture-1", + cause: { name: "GuestCliInstallError", outcome: "guest_exit", exitCode: 1 }, + cleanupCause: { name: "FreestyleApiError", status: 503 }, + }); + expect(fixture.liveVms.size).toBe(1); + }); + + test("keeps the guest install error when rollback also fails", async () => { + const fixture = freestyleGuestFixture({ + exec: async () => Response.json({ + statusCode: 1, + stderr: `CMUX_GUEST_INSTALL_FAILURE=${JSON.stringify({ + stage: "publish", error: "OSError", rollbackError: "PermissionError", errno: 13, + })}`, + }), + }); + const error = await fixture.createWithGuestInstall(guestCreateOptions).catch((error: unknown) => error); + expect((error as { cause?: unknown }).cause).toMatchObject({ + name: "GuestCliInstallError", + stage: "publish", + diagnostic: "OSError errno=13", + rollbackError: "PermissionError", + }); + }); +}); diff --git a/web/tests/vm-provider-errors.test.ts b/web/tests/vm-provider-errors.test.ts index 66afac075b8f..0a11777181fc 100644 --- a/web/tests/vm-provider-errors.test.ts +++ b/web/tests/vm-provider-errors.test.ts @@ -2,6 +2,7 @@ import { describe, expect, test } from "bun:test"; import { FreestyleApiError } from "freestyle"; import { ProviderError } from "../services/vms/drivers/types"; import { + isProviderDeletionConfirmed, isProviderIdentityNotFoundError, isProviderNotFoundError, } from "../services/vms/providerErrors"; @@ -51,4 +52,10 @@ describe("provider error classification", () => { expect(isProviderIdentityNotFoundError(err)).toBe(true); expect(isProviderNotFoundError(err)).toBe(false); }); + + test("background deletion confirmation requires the explicit NOT_FOUND code", () => { + expect(isProviderDeletionConfirmed({ cause: { status: 404, code: "NOT_FOUND" } })).toBe(true); + expect(isProviderDeletionConfirmed({ cause: { status: 404, code: "UNKNOWN" } })).toBe(false); + expect(isProviderDeletionConfirmed({ cause: { status: 503, code: "NOT_FOUND" } })).toBe(false); + }); }); diff --git a/web/tests/vm-workflows.test.ts b/web/tests/vm-workflows.test.ts index 78812173198a..1ab80fc6c902 100644 --- a/web/tests/vm-workflows.test.ts +++ b/web/tests/vm-workflows.test.ts @@ -1,4 +1,4 @@ -import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { afterAll, beforeAll, describe, expect, setSystemTime, test } from "bun:test"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import postgres, { type Sql } from "postgres"; @@ -41,6 +41,7 @@ import { } from "../services/vms/errors"; import { accountDeletionUserHash } from "../services/account/deletionLock"; import { isVmAttachTransportUnsupportedError } from "../services/vms/errors"; +import { freestyleGuestFixture, guestCreateOptions } from "./fixtures/freestyleGuest"; import { VM_DISK_MB_MAX, VM_RESOURCE_RESIZE_PENDING_METADATA_KEY, @@ -196,6 +197,343 @@ afterAll(async () => { }); describe("VM Effect workflows", () => { + dbTest("guest bootstrap retry allocates once after confirmed rollback and reuses the ready result", async () => { + if (!sql) throw new Error("test database not initialized"); + let fail = true; + const fixture = freestyleGuestFixture({ exec: async () => Response.json({ statusCode: fail ? 124 : 0 }) }); + const layer = providerLayer({ + ...unusedProviderGateway(), + create: (_, options) => Effect.tryPromise({ + try: () => fixture.createWithGuestInstall(options), + catch: (cause) => new VmProviderOperationError({ provider: "freestyle", operation: "create", cause }), + }), + }); + const program = createVm({ + ...guestCreateOptions, userId: "user-guest-retry", billingTeamId: "team-guest-retry", + billingCustomerType: "team", billingPlanId: "pro", provider: "freestyle", + idempotencyKey: "guest-retry", maxActiveVms: 1, + }).pipe(Effect.provide(layer)); + const first = await Effect.runPromise(Effect.either(program)); + expect(first._tag).toBe("Left"); + expect(fixture.allocations()).toBe(1); + expect(fixture.liveVms.size).toBe(0); + fail = false; + const attempts = await Promise.all([Effect.runPromise(Effect.either(program)), Effect.runPromise(Effect.either(program))]); + expect(attempts.some((attempt) => attempt._tag === "Right")).toBe(true); + for (const attempt of attempts) { + if (attempt._tag === "Left") expect(attempt.left._tag).toBe("VmCreateInProgressError"); + else expect(attempt.right.providerVmId).toBe("vm-fixture-2"); + } + const replay = await Effect.runPromise(program); + expect(replay.providerVmId).toBe("vm-fixture-2"); + expect(fixture.allocations()).toBe(2); + expect(fixture.liveVms.size).toBe(1); + const [events] = await sql<{ count: string }[]>`select count(*)::text as count from cloud_vm_usage_events where event_type = 'vm.created' and user_id = 'user-guest-retry'`; + expect(events.count).toBe("1"); + }); + + dbTest("unconfirmed guest rollback stays reserved and cannot become ready or allocate on retry", async () => { + if (!sql) throw new Error("test database not initialized"); + const fixture = freestyleGuestFixture({ exec: async () => Response.json({ statusCode: 1 }), deleteFailure: true }); + const layer = providerLayer({ + ...unusedProviderGateway(), + create: (_, options) => Effect.tryPromise({ + try: () => fixture.createWithGuestInstall(options), + catch: (cause) => new VmProviderOperationError({ provider: "freestyle", operation: "create", cause }), + }), + }); + const input = { + ...guestCreateOptions, userId: "user-guest-cleanup", billingTeamId: "team-guest-cleanup", + billingCustomerType: "team" as const, billingPlanId: "pro", provider: "freestyle" as const, + idempotencyKey: "guest-cleanup", maxActiveVms: 1, + }; + const run = (key = input.idempotencyKey) => Effect.runPromise(Effect.either(createVm({ ...input, idempotencyKey: key }).pipe(Effect.provide(layer)))); + expect((await run())._tag).toBe("Left"); + await sql`update cloud_vms set updated_at = now() - interval '2 days' where user_id = 'user-guest-cleanup'`; + const retry = await run(); + expect(retry._tag).toBe("Left"); + if (retry._tag === "Left") expect(retry.left).toMatchObject({ _tag: "VmCreateFailedError", code: "provider_create_cleanup_pending" }); + expect((await run("different-key"))._tag).toBe("Left"); + const [row] = await sql<{ status: string; providerVmId: string | null; metadata: Record }[]>` + select status, provider_vm_id as "providerVmId", provider_metadata as metadata from cloud_vms where user_id = 'user-guest-cleanup' + `; + expect(row.status).toBe("provisioning"); + expect(row.providerVmId).toBeNull(); + expect(row.metadata.createCleanupProviderVmId).toBe("vm-fixture-1"); + expect(await Effect.runPromise(listUserVms(input.userId).pipe(Effect.provide(layer)))).toEqual([]); + expect(fixture.allocations()).toBe(1); + expect(fixture.liveVms.size).toBe(1); + }); + + dbTest("reconciles retained guest allocations before allowing a retry", async () => { + if (!sql) throw new Error("test database not initialized"); + // The preceding cleanup-pending case intentionally leaves its row + // reserved. Remove those fixture rows so this race assertion measures one + // provider allocation, while production reconciliation still scans all + // eligible rows in order. + await sql`delete from cloud_vms where user_id like 'user-guest-cleanup%'`; + let failInstall = true; + const fixture = freestyleGuestFixture({ + exec: async () => Response.json({ statusCode: failInstall ? 1 : 0 }), + deleteFailure: true, + idPrefix: "vm-cleanup-reconcile", + }); + let destroyCalls = 0; + const layer = providerLayer({ + ...unusedProviderGateway(), + create: (_, options) => Effect.tryPromise({ + try: () => fixture.createWithGuestInstall(options), + catch: (cause) => new VmProviderOperationError({ provider: "freestyle", operation: "create", cause }), + }), + destroy: (_, providerVmId) => Effect.sync(() => { + destroyCalls += 1; + fixture.liveVms.delete(providerVmId); + }), + }); + const input = { + ...guestCreateOptions, userId: "user-guest-cleanup-reconcile", billingTeamId: "team-guest-cleanup-reconcile", + billingCustomerType: "team" as const, billingPlanId: "pro", provider: "freestyle" as const, + idempotencyKey: "guest-cleanup-reconcile", maxActiveVms: 1, + }; + + expect((await Effect.runPromise(Effect.either(createVm(input).pipe(Effect.provide(layer)))))._tag).toBe("Left"); + expect(fixture.liveVms.size).toBe(1); + await Promise.all([ + Effect.runPromise(reconcileVmProviderStatuses().pipe(Effect.provide(layer))), + Effect.runPromise(reconcileVmProviderStatuses().pipe(Effect.provide(layer))), + ]); + expect(destroyCalls).toBe(1); + const [resolved] = await sql<{ status: string; providerVmId: string | null; metadata: Record }[]>` + select status, provider_vm_id as "providerVmId", provider_metadata as metadata + from cloud_vms where user_id = 'user-guest-cleanup-reconcile' + `; + expect(resolved.status).toBe("failed"); + expect(resolved.providerVmId).toBeNull(); + expect(resolved.metadata.createCleanupProviderVmId).toBeUndefined(); + + failInstall = false; + const retry = await Effect.runPromise(createVm(input).pipe(Effect.provide(layer))); + expect(retry.providerVmId).toBe("vm-cleanup-reconcile-2"); + expect(fixture.liveVms).toEqual(new Set(["vm-cleanup-reconcile-2"])); + }); + + dbTest("retains failed cleanup with durable backoff and does not duplicate the provider delete", async () => { + if (!sql) throw new Error("test database not initialized"); + const nowMs = 1_800_000_000_000; + setSystemTime(nowMs); + try { + const fixture = freestyleGuestFixture({ exec: async () => Response.json({ statusCode: 1 }), deleteFailure: true }); + let destroyCalls = 0; + const layer = providerLayer({ + ...unusedProviderGateway(), + create: (_, options) => Effect.tryPromise({ + try: () => fixture.createWithGuestInstall(options), + catch: (cause) => new VmProviderOperationError({ provider: "freestyle", operation: "create", cause }), + }), + destroy: () => Effect.sync(() => { + destroyCalls += 1; + }).pipe(Effect.andThen(Effect.fail(new VmProviderOperationError({ + provider: "freestyle", + operation: "destroy", + cause: new Error("synthetic cleanup outage"), + })))), + }); + const input = { + ...guestCreateOptions, userId: "user-guest-cleanup-backoff", billingTeamId: "team-guest-cleanup-backoff", + billingCustomerType: "team" as const, billingPlanId: "pro", provider: "freestyle" as const, + idempotencyKey: "guest-cleanup-backoff", maxActiveVms: 1, + }; + + expect((await Effect.runPromise(Effect.either(createVm(input).pipe(Effect.provide(layer)))))._tag).toBe("Left"); + await Effect.runPromise(reconcileVmProviderStatuses().pipe(Effect.provide(layer))); + await Effect.runPromise(reconcileVmProviderStatuses().pipe(Effect.provide(layer))); + expect(destroyCalls).toBe(1); + const [row] = await sql<{ status: string; metadata: Record }[]>` + select status, provider_metadata as metadata from cloud_vms where user_id = 'user-guest-cleanup-backoff' + `; + expect(row.status).toBe("provisioning"); + expect(row.metadata.createCleanupProviderVmId).toBe("vm-fixture-1"); + expect(Number(row.metadata.createCleanupNextAttemptAtMs)).toBe(nowMs + 5_000); + expect(row.metadata.createCleanupLeaseId).toBeUndefined(); + } finally { + setSystemTime(); + } + }); + + dbTest("treats malformed cleanup timestamps and oversized attempts as immediately eligible", async () => { + if (!sql) throw new Error("test database not initialized"); + const fixture = freestyleGuestFixture({ + exec: async () => Response.json({ statusCode: 1 }), + deleteFailure: true, + idPrefix: "vm-cleanup-timestamp", + }); + let destroyCalls = 0; + const layer = providerLayer({ + ...unusedProviderGateway(), + create: (_, options) => Effect.tryPromise({ + try: () => fixture.createWithGuestInstall(options), + catch: (cause) => new VmProviderOperationError({ provider: "freestyle", operation: "create", cause }), + }), + destroy: (_, providerVmId) => Effect.sync(() => { + destroyCalls += 1; + fixture.liveVms.delete(providerVmId); + }), + }); + const input = { + ...guestCreateOptions, userId: "user-guest-cleanup-timestamp", billingTeamId: "team-guest-cleanup-timestamp", + billingCustomerType: "team" as const, billingPlanId: "pro", provider: "freestyle" as const, + idempotencyKey: "guest-cleanup-timestamp", maxActiveVms: 1, + }; + expect((await Effect.runPromise(Effect.either(createVm(input).pipe(Effect.provide(layer)))))._tag).toBe("Left"); + await sql` + update cloud_vms + set provider_metadata = provider_metadata || jsonb_build_object( + 'createCleanupNextAttemptAtMs', '999999999999999999999999999999', + 'createCleanupLeaseExpiresAtMs', 'not-a-timestamp', + 'createCleanupAttempt', '999999999999999999999999999999' + ) + where user_id = 'user-guest-cleanup-timestamp' + `; + await Effect.runPromise(reconcileVmProviderStatuses().pipe(Effect.provide(layer))); + expect(destroyCalls).toBe(1); + const [row] = await sql<{ status: string; metadata: Record }[]>` + select status, provider_metadata as metadata from cloud_vms where user_id = 'user-guest-cleanup-timestamp' + `; + expect(row.status).toBe("failed"); + expect(row.metadata.createCleanupProviderVmId).toBeUndefined(); + }); + + dbTest("does not resolve a replaced cleanup marker with a stale lease", async () => { + if (!sql) throw new Error("test database not initialized"); + const fixture = freestyleGuestFixture({ + exec: async () => Response.json({ statusCode: 1 }), + deleteFailure: true, + idPrefix: "vm-cleanup-cas", + }); + const layer = providerLayer({ + ...unusedProviderGateway(), + create: (_, options) => Effect.tryPromise({ + try: () => fixture.createWithGuestInstall(options), + catch: (cause) => new VmProviderOperationError({ provider: "freestyle", operation: "create", cause }), + }), + }); + const input = { + ...guestCreateOptions, userId: "user-guest-cleanup-cas", billingTeamId: "team-guest-cleanup-cas", + billingCustomerType: "team" as const, billingPlanId: "pro", provider: "freestyle" as const, + idempotencyKey: "guest-cleanup-cas", maxActiveVms: 1, + }; + expect((await Effect.runPromise(Effect.either(createVm(input).pipe(Effect.provide(layer)))))._tag).toBe("Left"); + const [row] = await sql<{ id: string; providerVmId: string }[]>` + select id, provider_metadata->>'createCleanupProviderVmId' as "providerVmId" + from cloud_vms where user_id = 'user-guest-cleanup-cas' + `; + const now = new Date("2026-09-20T00:00:00.000Z"); + const leaseExpiresAt = new Date(now.getTime() + 60_000); + const claim = await Effect.runPromise(vmRepositoryLiveShape.claimCreateCleanup!({ + id: row.id, + providerVmId: row.providerVmId, + leaseId: "lease-a", + now, + leaseExpiresAt, + })); + expect(claim?.attempt).toBe(1); + const duplicateClaim = await Effect.runPromise(vmRepositoryLiveShape.claimCreateCleanup!({ + id: row.id, + providerVmId: row.providerVmId, + leaseId: "lease-b", + now, + leaseExpiresAt, + })); + expect(duplicateClaim).toBeNull(); + await sql` + update cloud_vms + set provider_metadata = provider_metadata || jsonb_build_object('createCleanupProviderVmId', 'replacement-provider-vm') + where id = ${row.id} + `; + const resolved = await Effect.runPromise(vmRepositoryLiveShape.resolveCreateCleanup!({ + id: row.id, + providerVmId: row.providerVmId, + leaseId: "lease-a", + })); + expect(resolved).toBe(false); + const [stillPending] = await sql<{ status: string; providerVmId: string }[]>` + select status, provider_metadata->>'createCleanupProviderVmId' as "providerVmId" + from cloud_vms where id = ${row.id} + `; + expect(stillPending.status).toBe("provisioning"); + expect(stillPending.providerVmId).toBe("replacement-provider-vm"); + }); + + dbTest("unconfirmed guest rollback keeps the Base generation reserved", async () => { + if (!sql) throw new Error("test database not initialized"); + const fixture = freestyleGuestFixture({ exec: async () => Response.json({ statusCode: 1 }), deleteFailure: true }); + const layer = providerLayer({ + ...unusedProviderGateway(), + create: (_, options) => Effect.tryPromise({ + try: () => fixture.createWithGuestInstall({ ...options, imageSize: guestCreateOptions.imageSize }), + catch: (cause) => new VmProviderOperationError({ provider: "freestyle", operation: "create", cause }), + }), + }); + const input = { + userId: "user-guest-base-cleanup", billingTeamId: "team-guest-base-cleanup", + billingCustomerType: "team" as const, billingPlanId: "pro", provider: "freestyle" as const, + image: "sh-synthetic", baseName: "guest-cleanup", maxActiveVms: 1, + }; + const first = await Effect.runPromise(Effect.either(openBaseVm(input).pipe(Effect.provide(layer)))); + expect(first._tag).toBe("Left"); + const retry = await Effect.runPromise(Effect.either(openBaseVm(input).pipe(Effect.provide(layer)))); + if (retry._tag === "Left") expect(retry.left).toMatchObject({ _tag: "VmCreateFailedError", code: "provider_create_cleanup_pending" }); + else throw new Error("Base must not report a failed bootstrap ready"); + const reset = await Effect.runPromise(Effect.either(resetBaseVm(input).pipe(Effect.provide(layer)))); + expect(reset._tag).toBe("Left"); + expect(fixture.allocations()).toBe(1); + const [row] = await sql<{ status: string; providerVmId: string | null; metadata: Record }[]>` + select status, provider_vm_id as "providerVmId", provider_metadata as metadata from cloud_vms where user_id = 'user-guest-base-cleanup' + `; + expect(row.status).toBe("provisioning"); + expect(row.providerVmId).toBeNull(); + expect(row.metadata.createCleanupProviderVmId).toBe("vm-fixture-1"); + }); + + dbTest("confirmed Base cleanup restores the retained generation", async () => { + if (!sql) throw new Error("test database not initialized"); + let failInstall = false; + const fixture = freestyleGuestFixture({ + exec: async () => Response.json({ statusCode: failInstall ? 1 : 0 }), + deleteFailure: true, + idPrefix: "vm-base-cleanup-recovery", + }); + const layer = providerLayer({ + ...unusedProviderGateway(), + create: (_, options) => Effect.tryPromise({ + try: () => fixture.createWithGuestInstall(options), + catch: (cause) => new VmProviderOperationError({ provider: "freestyle", operation: "create", cause }), + }), + destroy: (_, providerVmId) => Effect.sync(() => { + fixture.liveVms.delete(providerVmId); + }), + }); + const input = { + userId: "user-base-cleanup-recovery", billingTeamId: "team-base-cleanup-recovery", + billingCustomerType: "team" as const, billingPlanId: "pro", provider: "freestyle" as const, + image: "sh-synthetic", baseName: "cleanup-recovery", maxActiveVms: 2, + }; + const first = await Effect.runPromise(openBaseVm(input).pipe(Effect.provide(layer))); + expect(first.providerVmId).toBe("vm-base-cleanup-recovery-1"); + failInstall = true; + const reset = await Effect.runPromise(Effect.either(resetBaseVm(input).pipe(Effect.provide(layer)))); + expect(reset._tag).toBe("Left"); + await Effect.runPromise(reconcileVmProviderStatuses().pipe(Effect.provide(layer))); + const recovered = await Effect.runPromise(openBaseVm(input).pipe(Effect.provide(layer))); + expect(recovered.providerVmId).toBe("vm-base-cleanup-recovery-1"); + const [base] = await sql<{ state: string; activeProviderVmId: string | null }[]>` + select state, active_provider_vm_id as "activeProviderVmId" + from cloud_vm_bases where scope_id = 'team-base-cleanup-recovery' and name = 'cleanup-recovery' + `; + expect(base.state).toBe("ready"); + expect(base.activeProviderVmId).toBe("vm-base-cleanup-recovery-1"); + }); + dbTest("keeps prompt revisions ordered across rapid renames and clock skew", async () => { if (!sql) throw new Error("test database not initialized"); const userId = "user-prompt-revisions";