From 25f1b521e247c7f153a984eed60c607a6d82c373 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sat, 19 Sep 2026 22:34:46 -0700 Subject: [PATCH 01/92] test: reproduce dropped Cloud placement diagnostics --- .../cloud-diagnostics-db-behavior.test.ts | 14 ++++++ web/tests/cloud-telemetry.test.ts | 47 +++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/web/tests/cloud-diagnostics-db-behavior.test.ts b/web/tests/cloud-diagnostics-db-behavior.test.ts index d7a6415b6c40..00c7babeffaa 100644 --- a/web/tests/cloud-diagnostics-db-behavior.test.ts +++ b/web/tests/cloud-diagnostics-db-behavior.test.ts @@ -6,6 +6,7 @@ import { acceptCloudTelemetry, claimCloudDiagnostics, finishCloudDiagnostics } f import { CloudTelemetryConflictError } from "../services/observability/cloudTelemetryIngest"; import { CloudOperationProgress, readCloudOperationProgress } from "../services/observability/cloudOperationProgress"; import type { CloudTelemetryBatch } from "../services/observability/cloudTelemetryContract"; +import { parseCloudTelemetryBatch } from "../services/observability/cloudTelemetryContract"; const enabled = process.env.CMUX_DB_TEST === "1"; const dbTest = enabled ? test : test.skip; @@ -32,6 +33,19 @@ afterEach(async () => { }); describe("Cloud diagnostic durable storage", () => { + dbTest("accepted placement diagnostics retain their category through the durable outbox", async () => { + const { owner, batch } = fixture(); + const submitted = { ...batch, spans: [{ ...batch.spans[0]!, failure: "placement" }] }; + const parsed = parseCloudTelemetryBatch(submitted); + expect(parsed).not.toBeNull(); + expect(await acceptCloudTelemetry(owner, parsed!)).toBe(1); + const claimed = await claimCloudDiagnostics(100, owner); + expect(claimed.rows).toHaveLength(1); + expect(claimed.rows[0]!.payload.span).toEqual(submitted.spans[0]); + await finishCloudDiagnostics(claimed.leaseId, true); + expect((await claimCloudDiagnostics(100, owner)).rows).toEqual([]); + }); + dbTest("concurrent retry receipts store one event and charge once", async () => { const { owner, batch } = fixture(); expect(await Promise.all(Array.from({ length: 6 }, () => acceptCloudTelemetry(owner, batch)))).toEqual([1, 1, 1, 1, 1, 1]); diff --git a/web/tests/cloud-telemetry.test.ts b/web/tests/cloud-telemetry.test.ts index 404ca418b8ea..5eb64ef0969a 100644 --- a/web/tests/cloud-telemetry.test.ts +++ b/web/tests/cloud-telemetry.test.ts @@ -30,6 +30,53 @@ function request(body: unknown = batch(), headers: Record = {}) } describe("Cloud diagnostic boundary", () => { + test("acknowledges and exports placement failures without dropping neighboring spans", async () => { + const { exportCloudDiagnostics } = await import("../services/observability/cloudTelemetryExport"); + const payload = batch([ + span({ operation: "terminal", phase: "materialize", failure: "placement" }), + span({ eventId: "8cc333de-a1bc-4eb1-8d69-1decd01e17a9" }), + ]); + const accepted: unknown[] = []; + const sent: { url: string; body: any }[] = []; + const handler = makeCloudTelemetryHandler({ + authenticate: async () => ({ id: "synthetic-owner" }), + checkIngress: async () => true, + accept: async (userId, value) => { + accepted.push(value); + await exportCloudDiagnostics(value.spans.map((span) => ({ + userId, eventId: span.eventId, attempts: 1, payload: { client: value.client, span }, + })), { + origin: "https://us-east-1.aws.edge.axiom.co", token: "test-only", + identityKey: "test-only-identity-key".repeat(2), tracesDataset: "test-traces", + errorsDataset: "test-errors", environment: "preview", revision: "abcdef123", + }, (async (url, init) => { + sent.push({ url: String(url), body: JSON.parse(String(init?.body)) }); + return new Response("{}"); + }) as typeof fetch); + return value.spans.length; + }, + scheduleDrain: () => {}, now: () => now, + }); + const response = await handler(request(payload)); + expect(response.status).toBe(202); + expect(await response.json()).toEqual({ + accepted: 2, eventIds: payload.spans.map((span) => span.eventId), + }); + expect(accepted).toEqual([payload]); + const errors = sent.find((item) => item.url.endsWith("/v1/ingest/test-errors"))!.body; + expect(errors.map((item: any) => item.failure)).toEqual(["placement", "network"]); + expect(errors[0]).toMatchObject({ + event_id: payload.spans[0]!.eventId, operation_id: payload.spans[0]!.operationId, + trace_id: payload.spans[0]!.traceId, span_id: payload.spans[0]!.spanId, + operation: "terminal", phase: "materialize", outcome: "failure", + }); + const spans = sent.find((item) => item.url.endsWith("/v1/traces"))!.body.resourceSpans; + expect(spans[0].scopeSpans[0].spans[0].attributes).toContainEqual({ + key: "error.type", value: { stringValue: "placement" }, + }); + expect(JSON.stringify(sent)).not.toContain("synthetic-owner"); + }); + test("accepts a timed operation with a real parent span", () => { expect(parseCloudTelemetryBatch(batch(), now)?.spans[0]).toEqual(span()); }); From 720dbe0e380046556af85580376e44977db4f9dd Mon Sep 17 00:00:00 2001 From: Austin Wang Date: Mon, 14 Sep 2026 02:31:35 -0700 Subject: [PATCH 02/92] test: reproduce structured provider failure misclassification (cherry picked from commit 091789dc6debadfe5fba24d93f7cc62f11dfa896) --- web/tests/vm-provider-errors.test.ts | 16 +++ web/tests/vm-stats-not-found.test.ts | 148 +++++++++++++++++++++++++++ 2 files changed, 164 insertions(+) create mode 100644 web/tests/vm-stats-not-found.test.ts diff --git a/web/tests/vm-provider-errors.test.ts b/web/tests/vm-provider-errors.test.ts index fb797f6d63f7..a174e9d85f27 100644 --- a/web/tests/vm-provider-errors.test.ts +++ b/web/tests/vm-provider-errors.test.ts @@ -1,10 +1,26 @@ import { describe, expect, test } from "bun:test"; +import { FreestyleApiError } from "freestyle"; +import { ProviderError } from "../services/vms/drivers/types"; import { isProviderIdentityNotFoundError, isProviderNotFoundError, } from "../services/vms/providerErrors"; describe("provider error classification", () => { + test.each([401, 403, 429, 500, 502, 503, 504])("HTTP %s takes precedence over wrapped missing-resource diagnostics", (status) => { + const error = new ProviderError("freestyle", "VM not found while reading stats", new FreestyleApiError( + status, { code: "INTERNAL", message: "VM not found in upstream cache" }, + )); + expect(isProviderNotFoundError(error)).toBe(false); + }); + + test("nested structured status takes precedence over a conflicting not-found code", () => { + const error = { code: "NOT_FOUND", cause: { response: { status: 502 } } }; + expect(isProviderNotFoundError(error)).toBe(false); + expect(isProviderNotFoundError({ code: "NOT_FOUND" })).toBe(true); + expect(isProviderNotFoundError({ cause: { statusCode: 404 } })).toBe(true); + }); + test("keeps identity deletion errors out of VM not-found classification", () => { expect(isProviderNotFoundError(new Error("identity does not exist"))).toBe(false); expect(isProviderIdentityNotFoundError(new Error("identity does not exist"))).toBe(true); diff --git a/web/tests/vm-stats-not-found.test.ts b/web/tests/vm-stats-not-found.test.ts new file mode 100644 index 000000000000..4d60ea8c807d --- /dev/null +++ b/web/tests/vm-stats-not-found.test.ts @@ -0,0 +1,148 @@ +import { describe, expect, spyOn, test } from "bun:test"; +import { Effect, Layer } from "effect"; +import { Freestyle } from "freestyle"; +import { VmBillingGateway, noOpVmBillingGateway } from "../services/vms/billingGateway"; +import { getProvider } from "../services/vms/drivers"; +import { FreestyleProvider } from "../services/vms/drivers/freestyle"; +import { VmDatabaseError } from "../services/vms/errors"; +import { isOperatorFaultVmError } from "../services/vms/observability"; +import { VmProviderGatewayLive } from "../services/vms/providerGateway"; +import { VmRepository, type CloudVmRow, type VmRepositoryShape } from "../services/vms/repository"; +import { vmWorkflowErrorResponse } from "../services/vms/routeHelpers"; +import { getVmStats } from "../services/vms/workflows"; + +// Real SDK response decoding, driver wrapping, Effect gateway and workflow. +// Only the provider HTTP transport and owned-row repository are fixtures. +async function withStatsFixture( + reply: () => Response, + run: (fixture: { + poll: (userId?: string, teamIds?: readonly string[]) => Promise<{ + tag: string; + response: Response; + payload: { error: string; action: string; retryable?: boolean; ui: { title: string; retryable: boolean } }; + }>; + requests: string[]; + writes: unknown[]; + row: CloudVmRow; + }) => Promise, + options: { failObservation?: boolean; team?: boolean } = {}, +) { + const requests: string[] = []; + const writes: unknown[] = []; + const row = { + id: "fixture-row", userId: "fixture-owner", billingTeamId: options.team ? "fixture-team" : null, + provider: "freestyle", providerVmId: "vm-fixture", providerMetadata: {}, status: "running", + } as CloudVmRow; + const client = new Freestyle({ + apiKey: "test-only", + fetch: (async (input, init) => { + const path = new URL(String(input)).pathname; + requests.push(`${init?.method} ${path}`); + expect(`${init?.method} ${path}`).toBe("GET /v5/vms/vm-fixture"); + return reply(); + }) as typeof fetch, + }); + const driver = new FreestyleProvider({ + client: () => client, + resolveDaemonSource: async () => { throw new Error("Stats must not install anything"); }, + }); + const getStats = spyOn(getProvider("freestyle"), "getStats").mockImplementation((id) => driver.getStats(id)); + const repo = new Proxy({ + findUserVm: (input: { userId: string; providerVmId: string }) => Effect.sync(() => + input.userId === row.userId && input.providerVmId === row.providerVmId && row.status !== "destroyed" ? row : null), + markProviderObservedStatus: (input: { id: string; providerVmId: string; status: "destroyed" }) => Effect.suspend(() => { + writes.push(input); + if (options.failObservation) return Effect.fail(new VmDatabaseError({ operation: "fixture", cause: new Error("offline") })); + expect(input).toEqual({ id: row.id, providerVmId: row.providerVmId, status: "destroyed" }); + row.status = input.status; + return Effect.succeed(true); + }), + }, { + get(target, key) { + if (key in target) return Reflect.get(target, key); + throw new Error(`Unexpected repository operation: ${String(key)}`); + }, + }) as unknown as VmRepositoryShape; + const layer = Layer.mergeAll( + Layer.succeed(VmRepository, repo), VmProviderGatewayLive, + Layer.succeed(VmBillingGateway, noOpVmBillingGateway()), + ); + try { + await run({ + row, requests, writes, + poll: async (userId = row.userId, teamIds = options.team ? ["fixture-team"] : []) => { + const result = await Effect.runPromise(Effect.either(getVmStats({ + userId, teamIds, providerVmId: row.providerVmId!, + }).pipe(Effect.provide(layer)))); + expect(result._tag).toBe("Left"); + if (result._tag !== "Left") throw new Error("Expected a fixture failure"); + const response = await vmWorkflowErrorResponse(result.left); + if (!response) throw new Error("Expected the shared HTTP error contract"); + return { tag: result.left._tag, response, payload: await response.json() }; + }, + }); + } finally { + getStats.mockRestore(); + } +} + +const missing = () => Response.json({ code: "NOT_FOUND", message: "not found: vm vm-fixture" }, { status: 404 }); + +describe("stats provider missing classification", () => { + test("typed NOT_FOUND becomes a terminal 404 across repeated polling, retaining the row", async () => { + await withStatsFixture(missing, async ({ poll, row, requests, writes }) => { + for (let attempt = 0; attempt < 3; attempt += 1) { + const { tag, response, payload } = await poll(); + expect(tag).toBe("VmNotFoundError"); + expect(response.status).toBe(404); + expect(response.headers.get("retry-after")).toBeNull(); + expect(payload.error).toBe("vm_not_found"); + expect(payload.ui).toMatchObject({ title: "Cloud VM not found", retryable: false }); + expect(payload.action).not.toMatch(/temporarily|retry|try again/i); + expect(isOperatorFaultVmError({ error: payload.error, status: response.status })).toBe(false); + } + expect(requests).toHaveLength(1); + expect(writes).toHaveLength(1); + expect(row).toMatchObject({ id: "fixture-row", providerVmId: "vm-fixture", status: "destroyed" }); + }); + }); + + test("a failed observation write never changes missing VM guidance to a retryable outage", async () => { + await withStatsFixture(missing, async ({ poll, row, requests }) => { + for (let attempt = 0; attempt < 3; attempt += 1) { + const { response, payload } = await poll(); + expect(response.status).toBe(404); + expect(payload.ui.retryable).toBe(false); + } + expect(requests).toHaveLength(3); + expect(row.status).toBe("running"); + }, { failObservation: true }); + }); + + test.each([false, true])("ownership is checked before the provider (team=%s)", async (team) => { + await withStatsFixture(missing, async ({ poll, requests, writes }) => { + const { response, payload } = await (team ? poll("fixture-owner", []) : poll("another-user")); + expect(response.status).toBe(404); + expect(payload.error).toBe("vm_not_found"); + expect(JSON.stringify(payload)).not.toMatch(/fixture-owner|fixture-team|fixture-row|freestyle/); + expect(requests).toHaveLength(0); + expect(writes).toHaveLength(0); + }, { team }); + }); + + test.each([ + { name: "typed upstream 502", reply: () => Response.json({ code: "INTERNAL", message: "upstream unavailable" }, { status: 502 }) }, + { name: "typed upstream 502 mentioning a missing VM", reply: () => Response.json({ code: "INTERNAL", message: "VM not found in upstream cache; retry" }, { status: 502 }) }, + { name: "transport failure", reply: () => { throw new TypeError("fetch failed"); } }, + ])("$name remains retryable and cannot mark the machine destroyed", async ({ reply }) => { + await withStatsFixture(reply, async ({ poll, row, writes }) => { + const { tag, response, payload } = await poll(); + expect(tag).toBe("VmProviderOperationError"); + expect(response.status).toBe(502); + expect(payload).toMatchObject({ error: "vm_cloud_service_unavailable", retryable: true, ui: { retryable: true } }); + expect(isOperatorFaultVmError({ error: payload.error, status: response.status })).toBe(true); + expect(writes).toHaveLength(0); + expect(row.status).toBe("running"); + }); + }); +}); From abf81ca74a49abc794991c0333d1ea691252170a Mon Sep 17 00:00:00 2001 From: Austin Wang Date: Mon, 14 Sep 2026 02:38:41 -0700 Subject: [PATCH 03/92] fix: prioritize HTTP status over provider missing-message heuristics (cherry picked from commit af20ca6442015f7b330600cf7d190305d39e41ed) --- web/oxlint-complexity-baseline.txt | 1 - web/services/vms/providerErrors.ts | 45 ++++++++++++++++++------------ 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/web/oxlint-complexity-baseline.txt b/web/oxlint-complexity-baseline.txt index 0b4e6ca297dd..cef9700bbd38 100644 --- a/web/oxlint-complexity-baseline.txt +++ b/web/oxlint-complexity-baseline.txt @@ -45,7 +45,6 @@ services/subrouter/accountInput.ts 4938ed3780d6d4a540c37a67a8c6120b03ba988e82db4 services/subrouter/hostedClient.ts 82062f7eeda90670adea4a1619e65dbf2cad2d1e05d02196fc191867b36ff7c4 function `parseCredentialLease` has a complexity of 22. Maximum allowed is 20. services/vms/observability.ts b9d616cc0a88f719e82b2005194d7fbab28058e34613a794738972deade85334 function `captureVmRequestOutcome` has a complexity of 38. Maximum allowed is 20. services/vms/observability.ts d15e28b21477d7ce31b23438b3d7640a95fb8313bbe77825400a6380617eed89 function `captureVmProvisionOutcome` has a complexity of 24. Maximum allowed is 20. -services/vms/providerErrors.ts 5c4e725a638f14da834c9943f2e0bf129c59471a1a8399a83fe73150c9fce20c function `isProviderNotFoundError` has a complexity of 24. Maximum allowed is 20. services/vms/providerErrors.ts bdc837c342072c514341cf5deaf8a0b169fe0ab6daf23399580ea83da8fca688 function `isProviderIdentityNotFoundError` has a complexity of 22. Maximum allowed is 20. services/vms/reaper.ts 05bb1f2917bd7a0c88eda164c98509e4b319efca1d398481080e8e15e3310c0b generator function has a complexity of 27. Maximum allowed is 20. services/vms/reaper.ts d59e84b06bcfd550d4e36fd741283757c08519f7cebaed05e3ade6e0cd04c6ed generator function has a complexity of 21. Maximum allowed is 20. diff --git a/web/services/vms/providerErrors.ts b/web/services/vms/providerErrors.ts index 9714cb85ff48..7003aab8efe7 100644 --- a/web/services/vms/providerErrors.ts +++ b/web/services/vms/providerErrors.ts @@ -28,24 +28,17 @@ function hasProviderMissingMessage( ); } -export function isProviderNotFoundError(err: unknown): boolean { - if (!err || typeof err !== "object") return false; - const candidate = err as { - code?: string | number; - name?: string; - status?: number; - statusCode?: number; - response?: { status?: number; data?: unknown }; - message?: string; - cause?: unknown; - }; - const status = - candidate.status ?? - candidate.statusCode ?? - candidate.response?.status ?? - undefined; - if (status === 404) return true; +type ProviderFailure = { + code?: string | number; + name?: string; + status?: number; + statusCode?: number; + response?: { status?: number; data?: unknown }; + message?: string; + cause?: unknown; +}; +function hasLegacyProviderNotFoundDetail(candidate: ProviderFailure): boolean { const code = String(candidate.code ?? candidate.name ?? "").toLowerCase(); if ( code === "not_found" || @@ -69,10 +62,26 @@ export function isProviderNotFoundError(err: unknown): boolean { return true; } - if (candidate.cause) return isProviderNotFoundError(candidate.cause); return false; } +export function isProviderNotFoundError(err: unknown): boolean { + const seen = new Set(); + let legacyNotFound = false; + let current = err; + while (current && typeof current === "object" && !seen.has(current)) { + seen.add(current); + const candidate = current as ProviderFailure; + const status = candidate.status ?? candidate.statusCode ?? candidate.response?.status; + // The concrete HTTP failure wins over wrapper/code/message heuristics. + // A 502 mentioning a missing VM must never mark the machine destroyed. + if (typeof status === "number" && status >= 400 && status <= 599) return status === 404; + legacyNotFound ||= hasLegacyProviderNotFoundDetail(candidate); + current = candidate.cause; + } + return legacyNotFound; +} + export function isProviderIdentityNotFoundError(err: unknown): boolean { if (!err || typeof err !== "object") return false; const candidate = err as { From 8b5a2093e35a0105106b0638445541e6c32e0943 Mon Sep 17 00:00:00 2001 From: Austin Wang Date: Fri, 18 Sep 2026 19:59:33 -0700 Subject: [PATCH 04/92] test: align VM stats fixture with ownership lookup (cherry picked from commit f9b39040002626c53684d96f985343dbc3a86a3e) --- web/tests/vm-stats-not-found.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/web/tests/vm-stats-not-found.test.ts b/web/tests/vm-stats-not-found.test.ts index 4d60ea8c807d..2571f9ee1078 100644 --- a/web/tests/vm-stats-not-found.test.ts +++ b/web/tests/vm-stats-not-found.test.ts @@ -30,7 +30,8 @@ async function withStatsFixture( const requests: string[] = []; const writes: unknown[] = []; const row = { - id: "fixture-row", userId: "fixture-owner", billingTeamId: options.team ? "fixture-team" : null, + id: "fixture-row", userId: "fixture-owner", ownerTeamId: options.team ? "fixture-team" : "fixture-owner", + billingTeamId: options.team ? "fixture-team" : null, provider: "freestyle", providerVmId: "vm-fixture", providerMetadata: {}, status: "running", } as CloudVmRow; const client = new Freestyle({ From 05ce6e3d7ad1dec2ab10338837b5c355bd247fa0 Mon Sep 17 00:00:00 2001 From: Austin Wang Date: Sat, 19 Sep 2026 22:16:57 -0700 Subject: [PATCH 05/92] fix: ignore invalid provider status sentinels (cherry picked from commit b015f9cb8190859a2043758658156ec5b37ca4d6) --- web/services/vms/providerErrors.ts | 11 +++++++++-- web/tests/vm-provider-errors.test.ts | 8 ++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/web/services/vms/providerErrors.ts b/web/services/vms/providerErrors.ts index 7003aab8efe7..cc121361e52b 100644 --- a/web/services/vms/providerErrors.ts +++ b/web/services/vms/providerErrors.ts @@ -65,6 +65,13 @@ function hasLegacyProviderNotFoundDetail(candidate: ProviderFailure): boolean { return false; } +function httpStatus(candidate: ProviderFailure): number | undefined { + const candidates = [candidate.status, candidate.statusCode, candidate.response?.status]; + return candidates.find((status): status is number => + typeof status === "number" && status >= 400 && status <= 599, + ); +} + export function isProviderNotFoundError(err: unknown): boolean { const seen = new Set(); let legacyNotFound = false; @@ -72,10 +79,10 @@ export function isProviderNotFoundError(err: unknown): boolean { while (current && typeof current === "object" && !seen.has(current)) { seen.add(current); const candidate = current as ProviderFailure; - const status = candidate.status ?? candidate.statusCode ?? candidate.response?.status; + const status = httpStatus(candidate); // The concrete HTTP failure wins over wrapper/code/message heuristics. // A 502 mentioning a missing VM must never mark the machine destroyed. - if (typeof status === "number" && status >= 400 && status <= 599) return status === 404; + if (status !== undefined) return status === 404; legacyNotFound ||= hasLegacyProviderNotFoundDetail(candidate); current = candidate.cause; } diff --git a/web/tests/vm-provider-errors.test.ts b/web/tests/vm-provider-errors.test.ts index a174e9d85f27..66afac075b8f 100644 --- a/web/tests/vm-provider-errors.test.ts +++ b/web/tests/vm-provider-errors.test.ts @@ -21,6 +21,14 @@ describe("provider error classification", () => { expect(isProviderNotFoundError({ cause: { statusCode: 404 } })).toBe(true); }); + test.each([ + { status: 0, response: { status: 502 } }, + { status: 600, response: { status: 503 } }, + { status: -1, statusCode: 502 }, + ])("ignores invalid status sentinels before selecting a retryable HTTP status: %j", (error) => { + expect(isProviderNotFoundError({ ...error, message: "VM not found while reading stats" })).toBe(false); + }); + test("keeps identity deletion errors out of VM not-found classification", () => { expect(isProviderNotFoundError(new Error("identity does not exist"))).toBe(false); expect(isProviderIdentityNotFoundError(new Error("identity does not exist"))).toBe(true); From 937d93677415916f76c13217da8d2047838a9d16 Mon Sep 17 00:00:00 2001 From: Austin Wang Date: Mon, 14 Sep 2026 02:20:00 -0700 Subject: [PATCH 06/92] test: reproduce ambiguous guest install exits and lost rollback failures (cherry picked from commit 0bfce3fa2c2126cfd27aaec79d6cf3972c826200) --- web/tests/fixtures/freestyleGuest.ts | 74 ++++++++++++++++++++++++++++ web/tests/vm-guest-install.test.ts | 59 ++++++++++++++++++++++ 2 files changed, 133 insertions(+) create mode 100644 web/tests/fixtures/freestyleGuest.ts create mode 100644 web/tests/vm-guest-install.test.ts diff --git a/web/tests/fixtures/freestyleGuest.ts b/web/tests/fixtures/freestyleGuest.ts new file mode 100644 index 000000000000..3b42c81d7e1a --- /dev/null +++ b/web/tests/fixtures/freestyleGuest.ts @@ -0,0 +1,74 @@ +import { Freestyle } from "freestyle"; +import { FreestyleProvider } from "../../services/vms/drivers/freestyle"; + +export type GuestExecRequest = { + command: string; + timeoutMs: number; + linuxUser: string; +}; + +/** Real pinned SDK, synthetic HTTP only. No provider credentials or network. */ +export function freestyleGuestFixture(options: { + exec?: (request: GuestExecRequest, signal?: AbortSignal | null) => Promise; + write?: (path: string, bytes: Uint8Array) => void; + remove?: (path: string) => void; + deleteFailure?: boolean; +} = {}) { + const requests: Array<{ method: string; path: string }> = []; + const writes: string[] = []; + const removals: string[] = []; + const liveVms = new Set(); + let allocations = 0; + let installPending = false; + const client = (_timeoutMs?: number, signal?: AbortSignal) => new Freestyle({ + apiKey: "synthetic-test-key", + baseUrl: "https://provider.invalid", + fetch: (async (input: RequestInfo | URL, init?: RequestInit) => { + const url = new URL(String(input)); + const method = init?.method ?? "GET"; + requests.push({ method, path: url.pathname }); + if (method === "POST" && url.pathname === "/v5/vms") { + const id = `vm-fixture-${++allocations}`; + liveVms.add(id); + return Response.json({ id, state: "running", resources: { cpu: 2, memory: 8192, storage: 32768 }, vpcs: [{ ipv4: "192.0.2.10" }] }); + } + if (url.pathname.endsWith("/fs/write")) { + const path = url.searchParams.get("path")!; + writes.push(path); + options.write?.(path, new Uint8Array(await new Response(init?.body).arrayBuffer())); + installPending = true; + return Response.json({}); + } + if (url.pathname.endsWith("/fs/remove")) { + const path = url.searchParams.get("path")!; + removals.push(path); + options.remove?.(path); + return Response.json({}); + } + if (url.pathname.endsWith("/exec-await")) { + const request = JSON.parse(String(init?.body)) as GuestExecRequest; + if (installPending) { + installPending = false; + return options.exec?.(request, signal ?? init?.signal) ?? Response.json({ statusCode: 0 }); + } + return Response.json({ statusCode: 0, stdout: "", stderr: "" }); + } + if (method === "DELETE" && /^\/v5\/vms\/vm-fixture-\d+$/.test(url.pathname)) { + if (options.deleteFailure) return Response.json({ code: "UNAVAILABLE", message: "synthetic delete failure" }, { status: 503 }); + liveVms.delete(url.pathname.split("/").at(-1)!); + return new Response(null, { status: 204 }); + } + throw new Error(`Unexpected synthetic provider request: ${method} ${url.pathname}`); + }) as typeof fetch, + }); + const provider = new FreestyleProvider({ + client, + resolveDaemonSource: async () => { throw new Error("fixture must never resolve a live daemon"); }, + }); + return { provider, client, requests, writes, removals, liveVms, allocations: () => allocations }; +} + +export const guestCreateOptions = { + image: "sh-synthetic", + imageSize: { name: "md", cpu: 2, memoryMb: 8192, storageMb: 32768 }, +} as const; diff --git a/web/tests/vm-guest-install.test.ts b/web/tests/vm-guest-install.test.ts new file mode 100644 index 000000000000..b8905bd5be65 --- /dev/null +++ b/web/tests/vm-guest-install.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, test } from "bun:test"; +import { freestyleGuestFixture, guestCreateOptions } from "./fixtures/freestyleGuest"; + +describe("Freestyle guest install result contract (SDK 0.2.10)", () => { + test("a reported zero exit publishes a ready machine", async () => { + const fixture = freestyleGuestFixture(); + const handle = await fixture.provider.create(guestCreateOptions); + expect(handle.status).toBe("running"); + expect(fixture.liveVms.size).toBe(1); + expect(fixture.allocations()).toBe(1); + }); + + // The incident preserved only the old 'exited 124' message. These are + // synthetic candidate responses, not invented copies of the lost body. + test.each([ + [{ stdout: "", stderr: "" }, "missing_status", undefined], + [{ statusCode: null, stdout: "", stderr: "" }, "provider_timeout", null], + [{ statusCode: 124, stderr: "synthetic guest exit" }, "guest_exit", 124], + [{ statusCode: 1, stderr: "synthetic rename failure" }, "guest_exit", 1], + [{ statusCode: "0" }, "invalid_status", undefined], + ])("preserves outcome %s without a false ready result", async (body, outcome, statusCode) => { + const fixture = freestyleGuestFixture({ exec: async () => Response.json(body) }); + const error = await fixture.provider.create(guestCreateOptions).catch((error: unknown) => error); + expect(error).toMatchObject({ + name: "ProviderError", + cause: { name: "GuestCliInstallError", outcome, statusCode }, + }); + expect(fixture.allocations()).toBe(1); + expect(fixture.liveVms.size).toBe(0); + expect(fixture.removals).toEqual(fixture.writes); + }); + + test.each([ + ["AbortError", "cancelled"], + ["TimeoutError", "transport_timeout"], + ])("preserves transport %s separately from a guest exit", async (name, outcome) => { + const cause = new DOMException("synthetic transport interruption", name); + const fixture = freestyleGuestFixture({ exec: async () => { throw cause; } }); + const error = await fixture.provider.create(guestCreateOptions).catch((error: unknown) => error); + expect(error).toMatchObject({ cause: { name: "GuestCliInstallError", outcome, cause } }); + expect(fixture.liveVms.size).toBe(0); + expect(fixture.removals).toEqual(fixture.writes); + }); + + test("failed rollback retains the allocated machine and both failures", async () => { + const fixture = freestyleGuestFixture({ + exec: async () => Response.json({ statusCode: 1, stderr: "synthetic install failure" }), + deleteFailure: true, + }); + const error = await fixture.provider.create(guestCreateOptions).catch((error: unknown) => error); + expect(error).toMatchObject({ + name: "ProviderCreateCleanupError", + providerVmId: "vm-fixture-1", + cause: { name: "GuestCliInstallError", outcome: "guest_exit", statusCode: 1 }, + cleanupCause: { name: "FreestyleApiError", status: 503 }, + }); + expect(fixture.liveVms.size).toBe(1); + }); +}); From 6441a49ebb323e71b3964044185683320c7d69b9 Mon Sep 17 00:00:00 2001 From: Austin Wang Date: Mon, 14 Sep 2026 02:23:30 -0700 Subject: [PATCH 07/92] test: reproduce unsafe guest shim publication in real filesystem (cherry picked from commit c80df5f190271520176cb6f78cfa74b7ac29f557) --- web/tests/vm-guest-install-runtime.test.ts | 82 ++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 web/tests/vm-guest-install-runtime.test.ts diff --git a/web/tests/vm-guest-install-runtime.test.ts b/web/tests/vm-guest-install-runtime.test.ts new file mode 100644 index 000000000000..fc3f91f14d7b --- /dev/null +++ b/web/tests/vm-guest-install-runtime.test.ts @@ -0,0 +1,82 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { spawnSync } from "node:child_process"; +import { mkdtempSync, mkdirSync, readFileSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { GUEST_CMUX_SHIM } from "../services/vms/guestCli"; +import { freestyleGuestFixture, guestCreateOptions } from "./fixtures/freestyleGuest"; + +const roots: string[] = []; +afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); + +/** Isolated guest filesystem; run the actual uploaded bytes and install command. */ +function guest(options: { corruptUpload?: boolean } = {}) { + const root = mkdtempSync(join(tmpdir(), "cmux-guest-install-")); + roots.push(root); + mkdirSync(join(root, "bin")); + const rebase = (value: string) => value.replaceAll("/usr/local/bin", join(root, "bin")) + .replaceAll("/etc/cmux", join(root, "etc")); + const target = join(root, "bin/cmux"); + const fixture = freestyleGuestFixture({ + write: (path, bytes) => writeFileSync(rebase(path), options.corruptUpload ? "#!/bin/sh\nexit 0\n" : bytes), + remove: (path) => rmSync(rebase(path), { force: true }), + exec: async (request) => { + const result = spawnSync("/bin/sh", ["-c", rebase(request.command)], { encoding: "utf8", timeout: 5_000 }); + return Response.json({ statusCode: result.status, stdout: result.stdout, stderr: result.stderr }); + }, + }); + return { root, target, fixture }; +} + +describe("guest CLI publication in an isolated filesystem", () => { + test("successful create installs the complete executable CLI and answers help", async () => { + const { fixture, target } = guest(); + const handle = await fixture.provider.create(guestCreateOptions); + expect(handle.status).toBe("running"); + expect(readFileSync(target, "utf8")).toBe(GUEST_CMUX_SHIM); + expect(statSync(target).mode & 0o777).toBe(0o755); + const result = spawnSync(target, ["--help"], { encoding: "utf8", timeout: 5_000 }); + expect(result.status).toBe(0); + expect(result.stdout).toContain("cmux"); + }); + + test("replaces a target symlink without modifying its referent", async () => { + const { fixture, root, target } = guest(); + const unrelated = join(root, "unrelated"); + writeFileSync(unrelated, "preserve me"); + symlinkSync(unrelated, target); + await fixture.provider.create(guestCreateOptions); + expect(readFileSync(unrelated, "utf8")).toBe("preserve me"); + expect(readFileSync(target, "utf8")).toBe(GUEST_CMUX_SHIM); + }); + + test("a directory destination fails instead of moving the shim inside and reporting ready", async () => { + const { fixture, target } = guest(); + mkdirSync(target); + const result = await fixture.provider.create(guestCreateOptions).then(() => "ready", () => "failed"); + expect(result).toBe("failed"); + expect(fixture.liveVms.size).toBe(0); + }); + + test("rejects a corrupted upload before replacing the previous generation", async () => { + const { fixture, target } = guest({ corruptUpload: true }); + writeFileSync(target, "previous generation"); + const result = await fixture.provider.create(guestCreateOptions).then(() => "ready", () => "failed"); + expect(result).toBe("failed"); + expect(readFileSync(target, "utf8")).toBe("previous generation"); + expect(fixture.liveVms.size).toBe(0); + }); + + test("prompt failure does not publish a new shim generation", async () => { + const { fixture, target, root } = guest(); + writeFileSync(target, "previous generation"); + writeFileSync(join(root, "etc"), "not a directory"); + const result = await fixture.provider.create({ + ...guestCreateOptions, + promptIdentity: { machineId: "synthetic", name: "synthetic", revision: 1 }, + }).then(() => "ready", () => "failed"); + expect(result).toBe("failed"); + expect(readFileSync(target, "utf8")).toBe("previous generation"); + expect(fixture.liveVms.size).toBe(0); + }); +}); From e1d7d3535327af2cc8dbe19ba76f9df2b2d2245f Mon Sep 17 00:00:00 2001 From: Austin Wang Date: Mon, 14 Sep 2026 02:42:30 -0700 Subject: [PATCH 08/92] test: gate guest bootstrap rollback and idempotency on PostgreSQL (cherry picked from commit d651670e23005ba6d99a328081be61c0bb3e5690) --- .github/workflows/cloud-vm-guest-install.yml | 64 ++++++++++++++++++ web/tests/vm-workflows.test.ts | 69 ++++++++++++++++++++ 2 files changed, 133 insertions(+) create mode 100644 .github/workflows/cloud-vm-guest-install.yml diff --git a/.github/workflows/cloud-vm-guest-install.yml b/.github/workflows/cloud-vm-guest-install.yml new file mode 100644 index 000000000000..7f0ab4215b1a --- /dev/null +++ b/.github/workflows/cloud-vm-guest-install.yml @@ -0,0 +1,64 @@ +name: Cloud VM guest installation + +on: + pull_request: + paths: + - web/services/vms/drivers/** + - web/services/vms/guestCli*.ts + - web/services/vms/guestPrompt.ts + - web/services/vms/workflows.ts + - web/services/vms/repository.ts + - web/services/vms/routeHelpers.ts + - web/tests/vm-guest-*.test.ts + - web/tests/fixtures/freestyleGuest.ts + - web/tests/vm-workflows.test.ts + - web/package.json + - web/bun.lock + - .github/workflows/cloud-vm-guest-install.yml + workflow_dispatch: {} + +permissions: + contents: read + +concurrency: + group: cloud-vm-guest-install-${{ github.ref }} + cancel-in-progress: true + +jobs: + guest-install: + runs-on: ubuntu-latest + timeout-minutes: 15 + defaults: + run: + working-directory: web + services: + postgres: + image: postgres:16-alpine + env: + POSTGRES_DB: cmux_test + POSTGRES_PASSWORD: cmux + POSTGRES_USER: cmux + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U cmux -d cmux_test" + --health-interval 1s --health-timeout 5s --health-retries 30 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + persist-credentials: false + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 + with: + bun-version: "1.3.14" + - run: bun install --frozen-lockfile + - name: SDK and guest filesystem regressions + run: bun test tests/vm-guest-install.test.ts tests/vm-guest-install-runtime.test.ts + - name: Migrate isolated database + env: + DATABASE_URL: postgres://cmux:cmux@localhost:5432/cmux_test + run: bunx drizzle-kit migrate --config drizzle.config.ts + - name: Guest provisioning and idempotency in PostgreSQL + env: + CMUX_DB_TEST: "1" + DATABASE_URL: postgres://cmux:cmux@localhost:5432/cmux_test + run: bun test --timeout=30000 --test-name-pattern 'guest bootstrap|unconfirmed guest' tests/vm-workflows.test.ts diff --git a/web/tests/vm-workflows.test.ts b/web/tests/vm-workflows.test.ts index 78812173198a..0523b16a0ec7 100644 --- a/web/tests/vm-workflows.test.ts +++ b/web/tests/vm-workflows.test.ts @@ -41,6 +41,7 @@ import { } from "../services/vms/errors"; import { accountDeletionUserHash } from "../services/account/deletionLock"; import { isVmAttachTransportUnsupportedError } from "../services/vms/errors"; +import { freestyleGuestFixture, guestCreateOptions } from "./fixtures/freestyleGuest"; import { VM_DISK_MB_MAX, VM_RESOURCE_RESIZE_PENDING_METADATA_KEY, @@ -196,6 +197,74 @@ afterAll(async () => { }); describe("VM Effect workflows", () => { + dbTest("guest bootstrap retry allocates once after confirmed rollback and reuses the ready result", async () => { + if (!sql) throw new Error("test database not initialized"); + let fail = true; + const fixture = freestyleGuestFixture({ exec: async () => Response.json({ statusCode: fail ? 124 : 0 }) }); + const layer = providerLayer({ + ...unusedProviderGateway(), + create: (_, options) => Effect.tryPromise({ + try: () => fixture.provider.create(options), + catch: (cause) => new VmProviderOperationError({ provider: "freestyle", operation: "create", cause }), + }), + }); + const program = createVm({ + ...guestCreateOptions, userId: "user-guest-retry", billingTeamId: "team-guest-retry", + billingCustomerType: "team", billingPlanId: "pro", provider: "freestyle", + idempotencyKey: "guest-retry", maxActiveVms: 1, + }).pipe(Effect.provide(layer)); + const first = await Effect.runPromise(Effect.either(program)); + expect(first._tag).toBe("Left"); + expect(fixture.allocations()).toBe(1); + expect(fixture.liveVms.size).toBe(0); + fail = false; + const attempts = await Promise.all([Effect.runPromise(Effect.either(program)), Effect.runPromise(Effect.either(program))]); + expect(attempts.some((attempt) => attempt._tag === "Right")).toBe(true); + for (const attempt of attempts) { + if (attempt._tag === "Left") expect(attempt.left._tag).toBe("VmCreateInProgressError"); + else expect(attempt.right.providerVmId).toBe("vm-fixture-2"); + } + const replay = await Effect.runPromise(program); + expect(replay.providerVmId).toBe("vm-fixture-2"); + expect(fixture.allocations()).toBe(2); + expect(fixture.liveVms.size).toBe(1); + const [events] = await sql<{ count: string }[]>`select count(*)::text as count from cloud_vm_usage_events where event_type = 'vm.created' and user_id = 'user-guest-retry'`; + expect(events.count).toBe("1"); + }); + + dbTest("unconfirmed guest rollback stays reserved and cannot become ready or allocate on retry", async () => { + if (!sql) throw new Error("test database not initialized"); + const fixture = freestyleGuestFixture({ exec: async () => Response.json({ statusCode: 1 }), deleteFailure: true }); + const layer = providerLayer({ + ...unusedProviderGateway(), + create: (_, options) => Effect.tryPromise({ + try: () => fixture.provider.create(options), + catch: (cause) => new VmProviderOperationError({ provider: "freestyle", operation: "create", cause }), + }), + }); + const input = { + ...guestCreateOptions, userId: "user-guest-cleanup", billingTeamId: "team-guest-cleanup", + billingCustomerType: "team" as const, billingPlanId: "pro", provider: "freestyle" as const, + idempotencyKey: "guest-cleanup", maxActiveVms: 1, + }; + const run = (key = input.idempotencyKey) => Effect.runPromise(Effect.either(createVm({ ...input, idempotencyKey: key }).pipe(Effect.provide(layer)))); + expect((await run())._tag).toBe("Left"); + await sql`update cloud_vms set updated_at = now() - interval '2 days' where user_id = 'user-guest-cleanup'`; + const retry = await run(); + expect(retry._tag).toBe("Left"); + if (retry._tag === "Left") expect(retry.left).toMatchObject({ _tag: "VmCreateFailedError", code: "provider_create_cleanup_pending" }); + expect((await run("different-key"))._tag).toBe("Left"); + const [row] = await sql<{ status: string; providerVmId: string | null; metadata: Record }[]>` + select status, provider_vm_id as "providerVmId", provider_metadata as metadata from cloud_vms where user_id = 'user-guest-cleanup' + `; + expect(row.status).toBe("provisioning"); + expect(row.providerVmId).toBeNull(); + expect(row.metadata.createCleanupProviderVmId).toBe("vm-fixture-1"); + expect(await Effect.runPromise(listUserVms(input.userId).pipe(Effect.provide(layer)))).toEqual([]); + expect(fixture.allocations()).toBe(1); + expect(fixture.liveVms.size).toBe(1); + }); + dbTest("keeps prompt revisions ordered across rapid renames and clock skew", async () => { if (!sql) throw new Error("test database not initialized"); const userId = "user-prompt-revisions"; From 74e3decfcf254e30814dc2b229cd3762f0a14c7e Mon Sep 17 00:00:00 2001 From: Austin Wang Date: Mon, 14 Sep 2026 02:48:24 -0700 Subject: [PATCH 09/92] fix: verify guest CLI publication and retain unconfirmed allocations (cherry picked from commit c14551a9636d9fbd7633a4614e87564ff72b94f0) --- .github/workflows/cloud-vm-guest-install.yml | 4 +- web/messages/ar.json | 5 + web/messages/bs.json | 5 + web/messages/da.json | 5 + web/messages/de.json | 5 + web/messages/en.json | 5 + web/messages/es.json | 5 + web/messages/fr.json | 5 + web/messages/it.json | 5 + web/messages/ja.json | 5 + web/messages/km.json | 5 + web/messages/ko.json | 5 + web/messages/no.json | 5 + web/messages/pl.json | 5 + web/messages/pt-BR.json | 5 + web/messages/ru.json | 5 + web/messages/th.json | 5 + web/messages/tr.json | 5 + web/messages/uk.json | 5 + web/messages/zh-CN.json | 5 + web/messages/zh-TW.json | 5 + web/services/vms/drivers/freestyle.ts | 47 +++---- web/services/vms/drivers/freestyleGuestCli.ts | 119 +++++++++++++++++ .../vms/drivers/freestyleRequestTiming.ts | 9 +- .../vms/drivers/guestCliInstallCommand.ts | 45 +++++++ .../vms/drivers/providerCreateCleanup.ts | 34 +++++ web/services/vms/repository.ts | 18 ++- web/services/vms/routeHelpers.ts | 23 ++++ web/services/vms/vmErrorMessages.ts | 9 ++ web/services/vms/workflows.ts | 29 ++-- web/tests/fixtures/freestyleGuest.ts | 11 +- web/tests/vm-freestyle-provider.test.ts | 4 - web/tests/vm-guest-create-response.test.ts | 23 ++++ web/tests/vm-guest-install-deadline.test.ts | 126 ++++++++++++++++++ web/tests/vm-guest-install-runtime.test.ts | 27 +++- web/tests/vm-guest-install.test.ts | 17 ++- web/tests/vm-workflows.test.ts | 31 +++++ 37 files changed, 617 insertions(+), 59 deletions(-) create mode 100644 web/services/vms/drivers/freestyleGuestCli.ts create mode 100644 web/services/vms/drivers/guestCliInstallCommand.ts create mode 100644 web/services/vms/drivers/providerCreateCleanup.ts create mode 100644 web/tests/vm-guest-create-response.test.ts create mode 100644 web/tests/vm-guest-install-deadline.test.ts diff --git a/.github/workflows/cloud-vm-guest-install.yml b/.github/workflows/cloud-vm-guest-install.yml index 7f0ab4215b1a..7bccefeafef2 100644 --- a/.github/workflows/cloud-vm-guest-install.yml +++ b/.github/workflows/cloud-vm-guest-install.yml @@ -9,6 +9,8 @@ on: - web/services/vms/workflows.ts - web/services/vms/repository.ts - web/services/vms/routeHelpers.ts + - web/services/vms/vmErrorMessages.ts + - web/messages/** - web/tests/vm-guest-*.test.ts - web/tests/fixtures/freestyleGuest.ts - web/tests/vm-workflows.test.ts @@ -52,7 +54,7 @@ jobs: bun-version: "1.3.14" - run: bun install --frozen-lockfile - name: SDK and guest filesystem regressions - run: bun test tests/vm-guest-install.test.ts tests/vm-guest-install-runtime.test.ts + run: bun test tests/vm-guest-install.test.ts tests/vm-guest-install-runtime.test.ts tests/vm-guest-install-deadline.test.ts tests/vm-guest-create-response.test.ts - name: Migrate isolated database env: DATABASE_URL: postgres://cmux:cmux@localhost:5432/cmux_test diff --git a/web/messages/ar.json b/web/messages/ar.json index 41c5aba97849..06e3028fd64e 100644 --- a/web/messages/ar.json +++ b/web/messages/ar.json @@ -1,5 +1,10 @@ { "vmErrors": { + "createCleanupPending": { + "title": "تنظيف آلة Cloud الافتراضية معلق", + "message": "فشل إعداد آلة Cloud الافتراضية وما زال التنظيف معلقًا.", + "action": "تواصل مع الدعم وأرفق معرّف التتبع قبل إنشاء آلة أخرى." + }, "unsupported": { "title": "عملية Cloud VM غير متاحة", "reason": "هذا النوع من الأجهزة لا يوفر هذه العملية.", diff --git a/web/messages/bs.json b/web/messages/bs.json index 3a56d6ec762d..cefb50f4c225 100644 --- a/web/messages/bs.json +++ b/web/messages/bs.json @@ -1,5 +1,10 @@ { "vmErrors": { + "createCleanupPending": { + "title": "Čišćenje Cloud VM-a na čekanju", + "message": "Postavljanje Cloud VM-a nije uspjelo i čišćenje je još na čekanju.", + "action": "Obratite se podršci s ID-jem praćenja prije kreiranja druge mašine." + }, "unsupported": { "title": "Operacija Cloud VM-a nije dostupna", "reason": "Ova vrsta mašine ne pruža ovu operaciju.", diff --git a/web/messages/da.json b/web/messages/da.json index f73c28637f7e..d12d0a2b5932 100644 --- a/web/messages/da.json +++ b/web/messages/da.json @@ -1,5 +1,10 @@ { "vmErrors": { + "createCleanupPending": { + "title": "Oprydning af Cloud VM afventer", + "message": "Opsætningen af Cloud VM mislykkedes, og oprydningen afventer stadig.", + "action": "Kontakt support med sporings-ID’et, før du opretter en ny maskine." + }, "unsupported": { "title": "Cloud VM-handling er ikke tilgængelig", "reason": "Denne maskintype tilbyder ikke denne handling.", diff --git a/web/messages/de.json b/web/messages/de.json index 5717ce18a30f..7995d8380b50 100644 --- a/web/messages/de.json +++ b/web/messages/de.json @@ -1,5 +1,10 @@ { "vmErrors": { + "createCleanupPending": { + "title": "Cloud-VM-Bereinigung ausstehend", + "message": "Die Einrichtung der Cloud-VM ist fehlgeschlagen. Die Bereinigung steht noch aus.", + "action": "Wenden Sie sich mit der Trace-ID an den Support, bevor Sie eine weitere Maschine erstellen." + }, "unsupported": { "title": "Cloud-VM-Vorgang nicht verfügbar", "reason": "Dieser Maschinentyp stellt diesen Vorgang nicht bereit.", diff --git a/web/messages/en.json b/web/messages/en.json index 4adc17597865..893a6f8404c9 100644 --- a/web/messages/en.json +++ b/web/messages/en.json @@ -136,6 +136,11 @@ "expiredBody": "{machine} and everything on it are fine — only this link's session ended. Reopen the desktop from cmux (right-click the machine, then Open Desktop) to get a fresh one." }, "vmErrors": { + "createCleanupPending": { + "title": "Cloud VM cleanup pending", + "message": "Cloud VM setup failed and cleanup is still pending.", + "action": "Contact support with the trace ID before creating another machine." + }, "artifactUnavailable": { "title": "Cloud VM setup unavailable", "message": "Cloud VM setup is temporarily unavailable.", diff --git a/web/messages/es.json b/web/messages/es.json index c9852f8a6605..0599e6617e2c 100644 --- a/web/messages/es.json +++ b/web/messages/es.json @@ -1,5 +1,10 @@ { "vmErrors": { + "createCleanupPending": { + "title": "Limpieza de la VM Cloud pendiente", + "message": "La configuración de la VM Cloud falló y la limpieza sigue pendiente.", + "action": "Contacta con soporte e indica el ID de seguimiento antes de crear otra máquina." + }, "unsupported": { "title": "Operación de Cloud VM no disponible", "reason": "Este tipo de máquina no ofrece esta operación.", diff --git a/web/messages/fr.json b/web/messages/fr.json index cb4479e3a85f..7a38b2c06ccd 100644 --- a/web/messages/fr.json +++ b/web/messages/fr.json @@ -1,5 +1,10 @@ { "vmErrors": { + "createCleanupPending": { + "title": "Nettoyage de la VM Cloud en attente", + "message": "La configuration de la VM Cloud a échoué et le nettoyage est encore en attente.", + "action": "Contactez le support avec l’identifiant de trace avant de créer une autre machine." + }, "unsupported": { "title": "Opération de Cloud VM indisponible", "reason": "Ce type de machine ne propose pas cette opération.", diff --git a/web/messages/it.json b/web/messages/it.json index 09bcdde747fd..d6c5961332a9 100644 --- a/web/messages/it.json +++ b/web/messages/it.json @@ -1,5 +1,10 @@ { "vmErrors": { + "createCleanupPending": { + "title": "Pulizia della VM Cloud in sospeso", + "message": "La configurazione della VM Cloud non è riuscita e la pulizia è ancora in sospeso.", + "action": "Contatta il supporto con l’ID della traccia prima di creare un’altra macchina." + }, "unsupported": { "title": "Operazione Cloud VM non disponibile", "reason": "Questo tipo di macchina non offre questa operazione.", diff --git a/web/messages/ja.json b/web/messages/ja.json index 8c12df6104bb..60b5fdf51809 100644 --- a/web/messages/ja.json +++ b/web/messages/ja.json @@ -136,6 +136,11 @@ "expiredBody": "{machine} とその中身は無事です。このリンクのセッションだけが終了しました。cmux でマシンを右クリックし、「デスクトップを開く」から新しく開き直してください。" }, "vmErrors": { + "createCleanupPending": { + "title": "Cloud VM のクリーンアップが保留中です", + "message": "Cloud VM のセットアップに失敗し、クリーンアップがまだ完了していません。", + "action": "別のマシンを作成する前に、トレース ID を添えてサポートにお問い合わせください。" + }, "artifactUnavailable": { "title": "Cloud VM のセットアップを利用できません", "message": "Cloud VM のセットアップを一時的に利用できません。", diff --git a/web/messages/km.json b/web/messages/km.json index 818dd3144610..4c069d0699e7 100644 --- a/web/messages/km.json +++ b/web/messages/km.json @@ -1,5 +1,10 @@ { "vmErrors": { + "createCleanupPending": { + "title": "ការសម្អាត Cloud VM កំពុងរង់ចាំ", + "message": "ការដំឡើង Cloud VM បានបរាជ័យ ហើយការសម្អាតមិនទាន់បញ្ចប់ទេ។", + "action": "សូមទាក់ទងផ្នែកគាំទ្រដោយផ្ដល់លេខសម្គាល់តាមដាន មុនពេលបង្កើតម៉ាស៊ីនផ្សេងទៀត។" + }, "unsupported": { "title": "ប្រតិបត្តិការ Cloud VM មិនអាចប្រើបាន", "reason": "ប្រភេទម៉ាស៊ីននេះមិនផ្តល់ប្រតិបត្តិការនេះទេ។", diff --git a/web/messages/ko.json b/web/messages/ko.json index d99960a47a2b..cf6ae9cd65e8 100644 --- a/web/messages/ko.json +++ b/web/messages/ko.json @@ -1,5 +1,10 @@ { "vmErrors": { + "createCleanupPending": { + "title": "Cloud VM 정리 대기 중", + "message": "Cloud VM 설정에 실패했으며 정리가 아직 완료되지 않았습니다.", + "action": "다른 머신을 만들기 전에 추적 ID와 함께 지원팀에 문의하세요." + }, "unsupported": { "title": "Cloud VM 작업을 사용할 수 없음", "reason": "이 머신 종류는 이 작업을 제공하지 않습니다.", diff --git a/web/messages/no.json b/web/messages/no.json index 7ab503663984..c235cdb1187c 100644 --- a/web/messages/no.json +++ b/web/messages/no.json @@ -1,5 +1,10 @@ { "vmErrors": { + "createCleanupPending": { + "title": "Opprydding av Cloud VM venter", + "message": "Oppsettet av Cloud VM mislyktes, og oppryddingen venter fortsatt.", + "action": "Kontakt kundestøtte med sporings-ID-en før du oppretter en ny maskin." + }, "unsupported": { "title": "Cloud VM-operasjon er ikke tilgjengelig", "reason": "Denne maskintypen tilbyr ikke denne operasjonen.", diff --git a/web/messages/pl.json b/web/messages/pl.json index 7c2bd846e5d1..c436c4ee722b 100644 --- a/web/messages/pl.json +++ b/web/messages/pl.json @@ -1,5 +1,10 @@ { "vmErrors": { + "createCleanupPending": { + "title": "Oczekiwanie na usunięcie pozostałości maszyny Cloud VM", + "message": "Konfiguracja maszyny Cloud VM nie powiodła się, a usuwanie pozostałości nadal oczekuje.", + "action": "Przed utworzeniem kolejnej maszyny skontaktuj się z pomocą techniczną, podając identyfikator śledzenia." + }, "unsupported": { "title": "Operacja Cloud VM jest niedostępna", "reason": "Ten typ maszyny nie udostępnia tej operacji.", diff --git a/web/messages/pt-BR.json b/web/messages/pt-BR.json index 7f3ba1521569..d55fce70f594 100644 --- a/web/messages/pt-BR.json +++ b/web/messages/pt-BR.json @@ -1,5 +1,10 @@ { "vmErrors": { + "createCleanupPending": { + "title": "Limpeza da VM Cloud pendente", + "message": "A configuração da VM Cloud falhou e a limpeza ainda está pendente.", + "action": "Entre em contato com o suporte e informe o ID de rastreamento antes de criar outra máquina." + }, "unsupported": { "title": "Operação de Cloud VM indisponível", "reason": "Este tipo de máquina não oferece esta operação.", diff --git a/web/messages/ru.json b/web/messages/ru.json index fb4dd725c6a8..0d9e8589b41a 100644 --- a/web/messages/ru.json +++ b/web/messages/ru.json @@ -1,5 +1,10 @@ { "vmErrors": { + "createCleanupPending": { + "title": "Очистка Cloud VM ожидается", + "message": "Не удалось настроить Cloud VM. Очистка ещё не завершена.", + "action": "Перед созданием другой машины обратитесь в поддержку и укажите идентификатор трассировки." + }, "unsupported": { "title": "Операция Cloud VM недоступна", "reason": "Этот тип машины не поддерживает эту операцию.", diff --git a/web/messages/th.json b/web/messages/th.json index 9d4ea4578eae..e966debcf8ae 100644 --- a/web/messages/th.json +++ b/web/messages/th.json @@ -1,5 +1,10 @@ { "vmErrors": { + "createCleanupPending": { + "title": "รอการล้างข้อมูล Cloud VM", + "message": "ตั้งค่า Cloud VM ไม่สำเร็จและยังล้างข้อมูลไม่เสร็จ", + "action": "ติดต่อฝ่ายสนับสนุนพร้อมรหัสการติดตามก่อนสร้างเครื่องอื่น" + }, "unsupported": { "title": "ไม่สามารถใช้การดำเนินการ Cloud VM ได้", "reason": "ประเภทเครื่องนี้ไม่มีการดำเนินการนี้", diff --git a/web/messages/tr.json b/web/messages/tr.json index 78b0273d3b2d..8aaf856f7c7e 100644 --- a/web/messages/tr.json +++ b/web/messages/tr.json @@ -1,5 +1,10 @@ { "vmErrors": { + "createCleanupPending": { + "title": "Cloud VM temizliği bekleniyor", + "message": "Cloud VM kurulumu başarısız oldu ve temizlik henüz tamamlanmadı.", + "action": "Başka bir makine oluşturmadan önce izleme kimliğiyle destek ekibine başvurun." + }, "unsupported": { "title": "Cloud VM işlemi kullanılamıyor", "reason": "Bu makine türü bu işlemi sağlamıyor.", diff --git a/web/messages/uk.json b/web/messages/uk.json index ca97b1950f07..00f780c2f79d 100644 --- a/web/messages/uk.json +++ b/web/messages/uk.json @@ -1,5 +1,10 @@ { "vmErrors": { + "createCleanupPending": { + "title": "Очищення Cloud VM очікується", + "message": "Не вдалося налаштувати Cloud VM. Очищення ще не завершено.", + "action": "Перш ніж створювати іншу машину, зверніться до підтримки та вкажіть ідентифікатор трасування." + }, "unsupported": { "title": "Операція Cloud VM недоступна", "reason": "Цей тип машини не підтримує цю операцію.", diff --git a/web/messages/zh-CN.json b/web/messages/zh-CN.json index b52c1d3064a6..a9cd29807fef 100644 --- a/web/messages/zh-CN.json +++ b/web/messages/zh-CN.json @@ -1,5 +1,10 @@ { "vmErrors": { + "createCleanupPending": { + "title": "Cloud VM 清理待完成", + "message": "Cloud VM 设置失败,清理仍未完成。", + "action": "创建另一台机器之前,请提供跟踪 ID 联系支持团队。" + }, "unsupported": { "title": "云端虚拟机操作不可用", "reason": "此机器类型不提供此操作。", diff --git a/web/messages/zh-TW.json b/web/messages/zh-TW.json index e984e8f23abf..32cd19b69208 100644 --- a/web/messages/zh-TW.json +++ b/web/messages/zh-TW.json @@ -1,5 +1,10 @@ { "vmErrors": { + "createCleanupPending": { + "title": "Cloud VM 清理待完成", + "message": "Cloud VM 設定失敗,清理仍未完成。", + "action": "建立另一台機器之前,請提供追蹤 ID 聯絡支援團隊。" + }, "unsupported": { "title": "雲端虛擬機操作無法使用", "reason": "此機器類型不提供這項操作。", diff --git a/web/services/vms/drivers/freestyle.ts b/web/services/vms/drivers/freestyle.ts index e47a975e4c1c..1ab55984fe0b 100644 --- a/web/services/vms/drivers/freestyle.ts +++ b/web/services/vms/drivers/freestyle.ts @@ -17,6 +17,8 @@ import { FreestyleResourceStatsReader } from "./freestyleResourceStatsReader"; import { announceFreestyleNetwork } from "./freestyleNetworkAnnouncement"; import { freestyleRequestFetch } from "./freestyleRequestTiming"; import { currentVmRequestContext } from "../requestContext"; +import { installFreestyleGuestCli, type FreestyleClientFactory } from "./freestyleGuestCli"; +import { rollbackFreestyleCreate } from "./providerCreateCleanup"; import { guestResourceReporterInstallCommand } from "../guestResourceReporter"; import { ProviderError, @@ -180,7 +182,7 @@ const EDGE_DOMAIN = /^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9 * Production uses the env-configured client and the live manifest. */ export type FreestyleProviderDependencies = { - readonly client: (timeoutMs?: number) => Freestyle; + readonly client: FreestyleClientFactory; readonly resolveDaemonSource: typeof resolveCmuxTuiSource; }; @@ -202,9 +204,9 @@ export function preconnectFreestyle(): void { } /** Exported for the publication provider, which shares this account-wide client. */ -export function freestyleClient(timeoutMs = DEFAULT_TIMEOUT_MS): Freestyle { +export function freestyleClient(timeoutMs = DEFAULT_TIMEOUT_MS, signal?: AbortSignal): Freestyle { const longFetch = freestyleRequestFetch({ - timeoutMs, + timeoutMs, signal, record: process.env.NODE_ENV === "development" ? (event) => { const traceId = currentVmRequestContext()?.traceId; console.info("cmux.vm.freestyle.request", JSON.stringify({ @@ -1018,9 +1020,8 @@ export class FreestyleProvider implements VMProvider { // A VM that failed to size or configure must not survive as an // orphan, and an undersized machine must not ship as if it were // the plan machine. - await vm.delete().catch((cleanupErr) => { - console.error(`[freestyle] create rollback failed; VM ${vmId} may be orphaned`, cleanupErr); - }); + const rollback = await Effect.runPromise(Effect.either(rollbackFreestyleCreate(this.deps.client, vmId, err))); + if (rollback._tag === "Left") throw rollback.left; throw err; } return { @@ -1397,9 +1398,8 @@ export class FreestyleProvider implements VMProvider { await this.ensureCmuxTuiRunning(vm, vmId, false).catch(() => undefined); await this.announcePrivateAddresses(vm, data); } catch (err) { - await vm.delete().catch((cleanupErr) => { - console.error(`[freestyle] restore rollback failed; VM ${vmId} may be orphaned`, cleanupErr); - }); + const rollback = await Effect.runPromise(Effect.either(rollbackFreestyleCreate(this.deps.client, vmId, err))); + if (rollback._tag === "Left") throw rollback.left; throw err; } return { @@ -1732,23 +1732,20 @@ export class FreestyleProvider implements VMProvider { * as a failed heal. */ private async installGuestCli(vm: Vm, vmId: string, promptIdentity?: GuestPromptIdentity): Promise { - const temporaryPath = `${GUEST_CMUX_SHIM_PATH}.tmp-${randomBytes(12).toString("hex")}`; - try { - await vm.fs.writeTextFile(temporaryPath, GUEST_CMUX_SHIM, { mode: 0o755 }); - const result = await vm.exec({ - command: `${guestBrowserInstallCommand()} && chmod 0755 '${temporaryPath}' && mv -f '${temporaryPath}' '${GUEST_CMUX_SHIM_PATH}'` - + (promptIdentity ? ` && ${guestPromptInstallCommand(promptIdentity)}` : ""), - timeoutMs: 30_000, - linuxUser: GUEST_LINUX_USER, - }); - const exitCode = result.statusCode ?? 124; - if (exitCode !== 0) { - throw new Error(`guest cmux shim install exited ${exitCode}`); + await withVmSpan("cmux.vm.guest_cli.install", "provider", {}, async (span) => { + const result = await Effect.runPromise(Effect.either(installFreestyleGuestCli(this.deps.client, vmId, promptIdentity))); + if (result._tag === "Left") { + setSpanAttributes(span, { + "cmux.vm.guest_install.outcome": result.left.outcome, + "cmux.vm.guest_install.stage": result.left.stage, + "cmux.vm.guest_install.exit_code": result.left.exitCode, + "cmux.duration_ms": result.left.elapsedMs, + "cmux.vm.guest_install.cleanup_failed": result.left.cleanupCause !== undefined, + }); + throw result.left; } - } catch (error) { - await vm.fs.remove(temporaryPath).catch(() => undefined); - throw error; - } + setSpanAttributes(span, { "cmux.vm.guest_install.outcome": "success", "cmux.duration_ms": result.right.elapsedMs }); + }); await this.ensureResourceReporter(vm, vmId); } diff --git a/web/services/vms/drivers/freestyleGuestCli.ts b/web/services/vms/drivers/freestyleGuestCli.ts new file mode 100644 index 000000000000..6e0a141e05f6 --- /dev/null +++ b/web/services/vms/drivers/freestyleGuestCli.ts @@ -0,0 +1,119 @@ +import { randomBytes } from "node:crypto"; +import { Clock, Data, Effect, Exit } from "effect"; +import type { Freestyle } from "freestyle"; +import { GUEST_CMUX_SHIM, GUEST_CMUX_SHIM_PATH } from "../guestCli"; +import type { GuestPromptIdentity } from "../guestPrompt"; +import { guestCliInstallCommand } from "./guestCliInstallCommand"; + +export type FreestyleClientFactory = (timeoutMs?: number, signal?: AbortSignal) => Freestyle; +export const GUEST_INSTALL_DEADLINE_MS = 45_000; +const CLEANUP_DEADLINE_MS = 10_000; +type InstallStage = "upload" | "install" | "validate" | "verify" | "browser" | "prompt" | "publish"; +type InstallOutcome = "missing_status" | "invalid_status" | "provider_timeout" | "guest_exit" + | "cancelled" | "transport_timeout" | "transport" | "deadline"; + +export class GuestCliInstallError extends Data.TaggedError("GuestCliInstallError")<{ + readonly stage: InstallStage; + readonly outcome: InstallOutcome; + readonly exitCode?: number | null; + readonly diagnostic?: string; + readonly elapsedMs?: number; + readonly cause?: unknown; + readonly cleanupCause?: unknown; +}> { + get message(): string { + return `guest cmux shim ${this.stage}: ${this.outcome}` + + (typeof this.exitCode === "number" ? ` (exit ${this.exitCode})` : "") + + (this.diagnostic ? `: ${this.diagnostic}` : ""); + } +} + +function guestFailure(result: unknown): GuestCliInstallError | undefined { + const response = result && typeof result === "object" ? result as Record : {}; + const status = response.statusCode; + if (status === 0) return; + const outcome: InstallOutcome = status === undefined ? "missing_status" + : status === null ? "provider_timeout" + : typeof status === "number" && Number.isInteger(status) ? "guest_exit" : "invalid_status"; + const failure: { stage: InstallStage; diagnostic?: string } = { stage: "install" }; + // Only our own bounded diagnostic record enters telemetry. Arbitrary guest + // stdout/stderr may contain private data and is deliberately not logged. + const marker = typeof response.stderr === "string" + ? response.stderr.split("\n").find((line) => line.startsWith("CMUX_GUEST_INSTALL_FAILURE=")) : undefined; + if (marker && marker.length < 512) { + try { + const detail = JSON.parse(marker.slice("CMUX_GUEST_INSTALL_FAILURE=".length)); + if (["validate", "verify", "browser", "prompt", "publish"].includes(detail.stage)) failure.stage = detail.stage; + if (typeof detail.error === "string" && /^[A-Za-z]{1,40}$/.test(detail.error)) { + failure.diagnostic = detail.error + + (Number.isInteger(detail.errno) ? ` errno=${detail.errno}` : "") + + (Number.isInteger(detail.exitCode) ? ` exit=${detail.exitCode}` : ""); + } + } catch { /* Unknown provider output is not evidence of an install stage. */ } + } + return new GuestCliInstallError({ + ...failure, outcome, + exitCode: status === null || typeof status === "number" ? status : undefined, + }); +} + +function transportFailure(stage: InstallStage, cause: unknown): GuestCliInstallError { + if (cause instanceof GuestCliInstallError) return cause; + const name = cause instanceof Error ? cause.name : undefined; + return new GuestCliInstallError({ + stage, cause, + outcome: name === "AbortError" ? "cancelled" : name === "TimeoutError" ? "transport_timeout" : "transport", + }); +} + +/** One overall deadline includes upload, exec and SDK background polling. */ +export function installFreestyleGuestCli(client: FreestyleClientFactory, vmId: string, identity?: GuestPromptIdentity) { + return Effect.gen(function* () { + const started = yield* Clock.currentTimeMillis; + const temporaryPath = `${GUEST_CMUX_SHIM_PATH}.tmp-${randomBytes(12).toString("hex")}`; + let stage: InstallStage = "upload"; + let cleanupCause: unknown; + const install = Effect.tryPromise({ + try: async (signal) => { + const vm = client(GUEST_INSTALL_DEADLINE_MS, signal).vms.ref(vmId); + await vm.fs.writeTextFile(temporaryPath, GUEST_CMUX_SHIM, { mode: 0o755 }); + signal.throwIfAborted(); + stage = "install"; + const result = await vm.exec({ + command: guestCliInstallCommand(temporaryPath, identity), timeoutMs: 30_000, linuxUser: "root", + }); + const failure = guestFailure(result); + if (failure) throw failure; + }, + catch: (cause) => transportFailure(stage, cause), + }).pipe(Effect.timeoutFail({ + duration: GUEST_INSTALL_DEADLINE_MS, + onTimeout: () => new GuestCliInstallError({ stage, outcome: "deadline" }), + })); + const cleanup = Effect.tryPromise({ + // A fresh request owns cleanup; an aborted install signal cannot cancel it. + try: (signal) => client(CLEANUP_DEADLINE_MS, signal).vms.ref(vmId).fs.remove(temporaryPath), + catch: (cause) => cause, + }).pipe( + Effect.interruptible, + Effect.timeoutFail({ duration: CLEANUP_DEADLINE_MS, onTimeout: () => new Error("guest upload cleanup deadline") }), + Effect.catchAll((cause) => Effect.sync(() => { cleanupCause = cause; })), + ); + yield* Effect.acquireUseRelease( + Effect.void, + () => install, + (_, exit) => Exit.isFailure(exit) ? cleanup : Effect.void, + ).pipe(Effect.catchAll((error) => Clock.currentTimeMillis.pipe(Effect.flatMap((finished) => + Effect.fail(new GuestCliInstallError({ + stage: error.stage, + outcome: error.outcome, + elapsedMs: finished - started, + exitCode: error.exitCode, + diagnostic: error.diagnostic, + cause: error.cause, + cleanupCause, + })) + )))); + return { elapsedMs: (yield* Clock.currentTimeMillis) - started }; + }); +} diff --git a/web/services/vms/drivers/freestyleRequestTiming.ts b/web/services/vms/drivers/freestyleRequestTiming.ts index 07d06c91a838..b19d34830a21 100644 --- a/web/services/vms/drivers/freestyleRequestTiming.ts +++ b/web/services/vms/drivers/freestyleRequestTiming.ts @@ -12,6 +12,7 @@ export type FreestyleRequestTiming = { type Options = { readonly timeoutMs: number; + readonly signal?: AbortSignal; readonly fetch?: typeof fetch; readonly record?: (event: FreestyleRequestTiming) => void; readonly now?: () => number; @@ -22,9 +23,11 @@ export function freestyleRequestFetch(options: Options): typeof fetch { const fetchImpl = options.fetch ?? fetch; const timeoutSignal = () => AbortSignal.timeout(options.timeoutMs); const request = ((input, init) => fetchImpl(input, { - ...(init ?? {}), signal: init?.signal - ? AbortSignal.any([init.signal, timeoutSignal()]) - : timeoutSignal(), + ...(init ?? {}), signal: AbortSignal.any([ + timeoutSignal(), + ...[options.signal, init?.signal ?? (input instanceof Request ? input.signal : undefined)] + .filter((signal): signal is AbortSignal => !!signal), + ]), })) as typeof fetch; if (!options.record) return request; diff --git a/web/services/vms/drivers/guestCliInstallCommand.ts b/web/services/vms/drivers/guestCliInstallCommand.ts new file mode 100644 index 000000000000..793be970ec75 --- /dev/null +++ b/web/services/vms/drivers/guestCliInstallCommand.ts @@ -0,0 +1,45 @@ +import { createHash } from "node:crypto"; +import { GUEST_CMUX_SHIM, GUEST_CMUX_SHIM_PATH } from "../guestCli"; +import { guestBrowserInstallCommand } from "../guestBrowser"; +import { guestPromptInstallCommand, type GuestPromptIdentity } from "../guestPrompt"; +import { shellQuote } from "./cmuxTuiDaemon"; + +const digest = createHash("sha256").update(GUEST_CMUX_SHIM).digest("hex"); + +// Validate the uploaded generation and all required setup before publishing it. +// os.replace replaces a symlink itself and refuses a directory destination; +// `mv -f source destination` can instead move source *inside* a directory. +const install = String.raw` +import hashlib, json, os, stat, subprocess, sys +source, target, digest, browser, prompt = sys.argv[1:] +stage = "validate" +try: + fd = os.open(source, os.O_RDONLY | os.O_NOFOLLOW) + with os.fdopen(fd, "rb") as stream: + if not stat.S_ISREG(os.fstat(stream.fileno()).st_mode): + raise ValueError("upload is not a regular file") + if hashlib.sha256(stream.read()).hexdigest() != digest: + raise ValueError("upload checksum mismatch") + os.fchmod(stream.fileno(), 0o755) + stage = "verify" + subprocess.run([source, "--help"], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) + stage = "browser" + subprocess.run(["/bin/sh", "-c", browser], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) + if prompt: + stage = "prompt" + subprocess.run(["/bin/sh", "-c", prompt], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) + stage = "publish" + os.replace(source, target) +except Exception as error: + # Bounded, structured diagnostics: no prompt payload or guest output. + print("CMUX_GUEST_INSTALL_FAILURE=" + json.dumps({ + "stage": stage, "error": type(error).__name__, + "errno": getattr(error, "errno", None), + "exitCode": getattr(error, "returncode", None) + }), file=sys.stderr) + sys.exit(1) +`; + +export function guestCliInstallCommand(temporaryPath: string, identity?: GuestPromptIdentity): string { + return `python3 -c ${shellQuote(install)} ${shellQuote(temporaryPath)} ${shellQuote(GUEST_CMUX_SHIM_PATH)} ${shellQuote(digest)} ${shellQuote(guestBrowserInstallCommand())} ${shellQuote(identity ? guestPromptInstallCommand(identity) : "")}`; +} diff --git a/web/services/vms/drivers/providerCreateCleanup.ts b/web/services/vms/drivers/providerCreateCleanup.ts new file mode 100644 index 000000000000..aa010f4f8c77 --- /dev/null +++ b/web/services/vms/drivers/providerCreateCleanup.ts @@ -0,0 +1,34 @@ +import { Effect } from "effect"; +import { FreestyleApiError } from "freestyle"; +import { ProviderError } from "./types"; +import type { FreestyleClientFactory } from "./freestyleGuestCli"; + +export function isProviderCreateCleanupError(value: unknown): value is ProviderCreateCleanupError { + return value instanceof ProviderCreateCleanupError; +} + +/** Retains the allocation until deletion is confirmed, including both failures. */ +export class ProviderCreateCleanupError extends ProviderError { + readonly name = "ProviderCreateCleanupError"; + constructor( + readonly providerVmId: string, + cause: unknown, + readonly cleanupCause: unknown, + ) { + super("freestyle", "create rollback is unconfirmed", cause); + } +} + +export function rollbackFreestyleCreate(client: FreestyleClientFactory, vmId: string, cause: unknown) { + return Effect.tryPromise({ + try: (signal) => client(15_000, signal).vms.ref(vmId).delete(), + catch: (cleanupCause) => new ProviderCreateCleanupError(vmId, cause, cleanupCause), + }).pipe(Effect.catchAll((error) => + error.cleanupCause instanceof FreestyleApiError + && error.cleanupCause.status === 404 && error.cleanupCause.code === "NOT_FOUND" + ? Effect.void : Effect.fail(error) + ), Effect.timeoutFail({ + duration: 15_000, + onTimeout: () => new ProviderCreateCleanupError(vmId, cause, new Error("create rollback deadline")), + })); +} diff --git a/web/services/vms/repository.ts b/web/services/vms/repository.ts index 7eeb2dc3d594..f51fd17f2636 100644 --- a/web/services/vms/repository.ts +++ b/web/services/vms/repository.ts @@ -332,6 +332,7 @@ export type VmRepositoryShape = { readonly userId: string; readonly code: string; readonly message: string; + readonly cleanupProviderVmId?: string; }) => Effect.Effect; readonly activeLimitCandidates: (input: { readonly userId: string; @@ -451,6 +452,8 @@ export type VmRepositoryShape = { readonly id: string; readonly code: string; readonly message: string; + /** Keeps the allocation reserved and unready until an operator confirms cleanup. */ + readonly cleanupProviderVmId?: string; }) => Effect.Effect; /** Durable deletion intents not yet finalized, scoped to their source machine. */ readonly pendingSnapshotDeletions: (input: { @@ -650,6 +653,8 @@ export const FAILED_CREATE_RETRY_WINDOW_MS = 15 * 60 * 1000; * provider failure). */ export const PROVIDER_CREATE_UNAVAILABLE_FAILURE_CODE = "provider_create_unavailable"; +/** Provider allocation remains owned by the failed row until cleanup is confirmed. */ +export const PROVIDER_CREATE_CLEANUP_PENDING_FAILURE_CODE = "provider_create_cleanup_pending"; const RETRYABLE_FAILED_CREATE_CODES = new Set([ "billing_credits_insufficient", @@ -1955,12 +1960,18 @@ export const vmRepositoryLiveShape: VmRepositoryShape = { await tx .update(cloudVms) .set({ - status: "failed", + status: input.cleanupProviderVmId ? "provisioning" : "failed", failureCode: input.code, failureMessage: input.message, + ...(input.cleanupProviderVmId ? { + providerMetadata: sql`${cloudVms.providerMetadata} || jsonb_build_object('createCleanupProviderVmId', ${input.cleanupProviderVmId}::text)`, + } : {}), updatedAt: now, }) .where(eq(cloudVms.id, input.vmId)); + // Leave the Base generation reserved too; reset/open must not allocate + // a replacement while its failed guest still exists at the provider. + if (input.cleanupProviderVmId) return; await tx .update(cloudVmBaseGenerations) .set({ state: "failed", updatedAt: now }) @@ -2677,9 +2688,12 @@ export const vmRepositoryLiveShape: VmRepositoryShape = { await db .update(cloudVms) .set({ - status: "failed", + status: input.cleanupProviderVmId ? "provisioning" : "failed", failureCode: input.code, failureMessage: input.message, + ...(input.cleanupProviderVmId ? { + providerMetadata: sql`${cloudVms.providerMetadata} || jsonb_build_object('createCleanupProviderVmId', ${input.cleanupProviderVmId}::text)`, + } : {}), updatedAt: new Date(), }) .where(eq(cloudVms.id, input.id)); diff --git a/web/services/vms/routeHelpers.ts b/web/services/vms/routeHelpers.ts index 98b89933d502..0ed20b64b260 100644 --- a/web/services/vms/routeHelpers.ts +++ b/web/services/vms/routeHelpers.ts @@ -56,6 +56,7 @@ import { import { vmArtifactUnavailableCopy, vmDisplayNameCopy, + vmCreateCleanupPendingCopy, vmRequestLocale, vmRequiresProCopy, vmMemoryErrorCopy, @@ -65,6 +66,8 @@ import { } from "./vmErrorMessages"; import { DISPLAY_NAME_MAX_LENGTH } from "./displayName"; import { ProviderArtifactUnavailableError } from "./drivers/types"; +import { isProviderCreateCleanupError } from "./drivers/providerCreateCleanup"; +import { PROVIDER_CREATE_CLEANUP_PENDING_FAILURE_CODE } from "./repository"; import type { Locale } from "../../i18n/routing"; /** Bearer + refresh token pair the mac app stashes in keychain. */ @@ -753,6 +756,9 @@ export const vmWorkflowErrorResponders = { if (providerArtifactUnavailable(error.cause)) { return vmArtifactUnavailableResponse(error, context.locale); } + if (isProviderCreateCleanupError(error.cause)) { + return vmCreateCleanupPendingResponse(context.locale); + } return vmProviderOperationErrorResponse(error); }, VmAccountDeletionInProgressError: (error) => @@ -955,6 +961,9 @@ export async function respondVmWorkflowError( context: VmWorkflowErrorResponderContext, overrides?: VmWorkflowErrorOverrides, ): Promise { + if (error._tag === "VmCreateFailedError" && error.code === PROVIDER_CREATE_CLEANUP_PENDING_FAILURE_CODE) { + return vmCreateCleanupPendingResponse(context.locale); + } const responders: VmWorkflowErrorResponders = overrides ? { ...vmWorkflowErrorResponders, ...overrides } : vmWorkflowErrorResponders; @@ -1003,6 +1012,20 @@ async function vmArtifactUnavailableResponse(error: VmProviderOperationError, lo }); } +async function vmCreateCleanupPendingResponse(locale: Locale): Promise { + const copy = await vmCreateCleanupPendingCopy(locale); + return vmErrorResponse({ + error: "vm_cloud_create_cleanup_pending", + status: 503, + message: copy.message, + action: copy.action, + phase: "create", + retryable: false, + displayTitle: copy.title, + details: { operation: "create", cleanupPending: true, retryable: false }, + }); +} + function vmProviderOperationErrorResponse(error: VmProviderOperationError): Response { const providerCause = providerCauseSummary(error.cause); const phase = vmPhaseForOperation(error.operation); diff --git a/web/services/vms/vmErrorMessages.ts b/web/services/vms/vmErrorMessages.ts index 56e81adaa281..c7fb4f34ca39 100644 --- a/web/services/vms/vmErrorMessages.ts +++ b/web/services/vms/vmErrorMessages.ts @@ -102,6 +102,15 @@ export async function vmArtifactUnavailableCopy(locale: Locale): Promise { + const translator = createTranslator({ + locale, + messages: await loadMessages(locale), + namespace: "vmErrors.createCleanupPending", + }) as unknown as (key: string) => string; + return { title: translator("title"), message: translator("message"), action: translator("action") }; +} + /** Load and translate the `vm_requires_pro` response copy for the request locale. */ export async function vmRequiresProCopy( locale: Locale, diff --git a/web/services/vms/workflows.ts b/web/services/vms/workflows.ts index 5e207ca027b2..6ab16144c43a 100644 --- a/web/services/vms/workflows.ts +++ b/web/services/vms/workflows.ts @@ -96,8 +96,10 @@ import { GO_PAUSE_INTENT_KEY, pauseGoVm } from "./goPause"; import { networkSlugForUser, privateNetworkUnavailableReason, resolveOwnerNetwork } from "./privateNetwork"; import { isProviderIdentityNotFoundError, isProviderNotFoundError } from "./providerErrors"; import { VmProviderGateway, VmProviderGatewayLive, type VmProviderGatewayShape } from "./providerGateway"; +import { isProviderCreateCleanupError } from "./drivers/providerCreateCleanup"; import { withVmProductAnalytics } from "./productAnalytics"; import { + PROVIDER_CREATE_CLEANUP_PENDING_FAILURE_CODE, PROVIDER_CREATE_UNAVAILABLE_FAILURE_CODE, VmRepository, vmRepositoryLiveShape, @@ -497,6 +499,10 @@ function rollbackProviderCreate( }); } +function isFailedVmCreate(vm: Pick): boolean { + return vm.status === "failed" || vm.failureCode === PROVIDER_CREATE_CLEANUP_PENDING_FAILURE_CODE; +} + /** Check the copied or requested shape before provisioning side effects. */ function requireMemoryPlan(planId: string, memoryMb: number | null) { const maxMemoryMb = maxMemoryMbForPlan(planId); @@ -637,7 +643,7 @@ export function createVm(input: CreateVmInput): Effect.Effect Promise; - write?: (path: string, bytes: Uint8Array) => void; - remove?: (path: string) => void; + write?: (path: string, bytes: Uint8Array, signal?: AbortSignal | null) => void | Promise; + remove?: (path: string, signal?: AbortSignal | null) => void | Promise; deleteFailure?: boolean; } = {}) { const requests: Array<{ method: string; path: string }> = []; @@ -35,14 +35,14 @@ export function freestyleGuestFixture(options: { if (url.pathname.endsWith("/fs/write")) { const path = url.searchParams.get("path")!; writes.push(path); - options.write?.(path, new Uint8Array(await new Response(init?.body).arrayBuffer())); + await options.write?.(path, new Uint8Array(await new Response(init?.body).arrayBuffer()), signal ?? init?.signal); installPending = true; return Response.json({}); } if (url.pathname.endsWith("/fs/remove")) { const path = url.searchParams.get("path")!; removals.push(path); - options.remove?.(path); + await options.remove?.(path, signal ?? init?.signal); return Response.json({}); } if (url.pathname.endsWith("/exec-await")) { @@ -53,6 +53,9 @@ export function freestyleGuestFixture(options: { } return Response.json({ statusCode: 0, stdout: "", stderr: "" }); } + if (url.pathname.endsWith("/resize")) { + return Response.json({ id: url.pathname.split("/").at(-2), state: "running", resources: JSON.parse(String(init?.body)) }); + } if (method === "DELETE" && /^\/v5\/vms\/vm-fixture-\d+$/.test(url.pathname)) { if (options.deleteFailure) return Response.json({ code: "UNAVAILABLE", message: "synthetic delete failure" }, { status: 503 }); liveVms.delete(url.pathname.split("/").at(-1)!); diff --git a/web/tests/vm-freestyle-provider.test.ts b/web/tests/vm-freestyle-provider.test.ts index 87c19bb4eb1d..56f4c1ce6172 100644 --- a/web/tests/vm-freestyle-provider.test.ts +++ b/web/tests/vm-freestyle-provider.test.ts @@ -328,7 +328,6 @@ describe("Freestyle platform contract", () => { expect(result.exitCode).toBe(0); expect(fake.writes).toHaveLength(1); expect(fake.writes[0]?.content).toBe(GUEST_CMUX_SHIM); - expect(fake.execs[1]).toContain(`mv -f`); expect(fake.execs.at(-1)).toBe("cmux self --json"); }); @@ -459,7 +458,6 @@ describe("FreestyleProvider create with edge rules", () => { expect(fake.writes[0]?.path).toMatch(/^\/usr\/local\/bin\/cmux\.tmp-[0-9a-f]{24}$/); expect(fake.writes[0]?.content).toContain("cmux auth status"); expect(fake.writes[0]?.content).not.toContain("crt_secret-token"); - expect(fake.execs.some((command) => command.includes("mv -f") && command.includes("/usr/local/bin/cmux'"))).toBe(true); expect(fake.execs.some((command) => command.includes("/api/coderouter/vm-usage/self"))).toBe(false); expect(fake.deletes).toEqual([]); }); @@ -479,7 +477,6 @@ describe("FreestyleProvider create with edge rules", () => { expect(handle.providerMetadata).toMatchObject({ networkId: "vpc_1" }); expect(fake.writes).toHaveLength(1); expect(fake.writes[0]?.path).toMatch(/^\/usr\/local\/bin\/cmux\.tmp-[0-9a-f]{24}$/); - expect(fake.execs.some((command) => command.includes("mv -f") && command.includes("/usr/local/bin/cmux'"))).toBe(true); expect(fake.writes[0]?.content).not.toContain("crt_secret-token"); expect(handle.providerMetadata).toMatchObject({ networkId: "vpc_1", @@ -512,7 +509,6 @@ describe("FreestyleProvider create with edge rules", () => { expect(ok.writes).toHaveLength(1); expect(ok.writes[0]?.path).toMatch(/^\/usr\/local\/bin\/cmux\.tmp-[0-9a-f]{24}$/); expect(ok.writes[0]?.content).not.toContain("crt_secret-token"); - expect(ok.execs.some((command) => command.includes("mv -f") && command.includes("/usr/local/bin/cmux'"))).toBe(true); expect(ok.deletes).toEqual([]); }); }); diff --git a/web/tests/vm-guest-create-response.test.ts b/web/tests/vm-guest-create-response.test.ts new file mode 100644 index 000000000000..8f0c955d7dc2 --- /dev/null +++ b/web/tests/vm-guest-create-response.test.ts @@ -0,0 +1,23 @@ +import { expect, test } from "bun:test"; +import { VmCreateFailedError, VmProviderOperationError } from "../services/vms/errors"; +import { ProviderCreateCleanupError } from "../services/vms/drivers/providerCreateCleanup"; +import { vmWorkflowErrorResponse } from "../services/vms/routeHelpers"; +import { locales } from "../i18n/routing"; + +test.each(locales)("cleanup-pending response is localized and never suggests retry (%s)", async (locale) => { + const original = new VmProviderOperationError({ provider: "freestyle", operation: "create", cause: + new ProviderCreateCleanupError("private-provider-id", new Error("private guest output"), new Error("private delete output")), + }); + const retry = new VmCreateFailedError({ idempotencyKey: "synthetic", code: "provider_create_cleanup_pending", message: "private diagnostic" }); + const first = await vmWorkflowErrorResponse(original, { locale }); + const repeated = await vmWorkflowErrorResponse(retry, { locale }); + expect(first?.status).toBe(503); + expect(first?.headers.has("retry-after")).toBe(false); + const body = await first!.json(); + expect(body.error).toBe("vm_cloud_create_cleanup_pending"); + expect(body.retryable).toBe(false); + expect(body.ui.retryable).toBe(false); + expect(JSON.stringify(body)).not.toContain("private"); + expect(await repeated!.json()).toEqual(body); + if (locale !== "en") expect(body.message).not.toBe("Cloud VM setup failed and cleanup is still pending."); +}); diff --git a/web/tests/vm-guest-install-deadline.test.ts b/web/tests/vm-guest-install-deadline.test.ts new file mode 100644 index 000000000000..78538fa689c5 --- /dev/null +++ b/web/tests/vm-guest-install-deadline.test.ts @@ -0,0 +1,126 @@ +import { expect, test } from "bun:test"; +import { Deferred, Effect, Fiber, TestClock, TestContext } from "effect"; +import { GUEST_INSTALL_DEADLINE_MS, installFreestyleGuestCli } from "../services/vms/drivers/freestyleGuestCli"; +import { freestyleGuestFixture } from "./fixtures/freestyleGuest"; +import { Freestyle, FreestyleApiError } from "freestyle"; +import { rollbackFreestyleCreate } from "../services/vms/drivers/providerCreateCleanup"; + +test("an active stalled exec is aborted at the overall deadline, then cleaned with a fresh signal", async () => { + let active = 0; + let aborted = 0; + await Effect.runPromise(Effect.gen(function* () { + const entered = yield* Deferred.make(); + const fixture = freestyleGuestFixture({ + exec: async (_, signal) => { + active++; + Effect.runSync(Deferred.succeed(entered, undefined)); + return new Promise((_, reject) => signal!.addEventListener("abort", () => { + active--; aborted++; reject(signal!.reason); + }, { once: true })); + }, + remove: (_, signal) => { expect(signal?.aborted).toBe(false); expect(active).toBe(0); }, + }); + const work = yield* Effect.fork(Effect.either(installFreestyleGuestCli(fixture.client, "vm-fixture-1"))); + yield* Deferred.await(entered); + expect(active).toBe(1); + yield* TestClock.adjust(GUEST_INSTALL_DEADLINE_MS - 1); + expect((yield* Fiber.poll(work))._tag).toBe("None"); + expect(active).toBe(1); + yield* TestClock.adjust(1); + const outcome = yield* Fiber.join(work); + expect(outcome._tag).toBe("Left"); + if (outcome._tag === "Left") expect(outcome.left).toMatchObject({ outcome: "deadline", stage: "install" }); + expect(active).toBe(0); + expect(aborted).toBe(1); + expect(fixture.removals).toEqual(fixture.writes); + }).pipe(Effect.provide(TestContext.TestContext))); +}); + +test("cancelling during upload never starts exec and still cleans the upload", async () => { + let aborted = false; + await Effect.runPromise(Effect.gen(function* () { + const entered = yield* Deferred.make(); + const fixture = freestyleGuestFixture({ write: async (_, __, signal) => { + Effect.runSync(Deferred.succeed(entered, undefined)); + await new Promise((_, reject) => signal!.addEventListener("abort", () => { + aborted = true; reject(signal!.reason); + }, { once: true })); + } }); + const work = yield* Effect.fork(installFreestyleGuestCli(fixture.client, "vm-fixture-1")); + yield* Deferred.await(entered); + yield* Fiber.interrupt(work); + expect(aborted).toBe(true); + expect(fixture.requests.some(({ path }) => path.endsWith("exec-await"))).toBe(false); + expect(fixture.removals).toEqual(fixture.writes); + })); +}); + +test("a stalled cleanup finalizer is bounded and retains the original install failure", async () => { + let aborted = false; + await Effect.runPromise(Effect.gen(function* () { + const entered = yield* Deferred.make(); + const fixture = freestyleGuestFixture({ + exec: async () => Response.json({ statusCode: 124 }), + remove: async (_, signal) => { + Effect.runSync(Deferred.succeed(entered, undefined)); + await new Promise((_, reject) => signal!.addEventListener("abort", () => { + aborted = true; reject(signal!.reason); + }, { once: true })); + }, + }); + const work = yield* Effect.fork(Effect.either(installFreestyleGuestCli(fixture.client, "vm-fixture-1"))); + yield* Deferred.await(entered); + yield* TestClock.adjust(10_000); + const outcome = yield* Fiber.join(work); + expect(outcome._tag).toBe("Left"); + if (outcome._tag === "Left") { + expect(outcome.left).toMatchObject({ outcome: "guest_exit", exitCode: 124 }); + expect(outcome.left.cleanupCause).toBeInstanceOf(Error); + } + expect(aborted).toBe(true); + }).pipe(Effect.provide(TestContext.TestContext))); +}); + +test("rollback accepts typed absence, while missing endpoints and provider outages stay unconfirmed", async () => { + for (const [status, code] of [[404, "NOT_FOUND"], [404, "UNKNOWN"], [503, "UNAVAILABLE"]] as const) { + const cause = new Error("bootstrap failed"); + const client = () => new Freestyle({ apiKey: "synthetic", baseUrl: "https://provider.invalid", fetch: + (async () => Response.json({ code, message: "synthetic deletion response" }, { status })) as typeof fetch, + }); + const result = await Effect.runPromise(Effect.either(rollbackFreestyleCreate(client, "vm-synthetic", cause))); + if (code === "NOT_FOUND") expect(result._tag).toBe("Right"); + else { + expect(result._tag).toBe("Left"); + if (result._tag === "Left") { + expect(result.left.cause).toBe(cause); + expect(result.left.cleanupCause).toBeInstanceOf(FreestyleApiError); + } + } + } +}); + +test("a stalled VM rollback is cancelled and retains the allocated id at its deadline", async () => { + let aborted = false; + const cause = new Error("bootstrap failed"); + await Effect.runPromise(Effect.gen(function* () { + const entered = yield* Deferred.make(); + const client = (_?: number, signal?: AbortSignal) => new Freestyle({ + apiKey: "synthetic", baseUrl: "https://provider.invalid", fetch: (async () => { + Effect.runSync(Deferred.succeed(entered, undefined)); + return new Promise((_, reject) => signal!.addEventListener("abort", () => { + aborted = true; reject(signal!.reason); + }, { once: true })); + }) as typeof fetch, + }); + const work = yield* Effect.fork(Effect.either(rollbackFreestyleCreate(client, "vm-synthetic", cause))); + yield* Deferred.await(entered); + yield* TestClock.adjust(15_000); + const result = yield* Fiber.join(work); + expect(result._tag).toBe("Left"); + if (result._tag === "Left") { + expect(result.left.providerVmId).toBe("vm-synthetic"); + expect(result.left.cause).toBe(cause); + } + expect(aborted).toBe(true); + }).pipe(Effect.provide(TestContext.TestContext))); +}); diff --git a/web/tests/vm-guest-install-runtime.test.ts b/web/tests/vm-guest-install-runtime.test.ts index fc3f91f14d7b..8f33a90edaf2 100644 --- a/web/tests/vm-guest-install-runtime.test.ts +++ b/web/tests/vm-guest-install-runtime.test.ts @@ -1,6 +1,6 @@ import { afterEach, describe, expect, test } from "bun:test"; import { spawnSync } from "node:child_process"; -import { mkdtempSync, mkdirSync, readFileSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs"; +import { mkdtempSync, mkdirSync, readFileSync, readdirSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { GUEST_CMUX_SHIM } from "../services/vms/guestCli"; @@ -14,14 +14,22 @@ function guest(options: { corruptUpload?: boolean } = {}) { const root = mkdtempSync(join(tmpdir(), "cmux-guest-install-")); roots.push(root); mkdirSync(join(root, "bin")); + mkdirSync(join(root, "fixture-bin")); + writeFileSync(join(root, "fixture-bin/getent"), "#!/bin/sh\nexit 1\n", { mode: 0o755 }); + writeFileSync(join(root, "fixture-bin/xdg-mime"), "#!/bin/sh\nexit 0\n", { mode: 0o755 }); const rebase = (value: string) => value.replaceAll("/usr/local/bin", join(root, "bin")) - .replaceAll("/etc/cmux", join(root, "etc")); + .replaceAll("/usr/local/share", join(root, "share")) + .replaceAll("/etc/cmux", join(root, "etc")) + .replaceAll("/etc/", join(root, "system-etc") + "/"); const target = join(root, "bin/cmux"); const fixture = freestyleGuestFixture({ write: (path, bytes) => writeFileSync(rebase(path), options.corruptUpload ? "#!/bin/sh\nexit 0\n" : bytes), remove: (path) => rmSync(rebase(path), { force: true }), exec: async (request) => { - const result = spawnSync("/bin/sh", ["-c", rebase(request.command)], { encoding: "utf8", timeout: 5_000 }); + const result = spawnSync("/bin/sh", ["-c", rebase(request.command)], { + encoding: "utf8", timeout: 5_000, + env: { ...process.env, HOME: root, PATH: `${join(root, "fixture-bin")}:${process.env.PATH}` }, + }); return Response.json({ statusCode: result.status, stdout: result.stdout, stderr: result.stderr }); }, }); @@ -30,7 +38,7 @@ function guest(options: { corruptUpload?: boolean } = {}) { describe("guest CLI publication in an isolated filesystem", () => { test("successful create installs the complete executable CLI and answers help", async () => { - const { fixture, target } = guest(); + const { fixture, target, root } = guest(); const handle = await fixture.provider.create(guestCreateOptions); expect(handle.status).toBe("running"); expect(readFileSync(target, "utf8")).toBe(GUEST_CMUX_SHIM); @@ -38,6 +46,17 @@ describe("guest CLI publication in an isolated filesystem", () => { const result = spawnSync(target, ["--help"], { encoding: "utf8", timeout: 5_000 }); expect(result.status).toBe(0); expect(result.stdout).toContain("cmux"); + const daemon = join(root, "cmux-tui"); + writeFileSync(daemon, '#!/bin/sh\nprintf \'%s\\n\' "$@" > "$HOME/daemon-args"\nprintf \'%s\\n\' \'{"session":"cloud","workspaces":[]}\'\n', { mode: 0o755 }); + const tree = spawnSync(target, ["tree", "--json"], { + encoding: "utf8", timeout: 5_000, + env: { PATH: process.env.PATH, HOME: root, CMUX_TUI_BIN: daemon }, + }); + expect(tree.status).toBe(0); + expect(JSON.parse(tree.stdout)).toEqual({ session: "cloud", workspaces: [] }); + expect(readFileSync(join(root, "daemon-args"), "utf8").trim().split("\n")) + .toEqual(["--session", "cloud", "--json", "session", "current", "snapshot"]); + expect(readdirSync(join(root, "bin"))).toEqual(["cmux", "cmux-open-url", "sensible-browser", "x-www-browser", "xdg-open"]); }); test("replaces a target symlink without modifying its referent", async () => { diff --git a/web/tests/vm-guest-install.test.ts b/web/tests/vm-guest-install.test.ts index b8905bd5be65..18a82a7007d2 100644 --- a/web/tests/vm-guest-install.test.ts +++ b/web/tests/vm-guest-install.test.ts @@ -14,17 +14,20 @@ describe("Freestyle guest install result contract (SDK 0.2.10)", () => { // synthetic candidate responses, not invented copies of the lost body. test.each([ [{ stdout: "", stderr: "" }, "missing_status", undefined], + [{ exitCode: 0 }, "missing_status", undefined], [{ statusCode: null, stdout: "", stderr: "" }, "provider_timeout", null], [{ statusCode: 124, stderr: "synthetic guest exit" }, "guest_exit", 124], [{ statusCode: 1, stderr: "synthetic rename failure" }, "guest_exit", 1], [{ statusCode: "0" }, "invalid_status", undefined], - ])("preserves outcome %s without a false ready result", async (body, outcome, statusCode) => { + ] as Array<[Record, string, number | null | undefined]>) + ("preserves outcome %s without a false ready result", async (body, outcome, statusCode) => { const fixture = freestyleGuestFixture({ exec: async () => Response.json(body) }); const error = await fixture.provider.create(guestCreateOptions).catch((error: unknown) => error); - expect(error).toMatchObject({ - name: "ProviderError", - cause: { name: "GuestCliInstallError", outcome, statusCode }, - }); + expect(error).toBeInstanceOf(Error); + const installError = (error as { cause: { name: string; outcome: string; exitCode?: number | null } }).cause; + expect(installError.name).toBe("GuestCliInstallError"); + expect(installError.outcome).toBe(outcome); + expect(installError.exitCode).toBe(statusCode); expect(fixture.allocations()).toBe(1); expect(fixture.liveVms.size).toBe(0); expect(fixture.removals).toEqual(fixture.writes); @@ -37,7 +40,7 @@ describe("Freestyle guest install result contract (SDK 0.2.10)", () => { const cause = new DOMException("synthetic transport interruption", name); const fixture = freestyleGuestFixture({ exec: async () => { throw cause; } }); const error = await fixture.provider.create(guestCreateOptions).catch((error: unknown) => error); - expect(error).toMatchObject({ cause: { name: "GuestCliInstallError", outcome, cause } }); + expect((error as { cause?: unknown }).cause).toMatchObject({ name: "GuestCliInstallError", outcome, cause }); expect(fixture.liveVms.size).toBe(0); expect(fixture.removals).toEqual(fixture.writes); }); @@ -51,7 +54,7 @@ describe("Freestyle guest install result contract (SDK 0.2.10)", () => { expect(error).toMatchObject({ name: "ProviderCreateCleanupError", providerVmId: "vm-fixture-1", - cause: { name: "GuestCliInstallError", outcome: "guest_exit", statusCode: 1 }, + cause: { name: "GuestCliInstallError", outcome: "guest_exit", exitCode: 1 }, cleanupCause: { name: "FreestyleApiError", status: 503 }, }); expect(fixture.liveVms.size).toBe(1); diff --git a/web/tests/vm-workflows.test.ts b/web/tests/vm-workflows.test.ts index 0523b16a0ec7..b67201d61a27 100644 --- a/web/tests/vm-workflows.test.ts +++ b/web/tests/vm-workflows.test.ts @@ -265,6 +265,37 @@ describe("VM Effect workflows", () => { expect(fixture.liveVms.size).toBe(1); }); + dbTest("unconfirmed guest rollback keeps the Base generation reserved", async () => { + if (!sql) throw new Error("test database not initialized"); + const fixture = freestyleGuestFixture({ exec: async () => Response.json({ statusCode: 1 }), deleteFailure: true }); + const layer = providerLayer({ + ...unusedProviderGateway(), + create: (_, options) => Effect.tryPromise({ + try: () => fixture.provider.create({ ...options, imageSize: guestCreateOptions.imageSize }), + catch: (cause) => new VmProviderOperationError({ provider: "freestyle", operation: "create", cause }), + }), + }); + const input = { + userId: "user-guest-base-cleanup", billingTeamId: "team-guest-base-cleanup", + billingCustomerType: "team" as const, billingPlanId: "pro", provider: "freestyle" as const, + image: "sh-synthetic", baseName: "guest-cleanup", maxActiveVms: 1, + }; + const first = await Effect.runPromise(Effect.either(openBaseVm(input).pipe(Effect.provide(layer)))); + expect(first._tag).toBe("Left"); + const retry = await Effect.runPromise(Effect.either(openBaseVm(input).pipe(Effect.provide(layer)))); + if (retry._tag === "Left") expect(retry.left).toMatchObject({ _tag: "VmCreateFailedError", code: "provider_create_cleanup_pending" }); + else throw new Error("Base must not report a failed bootstrap ready"); + const reset = await Effect.runPromise(Effect.either(resetBaseVm(input).pipe(Effect.provide(layer)))); + expect(reset._tag).toBe("Left"); + expect(fixture.allocations()).toBe(1); + const [row] = await sql<{ status: string; providerVmId: string | null; metadata: Record }[]>` + select status, provider_vm_id as "providerVmId", provider_metadata as metadata from cloud_vms where user_id = 'user-guest-base-cleanup' + `; + expect(row.status).toBe("provisioning"); + expect(row.providerVmId).toBeNull(); + expect(row.metadata.createCleanupProviderVmId).toBe("vm-fixture-1"); + }); + dbTest("keeps prompt revisions ordered across rapid renames and clock skew", async () => { if (!sql) throw new Error("test database not initialized"); const userId = "user-prompt-revisions"; From 00702e6fe141753118f5bc6c71373d169be91693 Mon Sep 17 00:00:00 2001 From: Austin Wang Date: Mon, 14 Sep 2026 21:08:31 -0700 Subject: [PATCH 10/92] test: preserve full environment in guest CLI runtime fixture (cherry picked from commit 7440557d50a949b9e323315359a91e3554fa179e) --- web/tests/vm-guest-install-runtime.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/tests/vm-guest-install-runtime.test.ts b/web/tests/vm-guest-install-runtime.test.ts index 8f33a90edaf2..b78b46b66262 100644 --- a/web/tests/vm-guest-install-runtime.test.ts +++ b/web/tests/vm-guest-install-runtime.test.ts @@ -50,7 +50,7 @@ describe("guest CLI publication in an isolated filesystem", () => { writeFileSync(daemon, '#!/bin/sh\nprintf \'%s\\n\' "$@" > "$HOME/daemon-args"\nprintf \'%s\\n\' \'{"session":"cloud","workspaces":[]}\'\n', { mode: 0o755 }); const tree = spawnSync(target, ["tree", "--json"], { encoding: "utf8", timeout: 5_000, - env: { PATH: process.env.PATH, HOME: root, CMUX_TUI_BIN: daemon }, + env: { ...process.env, HOME: root, CMUX_TUI_BIN: daemon }, }); expect(tree.status).toBe(0); expect(JSON.parse(tree.stdout)).toEqual({ session: "cloud", workspaces: [] }); From dc6ce9a00db3d6c2142f85289d3c4d089710ebbb Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sat, 19 Sep 2026 22:38:04 -0700 Subject: [PATCH 11/92] fix: accept native Cloud placement diagnostics --- web/services/observability/cloudTelemetryContract.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/services/observability/cloudTelemetryContract.ts b/web/services/observability/cloudTelemetryContract.ts index 3015d855c17e..28458be63830 100644 --- a/web/services/observability/cloudTelemetryContract.ts +++ b/web/services/observability/cloudTelemetryContract.ts @@ -17,7 +17,7 @@ export const cloudPhases = [ export const cloudFailures = [ "authentication", "session_refresh", "permission", "plan", "rate_limit", "conflict", "network", "timeout", "server", "response", "unsupported", "process", "protocol", - "not_found", "resource_limit", "storage", "cancelled", "unknown", + "not_found", "placement", "resource_limit", "storage", "cancelled", "unknown", ] as const; export type CloudTelemetrySpan = { From 3f318ca11899e06c608869da3dc297e51ae2a3c4 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sat, 19 Sep 2026 22:38:04 -0700 Subject: [PATCH 12/92] test: verify guest installer integration with current browser and timing paths --- web/services/vms/drivers/freestyle.ts | 2 +- web/tests/freestyle-request-timing.test.ts | 17 +++++++++++++++ web/tests/vm-guest-install-runtime.test.ts | 25 ++++++++++++++-------- 3 files changed, 34 insertions(+), 10 deletions(-) diff --git a/web/services/vms/drivers/freestyle.ts b/web/services/vms/drivers/freestyle.ts index 1ab55984fe0b..ad1ca90f4c46 100644 --- a/web/services/vms/drivers/freestyle.ts +++ b/web/services/vms/drivers/freestyle.ts @@ -55,7 +55,7 @@ import { import { recordSpanError, setSpanAttributes, withVmSpan } from "../telemetry"; import { parseSshPublicKey, scpPrepareCommand, SCP_KEY_TTL_SECONDS } from "./scp"; import { GUEST_CMUX_SHIM, GUEST_CMUX_SHIM_PATH } from "../guestCli"; -import { guestBrowserInstallCommand, guestBrowserReadyCommand } from "../guestBrowser"; +import { guestBrowserReadyCommand } from "../guestBrowser"; import { guestPromptInstallCommand, type GuestPromptIdentity } from "../guestPrompt"; import { approveCmuxTuiEnrollment, diff --git a/web/tests/freestyle-request-timing.test.ts b/web/tests/freestyle-request-timing.test.ts index 41e50dc05659..f9e8259e8181 100644 --- a/web/tests/freestyle-request-timing.test.ts +++ b/web/tests/freestyle-request-timing.test.ts @@ -2,6 +2,23 @@ import { describe, expect, test } from "bun:test"; import { freestyleRequestFetch, type FreestyleRequestTiming } from "../services/vms/drivers/freestyleRequestTiming"; describe("Freestyle enrollment request timings", () => { + test.each(["workflow", "init", "request"])("forwards %s cancellation alongside the provider timeout", async (source) => { + const controller = new AbortController(); + const reason = new DOMException("synthetic cancellation", "AbortError"); + const traced = freestyleRequestFetch({ + timeoutMs: 60_000, signal: source === "workflow" ? controller.signal : undefined, + fetch: (async (_input, init) => { + controller.abort(reason); + init!.signal!.throwIfAborted(); + return new Response("must not complete"); + }) as typeof fetch, + }); + const request = new Request("https://example.test/provider", { + signal: source === "request" ? controller.signal : undefined, + }); + await expect(traced(request, source === "init" ? { signal: controller.signal } : undefined)).rejects.toBe(reason); + }); + test("distinguishes the initial request from background completion without logging access material", async () => { const events: FreestyleRequestTiming[] = []; const forwarded: Array<{ input: RequestInfo | URL; init?: RequestInit }> = []; diff --git a/web/tests/vm-guest-install-runtime.test.ts b/web/tests/vm-guest-install-runtime.test.ts index b78b46b66262..a78a07b5bcd3 100644 --- a/web/tests/vm-guest-install-runtime.test.ts +++ b/web/tests/vm-guest-install-runtime.test.ts @@ -4,6 +4,7 @@ import { mkdtempSync, mkdirSync, readFileSync, readdirSync, rmSync, statSync, sy import { tmpdir } from "node:os"; import { join } from "node:path"; import { GUEST_CMUX_SHIM } from "../services/vms/guestCli"; +import { GUEST_BROWSER_FILES } from "../services/vms/guestBrowser"; import { freestyleGuestFixture, guestCreateOptions } from "./fixtures/freestyleGuest"; const roots: string[] = []; @@ -17,10 +18,12 @@ function guest(options: { corruptUpload?: boolean } = {}) { mkdirSync(join(root, "fixture-bin")); writeFileSync(join(root, "fixture-bin/getent"), "#!/bin/sh\nexit 1\n", { mode: 0o755 }); writeFileSync(join(root, "fixture-bin/xdg-mime"), "#!/bin/sh\nexit 0\n", { mode: 0o755 }); - const rebase = (value: string) => value.replaceAll("/usr/local/bin", join(root, "bin")) - .replaceAll("/usr/local/share", join(root, "share")) - .replaceAll("/etc/cmux", join(root, "etc")) - .replaceAll("/etc/", join(root, "system-etc") + "/"); + const prefixes: Record = { + "/usr/local/bin": join(root, "bin"), "/usr/local/share": join(root, "share"), + "/etc/cmux": join(root, "etc"), "/etc": join(root, "system-etc"), + }; + const rebase = (value: string) => value.replace(/\/usr\/local\/bin|\/usr\/local\/share|\/etc\/cmux|\/etc(?=\/)/g, + (prefix) => prefixes[prefix]!); const target = join(root, "bin/cmux"); const fixture = freestyleGuestFixture({ write: (path, bytes) => writeFileSync(rebase(path), options.corruptUpload ? "#!/bin/sh\nexit 0\n" : bytes), @@ -42,6 +45,9 @@ describe("guest CLI publication in an isolated filesystem", () => { const handle = await fixture.provider.create(guestCreateOptions); expect(handle.status).toBe("running"); expect(readFileSync(target, "utf8")).toBe(GUEST_CMUX_SHIM); + for (const file of GUEST_BROWSER_FILES.filter((file) => file.path.startsWith("/usr/local/bin/"))) { + expect(readFileSync(join(root, "bin", file.path.split("/").at(-1)!), "utf8")).toBe(file.content); + } expect(statSync(target).mode & 0o777).toBe(0o755); const result = spawnSync(target, ["--help"], { encoding: "utf8", timeout: 5_000 }); expect(result.status).toBe(0); @@ -56,7 +62,7 @@ describe("guest CLI publication in an isolated filesystem", () => { expect(JSON.parse(tree.stdout)).toEqual({ session: "cloud", workspaces: [] }); expect(readFileSync(join(root, "daemon-args"), "utf8").trim().split("\n")) .toEqual(["--session", "cloud", "--json", "session", "current", "snapshot"]); - expect(readdirSync(join(root, "bin"))).toEqual(["cmux", "cmux-open-url", "sensible-browser", "x-www-browser", "xdg-open"]); + expect(readdirSync(join(root, "bin")).sort()).toEqual(["cmux", "cmux-open-url", "sensible-browser", "x-www-browser", "xdg-open"]); }); test("replaces a target symlink without modifying its referent", async () => { @@ -89,12 +95,13 @@ describe("guest CLI publication in an isolated filesystem", () => { test("prompt failure does not publish a new shim generation", async () => { const { fixture, target, root } = guest(); writeFileSync(target, "previous generation"); - writeFileSync(join(root, "etc"), "not a directory"); - const result = await fixture.provider.create({ + mkdirSync(join(root, "etc")); + mkdirSync(join(root, "etc/.prompt-lock")); + const failure = await fixture.provider.create({ ...guestCreateOptions, promptIdentity: { machineId: "synthetic", name: "synthetic", revision: 1 }, - }).then(() => "ready", () => "failed"); - expect(result).toBe("failed"); + }).then(() => undefined, (error) => error); + expect(failure?.cause?.stage).toBe("prompt"); expect(readFileSync(target, "utf8")).toBe("previous generation"); expect(fixture.liveVms.size).toBe(0); }); From 124e6ec993923a875a3b3fcfbf001c8758cb8bf6 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sat, 19 Sep 2026 22:38:28 -0700 Subject: [PATCH 13/92] integrate: consolidate Cloud authentication deadline fixes Reuse the scoped implementation and regression suites from #12628 at 32155cfaaf. Leave unrelated CI, signing, and Iroh changes on their original branch. --- .github/workflows/auth-refresh-tests.yml | 32 +++ .../AuthClient+ResolvedAccessToken.swift | 9 + .../CmuxAuthRuntime/Client/AuthClient.swift | 6 + .../Client/StackAuthClient.swift | 10 + .../AuthCoordinator+CurrentTokens.swift | 41 ++++ ...oordinator+TokenTouchingPhaseTimeout.swift | 16 +- .../Coordinator/AuthCoordinator+Tokens.swift | 74 ++---- .../Coordinator/AuthPhaseTimeout.swift | 12 +- .../Coordinator/AuthTokenDeadline.swift | 9 + .../Coordinator/Clock+AuthTokenDeadline.swift | 12 + .../AuthCoordinatorCloudTokenTests.swift | 141 +++++++++++ .../Tests/CmuxAuthRuntimeTests/Fakes.swift | 8 +- Sources/Cloud/CloudDiagnosticFailure.swift | 13 +- Sources/Cloud/VMClient.swift | 20 +- Sources/Cloud/VMReflectionResult.swift | 13 + cmux.xcodeproj/project.pbxproj | 4 + cmuxTests/CloudOperationRecorderTests.swift | 22 ++ .../Sources/StackAuth/APIClient.swift | 224 ++++++----------- .../StackAuth/StackClientApp+TokenPair.swift | 10 + .../StackAuth/TokenRefreshAttempt.swift | 22 ++ .../Sources/StackAuth/TokenRefreshClock.swift | 26 ++ .../StackAuth/TokenRefreshCoordinator.swift | 129 ++++++++++ .../StackAuth/TokenRefreshFailure.swift | 9 + .../Sources/StackAuth/TokenStore.swift | 3 + .../StackAuthTests/RefreshDeadlineTests.swift | 128 ++++++++++ .../StackAuthTests/RefreshExchangeGate.swift | 25 ++ .../RefreshFixtureCompletion.swift | 23 ++ .../StackAuthTests/RefreshFixtureRoutes.swift | 12 + .../RefreshFixtureURLProtocol.swift | 15 ++ .../RefreshLifecycleTests.swift | 83 +++++++ .../StackAuthTests/RefreshTestClock.swift | 109 +++++++++ .../RefreshTimerRegistrationGate.swift | 24 ++ .../RefreshTransportFixture.swift | 44 ++++ .../RefreshTransportTimingTests.swift | 32 +++ .../StackAuthTests/TokenRefreshTests.swift | 225 ------------------ 35 files changed, 1127 insertions(+), 458 deletions(-) create mode 100644 .github/workflows/auth-refresh-tests.yml create mode 100644 Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Client/AuthClient+ResolvedAccessToken.swift create mode 100644 Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Coordinator/AuthCoordinator+CurrentTokens.swift create mode 100644 Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Coordinator/AuthTokenDeadline.swift create mode 100644 Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Coordinator/Clock+AuthTokenDeadline.swift create mode 100644 Packages/Shared/CmuxAuthRuntime/Tests/CmuxAuthRuntimeTests/AuthCoordinatorCloudTokenTests.swift create mode 100644 Sources/Cloud/VMReflectionResult.swift create mode 100644 vendor/stack-auth-swift-sdk-prerelease/Sources/StackAuth/StackClientApp+TokenPair.swift create mode 100644 vendor/stack-auth-swift-sdk-prerelease/Sources/StackAuth/TokenRefreshAttempt.swift create mode 100644 vendor/stack-auth-swift-sdk-prerelease/Sources/StackAuth/TokenRefreshClock.swift create mode 100644 vendor/stack-auth-swift-sdk-prerelease/Sources/StackAuth/TokenRefreshCoordinator.swift create mode 100644 vendor/stack-auth-swift-sdk-prerelease/Sources/StackAuth/TokenRefreshFailure.swift create mode 100644 vendor/stack-auth-swift-sdk-prerelease/Tests/StackAuthTests/RefreshDeadlineTests.swift create mode 100644 vendor/stack-auth-swift-sdk-prerelease/Tests/StackAuthTests/RefreshExchangeGate.swift create mode 100644 vendor/stack-auth-swift-sdk-prerelease/Tests/StackAuthTests/RefreshFixtureCompletion.swift create mode 100644 vendor/stack-auth-swift-sdk-prerelease/Tests/StackAuthTests/RefreshFixtureRoutes.swift create mode 100644 vendor/stack-auth-swift-sdk-prerelease/Tests/StackAuthTests/RefreshFixtureURLProtocol.swift create mode 100644 vendor/stack-auth-swift-sdk-prerelease/Tests/StackAuthTests/RefreshLifecycleTests.swift create mode 100644 vendor/stack-auth-swift-sdk-prerelease/Tests/StackAuthTests/RefreshTestClock.swift create mode 100644 vendor/stack-auth-swift-sdk-prerelease/Tests/StackAuthTests/RefreshTimerRegistrationGate.swift create mode 100644 vendor/stack-auth-swift-sdk-prerelease/Tests/StackAuthTests/RefreshTransportFixture.swift create mode 100644 vendor/stack-auth-swift-sdk-prerelease/Tests/StackAuthTests/RefreshTransportTimingTests.swift diff --git a/.github/workflows/auth-refresh-tests.yml b/.github/workflows/auth-refresh-tests.yml new file mode 100644 index 000000000000..ca5c1ca99796 --- /dev/null +++ b/.github/workflows/auth-refresh-tests.yml @@ -0,0 +1,32 @@ +name: Auth and session lifecycle + +on: + pull_request: + paths: + - Packages/Shared/CmuxAuthRuntime/** + - Packages/Shared/CmuxIrxTransport/** + - vendor/stack-auth-swift-sdk-prerelease/** + - .github/workflows/auth-refresh-tests.yml + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: auth-refresh-${{ github.ref }} + cancel-in-progress: true + +jobs: + auth-refresh-tests: + runs-on: ${{ vars.MACOS_RUNNER_26 || 'blacksmith-6vcpu-macos-26' }} + timeout-minutes: 15 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + persist-credentials: false + - name: Exercise shared refresh transport and deadlines without credentials + run: swift test --package-path vendor/stack-auth-swift-sdk-prerelease --filter 'RefreshLifecycleTests|RefreshDeadlineTests|RefreshTransportTimingTests|RateLimitRetryTests' + - name: Exercise coordinator cancellation, sessions, and recovery + run: swift test --package-path Packages/Shared/CmuxAuthRuntime --filter 'AuthCoordinator|AuthError' + - name: Verify retired session watcher completion over loopback + run: swift test --package-path Packages/Shared/CmuxIrxTransport --filter 'IrxLiveQUICTests.keepaliveAndAutoRedial' diff --git a/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Client/AuthClient+ResolvedAccessToken.swift b/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Client/AuthClient+ResolvedAccessToken.swift new file mode 100644 index 000000000000..8a005c456380 --- /dev/null +++ b/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Client/AuthClient+ResolvedAccessToken.swift @@ -0,0 +1,9 @@ +extension AuthClient { + /// Preserves compatibility for clients whose token reads have no typed failure. + public func resolvedAccessToken(forceRefresh: Bool = false) async throws -> String? { + try Task.checkCancellation() + let token = await (forceRefresh ? forceRefreshAccessToken() : accessToken()) + try Task.checkCancellation() + return token + } +} diff --git a/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Client/AuthClient.swift b/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Client/AuthClient.swift index fb20b0a5a4ea..7328e802de44 100644 --- a/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Client/AuthClient.swift +++ b/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Client/AuthClient.swift @@ -11,6 +11,12 @@ public protocol AuthClient: Sendable { /// The current Stack access token, or `nil` when there is no live session. func accessToken() async -> String? + /// Resolves access with typed cancellation/deadline failures when supported. + /// - Parameter forceRefresh: Bypasses the cached token after server rejection. + /// - Returns: A usable access token, or nil when unavailable. + /// - Throws: Cancellation or a classified transient refresh failure. + func resolvedAccessToken(forceRefresh: Bool) async throws -> String? + /// The current Stack refresh token, or `nil` when there is no live session. func refreshToken() async -> String? diff --git a/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Client/StackAuthClient.swift b/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Client/StackAuthClient.swift index f3423f58e2ee..8f7cf5955fde 100644 --- a/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Client/StackAuthClient.swift +++ b/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Client/StackAuthClient.swift @@ -58,6 +58,16 @@ public struct StackAuthClient: AuthClient { await stack.getAccessToken() } + public func resolvedAccessToken(forceRefresh: Bool) async throws -> String? { + let pair = await stack.resolvedTokenPair(forceRefresh: forceRefresh) + try Task.checkCancellation() + switch pair.refreshFailure { + case .cancelled, .sessionChanged: throw CancellationError() + case .timedOut: throw AuthError.timedOut + case nil: return pair.accessToken + } + } + public func refreshToken() async -> String? { await stack.getRefreshToken() } diff --git a/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Coordinator/AuthCoordinator+CurrentTokens.swift b/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Coordinator/AuthCoordinator+CurrentTokens.swift new file mode 100644 index 000000000000..379e2968326e --- /dev/null +++ b/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Coordinator/AuthCoordinator+CurrentTokens.swift @@ -0,0 +1,41 @@ +import Foundation + +extension AuthCoordinator { + /// Resolves a coherent backend credential pair within one cancellable + /// network deadline, including launch restore. Session changes reject the + /// capture; transient refresh failures preserve the existing session. + /// - Returns: The current access and refresh tokens. + /// - Throws: Cancellation on cancellation or session replacement, timedOut + /// at the deadline, networkError on a recoverable failure, or unauthorized + /// when available storage has no recoverable session. + public func currentTokens() async throws -> (accessToken: String, refreshToken: String) { + try Task.checkCancellation() + let deadline = clock.authTokenDeadline(after: timeouts.network) + // Bootstrap owns sign-in, which joins existing token work. Waiting for + // bootstrap is read-only and must stay outside that ownership registry. + try await withAuthPhaseTimeout( + .accessToken, duration: timeouts.network, clock: clock, log: log, + registry: phaseTimeoutRegistry, + blocksRetriesWhileTimedOutOperationActive: false, + deadline: deadline + ) { + await self.awaitBootstrapped() + } + try Task.checkCancellation() + guard !deadline.hasExpired() else { throw AuthError.timedOut } + return try await runTokenTouchingPhase(.accessToken, timeout: deadline.remaining()) { + return try await self.captureCloudTokens() + } + } + + private func captureCloudTokens() async throws -> (accessToken: String, refreshToken: String) { + try Task.checkCancellation() + guard activeSignInFlows.isEmpty, !isCapturingSignOutCredentials else { throw AuthError.networkError } + let generation = sessionGeneration + let pair = try await coherentTokenPairWithoutStateClear() + try Task.checkCancellation() + guard sessionGeneration == generation else { throw CancellationError() } + guard activeSignInFlows.isEmpty, !isCapturingSignOutCredentials else { throw AuthError.networkError } + return pair + } +} diff --git a/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Coordinator/AuthCoordinator+TokenTouchingPhaseTimeout.swift b/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Coordinator/AuthCoordinator+TokenTouchingPhaseTimeout.swift index 4ea239075c34..dc0e55d1e1cc 100644 --- a/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Coordinator/AuthCoordinator+TokenTouchingPhaseTimeout.swift +++ b/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Coordinator/AuthCoordinator+TokenTouchingPhaseTimeout.swift @@ -12,6 +12,8 @@ extension AuthCoordinator { log.log("auth.phase=\(phase.rawValue) previous timed-out token work still active") throw AuthError.timedOut } + try Task.checkCancellation() + let deadline = clock.authTokenDeadline(after: timeout) let phaseID = UUID() let generation = sessionGeneration let signOutEpoch = signOutEpoch @@ -28,6 +30,8 @@ extension AuthCoordinator { signOutEpoch: signOutEpoch, storeWriteHighWater: storeWriteHighWater ) + try Task.checkCancellation() + guard !deadline.hasExpired() else { throw AuthError.timedOut } switch result { case .success(let value): return value @@ -54,7 +58,8 @@ extension AuthCoordinator { phaseTask, id: phaseID, phase: phase, - timeout: timeout + timeout: timeout, + deadline: deadline ) } onCancel: { phaseTask.cancel() @@ -68,7 +73,8 @@ extension AuthCoordinator { _ phaseTask: Task, id: UUID, phase: AuthPhase, - timeout: Duration + timeout: Duration, + deadline: AuthTokenDeadline ) async throws -> T { try Task.checkCancellation() let race = AuthPhaseTimeoutRace() @@ -84,9 +90,9 @@ extension AuthCoordinator { continuation.finish(throwing: error) } } - let deadline = Task { [clock, log] in + let timer = Task { [log] in do { - try await clock.sleep(for: timeout, tolerance: nil) + try await deadline.wait() try Task.checkCancellation() } catch { return @@ -101,7 +107,7 @@ extension AuthCoordinator { } continuation.onTermination = { _ in phaseWaiter.cancel() - deadline.cancel() + timer.cancel() } } diff --git a/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Coordinator/AuthCoordinator+Tokens.swift b/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Coordinator/AuthCoordinator+Tokens.swift index b41dd21f4ac6..ae52e261a82e 100644 --- a/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Coordinator/AuthCoordinator+Tokens.swift +++ b/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Coordinator/AuthCoordinator+Tokens.swift @@ -53,7 +53,7 @@ extension AuthCoordinator { private func accessTokenWithoutStateClear() async throws -> String { let storageWasAvailable = await isTokenStorageAvailable() - if let token = await client.accessToken() { + if let token = try await client.resolvedAccessToken(forceRefresh: false) { return token } #if DEBUG @@ -82,7 +82,7 @@ extension AuthCoordinator { password: credentials.password, setLoading: false ) - return await client.accessToken() + return try? await client.resolvedAccessToken(forceRefresh: false) #else return nil #endif @@ -94,53 +94,14 @@ extension AuthCoordinator { await client.refreshToken() } - /// Both tokens for the current session, for callers that talk to - /// cmux-owned backend endpoints (e.g. the cloud VM service) with the - /// `Authorization: Bearer ` + `X-Stack-Refresh-Token: ` - /// header pair. - /// - /// Awaits the launch restore first: RPCs firing before the restore - /// finishes could otherwise observe an empty token store on a - /// refresh-token-only start and report "Not signed in" even though a valid - /// session becomes available moments later. - /// - Returns: The access and refresh tokens. - /// - Throws: ``AuthError/networkError`` when the access token is missing - /// but a refresh token survives, meaning the refresh failed transiently, - /// or when token storage was unavailable because the device was locked; - /// ``AuthError/unauthorized`` when available storage is missing either an - /// access token with no refresh token to recover from, or the refresh - /// token required by backend requests. - public func currentTokens() async throws -> (accessToken: String, refreshToken: String) { - await awaitBootstrapped() - let storageWasAvailable = await isTokenStorageAvailable() - guard let access = await client.accessToken(), !access.isEmpty else { - if let refresh = await client.refreshToken(), !refresh.isEmpty { - throw AuthError.networkError - } - throw emptyTokenReadError(storageWasAvailable: storageWasAvailable) - } - guard let refresh = await client.refreshToken(), !refresh.isEmpty else { - throw emptyTokenReadError(storageWasAvailable: storageWasAvailable) - } - return (access, refresh) - } - /// Both tokens for the current session as ONE coherent pair, for callers that /// must never send a torn (old-access, new-refresh) credential set. /// - /// ``currentTokens()`` reads the access and refresh tokens through two - /// separate awaits, so a ``forceRefreshAccessToken()`` landing between them - /// can rotate the pair and return an old access token with a rotated refresh - /// token. This instead brackets the read with the refresh token: capture the - /// refresh, resolve the access token through the LIVE store — a still-fresh - /// stored token comes back without the network (an offline caller with a - /// valid stored pair succeeds), and a stale one is refreshed through the - /// SDK's own store, persisted and deduplicated with concurrent refreshes so - /// repeated captures never re-mint — then re-read the refresh token. An - /// unchanged refresh proves no rotation crossed the window, so the resolved - /// access belongs to the returned refresh; a changed one retries against - /// the new capture. The read runs inside the coordinator's bounded - /// token-touching phase like every other token accessor. + /// Shares the coherent capture used by ``currentTokens()``: a fresh token + /// resolves locally, while concurrent expired-token callers share the SDK + /// exchange. Bracketing the access read with the refresh token rejects a + /// rotation crossing the capture. The token-touching phase bounds the whole + /// read and preserves cancellation and timeout classifications. /// - Returns: The access and refresh tokens from one coherent capture. /// - Throws: ``AuthError/networkError`` when the refresh token survives but a /// usable access token cannot be resolved for it (transient), when token @@ -153,7 +114,7 @@ extension AuthCoordinator { } } - private func coherentTokenPairWithoutStateClear() async throws -> (accessToken: String, refreshToken: String) { + func coherentTokenPairWithoutStateClear() async throws -> (accessToken: String, refreshToken: String) { let storageWasAvailable = await isTokenStorageAvailable() for _ in 0..<3 { guard let refresh = await client.refreshToken(), !refresh.isEmpty else { @@ -164,11 +125,13 @@ extension AuthCoordinator { // refreshed through the SDK's own store — persisted, and // deduplicated with any concurrent refresh — so repeated captures // never re-mint a token the store already refreshed. - guard let access = await client.accessToken(), !access.isEmpty else { - // The refresh token survived but no usable access token could - // be resolved: stay retryable, matching currentTokens()'s - // classification of a surviving-refresh access miss. - throw AuthError.networkError + guard let access = try await client.resolvedAccessToken(forceRefresh: false), !access.isEmpty else { + // A definitive SDK rejection clears storage; only a surviving + // refresh token makes an access miss recoverable. + if let surviving = await client.refreshToken(), !surviving.isEmpty { + throw AuthError.networkError + } + throw emptyTokenReadError(storageWasAvailable: storageWasAvailable) } // The bracket: an unchanged refresh across the access resolution // proves no rotation crossed the window, so the pair is coherent. @@ -245,9 +208,8 @@ extension AuthCoordinator { throw AuthError.networkError } let generation = sessionGeneration - // Read both tokens as one coherent pair so a concurrent force refresh - // cannot pair an old access token with a rotated refresh token; the - // separately-read `currentTokens()` cannot make that guarantee. + // Use the same coherent capture as Cloud requests, pinned to this + // identity and session generation. let tokens = try await coherentTokenPair() guard sessionGeneration == generation, currentUser?.id == accountID else { @@ -374,7 +336,7 @@ extension AuthCoordinator { private func forceRefreshAccessTokenWithoutStateClear() async throws -> String { let storageWasAvailable = await isTokenStorageAvailable() - if let token = await client.forceRefreshAccessToken() { + if let token = try await client.resolvedAccessToken(forceRefresh: true) { return token } // A surviving refresh token means the failure was transient diff --git a/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Coordinator/AuthPhaseTimeout.swift b/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Coordinator/AuthPhaseTimeout.swift index fca2ec3c1447..de9ab74f9d2e 100644 --- a/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Coordinator/AuthPhaseTimeout.swift +++ b/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Coordinator/AuthPhaseTimeout.swift @@ -35,6 +35,7 @@ actor AuthPhaseTimeoutRace { /// - duration: The deadline. /// - clock: The clock the deadline sleeps on (virtual in tests). /// - log: Redacted diagnostics sink; timeouts log the phase and duration. +/// - deadline: An existing absolute deadline when this wait shares a budget with later work. /// - operation: The bounded work. /// - Returns: The operation's value when it beats the deadline. /// - Throws: ``AuthError/timedOut`` at the deadline; otherwise rethrows the @@ -46,6 +47,7 @@ func withAuthPhaseTimeout( log: AuthDebugLog, registry: AuthPhaseTimeoutRegistry, blocksRetriesWhileTimedOutOperationActive: Bool, + deadline: AuthTokenDeadline? = nil, operation: @escaping @Sendable () async throws -> T ) async throws -> T { try Task.checkCancellation() @@ -63,6 +65,10 @@ func withAuthPhaseTimeout( let operationTask = Task { do { let value = try await operation() + if let deadline { + try Task.checkCancellation() + guard !deadline.hasExpired() else { throw AuthError.timedOut } + } if blocksRetriesWhileTimedOutOperationActive { await registry.end(phase, id: phaseID) } @@ -79,7 +85,11 @@ func withAuthPhaseTimeout( } let deadlineTask = Task { do { - try await clock.sleep(for: duration, tolerance: nil) + if let deadline { + try await deadline.wait() + } else { + try await clock.sleep(for: duration, tolerance: nil) + } try Task.checkCancellation() } catch { return diff --git a/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Coordinator/AuthTokenDeadline.swift b/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Coordinator/AuthTokenDeadline.swift new file mode 100644 index 000000000000..afbefb51a7ab --- /dev/null +++ b/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Coordinator/AuthTokenDeadline.swift @@ -0,0 +1,9 @@ +import Foundation + +/// Captures an absolute deadline so a response scheduled ahead of the timer on +/// wake cannot publish credentials after its budget elapsed. +struct AuthTokenDeadline: Sendable { + let hasExpired: @Sendable () -> Bool + let remaining: @Sendable () -> Duration + let wait: @Sendable () async throws -> Void +} diff --git a/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Coordinator/Clock+AuthTokenDeadline.swift b/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Coordinator/Clock+AuthTokenDeadline.swift new file mode 100644 index 000000000000..0fbbb0d2c79d --- /dev/null +++ b/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Coordinator/Clock+AuthTokenDeadline.swift @@ -0,0 +1,12 @@ +import Foundation + +extension Clock where Duration == Swift.Duration { + func authTokenDeadline(after duration: Duration) -> AuthTokenDeadline { + let end = now.advanced(by: duration) + return AuthTokenDeadline( + hasExpired: { self.now >= end }, + remaining: { max(.zero, self.now.duration(to: end)) }, + wait: { try await self.sleep(until: end, tolerance: nil) } + ) + } +} diff --git a/Packages/Shared/CmuxAuthRuntime/Tests/CmuxAuthRuntimeTests/AuthCoordinatorCloudTokenTests.swift b/Packages/Shared/CmuxAuthRuntime/Tests/CmuxAuthRuntimeTests/AuthCoordinatorCloudTokenTests.swift new file mode 100644 index 000000000000..80ae6e535069 --- /dev/null +++ b/Packages/Shared/CmuxAuthRuntime/Tests/CmuxAuthRuntimeTests/AuthCoordinatorCloudTokenTests.swift @@ -0,0 +1,141 @@ +import CMUXAuthCore +import Foundation +import Testing +@testable import CmuxAuthRuntime + +@MainActor +@Suite struct AuthCoordinatorCloudTokenTests { + @Test func cloudRequestDuringBootstrapAllowsPersonalAutoLogin() async throws { + let user = CMUXAuthUser(id: "personal", primaryEmail: "person@example.invalid", displayName: nil) + let client = FakeAuthClient(user: user, signInRefreshToken: "personal-refresh") + let coordinator = makeCoordinator(client: client, launch: AuthLaunchOptions( + clearAuthRequested: false, + mockDataEnabled: false, + environment: [ + "CMUX_UITEST_STACK_EMAIL": "person@example.invalid", + "CMUX_UITEST_STACK_PASSWORD": "synthetic" + ], + includesDevAuth: true, + replaceStoredSessionWithAutoLogin: true + )) + + // Cloud starts synchronously after auth.start() on the launch actor. + // Its bootstrap wait must not become token work that sign-in joins. + coordinator.start() + let tokens = try await coordinator.currentTokens() + + #expect(coordinator.currentUser == user) + #expect(await client.signedInWithCredential?.email == "person@example.invalid") + #expect(!tokens.accessToken.isEmpty) + #expect(!tokens.refreshToken.isEmpty) + } + + @Test func definitivelyRejectedRefreshIsUnauthorized() async { + let client = FakeAuthClient(access: "expired", refresh: "rejected") + await client.setRejectsRefreshOnAccess(true) + let coordinator = makeCoordinator(client: client) + await #expect(throws: AuthError.unauthorized) { try await coordinator.currentTokens() } + } + + @Test(arguments: [false, true]) + func cloudBootstrapWaitEndsWithoutCancellingStartup(cancel: Bool) async throws { + let entered = TestPhaseSignal() + let release = TestPhaseSignal() + let clock = ManualTestClock() + let coordinator = makeCoordinator( + client: FakeAuthClient(), timeout: .seconds(2), clock: clock, + isTokenStorageAvailable: { + await entered.markStarted() + await release.waitUntilStarted() + return true + } + ) + coordinator.start() + let caller = Task { try await coordinator.currentTokens() } + await entered.waitUntilStarted() + await clock.waitUntilSleepers() + if cancel { + caller.cancel() + await #expect(throws: CancellationError.self) { try await caller.value } + } else { + clock.advance(by: .seconds(2)) + await #expect(throws: AuthError.timedOut) { try await caller.value } + } + await release.markStarted() + await coordinator.awaitBootstrapped() + #expect(!coordinator.isAuthenticated) + } + + @Test func cancelledCloudCallerDoesNotReturnCredentials() async throws { + let client = FakeAuthClient(access: "access", refresh: "refresh") + let coordinator = makeCoordinator(client: client) + let gate = TestPhaseSignal() + let caller = Task { + await gate.waitUntilStarted() + return try await coordinator.currentTokens() + } + caller.cancel() + await gate.markStarted() + await #expect(throws: CancellationError.self) { try await caller.value } + } + + @Test func cloudCallerCancellationDetachesFromStalledTokenWork() async throws { + let client = HangingLaunchTokenProbeAuthClient( + user: CMUXAuthUser(id: "fixture", primaryEmail: nil, displayName: nil) + ) + let coordinator = makeCoordinator(client: client) + let caller = Task { try await coordinator.currentTokens() } + await client.accessTokenDidStart() + caller.cancel() + await #expect(throws: CancellationError.self) { try await caller.value } + await client.releaseHangingAccessTokenProbe() + } + + @Test func cloudCallerDeadlinePreservesRecoverableSession() async throws { + let client = HangingLaunchTokenProbeAuthClient( + user: CMUXAuthUser(id: "fixture", primaryEmail: nil, displayName: nil) + ) + let clock = ManualTestClock() + let coordinator = makeCoordinator(client: client, timeout: .seconds(2), clock: clock) + let caller = Task { try await coordinator.currentTokens() } + await client.accessTokenDidStart() + await clock.waitUntilSleepers() + clock.advance(by: .seconds(2)) + await #expect(throws: AuthError.timedOut) { try await caller.value } + await client.releaseHangingAccessTokenProbe() + #expect(await client.refreshToken() == "refresh") + } + + @Test func cloudReaderDoesNotCaptureASignInOwnedTokenStore() async throws { + let client = GateableValidationAuthClient(user: CMUXAuthUser(id: "fixture", primaryEmail: nil, displayName: nil)) + let coordinator = makeCoordinator(client: client) + try await coordinator.signInWithPassword(email: "fixture@example.invalid", password: "synthetic") + await client.armCredentialGate() + let replacement = Task { try await coordinator.signInWithPassword(email: "replacement@example.invalid", password: "synthetic") } + await client.credentialDidPark() + await #expect(throws: AuthError.networkError) { try await coordinator.currentTokens() } + await client.releaseParkedCredential() + try await replacement.value + } + + @Test func deadlineDoesNotRestartWhenItsWaitBeginsAfterSuspension() async throws { + let clock = ManualTestClock() + let deadline = clock.authTokenDeadline(after: .seconds(2)) + clock.advance(by: .seconds(600)) + try await deadline.wait() + #expect(deadline.hasExpired()) + } + + private func makeCoordinator(client: any AuthClient, timeout: Duration = .seconds(1), clock: any Clock = ContinuousClock(), launch: AuthLaunchOptions = .plain(), isTokenStorageAvailable: @escaping @Sendable () async -> Bool = { true }) -> AuthCoordinator { + let store = FakeKeyValueStore() + return AuthCoordinator( + client: client, + sessionCache: CMUXAuthSessionCache(keyValueStore: store, key: "tokens"), + userCache: CMUXAuthIdentityStore(keyValueStore: store, key: "user"), + teamSelection: CMUXAuthTeamSelectionStore(keyValueStore: store, key: "team"), + anchor: FakeAnchor(), config: .test, launch: launch, + timeouts: AuthTimeouts(interactiveFlow: .seconds(1), network: timeout), clock: clock, + isTokenStorageAvailable: isTokenStorageAvailable + ) + } +} diff --git a/Packages/Shared/CmuxAuthRuntime/Tests/CmuxAuthRuntimeTests/Fakes.swift b/Packages/Shared/CmuxAuthRuntime/Tests/CmuxAuthRuntimeTests/Fakes.swift index a4ab23842258..f96f3ae06519 100644 --- a/Packages/Shared/CmuxAuthRuntime/Tests/CmuxAuthRuntimeTests/Fakes.swift +++ b/Packages/Shared/CmuxAuthRuntime/Tests/CmuxAuthRuntimeTests/Fakes.swift @@ -17,8 +17,10 @@ final class FakeKeyValueStore: CMUXAuthKeyValueStore, @unchecked Sendable { /// Scriptable ``AuthClient`` recording calls and returning canned results. actor FakeAuthClient: AuthClient { + var rejectsRefreshOnAccess = false var access: String? var refresh: String? + private let signInRefreshToken: String? /// Result of ``forceRefreshAccessToken()``. When `nil` (the default), the /// fake returns the current ``access`` to preserve the original behavior; /// set it explicitly to script a force-refresh outcome independent of the @@ -46,12 +48,14 @@ actor FakeAuthClient: AuthClient { var mintedAccessToken: String? private(set) var lastMintedRefreshToken: String? - init(access: String? = nil, refresh: String? = nil, user: CMUXAuthUser? = nil) { + init(access: String? = nil, refresh: String? = nil, user: CMUXAuthUser? = nil, signInRefreshToken: String? = nil) { self.access = access self.refresh = refresh self.user = user + self.signInRefreshToken = signInRefreshToken } + func setRejectsRefreshOnAccess(_ value: Bool) { rejectsRefreshOnAccess = value } func setUser(_ user: CMUXAuthUser?) { self.user = user } func setTokens(access: String?, refresh: String?) { self.access = access @@ -70,6 +74,7 @@ actor FakeAuthClient: AuthClient { /// recorded in ``lastMintedRefreshToken``, and PERSISTED into the store so /// a repeat read reuses it instead of re-minting. func accessToken() async -> String? { + if rejectsRefreshOnAccess { access = nil; refresh = nil; return nil } if storedAccessIsStale, let refresh { mintedAccessTokenCount += 1 lastMintedRefreshToken = refresh @@ -126,6 +131,7 @@ actor FakeAuthClient: AuthClient { func signInWithCredential(email: String, password: String) async throws { signedInWithCredential = (email, password) access = "access" + if let signInRefreshToken { refresh = signInRefreshToken } } func signInWithOAuth(provider: String, anchor: any AuthPresentationAnchoring) async throws { diff --git a/Sources/Cloud/CloudDiagnosticFailure.swift b/Sources/Cloud/CloudDiagnosticFailure.swift index e03ee10606af..b392b5ab9454 100644 --- a/Sources/Cloud/CloudDiagnosticFailure.swift +++ b/Sources/Cloud/CloudDiagnosticFailure.swift @@ -1,3 +1,4 @@ +import CmuxAuthRuntime import Foundation enum CloudDiagnosticFailure: String, Codable, Sendable, Error { @@ -8,7 +9,7 @@ enum CloudDiagnosticFailure: String, Codable, Sendable, Error { var label: String { switch self { - case .authentication, .sessionRefresh: + case .authentication: return String(localized: "cloud.operation.failure.auth", defaultValue: "Cloud could not verify your session. Sign in again.") case .permission: return String(localized: "cloud.operation.failure.permission", defaultValue: "Cloud access was denied. Check your permissions.") @@ -16,7 +17,7 @@ enum CloudDiagnosticFailure: String, Codable, Sendable, Error { return String(localized: "cloud.operation.failure.plan", defaultValue: "Your plan does not allow this Cloud operation.") case .rateLimit: return String(localized: "cloud.operation.failure.rateLimit", defaultValue: "Cloud received too many requests. Wait before you retry.") - case .network, .timeout: + case .network, .timeout, .sessionRefresh: return String(localized: "cloud.operation.failure.network", defaultValue: "The Cloud connection did not complete. Check your connection and try again.") case .conflict: return String(localized: "cloud.operation.failure.conflict", defaultValue: "Another operation changed this machine. Refresh its state.") @@ -36,6 +37,14 @@ enum CloudDiagnosticFailure: String, Codable, Sendable, Error { static func classify(_ error: Error) -> Self { if let failure = error as? Self { return failure } if error is CancellationError { return .cancelled } + if let error = error as? AuthError { + switch error { + case .cancelled: return .cancelled + case .timedOut: return .timeout + case .offline, .networkError, .serverError: return .sessionRefresh + default: return .authentication + } + } if let error = error as? URLError { if error.code == .cancelled { return .cancelled } return error.code == .timedOut ? .timeout : .network diff --git a/Sources/Cloud/VMClient.swift b/Sources/Cloud/VMClient.swift index 457cf3e0049f..1928580b838c 100644 --- a/Sources/Cloud/VMClient.swift +++ b/Sources/Cloud/VMClient.swift @@ -552,18 +552,6 @@ struct VMPublicationDomain: Equatable, Sendable { } -/// One reflection read (`GET /api/vm//reflection[/]`): the HTTP status and the -/// JSON body as sent. A 404 with `{error: "not_found", paths: […]}` is a normal result -/// (an unknown reflection path), so the CLI can print the paths that do exist. -struct VMReflectionResult: Sendable { - let statusCode: Int - let body: Data - - var object: [String: Any] { - ((try? JSONSerialization.jsonObject(with: body, options: [])) as? [String: Any]) ?? [:] - } -} - /// One row of `GET /api/vm//snapshots`: the provider snapshot id, its display name /// when one was given, and the creation time as the ISO-8601 string the server sent. struct VMSnapshotSummary: Sendable, Equatable { @@ -2201,7 +2189,9 @@ actor VMClient { let tokens: (accessToken: String, refreshToken: String) do { tokens = try await CloudOperationContext.phase(.authentication) { try await auth.currentTokens() } - } catch AuthError.networkError { + } catch is CancellationError { + throw CancellationError() + } catch AuthError.networkError, AuthError.timedOut { throw VMClientError.sessionRefreshFailed } catch { throw VMClientError.notSignedIn @@ -2725,7 +2715,9 @@ actor MachineUsageClient { let tokens: (accessToken: String, refreshToken: String) do { tokens = try await CloudOperationContext.phase(.authentication) { try await auth.currentTokens() } - } catch AuthError.networkError { + } catch is CancellationError { + throw CancellationError() + } catch AuthError.networkError, AuthError.timedOut { throw MachineUsageClientError.sessionRefreshFailed } catch { throw MachineUsageClientError.notSignedIn diff --git a/Sources/Cloud/VMReflectionResult.swift b/Sources/Cloud/VMReflectionResult.swift new file mode 100644 index 000000000000..1bca6e9da48d --- /dev/null +++ b/Sources/Cloud/VMReflectionResult.swift @@ -0,0 +1,13 @@ +import Foundation + +/// One reflection read (`GET /api/vm//reflection[/]`): the HTTP status and the +/// JSON body as sent. A 404 with `{error: "not_found", paths: […]}` is a normal result +/// (an unknown reflection path), so the CLI can print the paths that do exist. +struct VMReflectionResult: Sendable { + let statusCode: Int + let body: Data + + var object: [String: Any] { + ((try? JSONSerialization.jsonObject(with: body, options: [])) as? [String: Any]) ?? [:] + } +} diff --git a/cmux.xcodeproj/project.pbxproj b/cmux.xcodeproj/project.pbxproj index 8684766b85e0..f38abc81e15b 100644 --- a/cmux.xcodeproj/project.pbxproj +++ b/cmux.xcodeproj/project.pbxproj @@ -3687,6 +3687,7 @@ F1C699902D31503EE3D492DF /* VMMachineKind.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6EECEC1A2D3CFC13D4343F46 /* VMMachineKind.swift */; }; B81607322100000000000002 /* VMMachineTerminalResolution.swift in Sources */ = {isa = PBXBuildFile; fileRef = B81607322100000000000004 /* VMMachineTerminalResolution.swift */; }; B81607322100000000000003 /* VMMachineTerminalResolution.swift in Sources */ = {isa = PBXBuildFile; fileRef = B81607322100000000000004 /* VMMachineTerminalResolution.swift */; }; + CADD34A6A84246019F1EB5DA /* VMReflectionResult.swift in Sources */ = {isa = PBXBuildFile; fileRef = 01DB85E9217B428E8E41F463 /* VMReflectionResult.swift */; }; 65F8FEC592CFA22BD1F4FA29 /* VMRemoteWorkspaceResolver.swift in Sources */ = {isa = PBXBuildFile; fileRef = BA2691D26BF6676FF32FBC54 /* VMRemoteWorkspaceResolver.swift */; }; D017093861349E48BF816E02 /* VMRemoteWorkspaceResolver.swift in Sources */ = {isa = PBXBuildFile; fileRef = BA2691D26BF6676FF32FBC54 /* VMRemoteWorkspaceResolver.swift */; }; 142CC5D4A966D7EF32E2889F /* VMResourceStatsStore+Entry.swift in Sources */ = {isa = PBXBuildFile; fileRef = 69879F55D8ABC32BD1538457 /* VMResourceStatsStore+Entry.swift */; }; @@ -7606,6 +7607,7 @@ C37800000000000000000004 /* VMDefaultCloudCommandTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VMDefaultCloudCommandTests.swift; sourceTree = ""; }; 6EECEC1A2D3CFC13D4343F46 /* VMMachineKind.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = VMMachineKind.swift; sourceTree = ""; }; B81607322100000000000004 /* VMMachineTerminalResolution.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VMMachineTerminalResolution.swift; sourceTree = ""; }; + 01DB85E9217B428E8E41F463 /* VMReflectionResult.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VMReflectionResult.swift; sourceTree = ""; }; BA2691D26BF6676FF32FBC54 /* VMRemoteWorkspaceResolver.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VMRemoteWorkspaceResolver.swift; sourceTree = ""; }; 69879F55D8ABC32BD1538457 /* VMResourceStatsStore+Entry.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "VMResourceStatsStore+Entry.swift"; sourceTree = ""; }; E87AC4F0A7C5E79CCAA7AE47 /* VMResourceStatsStore+Request.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "VMResourceStatsStore+Request.swift"; sourceTree = ""; }; @@ -8487,6 +8489,7 @@ FA1253800000000000000006 /* CloudTreeRowIcon.swift */, 98F890512F8E425FAD71FE7C /* VMClientError.swift */, B12438030000000000000002 /* VMSnapshotResult.swift */, + 01DB85E9217B428E8E41F463 /* VMReflectionResult.swift */, 831145877779546B628FCAFC /* VMClientTelemetry.swift */, 64BF8B9B8328108560B40B7F /* CloudRemoteOperationStep.swift */, 3EBCFB7E8CD8163F74525C53 /* CloudTelemetrySending.swift */, @@ -14937,6 +14940,7 @@ ACE4FE2A8F1CC6C0B6A673AB /* VMClientSocketCommands.swift in Sources */, 93AF7E8EC956FE883E252322 /* VMClientTelemetry.swift in Sources */, F1C699902D31503EE3D492DF /* VMMachineKind.swift in Sources */, + CADD34A6A84246019F1EB5DA /* VMReflectionResult.swift in Sources */, 142CC5D4A966D7EF32E2889F /* VMResourceStatsStore+Entry.swift in Sources */, 141CC77C279D12D2CBAC100D /* VMResourceStatsStore+Request.swift in Sources */, F8F833CD21A74AE6B74391D4 /* VMResourceStatsStore+RetentionToken.swift in Sources */, diff --git a/cmuxTests/CloudOperationRecorderTests.swift b/cmuxTests/CloudOperationRecorderTests.swift index 0e186844ebf0..27b5b70d0379 100644 --- a/cmuxTests/CloudOperationRecorderTests.swift +++ b/cmuxTests/CloudOperationRecorderTests.swift @@ -8,6 +8,28 @@ import Testing @MainActor struct CloudOperationRecorderTests { + @Test func authenticationFailuresKeepTheirTelemetryClassification() async throws { + let identity = AuthenticatedSessionIdentity(generation: 1, accountID: "synthetic") + let sink = CapturedCloudDiagnostics() + let recorder = CloudOperationRecorder(uploader: sink, identity: { identity }) + let scenarios: [(AuthError, CloudDiagnosticFailure, CloudTelemetrySpan.Outcome)] = [ + (.timedOut, .timeout, .timeout), + (.networkError, .sessionRefresh, .failure), + (.cancelled, .cancelled, .cancelled), + (.unauthorized, .authentication, .failure) + ] + for (error, failure, outcome) in scenarios { + let root = recorder.begin(.list) + let auth = recorder.beginChild(of: root, phase: .authentication, attempt: 0, file: #fileID, line: #line) + await recorder.finish(auth, error: error) + let span = try #require(await sink.spans.last) + #expect(span.failure == failure) + #expect(span.outcome == outcome) + await recorder.finish(root) + } + #expect(CloudDiagnosticFailure.sessionRefresh.label == CloudDiagnosticFailure.network.label) + } + @Test func alertLabelsAndScrollableDetailsOfferCopyError() throws { for text in ["Cloud could not connect.", String(repeating: "Cloud connection failed\n", count: 100)] { let alert = NSAlert() diff --git a/vendor/stack-auth-swift-sdk-prerelease/Sources/StackAuth/APIClient.swift b/vendor/stack-auth-swift-sdk-prerelease/Sources/StackAuth/APIClient.swift index c4f24eb1f3b7..e4eeab2c6f43 100644 --- a/vendor/stack-auth-swift-sdk-prerelease/Sources/StackAuth/APIClient.swift +++ b/vendor/stack-auth-swift-sdk-prerelease/Sources/StackAuth/APIClient.swift @@ -87,45 +87,12 @@ func isTokenFreshEnough(_ accessToken: String?) -> Bool { return expiresInMoreThan20s && issuedLessThan75sAgo } -// MARK: - Refresh Lock Manager - -/// Manages per-token-store refresh locks to ensure only one refresh per store at a time. -/// Uses ObjectIdentifier to key locks since token stores no longer have an id property. -actor RefreshLockManager { - static let shared = RefreshLockManager() - - private var activeLocks: [ObjectIdentifier: Bool] = [:] - private var waiters: [ObjectIdentifier: [CheckedContinuation]] = [:] - - func acquireLock(for store: any TokenStoreProtocol) async { - let key = ObjectIdentifier(store) - // Use WHILE loop to re-check condition after waking up. - // Multiple waiters may be resumed at once, but only one should acquire the lock. - while activeLocks[key] == true { - // Wait for existing refresh to complete - await withCheckedContinuation { continuation in - waiters[key, default: []].append(continuation) - } - } - activeLocks[key] = true - } - - func releaseLock(for store: any TokenStoreProtocol) { - let key = ObjectIdentifier(store) - activeLocks[key] = false - if let storeWaiters = waiters[key] { - for waiter in storeWaiters { - waiter.resume() - } - waiters[key] = nil - } - } -} - /// Result of getOrFetchLikelyValidTokens public struct TokenPair: Sendable { public let refreshToken: String? public let accessToken: String? + /// Why refresh could not safely return credentials, if applicable. + public var refreshFailure: TokenRefreshFailure? = nil } /// Internal API client for making HTTP requests to Stack Auth @@ -136,6 +103,9 @@ actor APIClient { let secretServerKey: String? private let tokenStore: any TokenStoreProtocol private let session: URLSession + private let refreshCoordinator: TokenRefreshCoordinator + let refreshClock: TokenRefreshClock + let refreshTimeoutNanoseconds: UInt64 private static let sdkVersion = "1.0.0" @@ -145,7 +115,10 @@ actor APIClient { publishableClientKey: String, secretServerKey: String? = nil, tokenStore: any TokenStoreProtocol, - session: URLSession = .shared + session: URLSession = .shared, + refreshCoordinator: TokenRefreshCoordinator = TokenStoreRegistry.shared.refreshCoordinator, + refreshClock: TokenRefreshClock = .system, + refreshTimeoutNanoseconds: UInt64 = 30_000_000_000 ) { self.baseUrl = baseUrl.hasSuffix("/") ? String(baseUrl.dropLast()) : baseUrl self.projectId = projectId @@ -153,6 +126,9 @@ actor APIClient { self.secretServerKey = secretServerKey self.tokenStore = tokenStore self.session = session + self.refreshCoordinator = refreshCoordinator + self.refreshClock = refreshClock + self.refreshTimeoutNanoseconds = refreshTimeoutNanoseconds } // MARK: - Request Methods @@ -390,18 +366,22 @@ actor APIClient { func setTokens(accessToken: String?, refreshToken: String?) async { await tokenStore.setTokens(accessToken: accessToken, refreshToken: refreshToken) + await refreshCoordinator.tokensDidChange(store: tokenStore, refreshToken: refreshToken) } func setTokens(accessToken: String?, refreshToken: String?, tokenStoreOverride: any TokenStoreProtocol) async { await tokenStoreOverride.setTokens(accessToken: accessToken, refreshToken: refreshToken) + await refreshCoordinator.tokensDidChange(store: tokenStoreOverride, refreshToken: refreshToken) } func clearTokens() async { await tokenStore.clearTokens() + await refreshCoordinator.tokensDidChange(store: tokenStore, refreshToken: nil) } func clearTokens(tokenStoreOverride: any TokenStoreProtocol) async { await tokenStoreOverride.clearTokens() + await refreshCoordinator.tokensDidChange(store: tokenStoreOverride, refreshToken: nil) } /// Compare-and-clear: clears the stored tokens only while the stored @@ -413,8 +393,11 @@ actor APIClient { newRefreshToken: nil, newAccessToken: nil ) + await refreshCoordinator.tokensDidChange( + store: tokenStore, refreshToken: await tokenStore.getStoredRefreshToken() + ) } - + /// Gets tokens, refreshing if needed. See spec for algorithm. /// This is the main function to use for getting an access token. func getOrFetchLikelyValidTokens() async -> TokenPair { @@ -427,138 +410,71 @@ actor APIClient { /// Internal implementation of getOrFetchLikelyValidTokens algorithm. private func getOrFetchLikelyValidTokensFromStore(_ ts: any TokenStoreProtocol) async -> TokenPair { - // Acquire lock to ensure only one refresh per token store - await RefreshLockManager.shared.acquireLock(for: ts) - - let originalRefreshToken = await ts.getStoredRefreshToken() - let originalAccessToken = await ts.getStoredAccessToken() - - let result: TokenPair - - // Case 1: No refresh token - if originalRefreshToken == nil { - // If access token expires in > 0 seconds, return it - if let token = originalAccessToken, !isTokenExpired(token) { - result = TokenPair(refreshToken: nil, accessToken: token) - } else { - // Access token is expired or nil - result = TokenPair(refreshToken: nil, accessToken: nil) - } - } else { - // Case 2: Refresh token exists - let refreshToken = originalRefreshToken! - - // Check if token is fresh enough (expires in > 20s AND issued < 75s ago) - if isTokenFreshEnough(originalAccessToken) { - result = TokenPair(refreshToken: refreshToken, accessToken: originalAccessToken) - } else { - // Need to refresh - switch await refresh(refreshToken: refreshToken) { - case .success(let newToken): - // Refresh succeeded - update tokens atomically - await ts.compareAndSet( - compareRefreshToken: refreshToken, - newRefreshToken: refreshToken, - newAccessToken: newToken - ) - result = TokenPair(refreshToken: refreshToken, accessToken: newToken) - case .definitivelyRejected: - // The server rejected the refresh token; it will never work - // again. Clear both tokens so the user is prompted to re-auth. - await ts.compareAndSet( - compareRefreshToken: refreshToken, - newRefreshToken: nil, - newAccessToken: nil - ) - result = TokenPair(refreshToken: nil, accessToken: nil) - case .transientFailure: - // Network/server hiccup. PRESERVE the refresh token so a retry - // after recovery succeeds, and never silently sign the user out. - // Return the original access token only if it is still usable; - // a proactive refresh fires every 75s of token age (see - // `isTokenFreshEnough`) while the token is valid for ~1h, so the - // common transient-failure case still has a good token. When the - // token is genuinely expired, return nil access + non-nil refresh - // so the caller can classify "recoverable" without decoding a JWT. - let usableAccessToken = isTokenExpired(originalAccessToken) ? nil : originalAccessToken - result = TokenPair(refreshToken: refreshToken, accessToken: usableAccessToken) - } - } + guard !Task.isCancelled else { + return TokenPair(refreshToken: nil, accessToken: nil, refreshFailure: .cancelled) } - - // Release lock synchronously before returning - await RefreshLockManager.shared.releaseLock(for: ts) - return result + let refresh = await ts.getStoredRefreshToken() + let access = await ts.getStoredAccessToken() + guard let refresh else { + return TokenPair(refreshToken: nil, accessToken: isTokenExpired(access) ? nil : access) + } + guard await ts.getStoredRefreshToken() == refresh else { + return TokenPair(refreshToken: nil, accessToken: nil, refreshFailure: .sessionChanged) + } + if isTokenFreshEnough(access) { + return TokenPair(refreshToken: refresh, accessToken: access) + } + return await resolveRefresh(store: ts, refresh: refresh, observedAccess: access, allowFallback: true) } - + /// Forcefully fetches a new access token from the server if possible. func fetchNewAccessToken() async -> TokenPair { - return await fetchNewAccessToken(tokenStore: tokenStore) + await fetchNewAccessToken(tokenStore: tokenStore) } - + func fetchNewAccessToken(tokenStoreOverride: any TokenStoreProtocol) async -> TokenPair { - return await fetchNewAccessToken(tokenStore: tokenStoreOverride) + await fetchNewAccessToken(tokenStore: tokenStoreOverride) } - + private func fetchNewAccessToken( tokenStore ts: any TokenStoreProtocol, ifAccessTokenUnchangedFrom failedAccessToken: String? = nil ) async -> TokenPair { - // Acquire lock to ensure only one refresh per token store - await RefreshLockManager.shared.acquireLock(for: ts) - - let result: TokenPair - - let currentAccessToken = await ts.getStoredAccessToken() - let currentRefreshToken = await ts.getStoredRefreshToken() + let refresh = await ts.getStoredRefreshToken() + let access = await ts.getStoredAccessToken() + guard let refresh else { return TokenPair(refreshToken: nil, accessToken: nil) } + if let failedAccessToken, let access, access != failedAccessToken, !isTokenExpired(access) { + return TokenPair(refreshToken: refresh, accessToken: access) + } + // A forced refresh must not fall back to the token the server rejected. + return await resolveRefresh(store: ts, refresh: refresh, observedAccess: access, allowFallback: false) + } - // Several requests can receive the same expired access token before - // the first refresh finishes. Once the lock opens, reuse that newer - // access token instead of exchanging the same refresh token again. - if let failedAccessToken, - let currentAccessToken, - currentAccessToken != failedAccessToken, - !isTokenExpired(currentAccessToken) { - result = TokenPair( - refreshToken: currentRefreshToken, - accessToken: currentAccessToken - ) - } else if let refreshToken = currentRefreshToken { - switch await refresh(refreshToken: refreshToken) { - case .success(let newToken): - await ts.compareAndSet( - compareRefreshToken: refreshToken, - newRefreshToken: refreshToken, - newAccessToken: newToken - ) - result = TokenPair(refreshToken: refreshToken, accessToken: newToken) - case .definitivelyRejected: - await ts.compareAndSet( - compareRefreshToken: refreshToken, - newRefreshToken: nil, - newAccessToken: nil - ) - result = TokenPair(refreshToken: nil, accessToken: nil) - case .transientFailure: - // Preserve the refresh token on a network/server hiccup; the caller - // retries later instead of being signed out (compareAndSet is NOT - // called, so the store keeps both tokens). Return nil access - // deliberately: this "force a NEW token" path runs only after the - // stored access token was just rejected (401 invalid_access_token in - // sendWithRetry), so handing it back would make the caller re-send the - // same dead token in a tight 401 loop. nil means "no new token right - // now"; the still-valid refresh token stays in the store for next time. - result = TokenPair(refreshToken: refreshToken, accessToken: nil) - } - } else { - result = TokenPair(refreshToken: nil, accessToken: nil) + private func resolveRefresh(store: any TokenStoreProtocol, refresh: String, observedAccess: String?, allowFallback: Bool) async -> TokenPair { + let result = await refreshCoordinator.resolve( + store: store, refreshToken: refresh, accessToken: observedAccess, + clock: refreshClock, timeoutNanoseconds: refreshTimeoutNanoseconds + ) { await self.refresh(refreshToken: refresh) } + let storedRefresh = await store.getStoredRefreshToken() + guard !Task.isCancelled else { + return TokenPair(refreshToken: nil, accessToken: nil, refreshFailure: .cancelled) + } + // A proactive refresh timeout must retain the same offline fallback as + // a transport failure. Forced refresh never reuses a rejected token. + if result.refreshFailure == .timedOut, allowFallback, + storedRefresh == refresh, !isTokenExpired(observedAccess) { + return TokenPair(refreshToken: refresh, accessToken: observedAccess) + } + guard storedRefresh == result.refreshToken else { + return TokenPair(refreshToken: nil, accessToken: nil, refreshFailure: result.refreshFailure ?? .sessionChanged) + } + if result.refreshFailure == nil, result.refreshToken != nil, + result.accessToken == nil, allowFallback, !isTokenExpired(observedAccess) { + return TokenPair(refreshToken: refresh, accessToken: observedAccess) } - - // Release lock synchronously before returning - await RefreshLockManager.shared.releaseLock(for: ts) return result } - + /// Get access token, refreshing if needed. Convenience wrapper around getOrFetchLikelyValidTokens. func getAccessToken() async -> String? { let tokens = await getOrFetchLikelyValidTokens() diff --git a/vendor/stack-auth-swift-sdk-prerelease/Sources/StackAuth/StackClientApp+TokenPair.swift b/vendor/stack-auth-swift-sdk-prerelease/Sources/StackAuth/StackClientApp+TokenPair.swift new file mode 100644 index 000000000000..a2a1a4c53816 --- /dev/null +++ b/vendor/stack-auth-swift-sdk-prerelease/Sources/StackAuth/StackClientApp+TokenPair.swift @@ -0,0 +1,10 @@ +extension StackClientApp { + /// Resolves both credentials through the shared refresh owner, retaining + /// cancellation, deadline, and session-replacement failures for callers. + /// - Parameter forceRefresh: Bypasses cached-token freshness after rejection. + /// - Returns: A coherent token pair or a classified refresh failure. + public func resolvedTokenPair(forceRefresh: Bool = false) async -> TokenPair { + if forceRefresh { return await client.fetchNewAccessToken() } + return await client.getOrFetchLikelyValidTokens() + } +} diff --git a/vendor/stack-auth-swift-sdk-prerelease/Sources/StackAuth/TokenRefreshAttempt.swift b/vendor/stack-auth-swift-sdk-prerelease/Sources/StackAuth/TokenRefreshAttempt.swift new file mode 100644 index 000000000000..8f1a63e6cc16 --- /dev/null +++ b/vendor/stack-auth-swift-sdk-prerelease/Sources/StackAuth/TokenRefreshAttempt.swift @@ -0,0 +1,22 @@ +import Foundation + +/// Mutable attempt state is confined to TokenRefreshCoordinator's actor. +final class TokenRefreshAttempt { + let id = UUID() + let store: any TokenStoreProtocol + let refreshToken: String + let accessToken: String? + let clock: TokenRefreshClock + let deadline: UInt64 + var operation: Task? + var timer: Task? + var waiters: [UUID: AsyncStream.Continuation] = [:] + + init(store: any TokenStoreProtocol, refreshToken: String, accessToken: String?, clock: TokenRefreshClock, deadline: UInt64) { + self.store = store + self.refreshToken = refreshToken + self.accessToken = accessToken + self.clock = clock + self.deadline = deadline + } +} diff --git a/vendor/stack-auth-swift-sdk-prerelease/Sources/StackAuth/TokenRefreshClock.swift b/vendor/stack-auth-swift-sdk-prerelease/Sources/StackAuth/TokenRefreshClock.swift new file mode 100644 index 000000000000..fdc25538b80a --- /dev/null +++ b/vendor/stack-auth-swift-sdk-prerelease/Sources/StackAuth/TokenRefreshClock.swift @@ -0,0 +1,26 @@ +import Foundation + +/// Monotonic deadline seam. Modern platforms count system sleep; the legacy +/// SDK deployment fallback retains Task.sleep's original uptime semantics. +struct TokenRefreshClock: Sendable { + let now: @Sendable () -> UInt64 + let sleepUntil: @Sendable (UInt64) async throws -> Void + + static var system: Self { + if #available(macOS 13, iOS 16, tvOS 16, watchOS 9, *) { + let clock = ContinuousClock() + let origin = clock.now + return Self(now: { + let elapsed = origin.duration(to: clock.now).components + return UInt64(max(0, elapsed.seconds)) * 1_000_000_000 + + UInt64(max(0, elapsed.attoseconds / 1_000_000_000)) + }, sleepUntil: { nanoseconds in + try await clock.sleep(until: origin.advanced(by: .nanoseconds(Int64(nanoseconds)))) + }) + } + return Self(now: { DispatchTime.now().uptimeNanoseconds }, sleepUntil: { deadline in + let now = DispatchTime.now().uptimeNanoseconds + if deadline > now { try await Task.sleep(nanoseconds: deadline - now) } + }) + } +} diff --git a/vendor/stack-auth-swift-sdk-prerelease/Sources/StackAuth/TokenRefreshCoordinator.swift b/vendor/stack-auth-swift-sdk-prerelease/Sources/StackAuth/TokenRefreshCoordinator.swift new file mode 100644 index 000000000000..54145609aeb1 --- /dev/null +++ b/vendor/stack-auth-swift-sdk-prerelease/Sources/StackAuth/TokenRefreshCoordinator.swift @@ -0,0 +1,129 @@ +import Foundation + +/// Shares one exchange and its failure across callers of the same token store. +/// The SDK already shares stores across StackClientApp instances; this registry +/// preserves that ownership boundary without serially retrying every waiter. +actor TokenRefreshCoordinator { + private var attempts: [ObjectIdentifier: TokenRefreshAttempt] = [:] + + func resolve( + store: any TokenStoreProtocol, + refreshToken: String, + accessToken: String?, + clock: TokenRefreshClock, + timeoutNanoseconds: UInt64, + operation: @escaping @Sendable () async -> APIClient.RefreshOutcome + ) async -> TokenPair { + guard !Task.isCancelled else { return failed(.cancelled) } + let storedAccess = await store.getStoredAccessToken() + guard await store.getStoredRefreshToken() == refreshToken else { return failed(.sessionChanged) } + guard !Task.isCancelled else { return failed(.cancelled) } + let key = ObjectIdentifier(store) + if attempts[key] == nil, storedAccess != accessToken, !isTokenExpired(storedAccess) { + return TokenPair(refreshToken: refreshToken, accessToken: storedAccess) + } + if let previous = attempts[key], previous.refreshToken != refreshToken { + end(key, id: previous.id, result: failed(.sessionChanged)) + } + let attempt: TokenRefreshAttempt + if let existing = attempts[key] { + attempt = existing + } else { + attempt = TokenRefreshAttempt( + store: store, refreshToken: refreshToken, accessToken: accessToken, + clock: clock, deadline: clock.now() &+ timeoutNanoseconds + ) + attempts[key] = attempt + let id = attempt.id + let deadline = attempt.deadline + attempt.operation = Task { [weak self] in + let outcome = await operation() + await self?.complete(key, id: id, outcome: outcome) + } + attempt.timer = Task { [weak self] in + do { try await clock.sleepUntil(deadline) } + catch { return } + guard !Task.isCancelled else { return } + await self?.end(key, id: id, result: TokenPair(refreshToken: nil, accessToken: nil, refreshFailure: .timedOut)) + } + } + let waiterID = UUID() + let id = attempt.id + let stream = AsyncStream { continuation in + attempt.waiters[waiterID] = continuation + continuation.onTermination = { [weak self] _ in + Task { await self?.removeWaiter(key, id: id, waiterID: waiterID) } + } + } + for await result in stream { + removeWaiter(key, id: id, waiterID: waiterID) + return Task.isCancelled ? failed(.cancelled) : result + } + removeWaiter(key, id: id, waiterID: waiterID) + return failed(.cancelled) + } + + func tokensDidChange(store: any TokenStoreProtocol, refreshToken: String?) { + let key = ObjectIdentifier(store) + guard let attempt = attempts[key], attempt.refreshToken != refreshToken else { return } + end(key, id: attempt.id, result: failed(.sessionChanged)) + } + + private func complete(_ key: ObjectIdentifier, id: UUID, outcome: APIClient.RefreshOutcome) async { + guard let attempt = attempts[key], attempt.id == id else { return } + guard attempt.clock.now() < attempt.deadline else { + end(key, id: id, result: failed(.timedOut)) + return + } + guard await attempt.store.getStoredRefreshToken() == attempt.refreshToken else { + end(key, id: id, result: failed(.sessionChanged)) + return + } + guard attempts[key]?.id == id, !Task.isCancelled else { return } + guard attempt.clock.now() < attempt.deadline else { + end(key, id: id, result: failed(.timedOut)) + return + } + let result: TokenPair + switch outcome { + case .success(let access): + await attempt.store.compareAndSet(compareRefreshToken: attempt.refreshToken, + newRefreshToken: attempt.refreshToken, newAccessToken: access) + result = TokenPair(refreshToken: attempt.refreshToken, accessToken: access) + case .definitivelyRejected: + await attempt.store.compareAndSet(compareRefreshToken: attempt.refreshToken, + newRefreshToken: nil, newAccessToken: nil) + result = TokenPair(refreshToken: nil, accessToken: nil) + case .transientFailure: + result = TokenPair(refreshToken: attempt.refreshToken, + accessToken: nil) + } + // compareAndSet prevents stale writes; this guard also prevents returning + // a token whose compare-and-set lost to sign-out or account replacement. + let expectedRefresh = result.refreshToken + guard await attempt.store.getStoredRefreshToken() == expectedRefresh else { + end(key, id: id, result: failed(.sessionChanged)) + return + } + end(key, id: id, result: attempt.clock.now() < attempt.deadline ? result : failed(.timedOut)) + } + + private func removeWaiter(_ key: ObjectIdentifier, id: UUID, waiterID: UUID) { + guard let attempt = attempts[key], attempt.id == id else { return } + attempt.waiters[waiterID] = nil + if attempt.waiters.isEmpty { end(key, id: id, result: failed(.cancelled)) } + } + + private func end(_ key: ObjectIdentifier, id: UUID, result: TokenPair) { + guard let attempt = attempts[key], attempt.id == id else { return } + attempts[key] = nil + attempt.operation?.cancel() + attempt.timer?.cancel() + for waiter in attempt.waiters.values { waiter.yield(result); waiter.finish() } + attempt.waiters = [:] + } + + private nonisolated func failed(_ failure: TokenRefreshFailure) -> TokenPair { + TokenPair(refreshToken: nil, accessToken: nil, refreshFailure: failure) + } +} diff --git a/vendor/stack-auth-swift-sdk-prerelease/Sources/StackAuth/TokenRefreshFailure.swift b/vendor/stack-auth-swift-sdk-prerelease/Sources/StackAuth/TokenRefreshFailure.swift new file mode 100644 index 000000000000..129a86ddc5e4 --- /dev/null +++ b/vendor/stack-auth-swift-sdk-prerelease/Sources/StackAuth/TokenRefreshFailure.swift @@ -0,0 +1,9 @@ +/// A refresh that could not produce credentials without changing session identity. +public enum TokenRefreshFailure: Error, Sendable { + /// This caller cancelled its wait. + case cancelled + /// The shared exchange exceeded its total elapsed deadline. + case timedOut + /// The captured session was cleared or replaced while refreshing. + case sessionChanged +} diff --git a/vendor/stack-auth-swift-sdk-prerelease/Sources/StackAuth/TokenStore.swift b/vendor/stack-auth-swift-sdk-prerelease/Sources/StackAuth/TokenStore.swift index 49fc816fe5a8..cbaf912d8215 100644 --- a/vendor/stack-auth-swift-sdk-prerelease/Sources/StackAuth/TokenStore.swift +++ b/vendor/stack-auth-swift-sdk-prerelease/Sources/StackAuth/TokenStore.swift @@ -59,6 +59,9 @@ public enum TokenStoreInit: Sendable { /// own actor serialization. public final class TokenStoreRegistry: @unchecked Sendable { public static let shared = TokenStoreRegistry() + // The existing store registry owns cross-client refresh identity. APIClient + // injects this owner; independent stores never block one another. + let refreshCoordinator = TokenRefreshCoordinator() private let lock = NSLock() diff --git a/vendor/stack-auth-swift-sdk-prerelease/Tests/StackAuthTests/RefreshDeadlineTests.swift b/vendor/stack-auth-swift-sdk-prerelease/Tests/StackAuthTests/RefreshDeadlineTests.swift new file mode 100644 index 000000000000..b48d1db4529f --- /dev/null +++ b/vendor/stack-auth-swift-sdk-prerelease/Tests/StackAuthTests/RefreshDeadlineTests.swift @@ -0,0 +1,128 @@ +import Foundation +import Testing +@testable import StackAuth + +@Suite struct RefreshDeadlineTests { + @Test(arguments: [true, false]) + func elapsedDeadlineWinsEvenWhenResponseRunsBeforeWakeTimer(wakeTimer: Bool) async { + let clock = RefreshTestClock() + let owner = TokenRefreshCoordinator() + let store = MemoryTokenStore() + let gate = RefreshExchangeGate() + await store.setTokens(accessToken: "expired", refreshToken: "session") + let request = Task { + await owner.resolve(store: store, refreshToken: "session", accessToken: "expired", + clock: adapted(clock), timeoutNanoseconds: 2_000_000_000) { await gate.exchange() } + } + await gate.waitUntilStarted() + await clock.waitUntilSleepers() + // Models a wake where the response runs before the deadline task. It + // proves ordering independence; it does not claim a physical system sleep. + clock.advance(by: .seconds(600), wakeSleepers: wakeTimer) + if !wakeTimer { await gate.release(.success(accessToken: "late")) } + let result = await request.value + #expect(result.refreshFailure == .timedOut) + #expect(await store.getStoredAccessToken() == "expired") + #expect(await store.getStoredRefreshToken() == "session") + if wakeTimer { await gate.release(.success(accessToken: "late")) } + + let retry = await owner.resolve(store: store, refreshToken: "session", accessToken: "expired", + clock: adapted(clock), timeoutNanoseconds: 2_000_000_000) { .success(accessToken: "recovered") } + #expect(retry.accessToken == "recovered") + #expect(await store.getStoredAccessToken() == "recovered") + } + + @Test func lastCancelledWaiterReleasesAttemptAndLateWorkCannotPublish() async { + let clock = RefreshTestClock() + let owner = TokenRefreshCoordinator() + let store = MemoryTokenStore() + let gate = RefreshExchangeGate() + await store.setTokens(accessToken: "expired", refreshToken: "session") + let request = Task { + await owner.resolve(store: store, refreshToken: "session", accessToken: "expired", + clock: adapted(clock), timeoutNanoseconds: 2_000_000_000) { await gate.exchange() } + } + await gate.waitUntilStarted() + request.cancel() + #expect(await request.value.refreshFailure == .cancelled) + let retry = await owner.resolve(store: store, refreshToken: "session", accessToken: "expired", + clock: adapted(clock), timeoutNanoseconds: 2_000_000_000) { .success(accessToken: "recovered") } + await gate.release(.success(accessToken: "late")) + #expect(retry.accessToken == "recovered") + #expect(await store.getStoredAccessToken() == "recovered") + } + + @Test(arguments: [200, 401, 503]) + func refreshClassifiesSuccessRejectionAndTransientFailure(status: Int) async { + let fixture = RefreshTransportFixture() + let session = await fixture.session() + let store = MemoryTokenStore() + await store.setTokens(accessToken: "expired", refreshToken: "session") + let client = APIClient(baseUrl: "https://" + fixture.host, projectId: "fixture", publishableClientKey: "synthetic", tokenStore: store, session: session) + await fixture.release(status: status, token: RefreshLifecycleTests.fresh) + let pair = await client.getOrFetchLikelyValidTokens() + #expect(pair.accessToken == (status == 200 ? RefreshLifecycleTests.fresh : nil)) + #expect(await store.getStoredRefreshToken() == (status == 401 ? nil : "session")) + #expect(await fixture.count == 1) + session.invalidateAndCancel() + await fixture.close() + } + + @Test(arguments: [true, false]) + func deadlinePreservesUsableTokenOnlyForProactiveRefresh(force: Bool) async { + let clock = RefreshTestClock() + let fixture = RefreshTransportFixture() + let session = await fixture.session() + let store = MemoryTokenStore() + // iat is old (proactive refresh needed), exp is still far in the future. + let usable = "eyJhbGciOiJIUzI1NiJ9.eyJpYXQiOjEsImV4cCI6OTk5OTk5OTk5OX0.synthetic" + await store.setTokens(accessToken: usable, refreshToken: "session") + let client = APIClient(baseUrl: "https://" + fixture.host, projectId: "fixture", publishableClientKey: "synthetic", + tokenStore: store, session: session, refreshClock: adapted(clock), refreshTimeoutNanoseconds: 2_000_000_000) + let request = Task { + await (force ? client.fetchNewAccessToken() : client.getOrFetchLikelyValidTokens()) + } + await fixture.waitForRequest() + await clock.waitUntilSleepers() + clock.advance(by: .seconds(2)) + let result = await request.value + #expect(result.accessToken == (force ? nil : usable)) + #expect(result.refreshFailure == (force ? .timedOut : nil)) + #expect(await store.getStoredAccessToken() == usable) + #expect(await store.getStoredRefreshToken() == "session") + session.invalidateAndCancel() + await fixture.close() + } + + @Test func delayedTimerRegistrationDoesNotRestartTheBudget() async { + let clock = RefreshTestClock() + let underlying = adapted(clock) + let registration = RefreshTimerRegistrationGate() + let delayedClock = TokenRefreshClock(now: underlying.now, sleepUntil: { deadline in + await registration.park() + try await underlying.sleepUntil(deadline) + }) + let owner = TokenRefreshCoordinator() + let store = MemoryTokenStore() + let exchange = RefreshExchangeGate() + await store.setTokens(accessToken: "expired", refreshToken: "session") + let request = Task { + await owner.resolve(store: store, refreshToken: "session", accessToken: "expired", + clock: delayedClock, timeoutNanoseconds: 2_000_000_000) { await exchange.exchange() } + } + await exchange.waitUntilStarted() + await registration.waitUntilParked() + clock.advance(by: .seconds(600)) + await registration.release() + #expect(await request.value.refreshFailure == .timedOut) + await exchange.release(.success(accessToken: "late")) + #expect(await store.getStoredAccessToken() == "expired") + } + + private func adapted(_ clock: RefreshTestClock) -> TokenRefreshClock { + TokenRefreshClock(now: { + let parts = clock.now.offset.components + return UInt64(parts.seconds) * 1_000_000_000 + UInt64(parts.attoseconds / 1_000_000_000) + }, sleepUntil: { try await clock.sleep(until: .init(offset: .nanoseconds(Int64($0))), tolerance: nil) }) + } +} diff --git a/vendor/stack-auth-swift-sdk-prerelease/Tests/StackAuthTests/RefreshExchangeGate.swift b/vendor/stack-auth-swift-sdk-prerelease/Tests/StackAuthTests/RefreshExchangeGate.swift new file mode 100644 index 000000000000..1a147db83d52 --- /dev/null +++ b/vendor/stack-auth-swift-sdk-prerelease/Tests/StackAuthTests/RefreshExchangeGate.swift @@ -0,0 +1,25 @@ +import Foundation +@testable import StackAuth + +actor RefreshExchangeGate { + private var continuations: [CheckedContinuation] = [] + private var started: [CheckedContinuation] = [] + private var calls = 0 + func exchange() async -> APIClient.RefreshOutcome { + calls += 1 + return await withCheckedContinuation { continuation in + continuations.append(continuation) + for waiter in started { waiter.resume() } + started = [] + } + } + func waitUntilStarted() async { + if calls > 0 { return } + await withCheckedContinuation { started.append($0) } + } + func release(_ result: APIClient.RefreshOutcome) { + let pending = continuations + continuations = [] + for continuation in pending { continuation.resume(returning: result) } + } +} diff --git a/vendor/stack-auth-swift-sdk-prerelease/Tests/StackAuthTests/RefreshFixtureCompletion.swift b/vendor/stack-auth-swift-sdk-prerelease/Tests/StackAuthTests/RefreshFixtureCompletion.swift new file mode 100644 index 000000000000..47b8d04921ac --- /dev/null +++ b/vendor/stack-auth-swift-sdk-prerelease/Tests/StackAuthTests/RefreshFixtureCompletion.swift @@ -0,0 +1,23 @@ +import Foundation + +actor RefreshFixtureCompletion { + private var count = 0 + private var waiter: AsyncStream.Continuation? + func finish() { + count += 1 + if count == 2 { waiter?.yield(true); waiter?.finish(); waiter = nil } + } + func withinDeadline() async -> Bool { + if count == 2 { return true } + let stream = AsyncStream { continuation in + waiter = continuation + let deadline = Task { + try? await Task.sleep(nanoseconds: 250_000_000) + if !Task.isCancelled { continuation.yield(false); continuation.finish() } + } + continuation.onTermination = { _ in deadline.cancel() } + } + for await value in stream { waiter = nil; return value } + return false + } +} diff --git a/vendor/stack-auth-swift-sdk-prerelease/Tests/StackAuthTests/RefreshFixtureRoutes.swift b/vendor/stack-auth-swift-sdk-prerelease/Tests/StackAuthTests/RefreshFixtureRoutes.swift new file mode 100644 index 000000000000..6c0c3a3e224b --- /dev/null +++ b/vendor/stack-auth-swift-sdk-prerelease/Tests/StackAuthTests/RefreshFixtureRoutes.swift @@ -0,0 +1,12 @@ +import Foundation + +actor RefreshFixtureRoutes { + private var routes: [String: RefreshTransportFixture] = [:] + func register(_ fixture: RefreshTransportFixture, host: String) { routes[host] = fixture } + func remove(host: String) { routes[host] = nil } + func route(_ request: RefreshFixtureURLProtocol) async { + if let host = request.request.url?.host, let fixture = routes[host] { + await fixture.receive(request) + } + } +} diff --git a/vendor/stack-auth-swift-sdk-prerelease/Tests/StackAuthTests/RefreshFixtureURLProtocol.swift b/vendor/stack-auth-swift-sdk-prerelease/Tests/StackAuthTests/RefreshFixtureURLProtocol.swift new file mode 100644 index 000000000000..7e8e0b04cc1d --- /dev/null +++ b/vendor/stack-auth-swift-sdk-prerelease/Tests/StackAuthTests/RefreshFixtureURLProtocol.swift @@ -0,0 +1,15 @@ +import Foundation + +/// URL loading owns initialization; the fixture actor serializes response delivery. +final class RefreshFixtureURLProtocol: URLProtocol, @unchecked Sendable { + override class func canInit(with request: URLRequest) -> Bool { request.url?.host?.hasSuffix(".invalid") == true } + override class func canonicalRequest(for request: URLRequest) -> URLRequest { request } + override func startLoading() { Task { await RefreshTransportFixture.routes.route(self) } } + override func stopLoading() {} + func respond(status: Int, token: String) { + let response = HTTPURLResponse(url: request.url!, statusCode: status, httpVersion: nil, headerFields: nil)! + client?.urlProtocol(self, didReceive: response, cacheStoragePolicy: .notAllowed) + client?.urlProtocol(self, didLoad: Data("{\"access_token\":\"\(token)\"}".utf8)) + client?.urlProtocolDidFinishLoading(self) + } +} diff --git a/vendor/stack-auth-swift-sdk-prerelease/Tests/StackAuthTests/RefreshLifecycleTests.swift b/vendor/stack-auth-swift-sdk-prerelease/Tests/StackAuthTests/RefreshLifecycleTests.swift new file mode 100644 index 000000000000..722b63f51d9b --- /dev/null +++ b/vendor/stack-auth-swift-sdk-prerelease/Tests/StackAuthTests/RefreshLifecycleTests.swift @@ -0,0 +1,83 @@ +import Foundation +import Testing +@testable import StackAuth + +@Suite struct RefreshLifecycleTests { + static let fresh = "eyJhbGciOiJIUzI1NiJ9.eyJleHAiOjk5OTk5OTk5OTl9.synthetic" + + @Test func freshStoredTokensDoNotStartARefresh() async { + let fixture = RefreshTransportFixture() + let session = await fixture.session() + let store = MemoryTokenStore() + await store.setTokens(accessToken: Self.fresh, refreshToken: "session") + await fixture.release(status: 503, token: "unused") + let client = APIClient(baseUrl: "https://" + fixture.host, projectId: "fixture", publishableClientKey: "synthetic", tokenStore: store, session: session) + let pair = await client.getOrFetchLikelyValidTokens() + #expect(pair.accessToken == Self.fresh) + #expect(pair.refreshToken == "session") + #expect(await fixture.count == 0) + session.invalidateAndCancel() + await fixture.close() + } + + @Test func concurrentFailureIsSharedAndCancelledWaitersDetach() async { + let fixture = RefreshTransportFixture() + let session = await fixture.session() + let store = MemoryTokenStore() + await store.setTokens(accessToken: "expired", refreshToken: "session-a") + let client = APIClient(baseUrl: "https://" + fixture.host, projectId: "fixture", publishableClientKey: "synthetic", tokenStore: store, session: session) + let completion = RefreshFixtureCompletion() + let calls = (0..<4).map { index in Task { + let result = await client.getOrFetchLikelyValidTokens() + if index < 2 { await completion.finish() } + return result + } } + await fixture.waitForRequest() + calls[0].cancel() + calls[1].cancel() + let detached = await completion.withinDeadline() + #expect(detached, "Cancelling two auth callers must detach them before transport completes") + await fixture.release(status: 503, token: "unused") + for call in calls { #expect(await call.value.accessToken == nil) } + #expect(await fixture.count == 1, "Concurrent failed refresh callers must share one exchange") + #expect(await store.getStoredRefreshToken() == "session-a") + session.invalidateAndCancel() + await fixture.close() + } + + @Test func lateRefreshNeverReturnsSignedOutCredentials() async { + let fixture = RefreshTransportFixture() + let session = await fixture.session() + let store = MemoryTokenStore() + await store.setTokens(accessToken: "expired", refreshToken: "session-a") + let client = APIClient(baseUrl: "https://" + fixture.host, projectId: "fixture", publishableClientKey: "synthetic", tokenStore: store, session: session) + let request = Task { await client.getOrFetchLikelyValidTokens() } + await fixture.waitForRequest() + await client.clearTokens() + await fixture.release(token: Self.fresh) + let pair = await request.value + #expect(pair.accessToken == nil, "A late refresh must not return credentials after sign-out") + #expect(await store.getStoredAccessToken() == nil) + #expect(await store.getStoredRefreshToken() == nil) + session.invalidateAndCancel() + await fixture.close() + } + + @Test func lateRefreshNeverReturnsPreviousAccountCredentials() async { + let fixture = RefreshTransportFixture() + let session = await fixture.session() + let store = MemoryTokenStore() + await store.setTokens(accessToken: "expired", refreshToken: "session-a") + let client = APIClient(baseUrl: "https://" + fixture.host, projectId: "fixture", publishableClientKey: "synthetic", tokenStore: store, session: session) + let request = Task { await client.getOrFetchLikelyValidTokens() } + await fixture.waitForRequest() + await client.setTokens(accessToken: "account-b", refreshToken: "session-b") + await fixture.release(token: Self.fresh) + let pair = await request.value + #expect(pair.accessToken == nil, "Account replacement must reject the old in-flight capture") + #expect(await store.getStoredAccessToken() == "account-b") + #expect(await store.getStoredRefreshToken() == "session-b") + session.invalidateAndCancel() + await fixture.close() + } +} diff --git a/vendor/stack-auth-swift-sdk-prerelease/Tests/StackAuthTests/RefreshTestClock.swift b/vendor/stack-auth-swift-sdk-prerelease/Tests/StackAuthTests/RefreshTestClock.swift new file mode 100644 index 000000000000..62a4ad229555 --- /dev/null +++ b/vendor/stack-auth-swift-sdk-prerelease/Tests/StackAuthTests/RefreshTestClock.swift @@ -0,0 +1,109 @@ +import Foundation + +/// A manually-advanced virtual clock for deadline tests: time only moves when +/// the test calls ``advance(by:)``, so timeout behavior is exercised without +/// real waiting. Sleeps are cancellation-responsive (resuming with +/// `CancellationError`), matching the contract `withAuthPhaseTimeout` relies on +/// to clean up the losing deadline child. +/// Clock.now is synchronous; the lock protects only the clock/continuation seam. +final class RefreshTestClock: Clock, @unchecked Sendable { + struct Instant: InstantProtocol, Sendable { + var offset: Duration + + func advanced(by duration: Duration) -> Instant { Instant(offset: offset + duration) } + func duration(to other: Instant) -> Duration { other.offset - offset } + static func < (lhs: Instant, rhs: Instant) -> Bool { lhs.offset < rhs.offset } + } + + private struct Sleeper { + let deadline: Instant + let continuation: CheckedContinuation + } + + private let lock = NSLock() + private var currentInstant = Instant(offset: .zero) + private var sleepers: [UUID: Sleeper] = [:] + /// Sleeps cancelled before their continuation was parked (the + /// `withTaskCancellationHandler` handler can run first). + private var cancelledSleeperIDs: Set = [] + private var parkWaiters: [(count: Int, continuation: CheckedContinuation)] = [] + + var now: Instant { + lock.lock() + defer { lock.unlock() } + return currentInstant + } + + var minimumResolution: Duration { .zero } + + func sleep(until deadline: Instant, tolerance: Duration?) async throws { + let id = UUID() + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + lock.lock() + if cancelledSleeperIDs.remove(id) != nil { + lock.unlock() + continuation.resume(throwing: CancellationError()) + return + } + if deadline <= currentInstant { + lock.unlock() + continuation.resume() + return + } + sleepers[id] = Sleeper(deadline: deadline, continuation: continuation) + let satisfied = takeSatisfiedParkWaitersLocked() + lock.unlock() + for waiter in satisfied { waiter.resume() } + } + } onCancel: { + lock.lock() + let sleeper = sleepers.removeValue(forKey: id) + if sleeper == nil { cancelledSleeperIDs.insert(id) } + lock.unlock() + sleeper?.continuation.resume(throwing: CancellationError()) + } + } + + /// Suspends until at least `count` sleepers are parked, so a test advances + /// time only once the deadline under test is actually waiting (instead of + /// racing its start). + func waitUntilSleepers(count: Int = 1) async { + await withCheckedContinuation { (continuation: CheckedContinuation) in + lock.lock() + if sleepers.count >= count { + lock.unlock() + continuation.resume() + return + } + parkWaiters.append((count, continuation)) + lock.unlock() + } + } + + /// Advance virtual time, resuming every sleeper whose deadline has passed. + func advance(by duration: Duration, wakeSleepers: Bool = true) { + lock.lock() + currentInstant = currentInstant.advanced(by: duration) + guard wakeSleepers else { lock.unlock(); return } + var due: [Sleeper] = [] + for (id, sleeper) in sleepers where sleeper.deadline <= currentInstant { + sleepers[id] = nil + due.append(sleeper) + } + lock.unlock() + for sleeper in due.sorted(by: { $0.deadline < $1.deadline }) { + sleeper.continuation.resume() + } + } + + private func takeSatisfiedParkWaitersLocked() -> [CheckedContinuation] { + var satisfied: [CheckedContinuation] = [] + parkWaiters.removeAll { waiter in + guard sleepers.count >= waiter.count else { return false } + satisfied.append(waiter.continuation) + return true + } + return satisfied + } +} diff --git a/vendor/stack-auth-swift-sdk-prerelease/Tests/StackAuthTests/RefreshTimerRegistrationGate.swift b/vendor/stack-auth-swift-sdk-prerelease/Tests/StackAuthTests/RefreshTimerRegistrationGate.swift new file mode 100644 index 000000000000..1def7c0ab18c --- /dev/null +++ b/vendor/stack-auth-swift-sdk-prerelease/Tests/StackAuthTests/RefreshTimerRegistrationGate.swift @@ -0,0 +1,24 @@ +import Foundation + +/// Holds clock registration without changing the requested absolute deadline. +actor RefreshTimerRegistrationGate { + private var parked = false + private var releaseWaiter: CheckedContinuation? + private var startWaiter: CheckedContinuation? + + func park() async { + parked = true + await withCheckedContinuation { continuation in + releaseWaiter = continuation + startWaiter?.resume() + startWaiter = nil + } + } + + func waitUntilParked() async { + if parked { return } + await withCheckedContinuation { startWaiter = $0 } + } + + func release() { releaseWaiter?.resume(); releaseWaiter = nil } +} diff --git a/vendor/stack-auth-swift-sdk-prerelease/Tests/StackAuthTests/RefreshTransportFixture.swift b/vendor/stack-auth-swift-sdk-prerelease/Tests/StackAuthTests/RefreshTransportFixture.swift new file mode 100644 index 000000000000..9e2a5125aa28 --- /dev/null +++ b/vendor/stack-auth-swift-sdk-prerelease/Tests/StackAuthTests/RefreshTransportFixture.swift @@ -0,0 +1,44 @@ +import Foundation +@testable import StackAuth + +/// Each ephemeral URLSession routes to its own actor through a unique synthetic +/// origin. URLProtocol has no instance injection API; this registry is only the +/// callback routing seam, and each fixture unregisters at teardown. +actor RefreshTransportFixture { + static let routes = RefreshFixtureRoutes() + let host = UUID().uuidString.lowercased() + ".invalid" + private var requests: [RefreshFixtureURLProtocol] = [] + private var started: [CheckedContinuation] = [] + private(set) var count = 0 + private var response: (Int, String)? + + func session() async -> URLSession { + await Self.routes.register(self, host: host) + let config = URLSessionConfiguration.ephemeral + config.protocolClasses = [RefreshFixtureURLProtocol.self] + return URLSession(configuration: config) + } + + func receive(_ request: RefreshFixtureURLProtocol) { + count += 1 + if let response { request.respond(status: response.0, token: response.1) } + else { requests.append(request) } + let waiters = started + started = [] + for waiter in waiters { waiter.resume() } + } + + func waitForRequest() async { + if count > 0 { return } + await withCheckedContinuation { started.append($0) } + } + + func release(status: Int = 200, token: String) { + response = (status, token) + let pending = requests + requests = [] + for request in pending { request.respond(status: status, token: token) } + } + + func close() async { await Self.routes.remove(host: host) } +} diff --git a/vendor/stack-auth-swift-sdk-prerelease/Tests/StackAuthTests/RefreshTransportTimingTests.swift b/vendor/stack-auth-swift-sdk-prerelease/Tests/StackAuthTests/RefreshTransportTimingTests.swift new file mode 100644 index 000000000000..d8bbd28b019d --- /dev/null +++ b/vendor/stack-auth-swift-sdk-prerelease/Tests/StackAuthTests/RefreshTransportTimingTests.swift @@ -0,0 +1,32 @@ +import Foundation +import Testing +@testable import StackAuth + +@Suite struct RefreshTransportTimingTests { + // Deliberate runtime timing smoke for the incident proof. Virtual deadline + // and wake ordering are covered separately by RefreshDeadlineTests. + @Test func realTransportWaitHasATotalDeadline() async { + let fixture = RefreshTransportFixture() + let session = await fixture.session() + let store = MemoryTokenStore() + await store.setTokens(accessToken: "expired", refreshToken: "session") + let client = APIClient(baseUrl: "https://" + fixture.host, projectId: "fixture", publishableClientKey: "synthetic", tokenStore: store, session: session, + refreshTimeoutNanoseconds: 100_000_000) + let start = ContinuousClock.now + let active = SuspendingClock.now + let request = Task { await client.getOrFetchLikelyValidTokens() } + await fixture.waitForRequest() + print("AUTH_TRANSPORT_PARKED pid=\(ProcessInfo.processInfo.processIdentifier)") + fflush(stdout) + let result = await request.value + let elapsed = start.duration(to: .now) + let awake = active.duration(to: .now) + #expect(result.refreshFailure == .timedOut) + #expect(elapsed >= .milliseconds(100)) + #expect(elapsed < .seconds(2)) + #expect(await store.getStoredRefreshToken() == "session") + print("AUTH_TIMING total=\(elapsed) awake=\(awake) outcome=timedOut requests=\(await fixture.count)") + session.invalidateAndCancel() + await fixture.close() + } +} diff --git a/vendor/stack-auth-swift-sdk-prerelease/Tests/StackAuthTests/TokenRefreshTests.swift b/vendor/stack-auth-swift-sdk-prerelease/Tests/StackAuthTests/TokenRefreshTests.swift index 2010eb4439e3..80121b8c62e1 100644 --- a/vendor/stack-auth-swift-sdk-prerelease/Tests/StackAuthTests/TokenRefreshTests.swift +++ b/vendor/stack-auth-swift-sdk-prerelease/Tests/StackAuthTests/TokenRefreshTests.swift @@ -310,228 +310,3 @@ struct TokenRefreshAlgorithmTests { } } } - -// MARK: - RefreshLockManager Concurrency Tests - -/// Minimal token store for lock testing - we only need an object identity for the lock key -private actor MockTokenStore: TokenStoreProtocol { - func getStoredAccessToken() async -> String? { nil } - func getStoredRefreshToken() async -> String? { nil } - func setTokens(accessToken: String?, refreshToken: String?) async {} - func clearTokens() async {} - func compareAndSet(compareRefreshToken: String, newRefreshToken: String?, newAccessToken: String?) async {} -} - -@Suite("RefreshLockManager Concurrency Tests") -struct RefreshLockManagerTests { - - @Test("Should serialize concurrent lock acquisitions") - func serializeConcurrentLocks() async { - let store = MockTokenStore() - var executionOrder: [Int] = [] - let orderLock = NSLock() - - func appendOrder(_ n: Int) { - orderLock.lock() - executionOrder.append(n) - orderLock.unlock() - } - - // Start 3 concurrent tasks that all try to acquire the lock - await withTaskGroup(of: Void.self) { group in - for i in 1...3 { - group.addTask { - await RefreshLockManager.shared.acquireLock(for: store) - appendOrder(i * 10) // Record entry: 10, 20, or 30 - // Simulate some work while holding the lock - try? await Task.sleep(nanoseconds: 10_000_000) // 10ms - appendOrder(i * 10 + 1) // Record exit: 11, 21, or 31 - await RefreshLockManager.shared.releaseLock(for: store) - } - } - } - - // Verify serialization: each task should complete (entry+exit) before next starts - // Valid patterns: [10,11,20,21,30,31], [10,11,30,31,20,21], [20,21,10,11,30,31], etc. - // Invalid: [10,20,11,21,...] - interleaved entries/exits - - #expect(executionOrder.count == 6) - - // Check that entries and exits are paired (no interleaving) - var inProgress: Int? = nil - for event in executionOrder { - let taskId = event / 10 - let isEntry = event % 10 == 0 - - if isEntry { - // Should not have another task in progress when entering - #expect(inProgress == nil, "Task \(taskId) entered while task \(inProgress ?? -1) was in progress") - inProgress = taskId - } else { - // Should be exiting the same task that entered - #expect(inProgress == taskId, "Task \(taskId) exited but task \(inProgress ?? -1) was in progress") - inProgress = nil - } - } - } - - @Test("Should allow different stores to lock concurrently") - func differentStoresCanLockConcurrently() async { - let store1 = MockTokenStore() - let store2 = MockTokenStore() - var concurrentCount = 0 - var maxConcurrent = 0 - let countLock = NSLock() - - func incrementConcurrent() { - countLock.lock() - concurrentCount += 1 - if concurrentCount > maxConcurrent { - maxConcurrent = concurrentCount - } - countLock.unlock() - } - - func decrementConcurrent() { - countLock.lock() - concurrentCount -= 1 - countLock.unlock() - } - - await withTaskGroup(of: Void.self) { group in - // Task for store1 - group.addTask { - await RefreshLockManager.shared.acquireLock(for: store1) - incrementConcurrent() - try? await Task.sleep(nanoseconds: 50_000_000) // 50ms - decrementConcurrent() - await RefreshLockManager.shared.releaseLock(for: store1) - } - - // Task for store2 - group.addTask { - await RefreshLockManager.shared.acquireLock(for: store2) - incrementConcurrent() - try? await Task.sleep(nanoseconds: 50_000_000) // 50ms - decrementConcurrent() - await RefreshLockManager.shared.releaseLock(for: store2) - } - } - - // Both stores should have been able to hold locks concurrently - #expect(maxConcurrent == 2, "Expected both stores to hold locks concurrently, but max concurrent was \(maxConcurrent)") - } - - @Test("Should handle high contention stress test") - func stressTestHighContention() async { - let store = MockTokenStore() - let taskCount = 50 - var executionOrder: [Int] = [] - let orderLock = NSLock() - - func appendOrder(_ n: Int) { - orderLock.lock() - executionOrder.append(n) - orderLock.unlock() - } - - // Launch 50 concurrent tasks all fighting for the same lock - await withTaskGroup(of: Void.self) { group in - for i in 1...taskCount { - group.addTask { - await RefreshLockManager.shared.acquireLock(for: store) - appendOrder(i * 10) // Entry - // Brief work while holding lock - try? await Task.sleep(nanoseconds: 1_000_000) // 1ms - appendOrder(i * 10 + 1) // Exit - await RefreshLockManager.shared.releaseLock(for: store) - } - } - } - - // Should have 100 events (50 entries + 50 exits) - #expect(executionOrder.count == taskCount * 2, "Expected \(taskCount * 2) events, got \(executionOrder.count)") - - // Verify serialization - no interleaving under high contention - var inProgress: Int? = nil - var interleaveCount = 0 - for event in executionOrder { - let taskId = event / 10 - let isEntry = event % 10 == 0 - - if isEntry { - if inProgress != nil { - interleaveCount += 1 - } - inProgress = taskId - } else { - if inProgress != taskId { - interleaveCount += 1 - } - inProgress = nil - } - } - - #expect(interleaveCount == 0, "Found \(interleaveCount) interleaving violations under high contention - LOCK BUG!") - } - - @Test("Should wake all waiters when lock is released and serialize their acquisition") - func wakeAllWaitersAndSerialize() async { - let store = MockTokenStore() - var executionOrder: [Int] = [] - let orderLock = NSLock() - - func appendOrder(_ n: Int) { - orderLock.lock() - executionOrder.append(n) - orderLock.unlock() - } - - // First task acquires lock and holds it - await RefreshLockManager.shared.acquireLock(for: store) - - // Start 3 tasks that will all wait for the lock - let waitingTasks = Task { - await withTaskGroup(of: Void.self) { group in - for i in 1...3 { - group.addTask { - await RefreshLockManager.shared.acquireLock(for: store) - appendOrder(i * 10) // Record entry: 10, 20, or 30 - // Hold the lock briefly to ensure we'd see interleaving if bug exists - try? await Task.sleep(nanoseconds: 10_000_000) // 10ms - appendOrder(i * 10 + 1) // Record exit: 11, 21, or 31 - await RefreshLockManager.shared.releaseLock(for: store) - } - } - } - } - - // Give tasks time to start waiting (all 3 should be blocked) - try? await Task.sleep(nanoseconds: 50_000_000) // 50ms - - // Release the lock - all waiters wake up, but only ONE should acquire - await RefreshLockManager.shared.releaseLock(for: store) - - // Wait for all tasks to complete - await waitingTasks.value - - // All 3 waiting tasks should have completed - #expect(executionOrder.count == 6, "Expected 6 events (3 entries + 3 exits), got \(executionOrder.count)") - - // CRITICAL: Verify no interleaving - this catches the while vs if bug - // If bug exists, multiple waiters acquire lock simultaneously after being resumed - var inProgress: Int? = nil - for event in executionOrder { - let taskId = event / 10 - let isEntry = event % 10 == 0 - - if isEntry { - #expect(inProgress == nil, "Task \(taskId) entered while task \(inProgress ?? -1) was in progress - LOCK BUG!") - inProgress = taskId - } else { - #expect(inProgress == taskId, "Task \(taskId) exited but task \(inProgress ?? -1) was in progress") - inProgress = nil - } - } - } -} From 5e40f87087f3a56565054e7e3936b01f78b76870 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sat, 19 Sep 2026 22:44:24 -0700 Subject: [PATCH 14/92] integrate: bound Cloud reads while retaining current resource and team ownership Adapt the read coordination and regression suites from #12636 at ec3f5bbe28 to current main. Retain main resource-stat reconciliation, pinning, team-scope fences, and already-landed Cloud split routing. Exclude unrelated renderer, Iroh, and signing changes. --- .github/workflows/auth-refresh-tests.yml | 3 - Sources/AppDelegate.swift | 4 +- Sources/Cloud/CloudReadNetworkMonitor.swift | 26 ++ Sources/Cloud/CloudReadRecoveryObserver.swift | 19 + .../Cloud/CloudReadRequestCoordinator.swift | 334 ++++++++++++++++++ Sources/Cloud/CloudRequestClock.swift | 13 + Sources/Cloud/MachineUsageSnapshot.swift | 15 + Sources/Cloud/MachineUsageTotals.swift | 15 + .../MachinesPanelViewModel+Refresh.swift | 162 +++++++++ Sources/Cloud/MachinesPanelViewModel.swift | 254 ++++--------- Sources/Cloud/TeamMachineUsage.swift | 30 ++ Sources/Cloud/VMCapabilities.swift | 76 ++++ Sources/Cloud/VMClient.swift | 231 +++++------- ...muxTuiSurfaceProviderRegistry+Routes.swift | 42 +++ .../CmuxTuiSurfaceProviderRegistry.swift | 50 +-- cmux.xcodeproj/project.pbxproj | 56 +++ cmuxTests/CloudReadManualClock.swift | 48 +++ .../CloudReadRequestCoordinatorTests.swift | 269 ++++++++++++++ cmuxTests/CloudRefreshURLProtocol.swift | 84 +++++ cmuxTests/VMClientReadCoalescingTests.swift | 286 +++++++++++++++ 20 files changed, 1636 insertions(+), 381 deletions(-) create mode 100644 Sources/Cloud/CloudReadNetworkMonitor.swift create mode 100644 Sources/Cloud/CloudReadRecoveryObserver.swift create mode 100644 Sources/Cloud/CloudReadRequestCoordinator.swift create mode 100644 Sources/Cloud/CloudRequestClock.swift create mode 100644 Sources/Cloud/MachineUsageSnapshot.swift create mode 100644 Sources/Cloud/MachineUsageTotals.swift create mode 100644 Sources/Cloud/MachinesPanelViewModel+Refresh.swift create mode 100644 Sources/Cloud/TeamMachineUsage.swift create mode 100644 Sources/Cloud/VMCapabilities.swift create mode 100644 Sources/Surfaces/CmuxTuiSurfaceProviderRegistry+Routes.swift create mode 100644 cmuxTests/CloudReadManualClock.swift create mode 100644 cmuxTests/CloudReadRequestCoordinatorTests.swift create mode 100644 cmuxTests/CloudRefreshURLProtocol.swift create mode 100644 cmuxTests/VMClientReadCoalescingTests.swift diff --git a/.github/workflows/auth-refresh-tests.yml b/.github/workflows/auth-refresh-tests.yml index ca5c1ca99796..fe37c2ebbea5 100644 --- a/.github/workflows/auth-refresh-tests.yml +++ b/.github/workflows/auth-refresh-tests.yml @@ -4,7 +4,6 @@ on: pull_request: paths: - Packages/Shared/CmuxAuthRuntime/** - - Packages/Shared/CmuxIrxTransport/** - vendor/stack-auth-swift-sdk-prerelease/** - .github/workflows/auth-refresh-tests.yml workflow_dispatch: @@ -28,5 +27,3 @@ jobs: run: swift test --package-path vendor/stack-auth-swift-sdk-prerelease --filter 'RefreshLifecycleTests|RefreshDeadlineTests|RefreshTransportTimingTests|RateLimitRetryTests' - name: Exercise coordinator cancellation, sessions, and recovery run: swift test --package-path Packages/Shared/CmuxAuthRuntime --filter 'AuthCoordinator|AuthError' - - name: Verify retired session watcher completion over loopback - run: swift test --package-path Packages/Shared/CmuxIrxTransport --filter 'IrxLiveQUICTests.keepaliveAndAutoRedial' diff --git a/Sources/AppDelegate.swift b/Sources/AppDelegate.swift index d3642b6a8e86..8bf0a0ed3051 100644 --- a/Sources/AppDelegate.swift +++ b/Sources/AppDelegate.swift @@ -2556,12 +2556,12 @@ final class AppDelegate: NSObject, NSApplicationDelegate, UNUserNotificationCent let cloudTunnel = makeCloudTunnelCoordinator() cloudTunnelCoordinator = cloudTunnel CmuxTuiSurfaceProviderRegistry.shared.portAccess.coordinator = cloudTunnel - VMClient.bootstrap(auth: auth.coordinator, operations: cloudOperations) + let cloudReads = VMClient.bootstrap(auth: auth.coordinator, operations: cloudOperations) TerminalController.shared.cloudTunnel = cloudTunnel RemotesClient.bootstrap(auth: auth.coordinator) AIAccountsClient.bootstrap(auth: auth.coordinator) CoderouterClient.bootstrap(auth: auth.coordinator) - MachineUsageClient.bootstrap(auth: auth.coordinator, operations: cloudOperations) + MachineUsageClient.bootstrap(auth: auth.coordinator, operations: cloudOperations, readRequests: cloudReads) PhonePushClient.shared.configure(auth: auth.coordinator) MobileHostService.shared.configure(auth: auth.coordinator) caffeineController.onStateChange = { [weak self] enabled in diff --git a/Sources/Cloud/CloudReadNetworkMonitor.swift b/Sources/Cloud/CloudReadNetworkMonitor.swift new file mode 100644 index 000000000000..59c83eb17948 --- /dev/null +++ b/Sources/Cloud/CloudReadNetworkMonitor.swift @@ -0,0 +1,26 @@ +import Foundation +@preconcurrency import Network + +/// One path observer for the VM client. A Network.framework delivery queue +/// feeds an AsyncStream; it never protects domain state or runs UI work. +final class CloudReadNetworkMonitor: Sendable { + private let monitor: NWPathMonitor + let updates: AsyncStream + + init() { + let monitor = NWPathMonitor() + self.monitor = monitor + let (updates, continuation) = AsyncStream.makeStream(bufferingPolicy: .bufferingNewest(1)) + self.updates = updates + monitor.pathUpdateHandler = { continuation.yield($0.status != .unsatisfied) } + continuation.onTermination = { _ in monitor.cancel() } + monitor.start(queue: DispatchQueue(label: "cmux.cloud.read-network")) + } + + deinit { monitor.cancel() } +} + +extension Notification.Name { + static let cmuxCloudReadNetworkChanged = Notification.Name("cmux.cloud.readNetworkChanged") + static let cmuxCloudReadNetworkRecovered = Notification.Name("cmux.cloud.readNetworkRecovered") +} diff --git a/Sources/Cloud/CloudReadRecoveryObserver.swift b/Sources/Cloud/CloudReadRecoveryObserver.swift new file mode 100644 index 000000000000..f49493f936be --- /dev/null +++ b/Sources/Cloud/CloudReadRecoveryObserver.swift @@ -0,0 +1,19 @@ +import Foundation + +/// Owns the legacy notification seam for the registry's network recovery refresh. +@MainActor +final class CloudReadRecoveryObserver { + private let notificationCenter: NotificationCenter + private var observer: NSObjectProtocol? + + init(notificationCenter: NotificationCenter, recover: @escaping @MainActor @Sendable () async -> Void) { + self.notificationCenter = notificationCenter + observer = notificationCenter.addObserver(forName: .cmuxCloudReadNetworkRecovered, object: nil, queue: .main) { _ in + Task { @MainActor in await recover() } + } + } + + deinit { + if let observer { notificationCenter.removeObserver(observer) } + } +} diff --git a/Sources/Cloud/CloudReadRequestCoordinator.swift b/Sources/Cloud/CloudReadRequestCoordinator.swift new file mode 100644 index 000000000000..ea6f2193aa6b --- /dev/null +++ b/Sources/Cloud/CloudReadRequestCoordinator.swift @@ -0,0 +1,334 @@ +import Foundation + +/// Owns overlapping read requests for one VM client. Caller cancellation releases +/// only that waiter; the last waiter cancels the transport. A cancelled request +/// keeps its slot until teardown completes, preventing replacement amplification. +actor CloudReadRequestCoordinator { + struct Key: Hashable, Sendable { + let path: String + let accountID: String? + let generation: UInt64? + let teamID: String? + } + + struct Response: Sendable { + let data: Data + let http: HTTPURLResponse + } + + struct Context: Sendable { + weak var owner: CloudReadRequestCoordinator? + let key: Key + } + + @TaskLocal static var current: Context? + + struct Waiter: Sendable { + let deadline: Duration + let continuation: CheckedContinuation + } + + struct Entry: Sendable { + let id: UUID + let deadline: Duration + var waiters: [UUID: Waiter] + var work: Task? + var timer: Task? + var terminalError: URLError? + var invalidated = false + let operation: @Sendable () async throws -> Response + var pending: Pending? + } + + struct Pending: Sendable { + let id: UUID + var waiters: [UUID: Waiter] + let operation: @Sendable () async throws -> Response + var timer: Task? + } + + private struct Cooldown { + let until: TimeInterval + let response: Response + } + + private nonisolated let clock: CloudRequestClock + private nonisolated let budget: Duration + private let onNetworkChange: @Sendable (Bool) async -> Void + private(set) var entries: [Key: Entry] = [:] + private var networkTask: Task? + private var isOnline: Bool? + private var cooldowns: [Key: Cooldown] = [:] + private var nextCooldownExpiry: TimeInterval? + + init(clock: CloudRequestClock = CloudRequestClock(ContinuousClock()), budget: Duration = .seconds(30), + onNetworkChange: @escaping @Sendable (Bool) async -> Void = { _ in }) { + self.clock = clock + self.budget = budget + self.onNetworkChange = onNetworkChange + } + + nonisolated func makeDeadline(elapsed: Duration = .zero, limit: Duration? = nil) -> Duration { + clock.now() + min(budget, limit ?? budget) - elapsed + } + + func read(_ key: Key, deadline: Duration? = nil, operation: @escaping @Sendable () async throws -> Response) async throws -> Response { + let waiter = UUID() + let response = try await withTaskCancellationHandler { + try Task.checkCancellation() + return try await withCheckedThrowingContinuation { continuation in + join(key, waiter: waiter, deadline: deadline ?? makeDeadline(), continuation: continuation, operation: operation) + } + } onCancel: { + Task { await self.cancel(key, waiter: waiter) } + } + try Task.checkCancellation() + return response + } + + private func join( + _ key: Key, waiter: UUID, deadline: Duration, continuation: CheckedContinuation, + operation: @escaping @Sendable () async throws -> Response + ) { + if clock.now() >= deadline { + continuation.resume(throwing: URLError(.timedOut)) + return + } + if isOnline == false { + continuation.resume(throwing: URLError(.notConnectedToInternet)) + return + } + if let entry = entries[key] { + if entry.terminalError != nil { + queueAfterTeardown(key, waiter: waiter, deadline: deadline, continuation: continuation, operation: operation) + } else if clock.now() >= entry.deadline { + expire(key, id: entry.id) + queueAfterTeardown(key, waiter: waiter, deadline: deadline, continuation: continuation, operation: operation) + } else { + entries[key]?.waiters[waiter] = Waiter(deadline: deadline, continuation: continuation) + armTimer(key, id: entry.id) + } + return + } + let now = seconds(clock.now()) + if let nextCooldownExpiry, now >= nextCooldownExpiry { + cooldowns = cooldowns.filter { $0.value.until > now } + self.nextCooldownExpiry = cooldowns.values.map(\.until).min() + } + if let cooldown = cooldowns[key] { + continuation.resume(returning: cooldown.response) + return + } + startEntry(key, id: UUID(), deadline: deadline, + waiters: [waiter: Waiter(deadline: deadline, continuation: continuation)], operation: operation) + } + + private func startEntry( + _ key: Key, id: UUID, deadline: Duration, waiters: [UUID: Waiter], + operation: @escaping @Sendable () async throws -> Response + ) { + let waiters = unexpired(waiters) + guard !waiters.isEmpty else { return } + if clock.now() >= deadline || isOnline == false { + let error = URLError(isOnline == false ? .notConnectedToInternet : .timedOut) + for waiter in waiters.values { waiter.continuation.resume(throwing: error) } + return + } + if let cooldown = cooldowns[key], cooldown.until > seconds(clock.now()) { + for waiter in waiters.values { waiter.continuation.resume(returning: cooldown.response) } + return + } + entries[key] = Entry(id: id, deadline: deadline, waiters: waiters, operation: operation) + startWork(key, id: id, operation: operation) + armTimer(key, id: id) + } + + private func armTimer(_ key: Key, id: UUID) { + guard let entry = entries[key], entry.terminalError == nil else { return } + entry.timer?.cancel() + let deadline = min(entry.deadline, entry.waiters.values.map(\.deadline).min() ?? entry.deadline) + entries[key]?.timer = Task { [weak self, clock] in + do { try await clock.sleepUntil(deadline) } catch { return } + await self?.expireDueWaiters(key, id: id) + } + } + + private func unexpired(_ waiters: [UUID: Waiter]) -> [UUID: Waiter] { + let now = clock.now() + return waiters.filter { _, waiter in + guard waiter.deadline > now else { + waiter.continuation.resume(throwing: URLError(.timedOut)) + return false + } + return true + } + } + + private func expireDueWaiters(_ key: Key, id: UUID) { + guard let entry = entries[key], entry.id == id, entry.terminalError == nil else { return } + if clock.now() >= entry.deadline { expire(key, id: id); return } + entries[key]?.waiters = unexpired(entry.waiters) + if entries[key]?.waiters.isEmpty == true { expire(key, id: id) } + else { armTimer(key, id: id) } + } + + private func queueAfterTeardown( + _ key: Key, waiter: UUID, deadline: Duration, continuation: CheckedContinuation, + operation: @escaping @Sendable () async throws -> Response + ) { + if let pending = entries[key]?.pending { + entries[key]?.pending?.waiters[waiter] = Waiter(deadline: deadline, continuation: continuation) + armPendingTimer(key, id: pending.id) + return + } + let id = UUID() + entries[key]?.pending = Pending(id: id, waiters: [waiter: Waiter(deadline: deadline, continuation: continuation)], operation: operation) + armPendingTimer(key, id: id) + } + + private func armPendingTimer(_ key: Key, id: UUID) { + guard let pending = entries[key]?.pending, pending.id == id else { return } + pending.timer?.cancel() + let waiters = unexpired(pending.waiters) + entries[key]?.pending?.waiters = waiters + guard let deadline = waiters.values.map(\.deadline).min() else { + entries[key]?.pending = nil + return + } + entries[key]?.pending?.timer = Task { [weak self, clock] in + do { try await clock.sleepUntil(deadline) } catch { return } + await self?.armPendingTimer(key, id: id) + } + } + + private func expirePending(_ key: Key, id: UUID, error: URLError = URLError(.timedOut)) { + guard let pending = entries[key]?.pending, pending.id == id else { return } + entries[key]?.pending = nil + pending.timer?.cancel() + for waiter in pending.waiters.values { waiter.continuation.resume(throwing: error) } + } + + private func startWork(_ key: Key, id: UUID, operation: @escaping @Sendable () async throws -> Response) { + let context = Context(owner: self, key: key) + entries[key]?.work = Task { [weak self] in + let result: Result + do { + try Task.checkCancellation() + result = .success(try await Self.$current.withValue(context, operation: operation)) + } catch { + result = .failure(error) + } + await self?.finish(key, id: id, result: result) + } + } + + /// A completed mutation invalidates any read that started before it. Its + /// readers share one trailing pass, within the original operation budget. + func invalidate() { + for key in entries.keys { entries[key]?.invalidated = true } + } + + /// Retains the server's minimum retry time across cancellation and later + /// polls. When it exceeds this operation's remaining budget, return the + /// original 429 now; future reads receive that response until retry is legal. + func noteRetryAfter(_ key: Key, seconds: TimeInterval, response: Response) -> Bool { + // Retry-After can contain Int.max seconds. Keep that distant deadline + // as a monotonic floating-point instant rather than overflowing Duration. + let until = self.seconds(clock.now()) + seconds + if until > (cooldowns[key]?.until ?? 0) { + cooldowns[key] = Cooldown(until: until, response: response) + nextCooldownExpiry = min(nextCooldownExpiry ?? until, until) + } + guard let entry = entries[key], entry.terminalError == nil else { return false } + return until < self.seconds(entry.deadline) + } + + private func seconds(_ duration: Duration) -> TimeInterval { + let parts = duration.components + return Double(parts.seconds) + Double(parts.attoseconds) / 1e18 + } + + private func cancel(_ key: Key, waiter: UUID) { + guard let removed = entries[key]?.waiters.removeValue(forKey: waiter) else { + if let removed = entries[key]?.pending?.waiters.removeValue(forKey: waiter) { + removed.continuation.resume(throwing: CancellationError()) + if entries[key]?.pending?.waiters.isEmpty == true { + entries[key]?.pending?.timer?.cancel() + entries[key]?.pending = nil + } + } + return + } + removed.continuation.resume(throwing: CancellationError()) + if entries[key]?.waiters.isEmpty == true { + entries[key]?.terminalError = URLError(.cancelled) + entries[key]?.timer?.cancel() + entries[key]?.work?.cancel() + } + } + + private func expire(_ key: Key, id: UUID, error: URLError = URLError(.timedOut)) { + guard let entry = entries[key], entry.id == id, entry.terminalError == nil else { return } + entries[key]?.terminalError = error + entries[key]?.waiters.removeAll() + entry.work?.cancel() + entry.timer?.cancel() + for waiter in entry.waiters.values { waiter.continuation.resume(throwing: error) } + } + + private func finish(_ key: Key, id: UUID, result: Result) { + guard let entry = entries[key], entry.id == id else { return } + // A response may run before the expired timer after sleep/wake. The + // monotonic deadline decides the outcome, not executor delivery order. + expireDueWaiters(key, id: id) + if entry.invalidated, entries[key]?.terminalError == nil, + case .success(let response) = result, (200...299).contains(response.http.statusCode) { + entries[key]?.invalidated = false + startWork(key, id: id, operation: entry.operation) + return + } + guard let completed = entries.removeValue(forKey: key) else { return } + completed.timer?.cancel() + for waiter in completed.waiters.values { waiter.continuation.resume(with: result) } + if let pending = completed.pending { + pending.timer?.cancel() + let deadline = min(makeDeadline(), pending.waiters.values.map(\.deadline).max() ?? clock.now()) + startEntry(key, id: pending.id, deadline: deadline, waiters: pending.waiters, operation: pending.operation) + } + } + + func observeNetwork(_ monitor: CloudReadNetworkMonitor) { + networkTask?.cancel() + networkTask = Task { [weak self, monitor] in + for await online in monitor.updates { + guard !Task.isCancelled else { return } + await self?.networkChanged(isOnline: online) + } + } + } + + func networkChanged(isOnline: Bool) async { + let changed = self.isOnline != isOnline && (self.isOnline != nil || !isOnline) + self.isOnline = isOnline + if !isOnline { + for (key, entry) in entries { + expire(key, id: entry.id, error: URLError(.notConnectedToInternet)) + if let pending = entry.pending { expirePending(key, id: pending.id, error: URLError(.notConnectedToInternet)) } + } + } + if changed { await onNetworkChange(isOnline) } + } + + deinit { + networkTask?.cancel() + for entry in entries.values { + entry.work?.cancel() + entry.timer?.cancel() + for waiter in entry.waiters.values { waiter.continuation.resume(throwing: CancellationError()) } + if let pending = entry.pending { + pending.timer?.cancel() + for waiter in pending.waiters.values { waiter.continuation.resume(throwing: CancellationError()) } + } + } + } +} diff --git a/Sources/Cloud/CloudRequestClock.swift b/Sources/Cloud/CloudRequestClock.swift new file mode 100644 index 000000000000..4b062e07db75 --- /dev/null +++ b/Sources/Cloud/CloudRequestClock.swift @@ -0,0 +1,13 @@ +import Foundation + +/// An injected monotonic clock shared by a read's deadline and Retry-After gate. +struct CloudRequestClock: Sendable { + let now: @Sendable () -> Duration + let sleepUntil: @Sendable (Duration) async throws -> Void + + init>(_ clock: C) { + let origin = clock.now + now = { origin.duration(to: clock.now) } + sleepUntil = { try await clock.sleep(until: origin.advanced(by: $0), tolerance: nil) } + } +} diff --git a/Sources/Cloud/MachineUsageSnapshot.swift b/Sources/Cloud/MachineUsageSnapshot.swift new file mode 100644 index 000000000000..3b82cf799c39 --- /dev/null +++ b/Sources/Cloud/MachineUsageSnapshot.swift @@ -0,0 +1,15 @@ +import Foundation + +/// One machine's usage readout: the row shows `totals` labeled with +/// `periodDays`. `vmID` is the id `GET /api/vm` returns as the machine id, so +/// it matches ``MachineSnapshot/id`` directly. +struct MachineUsageSnapshot: Equatable, Sendable { + let vmID: String + /// The provider machine id, the `id` that `GET /api/vm` lists. Rows key + /// on it when present because `vmID` is the backend's own uuid. + let providerVmID: String? + let displayName: String? + let periodDays: Int + let asOf: Date? + let totals: MachineUsageTotals +} diff --git a/Sources/Cloud/MachineUsageTotals.swift b/Sources/Cloud/MachineUsageTotals.swift new file mode 100644 index 000000000000..f2e0058e4f3e --- /dev/null +++ b/Sources/Cloud/MachineUsageTotals.swift @@ -0,0 +1,15 @@ +import Foundation + +/// Token and spend totals for one machine over the usage window, as +/// `GET /api/coderouter/vm-usage/team` reports them. +struct MachineUsageTotals: Equatable, Sendable { + let inputTokens: Int + let cachedInputTokens: Int + let outputTokens: Int + let totalTokens: Int + /// What the same traffic would have cost at list API prices. + let apiEquivalentUsd: Double + + /// Nothing to show for a machine that has not routed a single token. + var isEmpty: Bool { totalTokens <= 0 && apiEquivalentUsd <= 0 } +} diff --git a/Sources/Cloud/MachinesPanelViewModel+Refresh.swift b/Sources/Cloud/MachinesPanelViewModel+Refresh.swift new file mode 100644 index 000000000000..2438d654131f --- /dev/null +++ b/Sources/Cloud/MachinesPanelViewModel+Refresh.swift @@ -0,0 +1,162 @@ +import CmuxCloudMachines +import Foundation + +/// Refresh ownership stays on the panel; only immutable requests survive suspension. +extension MachinesPanelViewModel { + func refreshStats() { + guard isCloudEnabled(), statsTask == nil, let client = client ?? VMClient.shared else { return } + let ids = machines.filter { $0.capabilities.stats }.map(\.id) + guard !ids.isEmpty else { return } + let requestID = UUID() + statsID = requestID + statsTask = Task { [weak self] in + await withTaskGroup(of: Void.self) { group in + for id in ids { group.addTask { _ = try? await client.stats(id: id) } } + } + guard let self, self.statsID == requestID else { return } + self.statsTask = nil + self.statsID = nil + } + } + + /// Read the current shared resource owner, retaining its resize and list fences. + func applyResourceStats(machineIDs: Set?) { + guard isCloudEnabled(), let resourceStats else { return } + for id in machineIDs ?? Set(machineIndexByID.keys) { + guard let index = machineIndexByID[id], machines.indices.contains(index), + machines[index].id == id, machines[index].capabilities.stats else { continue } + let stats = resourceStats.stats(for: id) + if machines[index].stats != stats { machines[index].stats = stats } + } + } + + func refresh() { + guard isCloudEnabled(), let client = client ?? VMClient.shared else { return } + guard refreshTask == nil else { refreshRequestedWhileLoading = true; return } + isLoading = true + let generation = refreshGeneration + let scope = machinePinStore?.scopeIdentifier + refreshTask = Task { [weak self] in + let result: Result + do { result = .success(try await client.listPage()) } + catch { result = .failure(error) } + guard !Task.isCancelled, let self, generation == self.refreshGeneration else { return } + self.applyRefreshResult(result, generation: generation, scope: scope) + self.refreshTask = nil + if self.refreshRequestedWhileLoading { + self.refreshRequestedWhileLoading = false + self.refresh() + } + } + } + + func startPolling() { + wantsPolling = true + guard isCloudEnabled() else { pausePolling(); return } + refresh() + guard pollTask == nil else { return } + pollTask = Task { [weak self, pollingClock] in + while !Task.isCancelled { + do { try await pollingClock.sleep(for: Self.pollInterval) } catch { return } + guard !Task.isCancelled, let self else { return } + self.refresh() + } + } + } + + func stopPolling() { wantsPolling = false; pausePolling() } + + func pausePolling() { + pollTask?.cancel(); pollTask = nil + refreshTask?.cancel(); refreshTask = nil + refreshRequestedWhileLoading = false + refreshGeneration &+= 1 + isLoading = false + statsTask?.cancel(); statsTask = nil; statsID = nil + usageTask?.cancel(); usageTask = nil + usageFailureCount = 0 + usageRetryNotBefore = nil + treeTask?.cancel(); treeTask = nil + machineRefreshes.cancelAll() + freeAccessTransitionTask?.cancel(); freeAccessTransitionTask = nil + } + + func clearUnavailableMetrics() { + if let resourceStats { + for id in machineIndexByID.keys { + _ = resourceStats.finishRead(resourceStats.beginRead(machineID: id), stats: nil) + } + applyResourceStats(machineIDs: nil) + } + usageByMachineID = [:] + machines = MachineSnapshotBuilder.applyingUsage(to: machines, usage: [:]) + } + + func applyRefreshResult(_ result: Result, generation: UInt64, scope: String?) { + guard generation == refreshGeneration, scope == machinePinStore?.scopeIdentifier, isCloudEnabled() else { return } + do { + let page = try result.get() + try Task.checkCancellation() + guard generation == refreshGeneration, scope == machinePinStore?.scopeIdentifier, + isCloudEnabled() else { return } + let previous = resourceStats?.snapshot ?? [:] + let freeAccessWindowDays = page.limits?.freeAccessWindowDays ?? 0 + self.freeAccessWindowDays = freeAccessWindowDays + var snapshots = page.vms.map { + MachineSnapshotBuilder.snapshot( + from: $0, + freeAccessWindowDays: freeAccessWindowDays, + previousStats: previous[$0.id] + ) + } + snapshots = MachineSnapshotBuilder.applyingUsage(to: snapshots, usage: usageByMachineID) + let defaultMachineID = defaultMachineStore?.resolveMachineID( + from: snapshots.map { CloudMachineDescriptor(id: $0.id, isDesktop: $0.isDesktop) }, + isComplete: true + ) + snapshots = snapshots.map { snapshot in + var next = snapshot + next.isDefault = snapshot.id == defaultMachineID + return next + } + // The authoritative fleet plus catalog-only rows is the complete + // visible set: a pin whose machine is gone from both is pruned. + machinePinStore?.reconcile(machineIDs: MachineSnapshotBuilder.includingCatalogMachines(snapshots, catalog: scopedCatalogSnapshot()).map(\.id)) + machineIndexByID = Dictionary(uniqueKeysWithValues: snapshots.enumerated().map { ($0.element.id, $0.offset) }) + machines = snapshots + lastLimits = page.limits + scheduleFreeAccessTransition() + refreshStats() + refreshUsage() + readCatalog() + plan = MachineSnapshotBuilder.planSnapshot(activeCount: snapshots.count, limits: page.limits, machines: snapshots) + lastErrorDescription = nil + listProblem = nil + } catch is CancellationError { + return + } catch let error as VMClientError { + guard !Task.isCancelled, generation == refreshGeneration, + scope == machinePinStore?.scopeIdentifier else { return } + if case .notSignedIn = error { + machines = [] + machineIndexByID.removeAll() + plan = nil + activeOperation = nil + lastErrorDescription = nil + listProblem = nil + hasLoadedOnce = false + isLoading = false + return + } + lastErrorDescription = String(describing: error) + listProblem = Self.classifyListFailure(error) + } catch { + guard !Task.isCancelled, generation == refreshGeneration, + scope == machinePinStore?.scopeIdentifier else { return } + lastErrorDescription = String(describing: error) + listProblem = .unreachable + } + isLoading = false + hasLoadedOnce = true + } +} diff --git a/Sources/Cloud/MachinesPanelViewModel.swift b/Sources/Cloud/MachinesPanelViewModel.swift index 47ee519fc017..4c074e43eab5 100644 --- a/Sources/Cloud/MachinesPanelViewModel.swift +++ b/Sources/Cloud/MachinesPanelViewModel.swift @@ -11,21 +11,21 @@ extension Notification.Name { /// through this store. @MainActor final class MachinesPanelViewModel: ObservableObject { - @Published private(set) var machines: [MachineSnapshot] = [] - @Published private(set) var plan: MachinePlanSnapshot? - @Published private(set) var isLoading = false - @Published private(set) var hasLoadedOnce = false - @Published private(set) var lastErrorDescription: String? + @Published var machines: [MachineSnapshot] = [] + @Published var plan: MachinePlanSnapshot? + @Published var isLoading = false + @Published var hasLoadedOnce = false + @Published var lastErrorDescription: String? /// Why the machine list could not load, classified so the empty state can /// say the true thing: a server-rejected session needs a fresh sign-in, a /// plan gate needs an upgrade, and only genuinely transient failures get /// the retry-first "unreachable" presentation. - @Published private(set) var listProblem: CloudListProblem? + @Published var listProblem: CloudListProblem? /// Per-machine coderouter spend from the last successful usage fetch, /// keyed by machine id. Refreshed with every machine-list refresh (the /// slow poll and the explicit Refresh verb), never more often. Empty on /// backends without the usage route; a failed fetch keeps the last value. - @Published private(set) var usageByMachineID: [String: MachineUsageSnapshot] = [:] + @Published var usageByMachineID: [String: MachineUsageSnapshot] = [:] enum CloudListProblem: Equatable { /// HTTP 401: the Cloud service no longer accepts this session. @@ -53,7 +53,7 @@ final class MachinesPanelViewModel: ObservableObject { /// Human-readable label of the Cloud VM action currently running from this /// panel ("Checkpointing noble-wren…"). Replaces the plan meter in the /// header while set — the in-app substitute for a floating progress HUD. - @Published private(set) var activeOperation: String? + @Published var activeOperation: String? /// The surface catalog as one value: machines (this Mac first), their /// terminals/screens/browsers, and which local panes project them. @Published private(set) var catalog: SurfaceCatalogSnapshot = .empty @@ -102,51 +102,61 @@ final class MachinesPanelViewModel: ObservableObject { treeErrorDescription = description } - private var refreshTask: Task? - private var pollTask: Task? - private var statsTask: Task? + var refreshTask: Task? + var statsID: UUID? + let client: VMClient? + let isCloudEnabled: @MainActor () -> Bool + let pollingClock: any Clock + private var networkObserver: NSObjectProtocol? + var pollTask: Task? + var statsTask: Task? private var resourceUpdatesTask: Task? - private let resourceStats: VMResourceStatsStore? - private var machineIndexByID: [String: Int] = [:] - private var usageTask: Task? - private var usageFailureCount = 0 - private var usageRetryNotBefore: Date? + let resourceStats: VMResourceStatsStore? + var machineIndexByID: [String: Int] = [:] + var usageTask: Task? + var usageFailureCount = 0 + var usageRetryNotBefore: Date? /// One-shot timer armed at the exact next free-access transition (a /// countdown day-boundary or an expiry). Expiry is client-computable from /// createdAt + window, so rows flip at the boundary itself — scheduling, /// not polling; the slow poll only covers changes made elsewhere. - private var freeAccessTransitionTask: Task? - private var freeAccessWindowDays = 0 + var freeAccessTransitionTask: Task? + var freeAccessWindowDays = 0 /// Last plan limits the list returned; the banner countdown re-derives from /// these on every local recompute without another round trip. - private var lastLimits: VMPlanLimits? + var lastLimits: VMPlanLimits? var memoryOptionsMb: [Int] { lastLimits?.memoryOptionsMb ?? [] } var lockedMemoryOptionsMb: [Int]? { lastLimits?.lockedMemoryOptionsMb } var memoryUpgradePlanId: String? { lastLimits?.memoryUpgradePlanId } var memoryUpgradePlansByMb: [String: String]? { lastLimits?.memoryUpgradePlansByMb } private var authScopeObservers: [NSObjectProtocol] = [] private var featureFlagObserver: CloudFeatureAvailabilityObserver? - private var wantsPolling = false + var wantsPolling = false private var treeChangeObserver: NSObjectProtocol? private var createChangeObserver: NSObjectProtocol? - private var treeTask: Task? - private let machineRefreshes = CloudMachineRefreshCoordinator { await SurfaceCatalog.shared.refresh(machine: $0, force: true) } - private static let statsInterval: Duration = .seconds(20) + var treeTask: Task? + let machineRefreshes = CloudMachineRefreshCoordinator { await SurfaceCatalog.shared.refresh(machine: $0, force: true) } let defaultMachineStore: DefaultCloudMachineStore? /// Explicit machine pins and the stable fleet order; nil keeps fleet order. let machinePinStore: CloudMachinePinStore? private let catalogProvider: @MainActor () -> SurfaceCatalogSnapshot - private var awaitingCatalogScope = false + var awaitingCatalogScope = false init( createCoordinator: MachineCreateCoordinator? = nil, defaultMachineStore: DefaultCloudMachineStore? = nil, machinePinStore: CloudMachinePinStore? = nil, resourceStats: VMResourceStatsStore? = nil, + client: VMClient? = nil, + pollingClock: any Clock = ContinuousClock(), + isCloudEnabled: @escaping @MainActor () -> Bool = { CloudMachinesFeature.isEnabled }, catalogProvider: @escaping @MainActor () -> SurfaceCatalogSnapshot = { SurfaceCatalog.shared.snapshot } ) { - self.resourceStats = resourceStats ?? VMClient.shared?.resourceStats + self.client = client + self.pollingClock = pollingClock + self.isCloudEnabled = isCloudEnabled + self.resourceStats = resourceStats ?? client?.resourceStats ?? VMClient.shared?.resourceStats self.defaultMachineStore = defaultMachineStore self.machinePinStore = machinePinStore self.catalogProvider = catalogProvider @@ -172,7 +182,7 @@ final class MachinesPanelViewModel: ObservableObject { } } featureFlagObserver = CloudFeatureAvailabilityObserver( - isEnabled: { CloudMachinesFeature.isEnabled }, + isEnabled: isCloudEnabled, didChange: { [weak self] enabled in guard let self else { return } if enabled, self.wantsPolling { self.startPolling() } @@ -196,6 +206,19 @@ final class MachinesPanelViewModel: ObservableObject { MainActor.assumeIsolated { self?.readUnreadTerminalIDs() } } readUnreadTerminalIDs() + networkObserver = NotificationCenter.default.addObserver(forName: .cmuxCloudReadNetworkChanged, object: nil, queue: .main) { [weak self] notification in + guard let online = notification.userInfo?["isOnline"] as? Bool else { return } + MainActor.assumeIsolated { + guard let self else { return } + if online { + if self.wantsPolling { self.refresh() } + } else { + self.clearUnavailableMetrics() + self.lastErrorDescription = URLError(.notConnectedToInternet).localizedDescription + self.listProblem = .unreachable + } + } + } if let resourceStats = self.resourceStats { let changes = resourceStats.changes() resourceUpdatesTask = Task { [weak self] in @@ -235,6 +258,13 @@ final class MachinesPanelViewModel: ObservableObject { } } deinit { + refreshTask?.cancel() + pollTask?.cancel() + statsTask?.cancel() + usageTask?.cancel() + treeTask?.cancel() + freeAccessTransitionTask?.cancel() + if let networkObserver { NotificationCenter.default.removeObserver(networkObserver) } resourceUpdatesTask?.cancel() for observer in authScopeObservers { NotificationCenter.default.removeObserver(observer) @@ -313,30 +343,6 @@ final class MachinesPanelViewModel: ObservableObject { /// `asleep` without being woken, so polling never costs the user anything. /// Older servers omitting the flag retain the desktop-only polling policy /// through capability decoding; explicit support overrides that fallback. - func refreshStats() { - guard CloudMachinesFeature.isEnabled, let client = VMClient.shared else { return } - statsTask?.cancel() - let ids = machines.filter { $0.capabilities.stats }.map(\.id) - statsTask = Task { - await withTaskGroup(of: Void.self) { group in - for id in ids { - group.addTask { _ = try? await client.stats(id: id) } - } - } - } - } - - /// Read the shared owner's current snapshot, never a delayed poll's raw result. - private func applyResourceStats(machineIDs: Set?) { - guard CloudMachinesFeature.isEnabled, let resourceStats else { return } - for id in machineIDs ?? Set(machineIndexByID.keys) { - guard let index = machineIndexByID[id], machines.indices.contains(index), - machines[index].id == id, machines[index].capabilities.stats else { continue } - let stats = resourceStats.stats(for: id) - if machines[index].stats != stats { machines[index].stats = stats } - } - } - func refreshUsage() { guard CloudMachinesFeature.isEnabled, usageTask == nil else { return } if let retryNotBefore = usageRetryNotBefore, retryNotBefore > Date() { return } @@ -364,81 +370,18 @@ final class MachinesPanelViewModel: ObservableObject { usageByMachineID = usage machines = MachineSnapshotBuilder.applyingUsage(to: machines, usage: usage) } - private static let pollInterval: Duration = .seconds(45) + static let pollInterval: Duration = .seconds(45) /// A refresh asked for while one is in flight runs again afterwards: a /// create that lands mid-poll must still replace its pending row with the /// real machine now, not on the next 45 s sweep. - private var refreshRequestedWhileLoading = false + var refreshRequestedWhileLoading = false /// Invalidates refresh completions when the Cloud gate closes. A cancelled /// URLSession task may still resume on the main actor, so cancellation /// alone is not enough to prevent stale rows or follow-up work. - private(set) var refreshGeneration: UInt64 = 0 - func refresh() { - guard CloudMachinesFeature.isEnabled else { return } - guard refreshTask == nil else { - refreshRequestedWhileLoading = true - return - } - isLoading = true - let generation = refreshGeneration - refreshTask = Task { [weak self] in - await self?.performRefresh() - guard let self else { return } - guard generation == self.refreshGeneration else { return } - self.refreshTask = nil - if self.refreshRequestedWhileLoading { - self.refreshRequestedWhileLoading = false - self.refresh() - } - } - } - func startPolling() { - wantsPolling = true - guard CloudMachinesFeature.isEnabled else { - pausePolling() - return - } - refresh() - guard pollTask == nil else { return } - pollTask = Task { [weak self] in - while !Task.isCancelled { - try? await Task.sleep(for: Self.pollInterval) - guard !Task.isCancelled else { return } - guard let self else { return } - self.refresh() - } - } - } - - func stopPolling() { - wantsPolling = false - pausePolling() - } - - private func pausePolling() { - pollTask?.cancel() - pollTask = nil - refreshTask?.cancel() - refreshTask = nil - refreshRequestedWhileLoading = false - refreshGeneration &+= 1 - isLoading = false - statsTask?.cancel() - statsTask = nil - usageTask?.cancel() - usageTask = nil - usageFailureCount = 0 - usageRetryNotBefore = nil - treeTask?.cancel() - treeTask = nil - machineRefreshes.cancelAll() - freeAccessTransitionTask?.cancel() - freeAccessTransitionTask = nil - } - + var refreshGeneration: UInt64 = 0 /// Sleeps until the earliest upcoming transition across the fleet, then /// recomputes the free-access facet locally and re-arms for the next one. - private func scheduleFreeAccessTransition(now: Date = Date()) { + func scheduleFreeAccessTransition(now: Date = Date()) { freeAccessTransitionTask?.cancel() freeAccessTransitionTask = nil guard freeAccessWindowDays > 0 else { return } @@ -519,80 +462,5 @@ final class MachinesPanelViewModel: ObservableObject { return scoped } - private func performRefresh() async { - guard CloudMachinesFeature.isEnabled else { - isLoading = false - return - } - guard let client = VMClient.shared else { - isLoading = false - return - } - let generation = refreshGeneration - let scope = machinePinStore?.scopeIdentifier - do { - let page = try await client.listPage() - try Task.checkCancellation() - guard generation == refreshGeneration, scope == machinePinStore?.scopeIdentifier, - CloudMachinesFeature.isEnabled else { return } - let previous = resourceStats?.snapshot ?? [:] - let freeAccessWindowDays = page.limits?.freeAccessWindowDays ?? 0 - self.freeAccessWindowDays = freeAccessWindowDays - var snapshots = page.vms.map { - MachineSnapshotBuilder.snapshot( - from: $0, - freeAccessWindowDays: freeAccessWindowDays, - previousStats: previous[$0.id] - ) - } - snapshots = MachineSnapshotBuilder.applyingUsage(to: snapshots, usage: usageByMachineID) - let defaultMachineID = defaultMachineStore?.resolveMachineID( - from: snapshots.map { CloudMachineDescriptor(id: $0.id, isDesktop: $0.isDesktop) }, - isComplete: true - ) - snapshots = snapshots.map { snapshot in - var next = snapshot - next.isDefault = snapshot.id == defaultMachineID - return next - } - // The authoritative fleet plus catalog-only rows is the complete - // visible set: a pin whose machine is gone from both is pruned. - machinePinStore?.reconcile(machineIDs: MachineSnapshotBuilder.includingCatalogMachines(snapshots, catalog: scopedCatalogSnapshot()).map(\.id)) - machineIndexByID = Dictionary(uniqueKeysWithValues: snapshots.enumerated().map { ($0.element.id, $0.offset) }) - machines = snapshots - lastLimits = page.limits - scheduleFreeAccessTransition() - refreshStats() - refreshUsage() - readCatalog() - plan = MachineSnapshotBuilder.planSnapshot(activeCount: snapshots.count, limits: page.limits, machines: snapshots) - lastErrorDescription = nil - listProblem = nil - } catch is CancellationError { - return - } catch let error as VMClientError { - guard !Task.isCancelled, generation == refreshGeneration, - scope == machinePinStore?.scopeIdentifier else { return } - if case .notSignedIn = error { - machines = [] - machineIndexByID.removeAll() - plan = nil - activeOperation = nil - lastErrorDescription = nil - listProblem = nil - hasLoadedOnce = false - isLoading = false - return - } - lastErrorDescription = String(describing: error) - listProblem = Self.classifyListFailure(error) - } catch { - guard !Task.isCancelled, generation == refreshGeneration, - scope == machinePinStore?.scopeIdentifier else { return } - lastErrorDescription = String(describing: error) - listProblem = .unreachable - } - isLoading = false - hasLoadedOnce = true - } + } diff --git a/Sources/Cloud/TeamMachineUsage.swift b/Sources/Cloud/TeamMachineUsage.swift new file mode 100644 index 000000000000..fffd41f7287f --- /dev/null +++ b/Sources/Cloud/TeamMachineUsage.swift @@ -0,0 +1,30 @@ +import Foundation + +/// The team-wide usage payload. `kind == .unavailable` means the backend has no +/// usage store for this team (no rows are rendered, no error is surfaced). +struct TeamMachineUsage: Equatable, Sendable { + enum Kind: String, Sendable { + case ready + case unavailable + } + + let teamID: String + let periodDays: Int + let kind: Kind + let asOf: Date? + let machines: [MachineUsageSnapshot] + + /// The lookup the machines panel keys rows by. Empty when the backend says + /// usage is unavailable; blank ids are dropped; the first entry wins when + /// the backend repeats a machine. + var byMachineID: [String: MachineUsageSnapshot] { + guard kind == .ready else { return [:] } + var result: [String: MachineUsageSnapshot] = [:] + for machine in machines { + for key in [machine.providerVmID ?? "", machine.vmID] where !key.isEmpty && result[key] == nil { + result[key] = machine + } + } + return result + } +} diff --git a/Sources/Cloud/VMCapabilities.swift b/Sources/Cloud/VMCapabilities.swift new file mode 100644 index 000000000000..2956b442aaf1 --- /dev/null +++ b/Sources/Cloud/VMCapabilities.swift @@ -0,0 +1,76 @@ +import Foundation + +/// What a machine's provider can do; the app offers only verbs that can succeed +/// (Checkpoint/Fork disappear from menus when false — a verb that answers 502 +/// "not implemented" is not a verb). +struct VMCapabilities: Equatable, Sendable { + var snapshot: Bool + var restore: Bool + var fork: Bool + var exec: Bool + var stats: Bool + /// The provider can mint a browser preview URL for a machine port. + var ports: Bool + var desktop: Bool + var sizing: Bool + var persistentHome: Bool + var attachTransports: [String]? + + var ssh: Bool { attachTransports?.contains("ssh") ?? true } + var cmuxRemote: Bool { attachTransports?.contains("cmux-remote") ?? true } + + static let all = VMCapabilities( + snapshot: true, restore: true, fork: true, + exec: true, stats: true, ports: true, desktop: true, + sizing: true, persistentHome: true, attachTransports: nil) + + init( + snapshot: Bool, restore: Bool, fork: Bool, + exec: Bool = true, stats: Bool = true, ports: Bool = true, + desktop: Bool = true, sizing: Bool = true, persistentHome: Bool = true, + attachTransports: [String]? = nil + ) { + self.snapshot = snapshot + self.restore = restore + self.fork = fork + self.exec = exec + self.stats = stats + self.ports = ports + self.desktop = desktop + self.sizing = sizing + self.persistentHome = persistentHome + self.attachTransports = attachTransports + } + + /// Missing flags preserve legacy support; stats can use the historical kind fallback. + init(json: Any?, legacyStatsSupported: Bool = true) { + let dict = json as? [String: Any] + func flag(_ key: String, fallback: Bool = true) -> Bool { + if let value = dict?[key] as? Bool { return value } + if let number = dict?[key] as? NSNumber { return number.boolValue } + return fallback + } + let transports = (dict?["attachTransports"] as? [Any] ?? dict?["attach_transports"] as? [Any])? + .compactMap { $0 as? String } + self.init( + snapshot: flag("snapshot"), restore: flag("restore"), fork: flag("fork"), + exec: flag("exec"), stats: flag("stats", fallback: legacyStatsSupported), ports: flag("ports"), + desktop: flag("desktop"), sizing: flag("sizing"), + persistentHome: flag("persistentHome"), attachTransports: transports) + } + + var jsonObject: [String: Any] { + var object: [String: Any] = [ + "snapshot": snapshot, "restore": restore, "fork": fork, + "exec": exec, "stats": stats, "ports": ports, "desktop": desktop, + "sizing": sizing, "persistentHome": persistentHome, + ] + if let attachTransports { object["attach_transports"] = attachTransports } + return object + } + + init(vmResponse: [String: Any]) { + let kind = VMMachineKind.resolved(kind: vmResponse["kind"], image: vmResponse["image"]) + self.init(json: vmResponse["capabilities"], legacyStatsSupported: kind.hasDesktop) + } +} diff --git a/Sources/Cloud/VMClient.swift b/Sources/Cloud/VMClient.swift index 1928580b838c..b38fac3eece4 100644 --- a/Sources/Cloud/VMClient.swift +++ b/Sources/Cloud/VMClient.swift @@ -33,6 +33,7 @@ extension URLError.Code { } } + func formattedCloudVMHTTPError(status: Int, body: String) -> String { let trimmedBody = body.trimmingCharacters(in: .whitespacesAndNewlines) guard let data = trimmedBody.data(using: .utf8), @@ -347,81 +348,6 @@ struct VMExecResult: Sendable { let stderr: String } -/// What a machine's provider can do; the app offers only verbs that can succeed -/// (Checkpoint/Fork disappear from menus when false — a verb that answers 502 -/// "not implemented" is not a verb). -struct VMCapabilities: Equatable, Sendable { - var snapshot: Bool - var restore: Bool - var fork: Bool - var exec: Bool - var stats: Bool - /// The provider can mint a browser preview URL for a machine port. - var ports: Bool - var desktop: Bool - var sizing: Bool - var persistentHome: Bool - var attachTransports: [String]? - - var ssh: Bool { attachTransports?.contains("ssh") ?? true } - var cmuxRemote: Bool { attachTransports?.contains("cmux-remote") ?? true } - - static let all = VMCapabilities( - snapshot: true, restore: true, fork: true, - exec: true, stats: true, ports: true, desktop: true, - sizing: true, persistentHome: true, attachTransports: nil) - - init( - snapshot: Bool, restore: Bool, fork: Bool, - exec: Bool = true, stats: Bool = true, ports: Bool = true, - desktop: Bool = true, sizing: Bool = true, persistentHome: Bool = true, - attachTransports: [String]? = nil - ) { - self.snapshot = snapshot - self.restore = restore - self.fork = fork - self.exec = exec - self.stats = stats - self.ports = ports - self.desktop = desktop - self.sizing = sizing - self.persistentHome = persistentHome - self.attachTransports = attachTransports - } - - /// Missing flags preserve legacy support; stats can use the historical kind fallback. - init(json: Any?, legacyStatsSupported: Bool = true) { - let dict = json as? [String: Any] - func flag(_ key: String, fallback: Bool = true) -> Bool { - if let value = dict?[key] as? Bool { return value } - if let number = dict?[key] as? NSNumber { return number.boolValue } - return fallback - } - let transports = (dict?["attachTransports"] as? [Any] ?? dict?["attach_transports"] as? [Any])? - .compactMap { $0 as? String } - self.init( - snapshot: flag("snapshot"), restore: flag("restore"), fork: flag("fork"), - exec: flag("exec"), stats: flag("stats", fallback: legacyStatsSupported), ports: flag("ports"), - desktop: flag("desktop"), sizing: flag("sizing"), - persistentHome: flag("persistentHome"), attachTransports: transports) - } - - var jsonObject: [String: Any] { - var object: [String: Any] = [ - "snapshot": snapshot, "restore": restore, "fork": fork, - "exec": exec, "stats": stats, "ports": ports, "desktop": desktop, - "sizing": sizing, "persistentHome": persistentHome, - ] - if let attachTransports { object["attach_transports"] = attachTransports } - return object - } - - init(vmResponse: [String: Any]) { - let kind = VMMachineKind.resolved(kind: vmResponse["kind"], image: vmResponse["image"]) - self.init(json: vmResponse["capabilities"], legacyStatsSupported: kind.hasDesktop) - } -} - struct VMOpenPortEndpoint { let url: String let token: String @@ -551,7 +477,6 @@ struct VMPublicationDomain: Equatable, Sendable { } } - /// One row of `GET /api/vm//snapshots`: the provider snapshot id, its display name /// when one was given, and the creation time as the ISO-8601 string the server sent. struct VMSnapshotSummary: Sendable, Equatable { @@ -699,8 +624,17 @@ actor VMClient { /// Build the shared client with its injected auth dependency. Call once at /// the composition root. @MainActor - static func bootstrap(auth: AuthCoordinator, session: URLSession = .shared, operations: CloudOperationRecorder? = nil) { - shared = VMClient(session: session, auth: auth, resourceStats: VMResourceStatsStore(), checkpointRenames: SurfaceCatalog.shared.cloudRenameCoordinator, operations: operations, isCloudEnabled: { CloudMachinesFeature.offMainIsEnabled() }) + @discardableResult + static func bootstrap(auth: AuthCoordinator, session: URLSession = .shared, operations: CloudOperationRecorder? = nil) -> CloudReadRequestCoordinator { + let reads = CloudReadRequestCoordinator(onNetworkChange: { online in + await MainActor.run { + NotificationCenter.default.post(name: .cmuxCloudReadNetworkChanged, object: nil, userInfo: ["isOnline": online]) + if online { NotificationCenter.default.post(name: .cmuxCloudReadNetworkRecovered, object: nil) } + } + }) + shared = VMClient(session: session, auth: auth, resourceStats: VMResourceStatsStore(), checkpointRenames: SurfaceCatalog.shared.cloudRenameCoordinator, operations: operations, readRequests: reads, isCloudEnabled: { CloudMachinesFeature.offMainIsEnabled() }) + Task { await reads.observeNetwork(CloudReadNetworkMonitor()) } + return reads } /// Revoke endpoint credentials issued by the Cloud VM service during sign-out. @@ -744,6 +678,7 @@ actor VMClient { nonisolated let operations: CloudOperationRecorder? nonisolated let resourceStats: VMResourceStatsStore private let machineCache: CloudMachineCache + private let readRequests: CloudReadRequestCoordinator private let isCloudEnabled: @Sendable () -> Bool private let isDisabledByManagedPolicy: (@Sendable () -> Bool)? @@ -756,15 +691,17 @@ actor VMClient { operations: CloudOperationRecorder? = nil, machineCache: CloudMachineCache = CloudMachineCache(), isDisabledByManagedPolicy: (@Sendable () -> Bool)? = nil, + readRequests: CloudReadRequestCoordinator = CloudReadRequestCoordinator(), isCloudEnabled: @escaping @Sendable () -> Bool = { true } ) { self.session = session - self.resourceStats = resourceStats self.auth = auth + self.resourceStats = resourceStats self.checkpointRenames = checkpointRenames self.telemetry = telemetry self.operations = operations self.machineCache = machineCache + self.readRequests = readRequests self.isCloudEnabled = isCloudEnabled self.isDisabledByManagedPolicy = isDisabledByManagedPolicy } @@ -1998,6 +1935,7 @@ actor VMClient { // safety net when this tail cannot reach the API. } } + private func revokeCloudAccess( deviceID: String, accessToken: String?, @@ -2042,9 +1980,35 @@ actor VMClient { allowedUnderManagedPolicy: Bool = false ) async throws -> (Data, HTTPURLResponse) { let work = { - try await self.requestMeasured(method, path: path, jsonBody: jsonBody, extraHeaders: extraHeaders, - timeoutSeconds: timeoutSeconds, retryTransientServiceUnavailable: retryTransientServiceUnavailable, - allowedUnderManagedPolicy: allowedUnderManagedPolicy) + let isSharedRead = method == "GET" && (path == "/api/vm" || path.hasSuffix("/stats")) + if !isSharedRead { + return try await self.requestMeasured(method, path: path, jsonBody: jsonBody, extraHeaders: extraHeaders, + timeoutSeconds: timeoutSeconds, retryTransientServiceUnavailable: retryTransientServiceUnavailable, + allowedUnderManagedPolicy: allowedUnderManagedPolicy) + } + try Task.checkCancellation() + if !allowedUnderManagedPolicy, self.isDisabledByManagedPolicy?() == true { + throw VMClientError.disabledByManagedPolicy + } + let context = CloudOperationContext.current + let elapsed = context.map { $0.operation == .list || $0.operation == .stats ? $0.clock.duration(to: .now) : .zero } ?? .zero + let deadline = self.readRequests.makeDeadline(elapsed: elapsed) + let identity = await self.auth.authenticatedSessionIdentity + let teamID = await self.auth.resolvedTeamID + let key = CloudReadRequestCoordinator.Key(path: path, accountID: identity?.accountID, + generation: identity?.generation, teamID: teamID) + let value = try await self.readRequests.read(key, deadline: deadline) { + try await CloudOperationContext.$current.withValue(context) { + let (data, http) = try await self.requestMeasured(method, path: path, timeoutSeconds: timeoutSeconds) + return CloudReadRequestCoordinator.Response(data: data, http: http) + } + } + try Task.checkCancellation() + if let identity { + guard await self.auth.isAuthenticatedSessionIdentityCurrent(identity), + await self.auth.resolvedTeamID == teamID else { throw CancellationError() } + } + return (value.data, value.http) } if CloudOperationContext.current != nil || operations == nil { return try await work() } let kind: CloudOperationKind = path == "/api/vm" ? (method == "GET" ? .list : .create) : .resolve(path) @@ -2109,6 +2073,7 @@ actor VMClient { errorCode: http.statusCode >= 400 ? Self.cloudVMErrorCode(http: http, data: data) : nil, serverTraceId: Self.cloudVMServerTraceId(http: http, data: data) )) + if method != "GET", (200...299).contains(http.statusCode) { await readRequests.invalidate() } return (data, http) } catch let error as VMClientError { record(.transportFailure(kind: Self.transportFailureKind(error), detail: Self.transportFailureDetail(error))) @@ -2166,6 +2131,7 @@ actor VMClient { } return traceId } + private func performRequest( _ method: String, path: String, @@ -2265,6 +2231,12 @@ actor VMClient { guard let http = response as? HTTPURLResponse else { throw VMClientError.malformedResponse("non-HTTP response") } + if http.statusCode == 429, let read = CloudReadRequestCoordinator.current, let owner = read.owner { + let seconds = Self.retryDelaySeconds(statusCode: 429, retryAfterHeader: http.value(forHTTPHeaderField: "Retry-After")) ?? 60 + let canRetry = await owner.noteRetryAfter(read.key, seconds: seconds, + response: .init(data: data, http: http)) + if !canRetry { return (data, http) } + } if http.statusCode == 429, retriesLeft > 0 { retriesLeft -= 1 onRetry() @@ -2521,63 +2493,6 @@ actor VMClient { // MARK: - Per-machine coderouter usage -/// Token and spend totals for one machine over the usage window, as -/// `GET /api/coderouter/vm-usage/team` reports them. -struct MachineUsageTotals: Equatable, Sendable { - let inputTokens: Int - let cachedInputTokens: Int - let outputTokens: Int - let totalTokens: Int - /// What the same traffic would have cost at list API prices. - let apiEquivalentUsd: Double - - /// Nothing to show for a machine that has not routed a single token. - var isEmpty: Bool { totalTokens <= 0 && apiEquivalentUsd <= 0 } -} - -/// One machine's usage readout: the row shows `totals` labeled with -/// `periodDays`. `vmID` is the id `GET /api/vm` returns as the machine id, so -/// it matches ``MachineSnapshot/id`` directly. -struct MachineUsageSnapshot: Equatable, Sendable { - let vmID: String - /// The provider machine id, the `id` that `GET /api/vm` lists. Rows key - /// on it when present because `vmID` is the backend's own uuid. - let providerVmID: String? - let displayName: String? - let periodDays: Int - let asOf: Date? - let totals: MachineUsageTotals -} - -/// The team-wide usage payload. `kind == .unavailable` means the backend has no -/// usage store for this team (no rows are rendered, no error is surfaced). -struct TeamMachineUsage: Equatable, Sendable { - enum Kind: String, Sendable { - case ready - case unavailable - } - - let teamID: String - let periodDays: Int - let kind: Kind - let asOf: Date? - let machines: [MachineUsageSnapshot] - - /// The lookup the machines panel keys rows by. Empty when the backend says - /// usage is unavailable; blank ids are dropped; the first entry wins when - /// the backend repeats a machine. - var byMachineID: [String: MachineUsageSnapshot] { - guard kind == .ready else { return [:] } - var result: [String: MachineUsageSnapshot] = [:] - for machine in machines { - for key in [machine.providerVmID ?? "", machine.vmID] where !key.isEmpty && result[key] == nil { - result[key] = machine - } - } - return result - } -} - enum MachineUsageClientError: Error, CustomStringConvertible { case notSignedIn case sessionRefreshFailed @@ -2610,18 +2525,23 @@ actor MachineUsageClient { @MainActor private(set) static var shared: MachineUsageClient? @MainActor - static func bootstrap(auth: AuthCoordinator, session: URLSession = .shared, operations: CloudOperationRecorder? = nil) { - shared = MachineUsageClient(session: session, auth: auth, operations: operations) + static func bootstrap(auth: AuthCoordinator, session: URLSession = .shared, operations: CloudOperationRecorder? = nil, + readRequests: CloudReadRequestCoordinator? = nil) { + shared = MachineUsageClient(session: session, auth: auth, operations: operations, + readRequests: readRequests ?? CloudReadRequestCoordinator(budget: .seconds(15))) } private let session: URLSession private let auth: AuthCoordinator + private let readRequests: CloudReadRequestCoordinator nonisolated let operations: CloudOperationRecorder? - init(session: URLSession = .shared, auth: AuthCoordinator, operations: CloudOperationRecorder? = nil) { + init(session: URLSession = .shared, auth: AuthCoordinator, operations: CloudOperationRecorder? = nil, + readRequests: CloudReadRequestCoordinator = CloudReadRequestCoordinator(budget: .seconds(15))) { self.session = session self.auth = auth self.operations = operations + self.readRequests = readRequests } private func withOperation(_ kind: CloudOperationKind, foreground: Bool, _ work: () async throws -> T) async rethrows -> T { @@ -2631,7 +2551,31 @@ actor MachineUsageClient { func teamUsage(teamID: String? = nil) async throws -> TeamMachineUsage { return try await withOperation(.stats, foreground: false) { - let (data, _) = try await request("GET", path: "/api/coderouter/vm-usage/team", teamID: teamID) + let context = CloudOperationContext.current + let deadline = readRequests.makeDeadline(elapsed: context.map { $0.operation == .stats ? $0.clock.duration(to: .now) : .zero } ?? .zero, limit: .seconds(15)) + let identity = await auth.authenticatedSessionIdentity + let selectedTeam = await auth.resolvedTeamID + let explicitTeam = teamID?.trimmingCharacters(in: .whitespacesAndNewlines) + let key = CloudReadRequestCoordinator.Key(path: "/api/coderouter/vm-usage/team", accountID: identity?.accountID, + generation: identity?.generation, teamID: explicitTeam?.isEmpty == false ? explicitTeam : selectedTeam) + let response = try await readRequests.read(key, deadline: deadline) { + try await CloudOperationContext.$current.withValue(context) { + let (data, http) = try await self.request("GET", path: key.path, teamID: teamID) + if http.statusCode == 429 { + let seconds = TimeInterval(CmxRetryAfterPolicy.seconds(from: http) ?? CmxRetryAfterPolicy.defaultRateLimitSeconds) + _ = await self.readRequests.noteRetryAfter(key, seconds: seconds, response: .init(data: data, http: http)) + } + return CloudReadRequestCoordinator.Response(data: data, http: http) + } + } + if let identity { + guard await auth.isAuthenticatedSessionIdentityCurrent(identity), + await auth.resolvedTeamID == selectedTeam else { throw CancellationError() } + } + let (data, http) = (response.data, response.http) + guard (200...299).contains(http.statusCode) else { + throw MachineUsageClientError.httpStatus(http.statusCode, String(data: data, encoding: .utf8) ?? "") + } return try Self.decodeTeamUsage(data) } } @@ -2722,6 +2666,7 @@ actor MachineUsageClient { } catch { throw MachineUsageClientError.notSignedIn } + try Task.checkCancellation() let resolvedTeamID = await auth.resolvedTeamID guard var comps = URLComponents(url: AuthEnvironment.vmAPIBaseURL, resolvingAgainstBaseURL: false) else { diff --git a/Sources/Surfaces/CmuxTuiSurfaceProviderRegistry+Routes.swift b/Sources/Surfaces/CmuxTuiSurfaceProviderRegistry+Routes.swift new file mode 100644 index 000000000000..8ef401521856 --- /dev/null +++ b/Sources/Surfaces/CmuxTuiSurfaceProviderRegistry+Routes.swift @@ -0,0 +1,42 @@ +import CmuxSettings +import Foundation + +/// Routes Cloud traffic through the current account-owned registry. +extension CmuxTuiSurfaceProviderRegistry { + /// The headless link's local mux socket for a machine, connecting if needed. + func linkSocketPath(machineID: String) async throws -> (socketPath: String, session: String) { + guard isCloudEnabled() else { + throw CloudMachineLinkManager.ManagerError.retryLater(String( + localized: "cloud.feature.disabled", + defaultValue: "Cloud Machines are temporarily unavailable." + )) + } + let connected = try await links.connected(machineID: machineID) + return (connected.socketPath, connected.session) + } + + func privateRoute(machineID: String) async -> String? { + guard !isRetired, !ManagedDevicePolicy().isEnforced(.disableCloud), isCloudEnabled(), !Task.isCancelled else { + return nil + } + let epoch = accessEpoch + // The persisted device marker outlives this in-memory registry. An + // explicit open must discover its machine before using the saved-device + // shortcut, even when the first background fleet read has not run. + guard await providerRefreshingIfMissing(machineID: machineID) != nil else { return nil } + let route = await links.privateRoute(for: machineID) + guard !isRetired, epoch == accessEpoch, isCloudEnabled(), !Task.isCancelled else { return nil } + return route + } + + func resolvedPrivateRoute(machineID: String, through hub: CloudWireGuardHub.Ready, fallbackRoute: String, addresses: [String]) async throws -> String { + guard isCloudEnabled() else { + throw CloudMachineLinkManager.ManagerError.retryLater(String( + localized: "cloud.feature.disabled", + defaultValue: "Cloud Machines are temporarily unavailable." + )) + } + return try await links.resolvedPrivateRoute(machineID: machineID, through: hub, fallbackRoute: fallbackRoute, addresses: addresses) + } + +} diff --git a/Sources/Surfaces/CmuxTuiSurfaceProviderRegistry.swift b/Sources/Surfaces/CmuxTuiSurfaceProviderRegistry.swift index 0a19bd4b36d6..494033d3b672 100644 --- a/Sources/Surfaces/CmuxTuiSurfaceProviderRegistry.swift +++ b/Sources/Surfaces/CmuxTuiSurfaceProviderRegistry.swift @@ -16,7 +16,7 @@ final class CmuxTuiSurfaceProviderRegistry { private var catalog: SurfaceCatalog? private var providers: [String: CmuxTuiSurfaceProvider] = [:] - private let links: CloudMachineLinkManager + let links: CloudMachineLinkManager /// The app's one WireGuard hub for private-network machines; nil when no cmux-tui /// client is bundled (then no link can be made at all). nonisolated let wireGuardHub: CloudWireGuardHub? @@ -29,10 +29,11 @@ final class CmuxTuiSurfaceProviderRegistry { private var accessObserver: NSObjectProtocol? private var themeObserver: NSObjectProtocol? private var activationObserver: NSObjectProtocol? + private var networkObserver: CloudReadRecoveryObserver? private var featureFlagObserver: NSObjectProtocol? private let notificationCenter: NotificationCenter /// Whether the periodic fleet read may run right now. - private let isCloudEnabled: @MainActor () -> Bool + let isCloudEnabled: @MainActor () -> Bool private let allowsBackgroundWork: @MainActor () -> Bool private let listPage: @MainActor () async -> VMListPage? private let refreshProvider: @MainActor (CmuxTuiSurfaceProvider, Bool) async -> Void @@ -51,10 +52,10 @@ final class CmuxTuiSurfaceProviderRegistry { /// carries the epoch it was registered with and a stale one is dropped: /// without this, a `DisableCloud` teardown that lands just after the /// policy lifts would clear the freshly restarted registry. - private var accessEpoch: UInt64 = 0 + var accessEpoch: UInt64 = 0 /// Whether account access has ended. Retired registries reject all new Cloud work /// until ``start(catalog:)`` reactivates them for the next account. - private var isRetired = true + var isRetired = true /// Same cadence as the Machines panel's list refresh. private let pollInterval: Duration = .seconds(45) /// In-flight forward and link teardowns for deleted machines, keyed by @@ -162,6 +163,10 @@ final class CmuxTuiSurfaceProviderRegistry { ) { [weak self] _ in MainActor.assumeIsolated { self?.syncPollingToActivationPolicy() } } + networkObserver = CloudReadRecoveryObserver(notificationCenter: notificationCenter) { [weak self] in + guard let self, !self.isRetired, self.allowsBackgroundWork() else { return } + _ = await self.refresh(force: false) + } syncPollingToActivationPolicy() } @@ -354,42 +359,6 @@ final class CmuxTuiSurfaceProviderRegistry { return candidates.first { $0.caseInsensitiveCompare(rawID) == .orderedSame } ?? rawID } - /// The headless link's local mux socket for a machine, connecting if needed. - func linkSocketPath(machineID: String) async throws -> (socketPath: String, session: String) { - guard isCloudEnabled() else { - throw CloudMachineLinkManager.ManagerError.retryLater(String( - localized: "cloud.feature.disabled", - defaultValue: "Cloud Machines are temporarily unavailable." - )) - } - let connected = try await links.connected(machineID: machineID) - return (connected.socketPath, connected.session) - } - - func privateRoute(machineID: String) async -> String? { - guard !isRetired, !ManagedDevicePolicy().isEnforced(.disableCloud), isCloudEnabled(), !Task.isCancelled else { - return nil - } - let epoch = accessEpoch - // The persisted device marker outlives this in-memory registry. An - // explicit open must discover its machine before using the saved-device - // shortcut, even when the first background fleet read has not run. - guard await providerRefreshingIfMissing(machineID: machineID) != nil else { return nil } - let route = await links.privateRoute(for: machineID) - guard !isRetired, epoch == accessEpoch, isCloudEnabled(), !Task.isCancelled else { return nil } - return route - } - - func resolvedPrivateRoute(machineID: String, through hub: CloudWireGuardHub.Ready, fallbackRoute: String, addresses: [String]) async throws -> String { - guard isCloudEnabled() else { - throw CloudMachineLinkManager.ManagerError.retryLater(String( - localized: "cloud.feature.disabled", - defaultValue: "Cloud Machines are temporarily unavailable." - )) - } - return try await links.resolvedPrivateRoute(machineID: machineID, through: hub, fallbackRoute: fallbackRoute, addresses: addresses) - } - /// Cancels Cloud-only transport work when the remote gate closes while /// retaining providers, catalog resources, and persisted pane identities. /// Re-enabling the gate reuses those providers on the next discovery pass. @@ -471,6 +440,7 @@ final class CmuxTuiSurfaceProviderRegistry { } func accessDidEnd() async { + networkObserver = nil isRetired = true accessEpoch &+= 1 refreshGeneration &+= 1 diff --git a/cmux.xcodeproj/project.pbxproj b/cmux.xcodeproj/project.pbxproj index f38abc81e15b..8de27c66417b 100644 --- a/cmux.xcodeproj/project.pbxproj +++ b/cmux.xcodeproj/project.pbxproj @@ -838,10 +838,17 @@ 0C45DC8D9BE54DDEA2397892 /* CloudPrivateRouteSelectionTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1E6077A4B6CC441EB3668651 /* CloudPrivateRouteSelectionTests.swift */; }; D1AC3F6282A94EAEB79D160B /* CloudProviderRefreshCoordinator.swift in Sources */ = {isa = PBXBuildFile; fileRef = A09B853868B54438A5B14F26 /* CloudProviderRefreshCoordinator.swift */; }; AC38407F09E94DA38B94F168 /* CloudProviderRefreshCoordinatorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5C2A7143303749BE82E51F69 /* CloudProviderRefreshCoordinatorTests.swift */; }; + C12625B00000000000000006 /* CloudReadManualClock.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12625B00000000000000005 /* CloudReadManualClock.swift */; }; + C12625B00000000000000002 /* CloudReadNetworkMonitor.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12625B00000000000000001 /* CloudReadNetworkMonitor.swift */; }; + C12636A00000000000000004 /* CloudReadRecoveryObserver.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12636A00000000000000003 /* CloudReadRecoveryObserver.swift */; }; + C12625A00000000000000004 /* CloudReadRequestCoordinator.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12625A00000000000000003 /* CloudReadRequestCoordinator.swift */; }; + C12625B00000000000000004 /* CloudReadRequestCoordinatorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12625B00000000000000003 /* CloudReadRequestCoordinatorTests.swift */; }; + C12625000000000000000004 /* CloudRefreshURLProtocol.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12625000000000000000003 /* CloudRefreshURLProtocol.swift */; }; C12449010000000000000001 /* CloudRemoteColorOwnershipTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12449020000000000000001 /* CloudRemoteColorOwnershipTests.swift */; }; 595EC249E2E015ACE304F0EE /* CloudRemoteOperationStep.swift in Sources */ = {isa = PBXBuildFile; fileRef = 64BF8B9B8328108560B40B7F /* CloudRemoteOperationStep.swift */; }; B12438020000000000000001 /* CloudRenameCoordinator.swift in Sources */ = {isa = PBXBuildFile; fileRef = B12438020000000000000002 /* CloudRenameCoordinator.swift */; }; C12575000000000000000050 /* CloudRenameOptimismTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12575000000000000000051 /* CloudRenameOptimismTests.swift */; }; + C12625A00000000000000002 /* CloudRequestClock.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12625A00000000000000001 /* CloudRequestClock.swift */; }; 46383D21CF46427F36EBEB7B /* CloudSidebarAcceptanceTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8150EA26C949866059F3B6EA /* CloudSidebarAcceptanceTests.swift */; }; 5E3484F75CE048CC95018CF1 /* CloudSidebarAttentionLayoutTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5DB4F197C29A4C03B61418A1 /* CloudSidebarAttentionLayoutTests.swift */; }; D075FB64A68C4CA0A4042528 /* CloudSidebarConsistencyTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = B27A4938294E4A109BC6D2B3 /* CloudSidebarConsistencyTests.swift */; }; @@ -1463,6 +1470,7 @@ 646ACD2F9B5044B8A2AB6C3A /* CmuxTuiSurfaceProvider+TerminalPrimitives.swift in Sources */ = {isa = PBXBuildFile; fileRef = 122E7DF3DF3044E68322B11C /* CmuxTuiSurfaceProvider+TerminalPrimitives.swift */; }; D12439050000000000000001 /* CmuxTuiSurfaceProvider+WorkspaceLifecycle.swift in Sources */ = {isa = PBXBuildFile; fileRef = D12439050000000000000002 /* CmuxTuiSurfaceProvider+WorkspaceLifecycle.swift */; }; 175CFE773740410786E08A2B /* CmuxTuiSurfaceProviderRegistry+Production.swift in Sources */ = {isa = PBXBuildFile; fileRef = 95278207545546F08C9A325F /* CmuxTuiSurfaceProviderRegistry+Production.swift */; }; + DAE9491D73F526601DFC97A2 /* CmuxTuiSurfaceProviderRegistry+Routes.swift in Sources */ = {isa = PBXBuildFile; fileRef = A6040C2AA5B187C05EEC1A67 /* CmuxTuiSurfaceProviderRegistry+Routes.swift */; }; A1B2C3D4E5F6071827364961 /* CmuxTuiSurfaceProviderRegistry.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1B2C3D4E5F6071827364960 /* CmuxTuiSurfaceProviderRegistry.swift */; }; D01100800000000000000008 /* CmuxTuiSurfaceProviderRegistryDiscoveryTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D01100800000000000000007 /* CmuxTuiSurfaceProviderRegistryDiscoveryTests.swift */; }; 7A1D4C000000000000000106 /* CmuxTuiSurfaceProviderRegistryPollingTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7A1D4C000000000000000105 /* CmuxTuiSurfaceProviderRegistryPollingTests.swift */; }; 036359ABDB1F58B489D378A4 /* CmuxTuiSurfaceProviders.swift in Sources */ = {isa = PBXBuildFile; fileRef = 63F5BDFB110A8A96C9CA0A75 /* CmuxTuiSurfaceProviders.swift */; }; @@ -2100,8 +2108,11 @@ 98E6802001D61AC24F2ABB09 /* MachinesPanelUncappedPlanTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9A986C3063FCE5F28AA10ACA /* MachinesPanelUncappedPlanTests.swift */; }; E879C292A2D38E3DDBCB004D /* MachinesPanelView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 08610D114C8CF23877176D7E /* MachinesPanelView.swift */; }; C1257500000000000000001A /* MachinesPanelViewModel+MachinePins.swift in Sources */ = {isa = PBXBuildFile; fileRef = C1257500000000000000001B /* MachinesPanelViewModel+MachinePins.swift */; }; + F9ADD9F062DEA2BC5924BC38 /* MachinesPanelViewModel+Refresh.swift in Sources */ = {isa = PBXBuildFile; fileRef = E8DCA0BDC648E6CDCFD3E228 /* MachinesPanelViewModel+Refresh.swift */; }; D7E6477A2FC01DCCF353840D /* MachinesPanelViewModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 68672B3C63D4E348AFA4DEF7 /* MachinesPanelViewModel.swift */; }; 7A0CE1000000000000000724 /* MachinesTunnelBanner.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7A0CE1000000000000000723 /* MachinesTunnelBanner.swift */; }; + C12625D00000000000000004 /* MachineUsageSnapshot.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12625D00000000000000003 /* MachineUsageSnapshot.swift */; }; + C12625D00000000000000002 /* MachineUsageTotals.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12625D00000000000000001 /* MachineUsageTotals.swift */; }; D1F0A00600000000000000C1 /* MacPairedMacBackupBody.swift in Sources */ = {isa = PBXBuildFile; fileRef = D1F0A00600000000000000C2 /* MacPairedMacBackupBody.swift */; }; D1F0A00700000000000000C1 /* MacPairedMacBackupOpWire.swift in Sources */ = {isa = PBXBuildFile; fileRef = D1F0A00700000000000000C2 /* MacPairedMacBackupOpWire.swift */; }; D1F0A00100000000000000C1 /* MacPairedMacBackupPublisher.swift in Sources */ = {isa = PBXBuildFile; fileRef = D1F0A00100000000000000C2 /* MacPairedMacBackupPublisher.swift */; }; @@ -3272,6 +3283,7 @@ C7A502000000000000000002 /* TaskManagerWindowController.swift in Sources */ = {isa = PBXBuildFile; fileRef = C7A502000000000000000001 /* TaskManagerWindowController.swift */; }; 7490C00F7490C00F7490C00F /* TaskVMInfoMemoryPressureFootprintSampler.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7490D00F7490D00F7490D00F /* TaskVMInfoMemoryPressureFootprintSampler.swift */; }; B82090010000000000000001 /* TCCUsageDescriptionUITests.swift in Sources */ = {isa = PBXBuildFile; fileRef = B82090010000000000000002 /* TCCUsageDescriptionUITests.swift */; }; + C12625C00000000000000002 /* TeamMachineUsage.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12625C00000000000000001 /* TeamMachineUsage.swift */; }; 0044CBDD567D9CD29E6A6F12 /* TelemetrySettings.swift in Sources */ = {isa = PBXBuildFile; fileRef = 257DFD20F0961FE0BDAF70F2 /* TelemetrySettings.swift */; }; A10199010000000000000001 /* TerminalAdaptiveDefaultThemeSettings.swift in Sources */ = {isa = PBXBuildFile; fileRef = A10199010000000000000002 /* TerminalAdaptiveDefaultThemeSettings.swift */; }; C10190010000000000000002 /* TerminalAgentPanelInitialTitleTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C10190010000000000000001 /* TerminalAgentPanelInitialTitleTests.swift */; }; @@ -3675,9 +3687,11 @@ 66141EC09936A3AD5EC83307 /* View+CloudErrorCopy.swift in Sources */ = {isa = PBXBuildFile; fileRef = B51E0FBEDF07F4E6A6A792F7 /* View+CloudErrorCopy.swift */; }; A28B087F0000000000000005 /* ViewerNavigationKeyRouter.swift in Sources */ = {isa = PBXBuildFile; fileRef = A28B087F0000000000000006 /* ViewerNavigationKeyRouter.swift */; }; A3340002A3340002A3340002 /* ViewerNavigationTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = A3340001A3340001A3340001 /* ViewerNavigationTests.swift */; }; + C12625A00000000000000006 /* VMCapabilities.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12625A00000000000000005 /* VMCapabilities.swift */; }; 60AA592C22BFA579A064782E /* VMClient+ResourceStats.swift in Sources */ = {isa = PBXBuildFile; fileRef = 65A5357644AB29D74F4AC368 /* VMClient+ResourceStats.swift */; }; ACE4FE1A8F1CC6C0B6A673AB /* VMClient.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9CEC6EF35B71AE59FA45BA69 /* VMClient.swift */; }; EF56B870FD0D4AC490D99CD5 /* VMClientError.swift in Sources */ = {isa = PBXBuildFile; fileRef = 98F890512F8E425FAD71FE7C /* VMClientError.swift */; }; + C12625000000000000000002 /* VMClientReadCoalescingTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12625000000000000000001 /* VMClientReadCoalescingTests.swift */; }; 7A0CE1000000000000000044 /* VMClientSocketCommands+Tunnel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7A0CE1000000000000000018 /* VMClientSocketCommands+Tunnel.swift */; }; ACE4FE2A8F1CC6C0B6A673AB /* VMClientSocketCommands.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9CEC6F0035B71AE59FA45BA7 /* VMClientSocketCommands.swift */; }; 93AF7E8EC956FE883E252322 /* VMClientTelemetry.swift in Sources */ = {isa = PBXBuildFile; fileRef = 831145877779546B628FCAFC /* VMClientTelemetry.swift */; }; @@ -4887,10 +4901,17 @@ 1E6077A4B6CC441EB3668651 /* CloudPrivateRouteSelectionTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudPrivateRouteSelectionTests.swift; sourceTree = ""; }; A09B853868B54438A5B14F26 /* CloudProviderRefreshCoordinator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudProviderRefreshCoordinator.swift"; sourceTree = ""; }; 5C2A7143303749BE82E51F69 /* CloudProviderRefreshCoordinatorTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudProviderRefreshCoordinatorTests.swift; sourceTree = ""; }; + C12625B00000000000000005 /* CloudReadManualClock.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudReadManualClock.swift; sourceTree = ""; }; + C12625B00000000000000001 /* CloudReadNetworkMonitor.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudReadNetworkMonitor.swift; sourceTree = ""; }; + C12636A00000000000000003 /* CloudReadRecoveryObserver.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudReadRecoveryObserver.swift; sourceTree = ""; }; + C12625A00000000000000003 /* CloudReadRequestCoordinator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudReadRequestCoordinator.swift; sourceTree = ""; }; + C12625B00000000000000003 /* CloudReadRequestCoordinatorTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudReadRequestCoordinatorTests.swift; sourceTree = ""; }; + C12625000000000000000003 /* CloudRefreshURLProtocol.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudRefreshURLProtocol.swift; sourceTree = ""; }; C12449020000000000000001 /* CloudRemoteColorOwnershipTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudRemoteColorOwnershipTests.swift; sourceTree = ""; }; 64BF8B9B8328108560B40B7F /* CloudRemoteOperationStep.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudRemoteOperationStep.swift"; sourceTree = ""; }; B12438020000000000000002 /* CloudRenameCoordinator.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = CloudRenameCoordinator.swift; sourceTree = ""; }; C12575000000000000000051 /* CloudRenameOptimismTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudRenameOptimismTests.swift; sourceTree = ""; }; + C12625A00000000000000001 /* CloudRequestClock.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudRequestClock.swift; sourceTree = ""; }; 8150EA26C949866059F3B6EA /* CloudSidebarAcceptanceTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudSidebarAcceptanceTests.swift"; sourceTree = ""; }; 5DB4F197C29A4C03B61418A1 /* CloudSidebarAttentionLayoutTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudSidebarAttentionLayoutTests.swift"; sourceTree = ""; }; B27A4938294E4A109BC6D2B3 /* CloudSidebarConsistencyTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudSidebarConsistencyTests.swift; sourceTree = ""; }; @@ -5412,6 +5433,7 @@ 122E7DF3DF3044E68322B11C /* CmuxTuiSurfaceProvider+TerminalPrimitives.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = "CmuxTuiSurfaceProvider+TerminalPrimitives.swift"; sourceTree = ""; }; D12439050000000000000002 /* CmuxTuiSurfaceProvider+WorkspaceLifecycle.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = "CmuxTuiSurfaceProvider+WorkspaceLifecycle.swift"; sourceTree = ""; }; 95278207545546F08C9A325F /* CmuxTuiSurfaceProviderRegistry+Production.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = "CmuxTuiSurfaceProviderRegistry+Production.swift"; sourceTree = ""; }; + A6040C2AA5B187C05EEC1A67 /* CmuxTuiSurfaceProviderRegistry+Routes.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = "CmuxTuiSurfaceProviderRegistry+Routes.swift"; sourceTree = ""; }; A1B2C3D4E5F6071827364960 /* CmuxTuiSurfaceProviderRegistry.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = CmuxTuiSurfaceProviderRegistry.swift; sourceTree = ""; }; D01100800000000000000007 /* CmuxTuiSurfaceProviderRegistryDiscoveryTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CmuxTuiSurfaceProviderRegistryDiscoveryTests.swift; sourceTree = ""; }; 7A1D4C000000000000000105 /* CmuxTuiSurfaceProviderRegistryPollingTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CmuxTuiSurfaceProviderRegistryPollingTests.swift; sourceTree = ""; }; 63F5BDFB110A8A96C9CA0A75 /* CmuxTuiSurfaceProviders.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = CmuxTuiSurfaceProviders.swift; sourceTree = ""; }; @@ -6037,8 +6059,11 @@ 9A986C3063FCE5F28AA10ACA /* MachinesPanelUncappedPlanTests.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = MachinesPanelUncappedPlanTests.swift; sourceTree = ""; }; 08610D114C8CF23877176D7E /* MachinesPanelView.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = MachinesPanelView.swift; sourceTree = ""; }; C1257500000000000000001B /* MachinesPanelViewModel+MachinePins.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "MachinesPanelViewModel+MachinePins.swift"; sourceTree = ""; }; + E8DCA0BDC648E6CDCFD3E228 /* MachinesPanelViewModel+Refresh.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "MachinesPanelViewModel+Refresh.swift"; sourceTree = ""; }; 68672B3C63D4E348AFA4DEF7 /* MachinesPanelViewModel.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = MachinesPanelViewModel.swift; sourceTree = ""; }; 7A0CE1000000000000000723 /* MachinesTunnelBanner.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = MachinesTunnelBanner.swift; sourceTree = ""; }; + C12625D00000000000000003 /* MachineUsageSnapshot.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MachineUsageSnapshot.swift; sourceTree = ""; }; + C12625D00000000000000001 /* MachineUsageTotals.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MachineUsageTotals.swift; sourceTree = ""; }; D1F0A00600000000000000C2 /* MacPairedMacBackupBody.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = MacPairedMacBackupBody.swift; sourceTree = ""; }; D1F0A00700000000000000C2 /* MacPairedMacBackupOpWire.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = MacPairedMacBackupOpWire.swift; sourceTree = ""; }; D1F0A00100000000000000C2 /* MacPairedMacBackupPublisher.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = MacPairedMacBackupPublisher.swift; sourceTree = ""; }; @@ -7194,6 +7219,7 @@ C7A502000000000000000001 /* TaskManagerWindowController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TaskManagerWindowController.swift; sourceTree = ""; }; 7490D00F7490D00F7490D00F /* TaskVMInfoMemoryPressureFootprintSampler.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = App/TaskVMInfoMemoryPressureFootprintSampler.swift; sourceTree = ""; }; B82090010000000000000002 /* TCCUsageDescriptionUITests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TCCUsageDescriptionUITests.swift; sourceTree = ""; }; + C12625C00000000000000001 /* TeamMachineUsage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TeamMachineUsage.swift; sourceTree = ""; }; 257DFD20F0961FE0BDAF70F2 /* TelemetrySettings.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "TelemetrySettings.swift"; sourceTree = ""; }; A10199010000000000000002 /* TerminalAdaptiveDefaultThemeSettings.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TerminalAdaptiveDefaultThemeSettings.swift; sourceTree = ""; }; C10190010000000000000001 /* TerminalAgentPanelInitialTitleTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TerminalAgentPanelInitialTitleTests.swift; sourceTree = ""; }; @@ -7597,9 +7623,11 @@ B51E0FBEDF07F4E6A6A792F7 /* View+CloudErrorCopy.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "View+CloudErrorCopy.swift"; sourceTree = ""; }; A28B087F0000000000000006 /* ViewerNavigationKeyRouter.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Panels/ViewerNavigationKeyRouter.swift; sourceTree = ""; }; A3340001A3340001A3340001 /* ViewerNavigationTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ViewerNavigationTests.swift; sourceTree = ""; }; + C12625A00000000000000005 /* VMCapabilities.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VMCapabilities.swift; sourceTree = ""; }; 65A5357644AB29D74F4AC368 /* VMClient+ResourceStats.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "VMClient+ResourceStats.swift"; sourceTree = ""; }; 9CEC6EF35B71AE59FA45BA69 /* VMClient.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = VMClient.swift; sourceTree = ""; }; 98F890512F8E425FAD71FE7C /* VMClientError.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = VMClientError.swift; sourceTree = ""; }; + C12625000000000000000001 /* VMClientReadCoalescingTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VMClientReadCoalescingTests.swift; sourceTree = ""; }; 7A0CE1000000000000000018 /* VMClientSocketCommands+Tunnel.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = "VMClientSocketCommands+Tunnel.swift"; sourceTree = ""; }; 9CEC6F0035B71AE59FA45BA7 /* VMClientSocketCommands.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = VMClientSocketCommands.swift; sourceTree = ""; }; 831145877779546B628FCAFC /* VMClientTelemetry.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = VMClientTelemetry.swift; sourceTree = ""; }; @@ -8396,6 +8424,7 @@ 1EDD243C7C87A2AB78B12BAD /* LocalSurfaceProvider.swift */, A1B2C3D4E5F6071827364960 /* CmuxTuiSurfaceProviderRegistry.swift */, 95278207545546F08C9A325F /* CmuxTuiSurfaceProviderRegistry+Production.swift */, + A6040C2AA5B187C05EEC1A67 /* CmuxTuiSurfaceProviderRegistry+Routes.swift */, 63F5BDFB110A8A96C9CA0A75 /* CmuxTuiSurfaceProviders.swift */, D9F800000000000000000002 /* CmuxTuiRemoteRouting.swift */, 1FE7223A65E72C46927A4F02 /* SurfacePaneFactory.swift */, @@ -8429,6 +8458,7 @@ 590E4F5F342E5B27010ECC8D /* Cloud */ = { isa = PBXGroup; children = ( + C12636A00000000000000003 /* CloudReadRecoveryObserver.swift */, C11325A20000000000000001 /* CloudTuiTerminalProjectionTarget.swift */, C11324120000000000000001 /* CloudTuiManualIOCommand.swift */, C11324320000000000000001 /* CloudTuiManualIODescriptorLease.swift */, @@ -8485,6 +8515,14 @@ B12B8221BA9A4271E0919C72 /* MachinePlanSnapshot.swift */, BC1FC55E8DA750AD0DABABF4 /* VMResourceStatsStore.swift */, 65A5357644AB29D74F4AC368 /* VMClient+ResourceStats.swift */, + C12625A00000000000000005 /* VMCapabilities.swift */, + C12625C00000000000000001 /* TeamMachineUsage.swift */, + C12625D00000000000000001 /* MachineUsageTotals.swift */, + C12625D00000000000000003 /* MachineUsageSnapshot.swift */, + C12625A00000000000000001 /* CloudRequestClock.swift */, + C12625A00000000000000003 /* CloudReadRequestCoordinator.swift */, + C12625B00000000000000001 /* CloudReadNetworkMonitor.swift */, + E8DCA0BDC648E6CDCFD3E228 /* MachinesPanelViewModel+Refresh.swift */, FA1253800000000000000004 /* VMStats+State.swift */, FA1253800000000000000006 /* CloudTreeRowIcon.swift */, 98F890512F8E425FAD71FE7C /* VMClientError.swift */, @@ -10951,6 +10989,10 @@ B8E9500A0000000000000002 /* PostHogAnalyticsPropertiesTests.swift */, 8F8A0358C8DD0A7D998A86E9 /* VMClientTelemetryTests.swift */, 7B7B816D0E6DDD18F2D66C77 /* CloudOperationRecorderTests.swift */, + C12625000000000000000001 /* VMClientReadCoalescingTests.swift */, + C12625000000000000000003 /* CloudRefreshURLProtocol.swift */, + C12625B00000000000000003 /* CloudReadRequestCoordinatorTests.swift */, + C12625B00000000000000005 /* CloudReadManualClock.swift */, 604100010000000000000002 /* GhosttyDrawableSizeRetryTests.swift */, BCBC0A0E0000000000000D02 /* BrowserChromeMetricsTests.swift */, B5A5E55B0000000000000002 /* BrowserEphemeralRestorationTests.swift */, @@ -13139,8 +13181,12 @@ 7A0CE1000000000000000504 /* CloudPrivateNetworkPurpose.swift in Sources */, 7A0CE1000000000000000304 /* CloudPrivateNetworkUse.swift in Sources */, D1AC3F6282A94EAEB79D160B /* CloudProviderRefreshCoordinator.swift in Sources */, + C12625B00000000000000002 /* CloudReadNetworkMonitor.swift in Sources */, + C12636A00000000000000004 /* CloudReadRecoveryObserver.swift in Sources */, + C12625A00000000000000004 /* CloudReadRequestCoordinator.swift in Sources */, 595EC249E2E015ACE304F0EE /* CloudRemoteOperationStep.swift in Sources */, B12438020000000000000001 /* CloudRenameCoordinator.swift in Sources */, + C12625A00000000000000002 /* CloudRequestClock.swift in Sources */, A13086000000000000000012 /* CloudSidebarDropOperation.swift in Sources */, 1771C5B445A64CB486D8B7D7 /* CloudSidebarNotificationCoordinator.swift in Sources */, E37E404F1D6B458C8C79318A /* CloudSidebarOrganizationAction.swift in Sources */, @@ -13407,6 +13453,7 @@ 646ACD2F9B5044B8A2AB6C3A /* CmuxTuiSurfaceProvider+TerminalPrimitives.swift in Sources */, D12439050000000000000001 /* CmuxTuiSurfaceProvider+WorkspaceLifecycle.swift in Sources */, 175CFE773740410786E08A2B /* CmuxTuiSurfaceProviderRegistry+Production.swift in Sources */, + DAE9491D73F526601DFC97A2 /* CmuxTuiSurfaceProviderRegistry+Routes.swift in Sources */, A1B2C3D4E5F6071827364961 /* CmuxTuiSurfaceProviderRegistry.swift in Sources */, 036359ABDB1F58B489D378A4 /* CmuxTuiSurfaceProviders.swift in Sources */, 9520A0029520A0029520A002 /* CmuxVaultAgentRegistration+Hermes.swift in Sources */, @@ -13813,8 +13860,11 @@ 4A6A6A54919FEE7C66D41678 /* MachinesPanelBanners.swift in Sources */, E879C292A2D38E3DDBCB004D /* MachinesPanelView.swift in Sources */, C1257500000000000000001A /* MachinesPanelViewModel+MachinePins.swift in Sources */, + F9ADD9F062DEA2BC5924BC38 /* MachinesPanelViewModel+Refresh.swift in Sources */, D7E6477A2FC01DCCF353840D /* MachinesPanelViewModel.swift in Sources */, 7A0CE1000000000000000724 /* MachinesTunnelBanner.swift in Sources */, + C12625D00000000000000004 /* MachineUsageSnapshot.swift in Sources */, + C12625D00000000000000002 /* MachineUsageTotals.swift in Sources */, D1F0A00600000000000000C1 /* MacPairedMacBackupBody.swift in Sources */, D1F0A00700000000000000C1 /* MacPairedMacBackupOpWire.swift in Sources */, D1F0A00100000000000000C1 /* MacPairedMacBackupPublisher.swift in Sources */, @@ -14616,6 +14666,7 @@ C7A506000000000000000002 /* TaskManagerView.swift in Sources */, C7A502000000000000000002 /* TaskManagerWindowController.swift in Sources */, 7490C00F7490C00F7490C00F /* TaskVMInfoMemoryPressureFootprintSampler.swift in Sources */, + C12625C00000000000000002 /* TeamMachineUsage.swift in Sources */, 0044CBDD567D9CD29E6A6F12 /* TelemetrySettings.swift in Sources */, A10199010000000000000001 /* TerminalAdaptiveDefaultThemeSettings.swift in Sources */, 8362A0030000000000000003 /* TerminalCallerTTYBinding.swift in Sources */, @@ -14933,6 +14984,7 @@ C9A57209C9A57209C9A57209 /* VerticalTabsSidebar+WorkspaceGroups.swift in Sources */, 66141EC09936A3AD5EC83307 /* View+CloudErrorCopy.swift in Sources */, A28B087F0000000000000005 /* ViewerNavigationKeyRouter.swift in Sources */, + C12625A00000000000000006 /* VMCapabilities.swift in Sources */, 60AA592C22BFA579A064782E /* VMClient+ResourceStats.swift in Sources */, ACE4FE1A8F1CC6C0B6A673AB /* VMClient.swift in Sources */, EF56B870FD0D4AC490D99CD5 /* VMClientError.swift in Sources */, @@ -15727,6 +15779,9 @@ DBF72463931EE07F5759C449 /* CloudPortsVPNAffordanceTests.swift in Sources */, 0C45DC8D9BE54DDEA2397892 /* CloudPrivateRouteSelectionTests.swift in Sources */, AC38407F09E94DA38B94F168 /* CloudProviderRefreshCoordinatorTests.swift in Sources */, + C12625B00000000000000006 /* CloudReadManualClock.swift in Sources */, + C12625B00000000000000004 /* CloudReadRequestCoordinatorTests.swift in Sources */, + C12625000000000000000004 /* CloudRefreshURLProtocol.swift in Sources */, C12449010000000000000001 /* CloudRemoteColorOwnershipTests.swift in Sources */, C12575000000000000000050 /* CloudRenameOptimismTests.swift in Sources */, 46383D21CF46427F36EBEB7B /* CloudSidebarAcceptanceTests.swift in Sources */, @@ -16400,6 +16455,7 @@ 62F6C9E77A6A40B982D91040 /* VaultSessionSearchQueryTests.swift in Sources */, C5BEA2B0EA914759A8BFD577 /* VaultSocketPayloadTests.swift in Sources */, A3340002A3340002A3340002 /* ViewerNavigationTests.swift in Sources */, + C12625000000000000000002 /* VMClientReadCoalescingTests.swift in Sources */, 6EE115A36CAD9082FBC4593E /* VMClientTelemetryTests.swift in Sources */, C37800000000000000000003 /* VMDefaultCloudCommandTests.swift in Sources */, B81607322100000000000003 /* VMMachineTerminalResolution.swift in Sources */, diff --git a/cmuxTests/CloudReadManualClock.swift b/cmuxTests/CloudReadManualClock.swift new file mode 100644 index 000000000000..76316a9f8844 --- /dev/null +++ b/cmuxTests/CloudReadManualClock.swift @@ -0,0 +1,48 @@ +import Foundation + +/// Clock.now is synchronous. A lock protects this test clock's instant and +/// continuation registration; no production state uses this lock. +final class CloudReadManualClock: Clock, @unchecked Sendable { + struct Instant: InstantProtocol { + let offset: Duration + func advanced(by duration: Duration) -> Self { Self(offset: offset + duration) } + func duration(to other: Self) -> Duration { other.offset - offset } + static func < (lhs: Self, rhs: Self) -> Bool { lhs.offset < rhs.offset } + } + private struct Waiter { + let deadline: Instant + let continuation: CheckedContinuation + } + private let lock = NSLock() + private var instant = Instant(offset: .zero) + private var sleepers: [UUID: Waiter] = [:] + var now: Instant { lock.withLock { instant } } + var minimumResolution: Duration { .zero } + var pendingSleeperCount: Int { lock.withLock { sleepers.count } } + + func sleep(until deadline: Instant, tolerance: Duration?) async throws { + let id = UUID() + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + lock.withLock { + if Task.isCancelled { continuation.resume(throwing: CancellationError()) } + else if deadline <= instant { continuation.resume() } + else { sleepers[id] = Waiter(deadline: deadline, continuation: continuation) } + } + } + } onCancel: { + self.lock.withLock { self.sleepers.removeValue(forKey: id) }?.continuation.resume(throwing: CancellationError()) + } + } + + func advance(by duration: Duration, deliverTimers: Bool = true) { + let ready: [Waiter] = lock.withLock { + instant = instant.advanced(by: duration) + guard deliverTimers else { return [] } + let ready = sleepers.filter { $0.value.deadline <= instant } + for id in ready.keys { sleepers.removeValue(forKey: id) } + return Array(ready.values) + } + for waiter in ready { waiter.continuation.resume() } + } +} diff --git a/cmuxTests/CloudReadRequestCoordinatorTests.swift b/cmuxTests/CloudReadRequestCoordinatorTests.swift new file mode 100644 index 000000000000..3a0484ab4a47 --- /dev/null +++ b/cmuxTests/CloudReadRequestCoordinatorTests.swift @@ -0,0 +1,269 @@ +import Foundation +import Testing +#if canImport(cmux_DEV) +@testable import cmux_DEV +#elseif canImport(cmux) +@testable import cmux +#else +@testable import CloudReadFixture +#endif + +@Suite("Cloud read deadline and cancellation", .timeLimit(.minutes(1))) +struct CloudReadRequestCoordinatorTests { + private typealias Owner = CloudReadRequestCoordinator + private func key(_ id: String = "vm", account: String = "fixture", generation: UInt64 = 1) -> Owner.Key { + .init(path: id, accountID: account, generation: generation, teamID: "team") + } + private func response(_ status: Int = 200) -> Owner.Response { + .init(data: Data(), http: HTTPURLResponse(url: URL(string: "https://fixture.invalid")!, statusCode: status, httpVersion: nil, headerFields: nil)!) + } + + @Test("Four owners issue one request per machine", arguments: [1, 10, 100, 1000]) + func scale(machines: Int) async throws { + let gate = CloudReadResponseGate() + let owner = Owner() + let tasks = (0..<(machines * 4)).map { index in + Task { try await owner.read(key("vm-\(index % machines)")) { await gate.read(response()) } } + } + try await eventually { await owner.entries.values.reduce(0) { $0 + $1.waiters.count } == machines * 4 } + await gate.release() + for task in tasks { _ = try await task.value } + #expect(await gate.requests == machines) + #expect(await owner.entries.isEmpty) + print("cloud-read-scale machines=\(machines) owners=4 requests=\(await gate.requests) requests_per_machine=1") + } + + @Test("Cancelling one waiter preserves another; the last cancels and holds the draining slot") + func independentCancellation() async throws { + let gate = CloudReadResponseGate() + let owner = Owner() + let first = Task { try await owner.read(key()) { await gate.read(response()) } } + let second = Task { try await owner.read(key()) { await gate.read(response()) } } + try await eventually { await owner.entries.values.first?.waiters.count == 2 } + try await eventually { await gate.requests == 1 } + first.cancel() + do { _ = try await first.value; Issue.record("cancelled waiter returned a value") } catch is CancellationError {} catch { Issue.record("\(error)") } + #expect(await owner.entries.values.first?.waiters.count == 1) + second.cancel() + _ = await second.result + let replacement = Task { try await owner.read(key()) { await gate.read(response()) } } + try await eventually { await owner.entries.values.first?.pending?.waiters.count == 1 } + #expect(await gate.requests == 1) + await gate.release() + #expect(try await replacement.value.http.statusCode == 200) + #expect(await gate.requests == 2) + try await eventually { await owner.entries.isEmpty } + } + + @Test("Deadline returns before an uncooperative loader, retaining only its draining slot") + func deadlineDoesNotJoinStuckChild() async throws { + let clock = CloudReadManualClock() + let owner = Owner(clock: CloudRequestClock(clock), budget: .seconds(30)) + let gate = CloudReadResponseGate() + let task = Task { try await owner.read(key()) { await gate.read(response()) } } + try await eventually { await gate.requests == 1 } + clock.advance(by: .seconds(31)) + do { _ = try await task.value; Issue.record("expired request succeeded") } + catch { #expect((error as? URLError)?.code == .timedOut) } + #expect(await owner.entries.count == 1) + #expect(await owner.entries.values.first?.waiters.isEmpty == true) + await gate.release() + try await eventually { await owner.entries.isEmpty } + } + + @Test("A queued replacement has its own deadline while old cleanup stays held") + func queuedReplacementDeadline() async throws { + let clock = CloudReadManualClock() + let owner = Owner(clock: CloudRequestClock(clock), budget: .seconds(30)) + let gate = CloudReadResponseGate() + let first = Task { try await owner.read(key()) { await gate.read(response()) } } + try await eventually { await gate.requests == 1 } + first.cancel() + _ = await first.result + let replacement = Task { try await owner.read(key()) { await gate.read(response()) } } + try await eventually { await owner.entries.values.first?.pending?.waiters.count == 1 } + clock.advance(by: .seconds(31)) + do { _ = try await replacement.value; Issue.record("queued deadline missed") } + catch { #expect((error as? URLError)?.code == .timedOut) } + #expect(await gate.requests == 1) + await gate.release() + try await eventually { await owner.entries.isEmpty } + } + + @Test("Time spent acquiring read scope consumes the original budget") + func admissionUsesOriginalDeadline() async throws { + let clock = CloudReadManualClock() + let owner = Owner(clock: CloudRequestClock(clock)) + let deadline = owner.makeDeadline() + clock.advance(by: .seconds(600), deliverTimers: false) + do { + _ = try await owner.read(key(), deadline: deadline) { Issue.record("started after admission expired"); return response() } + } catch { #expect((error as? URLError)?.code == .timedOut) } + #expect(await owner.entries.isEmpty) + } + + @Test("A short waiter cannot shorten another caller's deadline", arguments: [false, true], [false, true]) + func independentDeadlines(queued: Bool, deliverTimers: Bool) async throws { + let clock = CloudReadManualClock() + let owner = Owner(clock: CloudRequestClock(clock)) + let gate = CloudReadResponseGate() + if queued { + let retired = Task { try await owner.read(key()) { await gate.read(response()) } } + try await eventually { await gate.requests == 1 } + retired.cancel() + _ = await retired.result + } + let long = Task { try await owner.read(key(), deadline: .seconds(20)) { await gate.read(response()) } } + try await eventually { + let entry = await owner.entries.values.first + return queued ? entry?.pending?.waiters.count == 1 : entry?.waiters.count == 1 + } + let short = Task { try await owner.read(key(), deadline: .seconds(1)) { await gate.read(response()) } } + try await eventually { + let entry = await owner.entries.values.first + return queued ? entry?.pending?.waiters.count == 2 : entry?.waiters.count == 2 + } + clock.advance(by: .seconds(2), deliverTimers: deliverTimers) + if deliverTimers { + _ = await short.result + let entry = await owner.entries.values.first + #expect(queued ? entry?.pending?.waiters.count == 1 : entry?.waiters.count == 1) + } + await gate.release() + do { _ = try await short.value; Issue.record("short waiter exceeded its deadline") } + catch { #expect((error as? URLError)?.code == .timedOut) } + #expect(try await long.value.http.statusCode == 200) + #expect(await gate.requests == (queued ? 2 : 1)) + #expect(await owner.entries.isEmpty) + } + + @Test("Response-first delivery after a simulated wake still expires") + func responseAfterDeadline() async throws { + let clock = CloudReadManualClock() + let owner = Owner(clock: CloudRequestClock(clock)) + let gate = CloudReadResponseGate() + let task = Task { try await owner.read(key()) { await gate.read(response()) } } + try await eventually { await gate.requests == 1 } + try await eventually { clock.pendingSleeperCount == 1 } + clock.advance(by: .seconds(600), deliverTimers: false) + await gate.release() + do { _ = try await task.value; Issue.record("late response succeeded") } + catch { #expect((error as? URLError)?.code == .timedOut) } + } + + @Test("A reader arriving after wake waits for a fresh pass instead of inheriting the old timeout") + func freshReaderAfterWake() async throws { + let clock = CloudReadManualClock() + let owner = Owner(clock: CloudRequestClock(clock)) + let gate = CloudReadResponseGate() + let old = Task { try await owner.read(key()) { await gate.read(response()) } } + try await eventually { await gate.requests == 1 && clock.pendingSleeperCount == 1 } + clock.advance(by: .seconds(600), deliverTimers: false) + let fresh = Task { try await owner.read(key()) { await gate.read(response()) } } + try await eventually { await owner.entries.values.first?.pending?.waiters.count == 1 } + await gate.release() + do { _ = try await old.value; Issue.record("old request survived its deadline") } + catch { #expect((error as? URLError)?.code == .timedOut) } + #expect(try await fresh.value.http.statusCode == 200) + #expect(await gate.requests == 2) + } + + @Test("Retry-After survives operation completion and offline recovery") + func serverCooldown() async throws { + let clock = CloudReadManualClock() + let owner = Owner(clock: CloudRequestClock(clock)) + let throttled = response(429) + let first = try await owner.read(key()) { + #expect(await owner.noteRetryAfter(key(), seconds: 60, response: throttled) == false) + return throttled + } + #expect(first.http.statusCode == 429) + await owner.networkChanged(isOnline: false) + await owner.networkChanged(isOnline: true) + let cached = try await owner.read(key()) { Issue.record("retried before the server allowed it"); return response() } + #expect(cached.http.statusCode == 429) + clock.advance(by: .seconds(60)) + let recovered = try await owner.read(key()) { response() } + #expect(recovered.http.statusCode == 200) + } + + @Test("An arbitrarily long Retry-After does not overflow or become an early retry") + func oversizedRetryAfter() async throws { + let owner = Owner() + let throttled = response(429) + _ = try await owner.read(key()) { + #expect(await owner.noteRetryAfter(key(), seconds: TimeInterval(Int.max), response: throttled) == false) + return throttled + } + #expect(try await owner.read(key()) { Issue.record("ignored long server cooldown"); return response() }.http.statusCode == 429) + } + + @Test("Offline cancels readers, refuses more work, and reconnect recovers") + func offlineRecovery() async throws { + let owner = Owner() + let gate = CloudReadResponseGate() + let task = Task { try await owner.read(key()) { await gate.read(response()) } } + try await eventually { await gate.requests == 1 } + await owner.networkChanged(isOnline: false) + do { _ = try await task.value; Issue.record("offline returned live data") } + catch { #expect((error as? URLError)?.code == .notConnectedToInternet) } + await gate.release() + try await eventually { await owner.entries.isEmpty } + do { _ = try await owner.read(key()) { Issue.record("started offline"); return response() } } + catch { #expect((error as? URLError)?.code == .notConnectedToInternet) } + await owner.networkChanged(isOnline: true) + #expect(try await owner.read(key()) { response() }.http.statusCode == 200) + } + + @Test("Account and session generations never share responses") + func identityIsolation() async throws { + let owner = Owner() + let gate = CloudReadResponseGate() + let first = Task { try await owner.read(key()) { await gate.read(response()) } } + let replacement = try await owner.read(key(generation: 2)) { response(201) } + #expect(replacement.http.statusCode == 201) + await gate.release() + #expect(try await first.value.http.statusCode == 200) + } + + @Test("A mutation landing during a read shares one fresh trailing pass") + func mutationInvalidatesRunningRead() async throws { + let owner = Owner() + let gate = CloudReadResponseGate() + let operation: @Sendable () async -> Owner.Response = { + await gate.read(response(await gate.requests == 0 ? 200 : 201)) + } + let first = Task { try await owner.read(key(), operation: operation) } + try await eventually { await gate.requests == 1 } + await owner.invalidate() + let second = Task { try await owner.read(key(), operation: operation) } + try await eventually { await owner.entries.values.first?.waiters.count == 2 } + await gate.release() + #expect(try await first.value.http.statusCode == 201) + #expect(try await second.value.http.statusCode == 201) + #expect(await gate.requests == 2) + } + + private func eventually(_ condition: @escaping () async -> Bool) async throws { + let deadline = ContinuousClock.now.advanced(by: .seconds(10)) + while !(await condition()), ContinuousClock.now < deadline { await Task.yield() } + try #require(await condition(), "Fixture did not reach its expected actor state") + } +} + +private actor CloudReadResponseGate { + private(set) var requests = 0 + private var released = false + private var waiting: [(CloudReadRequestCoordinator.Response, CheckedContinuation)] = [] + func read(_ response: CloudReadRequestCoordinator.Response) async -> CloudReadRequestCoordinator.Response { + requests += 1 + if released { return response } + return await withCheckedContinuation { waiting.append((response, $0)) } + } + func release() { + released = true + let pending = waiting + waiting.removeAll() + for (response, continuation) in pending { continuation.resume(returning: response) } + } +} diff --git a/cmuxTests/CloudRefreshURLProtocol.swift b/cmuxTests/CloudRefreshURLProtocol.swift new file mode 100644 index 000000000000..1f735e25dfe1 --- /dev/null +++ b/cmuxTests/CloudRefreshURLProtocol.swift @@ -0,0 +1,84 @@ +import Foundation + +/// URLProtocol's synchronous callbacks hand off to one actor; only that actor +/// reads the fixture state or calls the client, including after stopLoading. +final class CloudRefreshURLProtocol: URLProtocol, @unchecked Sendable { + enum Behavior: Sendable { case normal, statsUnavailable, listUnavailable, throttled } + private static let responses = Responses() + static func holdResponses() async { await responses.hold() } + static func releaseResponses() async { await responses.release() } + static func configure(_ behavior: Behavior) async { await responses.configure(behavior) } + static func waitUntilStarted(_ count: Int = 1) async { await responses.waitUntilStarted(count) } + static func waitUntilStopped() async { await responses.waitUntilStopped() } + static func reset() async { await responses.reset() } + static func requestCounts() async -> [String: Int] { await responses.counts } + override class func canInit(with request: URLRequest) -> Bool { true } + override class func canonicalRequest(for request: URLRequest) -> URLRequest { request } + override func startLoading() { Task { await Self.responses.start(self) } } + override func stopLoading() { Task { await Self.responses.stop(self) } } + + private actor Responses { + private(set) var counts: [String: Int] = [:] + private var tasks: [ObjectIdentifier: Task] = [:] + private var behavior = Behavior.normal + private var held = false + private var responseWaiters: [ObjectIdentifier: CheckedContinuation] = [:] + func hold() { held = true } + func release() { + held = false + let pending = responseWaiters + responseWaiters.removeAll() + for waiter in pending.values { waiter.resume() } + } + private var stopped = false + private var startWaiters: [(Int, CheckedContinuation)] = [] + private var stopWaiters: [CheckedContinuation] = [] + func configure(_ value: Behavior) { behavior = value } + func waitUntilStarted(_ count: Int) async { + guard counts.values.reduce(0, +) < count else { return } + await withCheckedContinuation { startWaiters.append((count, $0)) } + } + func waitUntilStopped() async { + guard !stopped else { return } + await withCheckedContinuation { stopWaiters.append($0) } + } + func reset() { + release() + for task in tasks.values { task.cancel() } + tasks.removeAll() + counts.removeAll() + behavior = .normal + stopped = false + } + func start(_ source: CloudRefreshURLProtocol) { + let key = ObjectIdentifier(source) + let path = source.request.url!.path + counts[path, default: 0] += 1 + let count = counts.values.reduce(0, +) + let ready = startWaiters.filter { $0.0 <= count } + startWaiters.removeAll { $0.0 <= count } + for (_, waiter) in ready { waiter.resume() } + let behavior = self.behavior + tasks[key] = Task { + guard !Task.isCancelled else { return } + if self.held { await withCheckedContinuation { self.responseWaiters[key] = $0 } } + guard self.tasks.removeValue(forKey: key) != nil else { return } + let unavailable = path.hasSuffix("/stats") ? behavior == .statsUnavailable : behavior == .listUnavailable + let response = HTTPURLResponse(url: source.request.url!, statusCode: behavior == .throttled ? 429 : unavailable ? 503 : 200, httpVersion: nil, + headerFields: behavior == .throttled ? ["Retry-After": "60"] : nil)! + source.client?.urlProtocol(source, didReceive: response, cacheStoragePolicy: .notAllowed) + let body = path.hasSuffix("/stats") ? #"{"state":"awake","cpus":2}"# : #"{"vms":[{"id":"fixture-0","provider":"fixture","image":"desktop-vnc","status":"running","createdAt":0,"capabilities":{"stats":true}}]}"# + source.client?.urlProtocol(source, didLoad: Data(body.utf8)) + source.client?.urlProtocolDidFinishLoading(source) + } + } + func stop(_ source: CloudRefreshURLProtocol) { + tasks.removeValue(forKey: ObjectIdentifier(source))?.cancel() + responseWaiters.removeValue(forKey: ObjectIdentifier(source))?.resume() + stopped = true + let waiters = stopWaiters + stopWaiters.removeAll() + for waiter in waiters { waiter.resume() } + } + } +} diff --git a/cmuxTests/VMClientReadCoalescingTests.swift b/cmuxTests/VMClientReadCoalescingTests.swift new file mode 100644 index 000000000000..fe01f11a247e --- /dev/null +++ b/cmuxTests/VMClientReadCoalescingTests.swift @@ -0,0 +1,286 @@ +import CMUXAuthCore +import CmuxAuthRuntime +import Foundation +import Testing + +#if canImport(cmux_DEV) +@testable import cmux_DEV +#elseif canImport(cmux) +@testable import cmux +#endif + +@MainActor +@Suite("Cloud read request ownership", .serialized) +struct VMClientReadCoalescingTests { + @Test("Overlapping machine stats callers share one HTTP request") + func statsReadersShareTransport() async throws { + let fixture = try await CloudRefreshFixture.make() + defer { fixture.session.invalidateAndCancel() } + await CloudRefreshURLProtocol.reset() + await CloudRefreshURLProtocol.holdResponses() + let requests = Task { + await withTaskGroup(of: Void.self) { group in + for _ in 0..<4 { + for machine in 0..<10 { + group.addTask { _ = try? await fixture.client.stats(id: "fixture-\(machine)") } + } + } + } + } + try await eventually { await fixture.readRequests.entries.values.reduce(0) { $0 + $1.waiters.count } == 40 } + await CloudRefreshURLProtocol.releaseResponses() + await requests.value + let counts = await CloudRefreshURLProtocol.requestCounts() + #expect(counts.count == 10) + #expect(counts.values.allSatisfy { $0 == 1 }, "Four owners must share one read per machine: \(counts.values.sorted())") + } + + @Test("Overlapping machine list callers share one HTTP request") + func listReadersShareTransport() async throws { + let fixture = try await CloudRefreshFixture.make() + defer { fixture.session.invalidateAndCancel() } + await CloudRefreshURLProtocol.reset() + await CloudRefreshURLProtocol.holdResponses() + let requests = Task { + await withTaskGroup(of: Void.self) { group in + for _ in 0..<4 { group.addTask { _ = try? await fixture.client.listPage() } } + } + } + try await eventually { await fixture.readRequests.entries.values.first?.waiters.count == 4 } + await CloudRefreshURLProtocol.releaseResponses() + await requests.value + let counts = await CloudRefreshURLProtocol.requestCounts() + #expect(counts.values.reduce(0, +) == 1) + } + @Test("A hidden panel cancels its list and cannot start stats from a late result") + func hiddenPanelCancelsFollowupWork() async throws { + let fixture = try await CloudRefreshFixture.make() + defer { fixture.session.invalidateAndCancel() } + await CloudRefreshURLProtocol.reset() + await CloudRefreshURLProtocol.holdResponses() + let model = MachinesPanelViewModel(client: fixture.client, isCloudEnabled: { true }) + model.startPolling() + await CloudRefreshURLProtocol.waitUntilStarted() + model.stopPolling() + await CloudRefreshURLProtocol.waitUntilStopped() + #expect(!model.isLoading) + #expect(model.machines.isEmpty) + NotificationCenter.default.post(name: .cmuxCloudReadNetworkChanged, object: nil, userInfo: ["isOnline": true]) + #expect(await CloudRefreshURLProtocol.requestCounts().values.reduce(0, +) == 1) + } + + @Test("Dropping a view model releases it and cancels its pending list") + func viewModelTeardown() async throws { + let fixture = try await CloudRefreshFixture.make() + defer { fixture.session.invalidateAndCancel() } + await CloudRefreshURLProtocol.reset() + await CloudRefreshURLProtocol.holdResponses() + var model: MachinesPanelViewModel? = MachinesPanelViewModel(client: fixture.client, isCloudEnabled: { true }) + weak var weakModel = model + model?.refresh() + await CloudRefreshURLProtocol.waitUntilStarted() + model = nil + #expect(weakModel == nil) + await CloudRefreshURLProtocol.waitUntilStopped() + } + + @Test("A failed stats sample clears the last live reading") + func failedStatsAreUnavailable() async throws { + let fixture = try await CloudRefreshFixture.make() + defer { fixture.session.invalidateAndCancel() } + await CloudRefreshURLProtocol.reset() + let model = MachinesPanelViewModel(client: fixture.client, isCloudEnabled: { true }) + defer { model.stopPolling() } + model.refresh() + try await eventually { model.machines.first?.stats?.state == .awake } + await CloudRefreshURLProtocol.configure(.statsUnavailable) + model.refresh() + try await eventually { !model.isLoading && model.machines.first?.stats?.state == .unavailable } + #expect(model.machines.count == 1) + #expect(model.listProblem == nil) + } + + @Test("Known offline state clears live samples without waiting for the next poll") + func offlinePresentation() async throws { + let fixture = try await CloudRefreshFixture.make() + defer { fixture.session.invalidateAndCancel() } + await CloudRefreshURLProtocol.reset() + let model = MachinesPanelViewModel(client: fixture.client, isCloudEnabled: { true }) + defer { model.stopPolling() } + model.refresh() + try await eventually { model.machines.first?.stats?.state == .awake } + NotificationCenter.default.post(name: .cmuxCloudReadNetworkChanged, object: nil, userInfo: ["isOnline": false]) + #expect(model.machines.first?.stats?.state == .unavailable) + #expect(model.listProblem == .unreachable) + #expect(model.lastErrorDescription == URLError(.notConnectedToInternet).localizedDescription) + } + + @Test("The VM operation budget cancels a slow transport") + func totalRequestBudget() async throws { + let clock = CloudReadManualClock() + let reads = CloudReadRequestCoordinator(clock: CloudRequestClock(clock), budget: .milliseconds(100)) + let fixture = try await CloudRefreshFixture.make(readRequests: reads) + defer { fixture.session.invalidateAndCancel() } + await CloudRefreshURLProtocol.reset() + await CloudRefreshURLProtocol.holdResponses() + let request = Task { try await fixture.client.stats(id: "fixture-0") } + await CloudRefreshURLProtocol.waitUntilStarted() + clock.advance(by: .milliseconds(101)) + do { _ = try await request.value; Issue.record("request exceeded its total budget") } + catch { #expect((error as? URLError)?.code == .timedOut) } + await CloudRefreshURLProtocol.waitUntilStopped() + await CloudRefreshURLProtocol.releaseResponses() + } + + @Test("HTTP Retry-After exceeds the budget without an early automatic retry") + func retryAfterAcrossCalls() async throws { + let clock = CloudReadManualClock() + let reads = CloudReadRequestCoordinator(clock: CloudRequestClock(clock)) + let fixture = try await CloudRefreshFixture.make(readRequests: reads) + defer { fixture.session.invalidateAndCancel() } + await CloudRefreshURLProtocol.reset() + await CloudRefreshURLProtocol.configure(.throttled) + for _ in 0..<2 { + do { _ = try await fixture.client.stats(id: "fixture-0"); Issue.record("throttle succeeded") } + catch VMClientError.httpStatus(429, _) {} catch { Issue.record("\(error)") } + } + #expect(await CloudRefreshURLProtocol.requestCounts().values.reduce(0, +) == 1) + await CloudRefreshURLProtocol.configure(.normal) + clock.advance(by: .seconds(60)) + #expect(try await fixture.client.stats(id: "fixture-0").state == .awake) + #expect(await CloudRefreshURLProtocol.requestCounts().values.reduce(0, +) == 2) + } + + @Test("A list delayed beyond a polling interval stays owned and stops when hidden") + func delayedListAcrossPoll() async throws { + let clock = CloudReadManualClock() + let reads = CloudReadRequestCoordinator(clock: CloudRequestClock(clock), budget: .seconds(90)) + let fixture = try await CloudRefreshFixture.make(readRequests: reads) + defer { fixture.session.invalidateAndCancel() } + await CloudRefreshURLProtocol.reset() + await CloudRefreshURLProtocol.holdResponses() + let model = MachinesPanelViewModel(client: fixture.client, pollingClock: clock, isCloudEnabled: { true }) + model.startPolling() + await CloudRefreshURLProtocol.waitUntilStarted() + try await eventually { clock.pendingSleeperCount == 2 } + clock.advance(by: .seconds(45)) + try await eventually { clock.pendingSleeperCount == 2 } + #expect(await CloudRefreshURLProtocol.requestCounts().values.reduce(0, +) == 1) + model.stopPolling() + await CloudRefreshURLProtocol.waitUntilStopped() + await CloudRefreshURLProtocol.releaseResponses() + #expect(!model.isLoading) + #expect(model.machines.isEmpty) + } + + @Test("Visible and hidden panels share the same machines without hidden follow-up work") + func visibleAndHiddenOwners() async throws { + let fixture = try await CloudRefreshFixture.make() + defer { fixture.session.invalidateAndCancel() } + await CloudRefreshURLProtocol.reset() + await CloudRefreshURLProtocol.holdResponses() + let models = (0..<4).map { _ in MachinesPanelViewModel(client: fixture.client, isCloudEnabled: { true }) } + defer { for model in models { model.stopPolling() } } + for model in models { model.startPolling() } + models[2].stopPolling() + models[3].stopPolling() + try await eventually { await fixture.readRequests.entries.values.first?.waiters.count == 2 } + await CloudRefreshURLProtocol.releaseResponses() + try await eventually { models[0].machines.first?.stats != nil && models[1].machines.first?.stats != nil } + let counts = await CloudRefreshURLProtocol.requestCounts() + #expect(counts["/api/vm"] == 1) + #expect(counts["/api/vm/fixture-0/stats"] == 1) + #expect(models[2].machines.isEmpty && models[3].machines.isEmpty) + } + + @Test("Team usage shares offline state with list and stats while keeping its shorter budget") + func teamUsageNetworkState() async throws { + let clock = CloudReadManualClock() + let reads = CloudReadRequestCoordinator(clock: CloudRequestClock(clock)) + let fixture = try await CloudRefreshFixture.make(readRequests: reads) + defer { fixture.session.invalidateAndCancel() } + await CloudRefreshURLProtocol.reset() + await CloudRefreshURLProtocol.holdResponses() + let usage = MachineUsageClient(session: fixture.session, auth: fixture.auth, readRequests: reads) + let request = Task { try await usage.teamUsage() } + await CloudRefreshURLProtocol.waitUntilStarted() + await reads.networkChanged(isOnline: false) + do { _ = try await request.value; Issue.record("usage survived offline") } + catch { #expect((error as? URLError)?.code == .notConnectedToInternet) } + await CloudRefreshURLProtocol.waitUntilStopped() + do { _ = try await usage.teamUsage(); Issue.record("usage started offline") } + catch { #expect((error as? URLError)?.code == .notConnectedToInternet) } + #expect(await CloudRefreshURLProtocol.requestCounts().values.reduce(0, +) == 1) + await reads.networkChanged(isOnline: true) + let recovery = Task { try await usage.teamUsage() } + await CloudRefreshURLProtocol.waitUntilStarted(2) + clock.advance(by: .seconds(16)) + do { _ = try await recovery.value; Issue.record("usage exceeded its 15 second budget") } + catch { #expect((error as? URLError)?.code == .timedOut) } + await CloudRefreshURLProtocol.releaseResponses() + } + + private func eventually(_ condition: () async -> Bool) async throws { + let deadline = ContinuousClock.now.advanced(by: .seconds(10)) + while !(await condition()), ContinuousClock.now < deadline { await Task.yield() } + try #require(await condition()) + } + +} + +@MainActor +struct CloudRefreshFixture { + let client: VMClient + let auth: AuthCoordinator + let session: URLSession + let readRequests: CloudReadRequestCoordinator + + static func make(readRequests: CloudReadRequestCoordinator = CloudReadRequestCoordinator()) async throws -> Self { + let defaults = try #require(UserDefaults(suiteName: "CloudRefreshFixture.\(UUID())")) + let auth = AuthCoordinator( + client: CloudRefreshAuthClient(), + sessionCache: CMUXAuthSessionCache(keyValueStore: defaults, key: "session"), + userCache: CMUXAuthIdentityStore(keyValueStore: defaults, key: "user"), + teamSelection: CMUXAuthTeamSelectionStore(keyValueStore: defaults, key: "team"), + anchor: AuthPresentationContextProvider(), + config: AuthConfig( + stack: CMUXAuthConfig(projectId: "fixture", publishableClientKey: "fixture"), + magicLinkCallbackURL: "http://127.0.0.1:1/callback", apiBaseURL: "http://127.0.0.1:1" + ), + launch: AuthLaunchOptions( + clearAuthRequested: false, mockDataEnabled: false, + environment: ["CMUX_UITEST_AUTH_FIXTURE": "1", "CMUX_UITEST_AUTH_USER_ID": "fixture"], + includesDevAuth: true + ) + ) + auth.start() + await auth.awaitBootstrapped() + try #require(auth.isAuthenticated) + let configuration = URLSessionConfiguration.ephemeral + configuration.protocolClasses = [CloudRefreshURLProtocol.self] + let session = URLSession(configuration: configuration) + return Self(client: VMClient( + session: session, auth: auth, resourceStats: VMResourceStatsStore(), checkpointRenames: CloudRenameCoordinator(), + machineCache: CloudMachineCache(defaults: defaults), readRequests: readRequests + ), auth: auth, session: session, readRequests: readRequests) + } +} + +private actor CloudRefreshAuthClient: AuthClient { + func accessToken() async -> String? { "fixture-access" } + func refreshToken() async -> String? { "fixture-refresh" } + func forceRefreshAccessToken() async -> String? { "fixture-access" } + func currentUser(throwOnMissing: Bool) async throws -> CMUXAuthUser? { + CMUXAuthUser(id: "fixture", primaryEmail: "fixture@example.test", displayName: "Fixture") + } + func listTeams() async throws -> [CMUXAuthTeam] { [] } + func sendMagicLinkEmail(email: String, callbackURL: String) async throws -> String { "fixture" } + func signInWithMagicLink(code: String) async throws {} + func signInWithCredential(email: String, password: String) async throws {} + func signInWithOAuth(provider: String, anchor: any AuthPresentationAnchoring) async throws {} + func storedAccessToken() async -> String? { "fixture-access" } + func clearLocalSession() async {} + func clearLocalSession(ifRefreshTokenMatches refreshToken: String) async {} + func revokeSession(accessToken: String?, refreshToken: String?) async throws {} + func freshAccessToken(accessToken: String?, refreshToken: String) async -> String? { accessToken } +} From 2725d74d547016bd6d5fe39154bb5196aa6b9811 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sat, 19 Sep 2026 23:00:35 -0700 Subject: [PATCH 15/92] fix: align guest regression fixtures and hosted runner selection --- .github/workflows/cloud-vm-guest-install.yml | 2 +- web/tests/freestyle-network-announcement.test.ts | 12 ++++++++++-- web/tests/vm-resource-reporter-install.test.ts | 11 ++++++++++- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cloud-vm-guest-install.yml b/.github/workflows/cloud-vm-guest-install.yml index 7bccefeafef2..5de15231591b 100644 --- a/.github/workflows/cloud-vm-guest-install.yml +++ b/.github/workflows/cloud-vm-guest-install.yml @@ -28,7 +28,7 @@ concurrency: jobs: guest-install: - runs-on: ubuntu-latest + runs-on: ${{ vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }} timeout-minutes: 15 defaults: run: diff --git a/web/tests/freestyle-network-announcement.test.ts b/web/tests/freestyle-network-announcement.test.ts index 73b3a60f42e7..aadbe787d79f 100644 --- a/web/tests/freestyle-network-announcement.test.ts +++ b/web/tests/freestyle-network-announcement.test.ts @@ -60,7 +60,7 @@ describe("Freestyle private network readiness", () => { exec: async ({ command }: { command: string }) => { // Adapter, reporter, and hook preparation can add probes. This test // guards publication/rollback ordering, not the number of setup execs. - if (command.startsWith("python3 -c ")) events.push("guest-network"); + if (command.includes("Private network addresses are not ready")) events.push("guest-network"); return { statusCode: 0, stdout: "", stderr: "" }; }, fs: { @@ -72,6 +72,10 @@ describe("Freestyle private network readiness", () => { const client = { vms: { create: async () => { events.push("allocated"); return { vm, vmId: data.id, data }; }, get: async () => data, + ref: (id: string) => { + if (id !== data.id) throw new Error(`unexpected VM ref: ${id}`); + return vm; + }, } } as unknown as Freestyle; const provider = new FreestyleProvider({ client: () => client, @@ -102,7 +106,7 @@ describe("Freestyle private network readiness", () => { }; const vm = { exec: async ({ command }: { command: string }) => { - if (command.startsWith("python3 -c ")) { + if (command.includes("Private network addresses are not ready")) { events.push("guest-network"); return { statusCode: 124, stdout: "", stderr: "network state probe timed out" }; } @@ -113,6 +117,10 @@ describe("Freestyle private network readiness", () => { }; const client = { vms: { create: async () => { events.push("allocated"); return { vm, vmId: data.id, data }; }, + ref: (id: string) => { + if (id !== data.id) throw new Error(`unexpected VM ref: ${id}`); + return vm; + }, } } as unknown as Freestyle; const provider = new FreestyleProvider({ client: () => client, diff --git a/web/tests/vm-resource-reporter-install.test.ts b/web/tests/vm-resource-reporter-install.test.ts index 92a2db38c26f..19a5b21acce1 100644 --- a/web/tests/vm-resource-reporter-install.test.ts +++ b/web/tests/vm-resource-reporter-install.test.ts @@ -17,7 +17,16 @@ function fixture(cliFails = false) { fs: { writeTextFile: async () => {}, remove: async () => {} }, delete: async () => { deleted.push(vmId); }, }; - const client = { vms: { create: async () => ({ vm, vmId, data }), get: async () => data } } as unknown as Freestyle; + const client = { + vms: { + create: async () => ({ vm, vmId, data }), + get: async () => data, + ref: (id: string) => { + if (id !== vmId) throw new Error(`unexpected VM ref: ${id}`); + return vm; + }, + }, + } as unknown as Freestyle; const provider = new FreestyleProvider({ client: () => client, resolveDaemonSource: async () => { throw new Error("No daemon install expected"); } }); return { provider, deleted, commands }; } From 3839b65cd62e7b426b9fa4027b8cb038af6bc351 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sat, 19 Sep 2026 23:06:03 -0700 Subject: [PATCH 16/92] ci: validate auth on the configured Swift package runner --- .github/workflows/auth-refresh-tests.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/auth-refresh-tests.yml b/.github/workflows/auth-refresh-tests.yml index fe37c2ebbea5..4c2f05df918e 100644 --- a/.github/workflows/auth-refresh-tests.yml +++ b/.github/workflows/auth-refresh-tests.yml @@ -17,12 +17,14 @@ concurrency: jobs: auth-refresh-tests: - runs-on: ${{ vars.MACOS_RUNNER_26 || 'blacksmith-6vcpu-macos-26' }} + runs-on: ${{ vars.MACOS_RUNNER_DUAL_XCODE || 'blacksmith-6vcpu-macos-15' }} timeout-minutes: 15 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd with: persist-credentials: false + - name: Select pinned Xcode + run: ./scripts/select-ci-xcode.sh - name: Exercise shared refresh transport and deadlines without credentials run: swift test --package-path vendor/stack-auth-swift-sdk-prerelease --filter 'RefreshLifecycleTests|RefreshDeadlineTests|RefreshTransportTimingTests|RateLimitRetryTests' - name: Exercise coordinator cancellation, sessions, and recovery From 72716d26090c3b5783e9832b69f7e9e4f3958a52 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sat, 19 Sep 2026 23:21:51 -0700 Subject: [PATCH 17/92] test: expose hidden Cloud refresh and mutation fanout regressions --- cmuxTests/CloudOperationRecorderTests.swift | 82 +++++++++++++++++++ .../CloudReadRequestCoordinatorTests.swift | 2 +- cmuxTests/VMClientReadCoalescingTests.swift | 42 +++++++++- 3 files changed, 122 insertions(+), 4 deletions(-) diff --git a/cmuxTests/CloudOperationRecorderTests.swift b/cmuxTests/CloudOperationRecorderTests.swift index 27b5b70d0379..4422ca665ce2 100644 --- a/cmuxTests/CloudOperationRecorderTests.swift +++ b/cmuxTests/CloudOperationRecorderTests.swift @@ -8,6 +8,46 @@ import Testing @MainActor struct CloudOperationRecorderTests { + @Test("Placement receipts distinguish acceptance from permanent rejection", arguments: [202, 400]) + func placementUploaderReceipt(status: Int) async throws { + let fixture = try await CloudRefreshFixture.make() + defer { fixture.session.invalidateAndCancel() } + let directory = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + defer { try? FileManager.default.removeItem(at: directory) } + let queueURL = directory.appendingPathComponent("queue.json") + let configuration = URLSessionConfiguration.ephemeral + configuration.protocolClasses = [PlacementReceiptURLProtocol.self] + let session = URLSession(configuration: configuration) + defer { session.invalidateAndCancel() } + let uploader = CloudTelemetryUploader( + auth: fixture.auth, + baseURL: try #require(URL(string: "https://receipt-\(status).test")), + client: CloudTelemetryClient.current(info: [:], flavor: .dev), + session: session, + queueURL: queueURL + ) + let identity = try #require(fixture.auth.authenticatedSessionIdentity) + let recorder = CloudOperationRecorder(uploader: uploader, identity: { identity }) + let operation = recorder.begin(.open) + await recorder.finish(operation, error: CmuxTuiSurfaceProvider.ProviderError.remotePlacementUnavailable("fixture-machine")) + let persisted = try #require(JSONSerialization.jsonObject(with: Data(contentsOf: queueURL)) as? [[String: Any]]) + let span = try #require(persisted.first?["span"] as? [String: Any]) + #expect(span["failure"] as? String == "placement") + #expect(span["traceId"] as? String == operation.traceID) + #expect(span["operationId"] as? String == operation.operationID.uuidString.lowercased()) + let deadline = ContinuousClock.now.advanced(by: .seconds(10)) + var acknowledged = false + while ContinuousClock.now < deadline { + let data = try Data(contentsOf: queueURL) + let entries = try #require(JSONSerialization.jsonObject(with: data) as? [[String: Any]]) + if entries.isEmpty { acknowledged = true; break } + try await Task.sleep(for: .milliseconds(10)) + } + #expect(acknowledged) + #expect(await uploader.droppedCount == (status == 400 ? 1 : 0)) + await uploader.clearForSignOut() + } + @Test func authenticationFailuresKeepTheirTelemetryClassification() async throws { let identity = AuthenticatedSessionIdentity(generation: 1, accountID: "synthetic") let sink = CapturedCloudDiagnostics() @@ -247,4 +287,46 @@ private actor CapturedCloudDiagnostics: CloudTelemetrySending { func enqueue(_ span: CloudTelemetrySpan, identity: AuthenticatedSessionIdentity) { spans.append(span) } func clearForSignOut() { spans.removeAll() } } + +/// The request is handled synchronously without shared mutable fixture state. +private final class PlacementReceiptURLProtocol: URLProtocol, @unchecked Sendable { + override class func canInit(with request: URLRequest) -> Bool { true } + override class func canonicalRequest(for request: URLRequest) -> URLRequest { request } + override func stopLoading() {} + override func startLoading() { + do { + var body = request.httpBody ?? Data() + if body.isEmpty, let stream = request.httpBodyStream { + stream.open() + defer { stream.close() } + var buffer = [UInt8](repeating: 0, count: 4096) + while stream.hasBytesAvailable { + let count = stream.read(&buffer, maxLength: buffer.count) + if count < 0 { throw URLError(.cannotDecodeContentData) } + if count == 0 { break } + body.append(contentsOf: buffer.prefix(count)) + } + } + guard let batch = try JSONSerialization.jsonObject(with: body) as? [String: Any], + let spans = batch["spans"] as? [[String: Any]], spans.count == 1, + let span = spans.first, let eventID = span["eventId"] as? String, + span["failure"] as? String == "placement", + (span["traceId"] as? String)?.count == 32, + (span["spanId"] as? String)?.count == 16 else { + throw URLError(.cannotDecodeContentData) + } + let status = request.url?.host == "receipt-202.test" ? 202 : 400 + guard let url = request.url, + let response = HTTPURLResponse(url: url, statusCode: status, httpVersion: nil, headerFields: nil) else { + throw URLError(.badURL) + } + let receipt = try JSONSerialization.data(withJSONObject: ["eventIds": [eventID]]) + client?.urlProtocol(self, didReceive: response, cacheStoragePolicy: .notAllowed) + client?.urlProtocol(self, didLoad: receipt) + client?.urlProtocolDidFinishLoading(self) + } catch { + client?.urlProtocol(self, didFailWithError: error) + } + } +} #endif diff --git a/cmuxTests/CloudReadRequestCoordinatorTests.swift b/cmuxTests/CloudReadRequestCoordinatorTests.swift index 3a0484ab4a47..6b1b9cf3ac1b 100644 --- a/cmuxTests/CloudReadRequestCoordinatorTests.swift +++ b/cmuxTests/CloudReadRequestCoordinatorTests.swift @@ -251,7 +251,7 @@ struct CloudReadRequestCoordinatorTests { } } -private actor CloudReadResponseGate { +actor CloudReadResponseGate { private(set) var requests = 0 private var released = false private var waiting: [(CloudReadRequestCoordinator.Response, CheckedContinuation)] = [] diff --git a/cmuxTests/VMClientReadCoalescingTests.swift b/cmuxTests/VMClientReadCoalescingTests.swift index fe01f11a247e..c244d5f1846e 100644 --- a/cmuxTests/VMClientReadCoalescingTests.swift +++ b/cmuxTests/VMClientReadCoalescingTests.swift @@ -61,7 +61,9 @@ struct VMClientReadCoalescingTests { let model = MachinesPanelViewModel(client: fixture.client, isCloudEnabled: { true }) model.startPolling() await CloudRefreshURLProtocol.waitUntilStarted() + model.beginOperation("fixture operation") model.stopPolling() + model.endOperation() await CloudRefreshURLProtocol.waitUntilStopped() #expect(!model.isLoading) #expect(model.machines.isEmpty) @@ -76,7 +78,8 @@ struct VMClientReadCoalescingTests { await CloudRefreshURLProtocol.reset() await CloudRefreshURLProtocol.holdResponses() var model: MachinesPanelViewModel? = MachinesPanelViewModel(client: fixture.client, isCloudEnabled: { true }) - weak var weakModel = model + weak var weakModel: MachinesPanelViewModel? + weakModel = model model?.refresh() await CloudRefreshURLProtocol.waitUntilStarted() model = nil @@ -95,9 +98,13 @@ struct VMClientReadCoalescingTests { try await eventually { model.machines.first?.stats?.state == .awake } await CloudRefreshURLProtocol.configure(.statsUnavailable) model.refresh() - try await eventually { !model.isLoading && model.machines.first?.stats?.state == .unavailable } + try await eventually { !model.isLoading && model.machines.first?.stats?.state == .unknown } #expect(model.machines.count == 1) #expect(model.listProblem == nil) + #expect(model.machines.first?.stats?.cpus == 2) + #expect(model.machines.first?.stats?.cpuPercent == nil) + #expect(model.machines.first?.stats?.memoryUsedMb == nil) + #expect(model.machines.first?.stats?.diskUsedMb == nil) } @Test("Known offline state clears live samples without waiting for the next poll") @@ -110,7 +117,9 @@ struct VMClientReadCoalescingTests { model.refresh() try await eventually { model.machines.first?.stats?.state == .awake } NotificationCenter.default.post(name: .cmuxCloudReadNetworkChanged, object: nil, userInfo: ["isOnline": false]) - #expect(model.machines.first?.stats?.state == .unavailable) + #expect(model.machines.first?.stats?.state == .unknown) + #expect(model.machines.first?.stats?.cpus == 2) + #expect(model.machines.first?.stats?.cpuPercent == nil) #expect(model.listProblem == .unreachable) #expect(model.lastErrorDescription == URLError(.notConnectedToInternet).localizedDescription) } @@ -220,6 +229,33 @@ struct VMClientReadCoalescingTests { await CloudRefreshURLProtocol.releaseResponses() } + @Test("Successful resize invalidates only its authenticated list and machine stats") + func mutationInvalidationUsesRequestScope() async throws { + let fixture = try await CloudRefreshFixture.make() + defer { fixture.session.invalidateAndCancel() } + await CloudRefreshURLProtocol.reset() + let identity = try #require(fixture.auth.authenticatedSessionIdentity) + let paths = ["/api/vm", "/api/vm/fixture-0/stats", "/api/vm/fixture-1/stats"] + let gates = paths.map { _ in CloudReadResponseGate() } + let requests = zip(paths, gates).map { path, gate in + let key = CloudReadRequestCoordinator.Key(path: path, accountID: identity.accountID, + generation: identity.generation, teamID: fixture.auth.resolvedTeamID) + return Task { try await fixture.readRequests.read(key) { + let status = await gate.requests == 0 ? 200 : 201 + return await gate.read(.init(data: Data(), http: HTTPURLResponse( + url: URL(string: "https://fixture.invalid")!, statusCode: status, httpVersion: nil, headerFields: nil + )!)) + } } + } + for gate in gates { try await eventually { await gate.requests == 1 } } + _ = try await fixture.client.request("POST", path: "/api/vm/fixture-0/resize", jsonBody: ["cpu": 4]) + for gate in gates { await gate.release() } + for (index, request) in requests.enumerated() { + #expect(try await request.value.http.statusCode == (index < 2 ? 201 : 200)) + #expect(await gates[index].requests == (index < 2 ? 2 : 1)) + } + } + private func eventually(_ condition: () async -> Bool) async throws { let deadline = ContinuousClock.now.advanced(by: .seconds(10)) while !(await condition()), ContinuousClock.now < deadline { await Task.yield() } From 26a3ef43aa806df5b8b33162bc37ed8035975c92 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sat, 19 Sep 2026 23:33:32 -0700 Subject: [PATCH 18/92] fix: bound Cloud read invalidation and cooldown ownership --- .../CloudReadCooldownStore+Cooldown.swift | 8 + .../CloudReadCooldownStore+Session.swift | 8 + Sources/Cloud/CloudReadCooldownStore.swift | 70 +++++++++ Sources/Cloud/CloudReadMutation.swift | 52 +++++++ .../CloudReadRequestCoordinator+Context.swift | 8 + .../CloudReadRequestCoordinator+Entry.swift | 15 ++ .../CloudReadRequestCoordinator+Key.swift | 10 ++ .../CloudReadRequestCoordinator+Pending.swift | 10 ++ ...CloudReadRequestCoordinator+Response.swift | 8 + .../CloudReadRequestCoordinator+Waiter.swift | 8 + .../Cloud/CloudReadRequestCoordinator.swift | 82 ++-------- .../MachinesPanelViewModel+Refresh.swift | 124 --------------- Sources/Cloud/MachinesPanelViewModel.swift | 143 ++++++++++++++++-- Sources/Cloud/TeamMachineUsage.swift | 5 - Sources/Cloud/TeamMachineUsageKind.swift | 7 + Sources/Cloud/VMClient.swift | 7 +- cmux.xcodeproj/project.pbxproj | 44 ++++++ cmuxTests/CloudOperationRecorderTests.swift | 13 +- .../CloudReadRequestCoordinatorTests.swift | 110 +++++++++++++- .../Sources/StackAuth/TokenRefreshClock.swift | 3 + 20 files changed, 519 insertions(+), 216 deletions(-) create mode 100644 Sources/Cloud/CloudReadCooldownStore+Cooldown.swift create mode 100644 Sources/Cloud/CloudReadCooldownStore+Session.swift create mode 100644 Sources/Cloud/CloudReadCooldownStore.swift create mode 100644 Sources/Cloud/CloudReadMutation.swift create mode 100644 Sources/Cloud/CloudReadRequestCoordinator+Context.swift create mode 100644 Sources/Cloud/CloudReadRequestCoordinator+Entry.swift create mode 100644 Sources/Cloud/CloudReadRequestCoordinator+Key.swift create mode 100644 Sources/Cloud/CloudReadRequestCoordinator+Pending.swift create mode 100644 Sources/Cloud/CloudReadRequestCoordinator+Response.swift create mode 100644 Sources/Cloud/CloudReadRequestCoordinator+Waiter.swift create mode 100644 Sources/Cloud/TeamMachineUsageKind.swift diff --git a/Sources/Cloud/CloudReadCooldownStore+Cooldown.swift b/Sources/Cloud/CloudReadCooldownStore+Cooldown.swift new file mode 100644 index 000000000000..b9fe83cbe1bc --- /dev/null +++ b/Sources/Cloud/CloudReadCooldownStore+Cooldown.swift @@ -0,0 +1,8 @@ +import Foundation + +extension CloudReadCooldownStore { + struct Cooldown: Sendable { + let until: TimeInterval + let response: CloudReadRequestCoordinator.Response + } +} diff --git a/Sources/Cloud/CloudReadCooldownStore+Session.swift b/Sources/Cloud/CloudReadCooldownStore+Session.swift new file mode 100644 index 000000000000..8df5a4ed5f1b --- /dev/null +++ b/Sources/Cloud/CloudReadCooldownStore+Session.swift @@ -0,0 +1,8 @@ +import Foundation + +extension CloudReadCooldownStore { + struct Session: Equatable, Sendable { + let accountID: String? + let generation: UInt64? + } +} diff --git a/Sources/Cloud/CloudReadCooldownStore.swift b/Sources/Cloud/CloudReadCooldownStore.swift new file mode 100644 index 000000000000..953214679e00 --- /dev/null +++ b/Sources/Cloud/CloudReadCooldownStore.swift @@ -0,0 +1,70 @@ +import Foundation + +/// Bounds retained throttling state without turning eviction into an early retry. +struct CloudReadCooldownStore: Sendable { + private let capacity: Int + private var session: Session? + private var cooldowns: [CloudReadRequestCoordinator.Key: Cooldown] = [:] + private var overflow: Cooldown? + + init(capacity: Int = 256) { + self.capacity = max(1, capacity) + } + + var retainedCount: Int { cooldowns.count + (overflow == nil ? 0 : 1) } + + mutating func activateSession(for key: CloudReadRequestCoordinator.Key) { + let incoming = Session(accountID: key.accountID, generation: key.generation) + guard incoming != session else { return } + if let session { + // AuthCoordinator generations increase across account replacement. + // An old read admitted late cannot reactivate a retired session. + guard let generation = incoming.generation else { return } + if let previousGeneration = session.generation, generation <= previousGeneration { return } + } + session = incoming + cooldowns.removeAll(keepingCapacity: true) + overflow = nil + } + + mutating func response(for key: CloudReadRequestCoordinator.Key, now: TimeInterval) -> CloudReadRequestCoordinator.Response? { + guard matchesSession(key) else { return nil } + prune(now: now) + return (cooldowns[key] ?? overflow)?.response + } + + mutating func record( + _ key: CloudReadRequestCoordinator.Key, until: TimeInterval, now: TimeInterval, + response: CloudReadRequestCoordinator.Response + ) { + guard matchesSession(key), until > now else { return } + prune(now: now) + let compactResponse = CloudReadRequestCoordinator.Response( + data: Data(#"{"error":"rate_limited"}"#.utf8), + http: HTTPURLResponse( + url: response.http.url ?? URL(fileURLWithPath: "/"), statusCode: 429, + httpVersion: nil, headerFields: nil + )! + ) + let cooldown = Cooldown(until: until, response: compactResponse) + if let existing = cooldowns[key] { + if until > existing.until { cooldowns[key] = cooldown } + } else if cooldowns.count < capacity { + cooldowns[key] = cooldown + } else if until > (overflow?.until ?? now) { + // Once full, retain one conservative session barrier instead of + // arbitrary keys/bodies. Unknown paths may wait longer at capacity; + // no forgotten path can retry before its server minimum. + overflow = cooldown + } + } + + private func matchesSession(_ key: CloudReadRequestCoordinator.Key) -> Bool { + session == Session(accountID: key.accountID, generation: key.generation) + } + + private mutating func prune(now: TimeInterval) { + cooldowns = cooldowns.filter { $0.value.until > now } + if let overflow, overflow.until <= now { self.overflow = nil } + } +} diff --git a/Sources/Cloud/CloudReadMutation.swift b/Sources/Cloud/CloudReadMutation.swift new file mode 100644 index 000000000000..6c8dd7ad9a72 --- /dev/null +++ b/Sources/Cloud/CloudReadMutation.swift @@ -0,0 +1,52 @@ +import Foundation + +/// Identifies the read snapshots changed by one successful control-plane mutation. +struct CloudReadMutation: Sendable { + let scope: CloudReadRequestCoordinator.Key + let affectedPaths: Set + + init(method: String, scope: CloudReadRequestCoordinator.Key, responseData: Data) { + self.scope = scope + var paths: Set = [] + let parts = scope.path.split(separator: "/").map(String.init) + let createsMachine = method == "POST" && [ + "/api/vm", "/api/vm/restore", "/api/vm/base/open", "/api/vm/base/reset" + ].contains(scope.path) + var forksMachine = false + if createsMachine { + paths.insert("/api/vm") + paths.insert("/api/coderouter/vm-usage/team") + } else if parts.count >= 3, parts[0] == "api", parts[1] == "vm", + !["base", "restore", "tunnel", "publications", "domains"].contains(parts[2]) { + let machinePath = "/api/vm/\(parts[2])" + if parts.count == 3, method == "PATCH" || method == "DELETE" { + paths.insert("/api/vm") + paths.insert("/api/coderouter/vm-usage/team") + if method == "DELETE" { paths.insert("\(machinePath)/stats") } + } else if parts.count == 4, method == "POST", [ + "pause", "resume", "resize", "fork", "attach-endpoint", "ssh-endpoint", + "scp-endpoint", "sessions", "exec", "open-port" + ].contains(parts[3]) { + // Open/exec/file-transfer routes can wake a suspended machine. + // Other machines' samples remain valid while this one changes. + paths.insert("/api/vm") + paths.insert("\(machinePath)/stats") + if parts[3] == "fork" { + forksMachine = true + paths.insert("/api/coderouter/vm-usage/team") + } + } + } + if createsMachine || forksMachine { + if let object = try? JSONSerialization.jsonObject(with: responseData) as? [String: Any], + let id = object["id"] as? String, !id.isEmpty { + var allowed = CharacterSet.urlPathAllowed + allowed.remove(charactersIn: "/?#") + if let encodedID = id.addingPercentEncoding(withAllowedCharacters: allowed) { + paths.insert("/api/vm/\(encodedID)/stats") + } + } + } + affectedPaths = paths + } +} diff --git a/Sources/Cloud/CloudReadRequestCoordinator+Context.swift b/Sources/Cloud/CloudReadRequestCoordinator+Context.swift new file mode 100644 index 000000000000..d15662c53d03 --- /dev/null +++ b/Sources/Cloud/CloudReadRequestCoordinator+Context.swift @@ -0,0 +1,8 @@ +import Foundation + +extension CloudReadRequestCoordinator { + struct Context: Sendable { + weak var owner: CloudReadRequestCoordinator? + let key: Key + } +} diff --git a/Sources/Cloud/CloudReadRequestCoordinator+Entry.swift b/Sources/Cloud/CloudReadRequestCoordinator+Entry.swift new file mode 100644 index 000000000000..b41b45936fe3 --- /dev/null +++ b/Sources/Cloud/CloudReadRequestCoordinator+Entry.swift @@ -0,0 +1,15 @@ +import Foundation + +extension CloudReadRequestCoordinator { + struct Entry: Sendable { + let id: UUID + let deadline: Duration + var waiters: [UUID: Waiter] + var work: Task? + var timer: Task? + var terminalError: URLError? + var invalidated = false + let operation: @Sendable () async throws -> Response + var pending: Pending? + } +} diff --git a/Sources/Cloud/CloudReadRequestCoordinator+Key.swift b/Sources/Cloud/CloudReadRequestCoordinator+Key.swift new file mode 100644 index 000000000000..c47bc0c4919c --- /dev/null +++ b/Sources/Cloud/CloudReadRequestCoordinator+Key.swift @@ -0,0 +1,10 @@ +import Foundation + +extension CloudReadRequestCoordinator { + struct Key: Hashable, Sendable { + let path: String + let accountID: String? + let generation: UInt64? + let teamID: String? + } +} diff --git a/Sources/Cloud/CloudReadRequestCoordinator+Pending.swift b/Sources/Cloud/CloudReadRequestCoordinator+Pending.swift new file mode 100644 index 000000000000..c437a23431db --- /dev/null +++ b/Sources/Cloud/CloudReadRequestCoordinator+Pending.swift @@ -0,0 +1,10 @@ +import Foundation + +extension CloudReadRequestCoordinator { + struct Pending: Sendable { + let id: UUID + var waiters: [UUID: Waiter] + let operation: @Sendable () async throws -> Response + var timer: Task? + } +} diff --git a/Sources/Cloud/CloudReadRequestCoordinator+Response.swift b/Sources/Cloud/CloudReadRequestCoordinator+Response.swift new file mode 100644 index 000000000000..d05563b58e15 --- /dev/null +++ b/Sources/Cloud/CloudReadRequestCoordinator+Response.swift @@ -0,0 +1,8 @@ +import Foundation + +extension CloudReadRequestCoordinator { + struct Response: Sendable { + let data: Data + let http: HTTPURLResponse + } +} diff --git a/Sources/Cloud/CloudReadRequestCoordinator+Waiter.swift b/Sources/Cloud/CloudReadRequestCoordinator+Waiter.swift new file mode 100644 index 000000000000..8d09730cef91 --- /dev/null +++ b/Sources/Cloud/CloudReadRequestCoordinator+Waiter.swift @@ -0,0 +1,8 @@ +import Foundation + +extension CloudReadRequestCoordinator { + struct Waiter: Sendable { + let deadline: Duration + let continuation: CheckedContinuation + } +} diff --git a/Sources/Cloud/CloudReadRequestCoordinator.swift b/Sources/Cloud/CloudReadRequestCoordinator.swift index ea6f2193aa6b..1e3bd0476b13 100644 --- a/Sources/Cloud/CloudReadRequestCoordinator.swift +++ b/Sources/Cloud/CloudReadRequestCoordinator.swift @@ -4,62 +4,15 @@ import Foundation /// only that waiter; the last waiter cancels the transport. A cancelled request /// keeps its slot until teardown completes, preventing replacement amplification. actor CloudReadRequestCoordinator { - struct Key: Hashable, Sendable { - let path: String - let accountID: String? - let generation: UInt64? - let teamID: String? - } - - struct Response: Sendable { - let data: Data - let http: HTTPURLResponse - } - - struct Context: Sendable { - weak var owner: CloudReadRequestCoordinator? - let key: Key - } - @TaskLocal static var current: Context? - struct Waiter: Sendable { - let deadline: Duration - let continuation: CheckedContinuation - } - - struct Entry: Sendable { - let id: UUID - let deadline: Duration - var waiters: [UUID: Waiter] - var work: Task? - var timer: Task? - var terminalError: URLError? - var invalidated = false - let operation: @Sendable () async throws -> Response - var pending: Pending? - } - - struct Pending: Sendable { - let id: UUID - var waiters: [UUID: Waiter] - let operation: @Sendable () async throws -> Response - var timer: Task? - } - - private struct Cooldown { - let until: TimeInterval - let response: Response - } - private nonisolated let clock: CloudRequestClock private nonisolated let budget: Duration private let onNetworkChange: @Sendable (Bool) async -> Void private(set) var entries: [Key: Entry] = [:] private var networkTask: Task? private var isOnline: Bool? - private var cooldowns: [Key: Cooldown] = [:] - private var nextCooldownExpiry: TimeInterval? + private var cooldowns = CloudReadCooldownStore() init(clock: CloudRequestClock = CloudRequestClock(ContinuousClock()), budget: Duration = .seconds(30), onNetworkChange: @escaping @Sendable (Bool) async -> Void = { _ in }) { @@ -90,6 +43,7 @@ actor CloudReadRequestCoordinator { _ key: Key, waiter: UUID, deadline: Duration, continuation: CheckedContinuation, operation: @escaping @Sendable () async throws -> Response ) { + cooldowns.activateSession(for: key) if clock.now() >= deadline { continuation.resume(throwing: URLError(.timedOut)) return @@ -110,13 +64,8 @@ actor CloudReadRequestCoordinator { } return } - let now = seconds(clock.now()) - if let nextCooldownExpiry, now >= nextCooldownExpiry { - cooldowns = cooldowns.filter { $0.value.until > now } - self.nextCooldownExpiry = cooldowns.values.map(\.until).min() - } - if let cooldown = cooldowns[key] { - continuation.resume(returning: cooldown.response) + if let response = cooldowns.response(for: key, now: seconds(clock.now())) { + continuation.resume(returning: response) return } startEntry(key, id: UUID(), deadline: deadline, @@ -134,8 +83,8 @@ actor CloudReadRequestCoordinator { for waiter in waiters.values { waiter.continuation.resume(throwing: error) } return } - if let cooldown = cooldowns[key], cooldown.until > seconds(clock.now()) { - for waiter in waiters.values { waiter.continuation.resume(returning: cooldown.response) } + if let response = cooldowns.response(for: key, now: seconds(clock.now())) { + for waiter in waiters.values { waiter.continuation.resume(returning: response) } return } entries[key] = Entry(id: id, deadline: deadline, waiters: waiters, operation: operation) @@ -222,23 +171,24 @@ actor CloudReadRequestCoordinator { } } - /// A completed mutation invalidates any read that started before it. Its - /// readers share one trailing pass, within the original operation budget. - func invalidate() { - for key in entries.keys { entries[key]?.invalidated = true } + /// A completed mutation invalidates affected reads from the captured auth + /// and team scope. Its readers share a trailing pass in the original budget. + func invalidate(_ mutation: CloudReadMutation) { + for path in mutation.affectedPaths { + let key = Key(path: path, accountID: mutation.scope.accountID, + generation: mutation.scope.generation, teamID: mutation.scope.teamID) + entries[key]?.invalidated = true + } } /// Retains the server's minimum retry time across cancellation and later /// polls. When it exceeds this operation's remaining budget, return the - /// original 429 now; future reads receive that response until retry is legal. + /// original 429 now; future reads receive a compact 429 until retry is legal. func noteRetryAfter(_ key: Key, seconds: TimeInterval, response: Response) -> Bool { // Retry-After can contain Int.max seconds. Keep that distant deadline // as a monotonic floating-point instant rather than overflowing Duration. let until = self.seconds(clock.now()) + seconds - if until > (cooldowns[key]?.until ?? 0) { - cooldowns[key] = Cooldown(until: until, response: response) - nextCooldownExpiry = min(nextCooldownExpiry ?? until, until) - } + cooldowns.record(key, until: until, now: self.seconds(clock.now()), response: response) guard let entry = entries[key], entry.terminalError == nil else { return false } return until < self.seconds(entry.deadline) } diff --git a/Sources/Cloud/MachinesPanelViewModel+Refresh.swift b/Sources/Cloud/MachinesPanelViewModel+Refresh.swift index 2438d654131f..ffbdff609e69 100644 --- a/Sources/Cloud/MachinesPanelViewModel+Refresh.swift +++ b/Sources/Cloud/MachinesPanelViewModel+Refresh.swift @@ -19,37 +19,6 @@ extension MachinesPanelViewModel { } } - /// Read the current shared resource owner, retaining its resize and list fences. - func applyResourceStats(machineIDs: Set?) { - guard isCloudEnabled(), let resourceStats else { return } - for id in machineIDs ?? Set(machineIndexByID.keys) { - guard let index = machineIndexByID[id], machines.indices.contains(index), - machines[index].id == id, machines[index].capabilities.stats else { continue } - let stats = resourceStats.stats(for: id) - if machines[index].stats != stats { machines[index].stats = stats } - } - } - - func refresh() { - guard isCloudEnabled(), let client = client ?? VMClient.shared else { return } - guard refreshTask == nil else { refreshRequestedWhileLoading = true; return } - isLoading = true - let generation = refreshGeneration - let scope = machinePinStore?.scopeIdentifier - refreshTask = Task { [weak self] in - let result: Result - do { result = .success(try await client.listPage()) } - catch { result = .failure(error) } - guard !Task.isCancelled, let self, generation == self.refreshGeneration else { return } - self.applyRefreshResult(result, generation: generation, scope: scope) - self.refreshTask = nil - if self.refreshRequestedWhileLoading { - self.refreshRequestedWhileLoading = false - self.refresh() - } - } - } - func startPolling() { wantsPolling = true guard isCloudEnabled() else { pausePolling(); return } @@ -66,97 +35,4 @@ extension MachinesPanelViewModel { func stopPolling() { wantsPolling = false; pausePolling() } - func pausePolling() { - pollTask?.cancel(); pollTask = nil - refreshTask?.cancel(); refreshTask = nil - refreshRequestedWhileLoading = false - refreshGeneration &+= 1 - isLoading = false - statsTask?.cancel(); statsTask = nil; statsID = nil - usageTask?.cancel(); usageTask = nil - usageFailureCount = 0 - usageRetryNotBefore = nil - treeTask?.cancel(); treeTask = nil - machineRefreshes.cancelAll() - freeAccessTransitionTask?.cancel(); freeAccessTransitionTask = nil - } - - func clearUnavailableMetrics() { - if let resourceStats { - for id in machineIndexByID.keys { - _ = resourceStats.finishRead(resourceStats.beginRead(machineID: id), stats: nil) - } - applyResourceStats(machineIDs: nil) - } - usageByMachineID = [:] - machines = MachineSnapshotBuilder.applyingUsage(to: machines, usage: [:]) - } - - func applyRefreshResult(_ result: Result, generation: UInt64, scope: String?) { - guard generation == refreshGeneration, scope == machinePinStore?.scopeIdentifier, isCloudEnabled() else { return } - do { - let page = try result.get() - try Task.checkCancellation() - guard generation == refreshGeneration, scope == machinePinStore?.scopeIdentifier, - isCloudEnabled() else { return } - let previous = resourceStats?.snapshot ?? [:] - let freeAccessWindowDays = page.limits?.freeAccessWindowDays ?? 0 - self.freeAccessWindowDays = freeAccessWindowDays - var snapshots = page.vms.map { - MachineSnapshotBuilder.snapshot( - from: $0, - freeAccessWindowDays: freeAccessWindowDays, - previousStats: previous[$0.id] - ) - } - snapshots = MachineSnapshotBuilder.applyingUsage(to: snapshots, usage: usageByMachineID) - let defaultMachineID = defaultMachineStore?.resolveMachineID( - from: snapshots.map { CloudMachineDescriptor(id: $0.id, isDesktop: $0.isDesktop) }, - isComplete: true - ) - snapshots = snapshots.map { snapshot in - var next = snapshot - next.isDefault = snapshot.id == defaultMachineID - return next - } - // The authoritative fleet plus catalog-only rows is the complete - // visible set: a pin whose machine is gone from both is pruned. - machinePinStore?.reconcile(machineIDs: MachineSnapshotBuilder.includingCatalogMachines(snapshots, catalog: scopedCatalogSnapshot()).map(\.id)) - machineIndexByID = Dictionary(uniqueKeysWithValues: snapshots.enumerated().map { ($0.element.id, $0.offset) }) - machines = snapshots - lastLimits = page.limits - scheduleFreeAccessTransition() - refreshStats() - refreshUsage() - readCatalog() - plan = MachineSnapshotBuilder.planSnapshot(activeCount: snapshots.count, limits: page.limits, machines: snapshots) - lastErrorDescription = nil - listProblem = nil - } catch is CancellationError { - return - } catch let error as VMClientError { - guard !Task.isCancelled, generation == refreshGeneration, - scope == machinePinStore?.scopeIdentifier else { return } - if case .notSignedIn = error { - machines = [] - machineIndexByID.removeAll() - plan = nil - activeOperation = nil - lastErrorDescription = nil - listProblem = nil - hasLoadedOnce = false - isLoading = false - return - } - lastErrorDescription = String(describing: error) - listProblem = Self.classifyListFailure(error) - } catch { - guard !Task.isCancelled, generation == refreshGeneration, - scope == machinePinStore?.scopeIdentifier else { return } - lastErrorDescription = String(describing: error) - listProblem = .unreachable - } - isLoading = false - hasLoadedOnce = true - } } diff --git a/Sources/Cloud/MachinesPanelViewModel.swift b/Sources/Cloud/MachinesPanelViewModel.swift index 4c074e43eab5..e009b61b5ad4 100644 --- a/Sources/Cloud/MachinesPanelViewModel.swift +++ b/Sources/Cloud/MachinesPanelViewModel.swift @@ -11,21 +11,21 @@ extension Notification.Name { /// through this store. @MainActor final class MachinesPanelViewModel: ObservableObject { - @Published var machines: [MachineSnapshot] = [] - @Published var plan: MachinePlanSnapshot? - @Published var isLoading = false - @Published var hasLoadedOnce = false - @Published var lastErrorDescription: String? + @Published private(set) var machines: [MachineSnapshot] = [] + @Published private(set) var plan: MachinePlanSnapshot? + @Published private(set) var isLoading = false + @Published private(set) var hasLoadedOnce = false + @Published private(set) var lastErrorDescription: String? /// Why the machine list could not load, classified so the empty state can /// say the true thing: a server-rejected session needs a fresh sign-in, a /// plan gate needs an upgrade, and only genuinely transient failures get /// the retry-first "unreachable" presentation. - @Published var listProblem: CloudListProblem? + @Published private(set) var listProblem: CloudListProblem? /// Per-machine coderouter spend from the last successful usage fetch, /// keyed by machine id. Refreshed with every machine-list refresh (the /// slow poll and the explicit Refresh verb), never more often. Empty on /// backends without the usage route; a failed fetch keeps the last value. - @Published var usageByMachineID: [String: MachineUsageSnapshot] = [:] + @Published private(set) var usageByMachineID: [String: MachineUsageSnapshot] = [:] enum CloudListProblem: Equatable { /// HTTP 401: the Cloud service no longer accepts this session. @@ -53,7 +53,7 @@ final class MachinesPanelViewModel: ObservableObject { /// Human-readable label of the Cloud VM action currently running from this /// panel ("Checkpointing noble-wren…"). Replaces the plan meter in the /// header while set — the in-app substitute for a floating progress HUD. - @Published var activeOperation: String? + @Published private(set) var activeOperation: String? /// The surface catalog as one value: machines (this Mac first), their /// terminals/screens/browsers, and which local panes project them. @Published private(set) var catalog: SurfaceCatalogSnapshot = .empty @@ -95,7 +95,7 @@ final class MachinesPanelViewModel: ObservableObject { func endOperation() { activeOperation = nil - refresh() + if wantsPolling { refresh() } } func noteTreeFailure(_ description: String) { @@ -295,7 +295,7 @@ final class MachinesPanelViewModel: ObservableObject { ) treeErrorDescription = String(format: format, machineID, MachineCreateOperation.headline(ofOutput: output) ?? output) } - refresh() + if wantsPolling { refresh() } } /// Publishes the catalog's current value and the local workspace list. Cheap /// (a value read), so every change notification may call it. @@ -462,5 +462,128 @@ final class MachinesPanelViewModel: ObservableObject { return scoped } + /// Read the current shared resource owner, retaining its resize and list fences. + func applyResourceStats(machineIDs: Set?) { + guard isCloudEnabled(), let resourceStats else { return } + for id in machineIDs ?? Set(machineIndexByID.keys) { + guard let index = machineIndexByID[id], machines.indices.contains(index), + machines[index].id == id, machines[index].capabilities.stats else { continue } + let stats = resourceStats.stats(for: id) + if machines[index].stats != stats { machines[index].stats = stats } + } + } + + func refresh() { + guard isCloudEnabled(), let client = client ?? VMClient.shared else { return } + guard refreshTask == nil else { refreshRequestedWhileLoading = true; return } + isLoading = true + let generation = refreshGeneration + let scope = machinePinStore?.scopeIdentifier + refreshTask = Task { [weak self] in + let result: Result + do { result = .success(try await client.listPage()) } + catch { result = .failure(error) } + guard !Task.isCancelled, let self, generation == self.refreshGeneration else { return } + self.applyRefreshResult(result, generation: generation, scope: scope) + self.refreshTask = nil + if self.refreshRequestedWhileLoading { + self.refreshRequestedWhileLoading = false + self.refresh() + } + } + } + + func pausePolling() { + pollTask?.cancel(); pollTask = nil + refreshTask?.cancel(); refreshTask = nil + refreshRequestedWhileLoading = false + refreshGeneration &+= 1 + isLoading = false + statsTask?.cancel(); statsTask = nil; statsID = nil + usageTask?.cancel(); usageTask = nil + usageFailureCount = 0 + usageRetryNotBefore = nil + treeTask?.cancel(); treeTask = nil + machineRefreshes.cancelAll() + freeAccessTransitionTask?.cancel(); freeAccessTransitionTask = nil + } + + func clearUnavailableMetrics() { + if let resourceStats { + for id in machineIndexByID.keys { + _ = resourceStats.finishRead(resourceStats.beginRead(machineID: id), stats: nil) + } + applyResourceStats(machineIDs: nil) + } + usageByMachineID = [:] + machines = MachineSnapshotBuilder.applyingUsage(to: machines, usage: [:]) + } + func applyRefreshResult(_ result: Result, generation: UInt64, scope: String?) { + guard generation == refreshGeneration, scope == machinePinStore?.scopeIdentifier, isCloudEnabled() else { return } + do { + let page = try result.get() + try Task.checkCancellation() + guard generation == refreshGeneration, scope == machinePinStore?.scopeIdentifier, + isCloudEnabled() else { return } + let previous = resourceStats?.snapshot ?? [:] + let freeAccessWindowDays = page.limits?.freeAccessWindowDays ?? 0 + self.freeAccessWindowDays = freeAccessWindowDays + var snapshots = page.vms.map { + MachineSnapshotBuilder.snapshot( + from: $0, + freeAccessWindowDays: freeAccessWindowDays, + previousStats: previous[$0.id] + ) + } + snapshots = MachineSnapshotBuilder.applyingUsage(to: snapshots, usage: usageByMachineID) + let defaultMachineID = defaultMachineStore?.resolveMachineID( + from: snapshots.map { CloudMachineDescriptor(id: $0.id, isDesktop: $0.isDesktop) }, + isComplete: true + ) + snapshots = snapshots.map { snapshot in + var next = snapshot + next.isDefault = snapshot.id == defaultMachineID + return next + } + // The authoritative fleet plus catalog-only rows is the complete + // visible set: a pin whose machine is gone from both is pruned. + machinePinStore?.reconcile(machineIDs: MachineSnapshotBuilder.includingCatalogMachines(snapshots, catalog: scopedCatalogSnapshot()).map(\.id)) + machineIndexByID = Dictionary(uniqueKeysWithValues: snapshots.enumerated().map { ($0.element.id, $0.offset) }) + machines = snapshots + lastLimits = page.limits + scheduleFreeAccessTransition() + refreshStats() + refreshUsage() + readCatalog() + plan = MachineSnapshotBuilder.planSnapshot(activeCount: snapshots.count, limits: page.limits, machines: snapshots) + lastErrorDescription = nil + listProblem = nil + } catch is CancellationError { + return + } catch let error as VMClientError { + guard !Task.isCancelled, generation == refreshGeneration, + scope == machinePinStore?.scopeIdentifier else { return } + if case .notSignedIn = error { + machines = [] + machineIndexByID.removeAll() + plan = nil + activeOperation = nil + lastErrorDescription = nil + listProblem = nil + hasLoadedOnce = false + isLoading = false + return + } + lastErrorDescription = String(describing: error) + listProblem = Self.classifyListFailure(error) + } catch { + guard !Task.isCancelled, generation == refreshGeneration, + scope == machinePinStore?.scopeIdentifier else { return } + lastErrorDescription = String(describing: error) + listProblem = .unreachable + } + isLoading = false + hasLoadedOnce = true + } } diff --git a/Sources/Cloud/TeamMachineUsage.swift b/Sources/Cloud/TeamMachineUsage.swift index fffd41f7287f..1c293941a4b8 100644 --- a/Sources/Cloud/TeamMachineUsage.swift +++ b/Sources/Cloud/TeamMachineUsage.swift @@ -3,11 +3,6 @@ import Foundation /// The team-wide usage payload. `kind == .unavailable` means the backend has no /// usage store for this team (no rows are rendered, no error is surfaced). struct TeamMachineUsage: Equatable, Sendable { - enum Kind: String, Sendable { - case ready - case unavailable - } - let teamID: String let periodDays: Int let kind: Kind diff --git a/Sources/Cloud/TeamMachineUsageKind.swift b/Sources/Cloud/TeamMachineUsageKind.swift new file mode 100644 index 000000000000..3361bfa792fc --- /dev/null +++ b/Sources/Cloud/TeamMachineUsageKind.swift @@ -0,0 +1,7 @@ +extension TeamMachineUsage { + /// Whether the backend has usage records available for this team. + enum Kind: String, Sendable { + case ready + case unavailable + } +} diff --git a/Sources/Cloud/VMClient.swift b/Sources/Cloud/VMClient.swift index b38fac3eece4..e61f8ec39f8c 100644 --- a/Sources/Cloud/VMClient.swift +++ b/Sources/Cloud/VMClient.swift @@ -2073,7 +2073,6 @@ actor VMClient { errorCode: http.statusCode >= 400 ? Self.cloudVMErrorCode(http: http, data: data) : nil, serverTraceId: Self.cloudVMServerTraceId(http: http, data: data) )) - if method != "GET", (200...299).contains(http.statusCode) { await readRequests.invalidate() } return (data, http) } catch let error as VMClientError { record(.transportFailure(kind: Self.transportFailureKind(error), detail: Self.transportFailureDetail(error))) @@ -2284,6 +2283,12 @@ actor VMClient { throw VMClientError.notSignedIn } } + if method != "GET", (200...299).contains(http.statusCode) { + await readRequests.invalidate(CloudReadMutation(method: method, scope: .init( + path: path, accountID: sessionIdentity?.accountID, + generation: sessionIdentity?.generation, teamID: requestedTeamID + ), responseData: data)) + } return (data, http) } } diff --git a/cmux.xcodeproj/project.pbxproj b/cmux.xcodeproj/project.pbxproj index 8de27c66417b..1dd366b9f18e 100644 --- a/cmux.xcodeproj/project.pbxproj +++ b/cmux.xcodeproj/project.pbxproj @@ -841,6 +841,16 @@ C12625B00000000000000006 /* CloudReadManualClock.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12625B00000000000000005 /* CloudReadManualClock.swift */; }; C12625B00000000000000002 /* CloudReadNetworkMonitor.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12625B00000000000000001 /* CloudReadNetworkMonitor.swift */; }; C12636A00000000000000004 /* CloudReadRecoveryObserver.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12636A00000000000000003 /* CloudReadRecoveryObserver.swift */; }; + C13151C00000000000000002 /* CloudReadCooldownStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C00000000000000001 /* CloudReadCooldownStore.swift */; }; + C13151C00000000000000004 /* CloudReadMutation.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C00000000000000003 /* CloudReadMutation.swift */; }; + C13151C00000000000000006 /* CloudReadRequestCoordinator+Key.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C00000000000000005 /* CloudReadRequestCoordinator+Key.swift */; }; + C13151C00000000000000008 /* CloudReadRequestCoordinator+Response.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C00000000000000007 /* CloudReadRequestCoordinator+Response.swift */; }; + C13151C0000000000000000A /* CloudReadRequestCoordinator+Context.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C00000000000000009 /* CloudReadRequestCoordinator+Context.swift */; }; + C13151C0000000000000000C /* CloudReadRequestCoordinator+Waiter.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C0000000000000000B /* CloudReadRequestCoordinator+Waiter.swift */; }; + C13151C0000000000000000E /* CloudReadRequestCoordinator+Entry.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C0000000000000000D /* CloudReadRequestCoordinator+Entry.swift */; }; + C13151C00000000000000010 /* CloudReadRequestCoordinator+Pending.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C0000000000000000F /* CloudReadRequestCoordinator+Pending.swift */; }; + C13151C00000000000000012 /* CloudReadCooldownStore+Session.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C00000000000000011 /* CloudReadCooldownStore+Session.swift */; }; + C13151C00000000000000014 /* CloudReadCooldownStore+Cooldown.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C00000000000000013 /* CloudReadCooldownStore+Cooldown.swift */; }; C12625A00000000000000004 /* CloudReadRequestCoordinator.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12625A00000000000000003 /* CloudReadRequestCoordinator.swift */; }; C12625B00000000000000004 /* CloudReadRequestCoordinatorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12625B00000000000000003 /* CloudReadRequestCoordinatorTests.swift */; }; C12625000000000000000004 /* CloudRefreshURLProtocol.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12625000000000000000003 /* CloudRefreshURLProtocol.swift */; }; @@ -3284,6 +3294,7 @@ 7490C00F7490C00F7490C00F /* TaskVMInfoMemoryPressureFootprintSampler.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7490D00F7490D00F7490D00F /* TaskVMInfoMemoryPressureFootprintSampler.swift */; }; B82090010000000000000001 /* TCCUsageDescriptionUITests.swift in Sources */ = {isa = PBXBuildFile; fileRef = B82090010000000000000002 /* TCCUsageDescriptionUITests.swift */; }; C12625C00000000000000002 /* TeamMachineUsage.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12625C00000000000000001 /* TeamMachineUsage.swift */; }; + C13151D00000000000000002 /* TeamMachineUsageKind.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151D00000000000000001 /* TeamMachineUsageKind.swift */; }; 0044CBDD567D9CD29E6A6F12 /* TelemetrySettings.swift in Sources */ = {isa = PBXBuildFile; fileRef = 257DFD20F0961FE0BDAF70F2 /* TelemetrySettings.swift */; }; A10199010000000000000001 /* TerminalAdaptiveDefaultThemeSettings.swift in Sources */ = {isa = PBXBuildFile; fileRef = A10199010000000000000002 /* TerminalAdaptiveDefaultThemeSettings.swift */; }; C10190010000000000000002 /* TerminalAgentPanelInitialTitleTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C10190010000000000000001 /* TerminalAgentPanelInitialTitleTests.swift */; }; @@ -4904,6 +4915,16 @@ C12625B00000000000000005 /* CloudReadManualClock.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudReadManualClock.swift; sourceTree = ""; }; C12625B00000000000000001 /* CloudReadNetworkMonitor.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudReadNetworkMonitor.swift; sourceTree = ""; }; C12636A00000000000000003 /* CloudReadRecoveryObserver.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudReadRecoveryObserver.swift; sourceTree = ""; }; + C13151C00000000000000001 /* CloudReadCooldownStore.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudReadCooldownStore.swift; sourceTree = ""; }; + C13151C00000000000000003 /* CloudReadMutation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudReadMutation.swift; sourceTree = ""; }; + C13151C00000000000000005 /* CloudReadRequestCoordinator+Key.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudReadRequestCoordinator+Key.swift"; sourceTree = ""; }; + C13151C00000000000000007 /* CloudReadRequestCoordinator+Response.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudReadRequestCoordinator+Response.swift"; sourceTree = ""; }; + C13151C00000000000000009 /* CloudReadRequestCoordinator+Context.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudReadRequestCoordinator+Context.swift"; sourceTree = ""; }; + C13151C0000000000000000B /* CloudReadRequestCoordinator+Waiter.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudReadRequestCoordinator+Waiter.swift"; sourceTree = ""; }; + C13151C0000000000000000D /* CloudReadRequestCoordinator+Entry.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudReadRequestCoordinator+Entry.swift"; sourceTree = ""; }; + C13151C0000000000000000F /* CloudReadRequestCoordinator+Pending.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudReadRequestCoordinator+Pending.swift"; sourceTree = ""; }; + C13151C00000000000000011 /* CloudReadCooldownStore+Session.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudReadCooldownStore+Session.swift"; sourceTree = ""; }; + C13151C00000000000000013 /* CloudReadCooldownStore+Cooldown.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudReadCooldownStore+Cooldown.swift"; sourceTree = ""; }; C12625A00000000000000003 /* CloudReadRequestCoordinator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudReadRequestCoordinator.swift; sourceTree = ""; }; C12625B00000000000000003 /* CloudReadRequestCoordinatorTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudReadRequestCoordinatorTests.swift; sourceTree = ""; }; C12625000000000000000003 /* CloudRefreshURLProtocol.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudRefreshURLProtocol.swift; sourceTree = ""; }; @@ -7220,6 +7241,7 @@ 7490D00F7490D00F7490D00F /* TaskVMInfoMemoryPressureFootprintSampler.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = App/TaskVMInfoMemoryPressureFootprintSampler.swift; sourceTree = ""; }; B82090010000000000000002 /* TCCUsageDescriptionUITests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TCCUsageDescriptionUITests.swift; sourceTree = ""; }; C12625C00000000000000001 /* TeamMachineUsage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TeamMachineUsage.swift; sourceTree = ""; }; + C13151D00000000000000001 /* TeamMachineUsageKind.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TeamMachineUsageKind.swift; sourceTree = ""; }; 257DFD20F0961FE0BDAF70F2 /* TelemetrySettings.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "TelemetrySettings.swift"; sourceTree = ""; }; A10199010000000000000002 /* TerminalAdaptiveDefaultThemeSettings.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TerminalAdaptiveDefaultThemeSettings.swift; sourceTree = ""; }; C10190010000000000000001 /* TerminalAgentPanelInitialTitleTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TerminalAgentPanelInitialTitleTests.swift; sourceTree = ""; }; @@ -8517,9 +8539,20 @@ 65A5357644AB29D74F4AC368 /* VMClient+ResourceStats.swift */, C12625A00000000000000005 /* VMCapabilities.swift */, C12625C00000000000000001 /* TeamMachineUsage.swift */, + C13151D00000000000000001 /* TeamMachineUsageKind.swift */, C12625D00000000000000001 /* MachineUsageTotals.swift */, C12625D00000000000000003 /* MachineUsageSnapshot.swift */, C12625A00000000000000001 /* CloudRequestClock.swift */, + C13151C00000000000000001 /* CloudReadCooldownStore.swift */, + C13151C00000000000000003 /* CloudReadMutation.swift */, + C13151C00000000000000005 /* CloudReadRequestCoordinator+Key.swift */, + C13151C00000000000000007 /* CloudReadRequestCoordinator+Response.swift */, + C13151C00000000000000009 /* CloudReadRequestCoordinator+Context.swift */, + C13151C0000000000000000B /* CloudReadRequestCoordinator+Waiter.swift */, + C13151C0000000000000000D /* CloudReadRequestCoordinator+Entry.swift */, + C13151C0000000000000000F /* CloudReadRequestCoordinator+Pending.swift */, + C13151C00000000000000011 /* CloudReadCooldownStore+Session.swift */, + C13151C00000000000000013 /* CloudReadCooldownStore+Cooldown.swift */, C12625A00000000000000003 /* CloudReadRequestCoordinator.swift */, C12625B00000000000000001 /* CloudReadNetworkMonitor.swift */, E8DCA0BDC648E6CDCFD3E228 /* MachinesPanelViewModel+Refresh.swift */, @@ -13183,6 +13216,16 @@ D1AC3F6282A94EAEB79D160B /* CloudProviderRefreshCoordinator.swift in Sources */, C12625B00000000000000002 /* CloudReadNetworkMonitor.swift in Sources */, C12636A00000000000000004 /* CloudReadRecoveryObserver.swift in Sources */, + C13151C00000000000000002 /* CloudReadCooldownStore.swift in Sources */, + C13151C00000000000000004 /* CloudReadMutation.swift in Sources */, + C13151C00000000000000006 /* CloudReadRequestCoordinator+Key.swift in Sources */, + C13151C00000000000000008 /* CloudReadRequestCoordinator+Response.swift in Sources */, + C13151C0000000000000000A /* CloudReadRequestCoordinator+Context.swift in Sources */, + C13151C0000000000000000C /* CloudReadRequestCoordinator+Waiter.swift in Sources */, + C13151C0000000000000000E /* CloudReadRequestCoordinator+Entry.swift in Sources */, + C13151C00000000000000010 /* CloudReadRequestCoordinator+Pending.swift in Sources */, + C13151C00000000000000012 /* CloudReadCooldownStore+Session.swift in Sources */, + C13151C00000000000000014 /* CloudReadCooldownStore+Cooldown.swift in Sources */, C12625A00000000000000004 /* CloudReadRequestCoordinator.swift in Sources */, 595EC249E2E015ACE304F0EE /* CloudRemoteOperationStep.swift in Sources */, B12438020000000000000001 /* CloudRenameCoordinator.swift in Sources */, @@ -14667,6 +14710,7 @@ C7A502000000000000000002 /* TaskManagerWindowController.swift in Sources */, 7490C00F7490C00F7490C00F /* TaskVMInfoMemoryPressureFootprintSampler.swift in Sources */, C12625C00000000000000002 /* TeamMachineUsage.swift in Sources */, + C13151D00000000000000002 /* TeamMachineUsageKind.swift in Sources */, 0044CBDD567D9CD29E6A6F12 /* TelemetrySettings.swift in Sources */, A10199010000000000000001 /* TerminalAdaptiveDefaultThemeSettings.swift in Sources */, 8362A0030000000000000003 /* TerminalCallerTTYBinding.swift in Sources */, diff --git a/cmuxTests/CloudOperationRecorderTests.swift b/cmuxTests/CloudOperationRecorderTests.swift index 4422ca665ce2..6ef6f666d56b 100644 --- a/cmuxTests/CloudOperationRecorderTests.swift +++ b/cmuxTests/CloudOperationRecorderTests.swift @@ -31,10 +31,13 @@ struct CloudOperationRecorderTests { let operation = recorder.begin(.open) await recorder.finish(operation, error: CmuxTuiSurfaceProvider.ProviderError.remotePlacementUnavailable("fixture-machine")) let persisted = try #require(JSONSerialization.jsonObject(with: Data(contentsOf: queueURL)) as? [[String: Any]]) - let span = try #require(persisted.first?["span"] as? [String: Any]) - #expect(span["failure"] as? String == "placement") - #expect(span["traceId"] as? String == operation.traceID) - #expect(span["operationId"] as? String == operation.operationID.uuidString.lowercased()) + // The transport may already have acknowledged the record before this + // actor resumes; its fixture validates the serialized span either way. + if let span = persisted.first?["span"] as? [String: Any] { + #expect(span["failure"] as? String == "placement") + #expect(span["traceId"] as? String == operation.traceID) + #expect(span["operationId"] as? String == operation.operationID.uuidString.lowercased()) + } let deadline = ContinuousClock.now.advanced(by: .seconds(10)) var acknowledged = false while ContinuousClock.now < deadline { @@ -300,7 +303,7 @@ private final class PlacementReceiptURLProtocol: URLProtocol, @unchecked Sendabl stream.open() defer { stream.close() } var buffer = [UInt8](repeating: 0, count: 4096) - while stream.hasBytesAvailable { + while true { let count = stream.read(&buffer, maxLength: buffer.count) if count < 0 { throw URLError(.cannotDecodeContentData) } if count == 0 { break } diff --git a/cmuxTests/CloudReadRequestCoordinatorTests.swift b/cmuxTests/CloudReadRequestCoordinatorTests.swift index 6b1b9cf3ac1b..036e1feedeb5 100644 --- a/cmuxTests/CloudReadRequestCoordinatorTests.swift +++ b/cmuxTests/CloudReadRequestCoordinatorTests.swift @@ -11,8 +11,8 @@ import Testing @Suite("Cloud read deadline and cancellation", .timeLimit(.minutes(1))) struct CloudReadRequestCoordinatorTests { private typealias Owner = CloudReadRequestCoordinator - private func key(_ id: String = "vm", account: String = "fixture", generation: UInt64 = 1) -> Owner.Key { - .init(path: id, accountID: account, generation: generation, teamID: "team") + private func key(_ id: String = "vm", account: String = "fixture", generation: UInt64 = 1, team: String = "team") -> Owner.Key { + .init(path: id, accountID: account, generation: generation, teamID: team) } private func response(_ status: Int = 200) -> Owner.Response { .init(data: Data(), http: HTTPURLResponse(url: URL(string: "https://fixture.invalid")!, statusCode: status, httpVersion: nil, headerFields: nil)!) @@ -230,13 +230,14 @@ struct CloudReadRequestCoordinatorTests { func mutationInvalidatesRunningRead() async throws { let owner = Owner() let gate = CloudReadResponseGate() + let list = key("/api/vm") let operation: @Sendable () async -> Owner.Response = { await gate.read(response(await gate.requests == 0 ? 200 : 201)) } - let first = Task { try await owner.read(key(), operation: operation) } + let first = Task { try await owner.read(list, operation: operation) } try await eventually { await gate.requests == 1 } - await owner.invalidate() - let second = Task { try await owner.read(key(), operation: operation) } + await owner.invalidate(CloudReadMutation(method: "POST", scope: list, responseData: Data())) + let second = Task { try await owner.read(list, operation: operation) } try await eventually { await owner.entries.values.first?.waiters.count == 2 } await gate.release() #expect(try await first.value.http.statusCode == 201) @@ -244,6 +245,105 @@ struct CloudReadRequestCoordinatorTests { #expect(await gate.requests == 2) } + @Test("Resize refreshes only the captured team's list and target machine") + func mutationScopeIsolation() async throws { + let owner = Owner() + let scenarios: [(Owner.Key, Bool)] = [ + (key("/api/vm"), true), (key("/api/vm/target/stats"), true), + (key("/api/vm/other/stats"), false), + (key("/api/coderouter/vm-usage/team"), false), + (key("/api/vm", account: "other"), false), + (key("/api/vm", generation: 2), false), + (key("/api/vm", team: "other"), false) + ] + let gates = scenarios.map { _ in CloudReadResponseGate() } + let requests = zip(scenarios, gates).map { scenario, gate in + Task { try await owner.read(scenario.0) { + await gate.read(response(await gate.requests == 0 ? 200 : 201)) + } } + } + try await eventually { await owner.entries.count == scenarios.count } + for gate in gates { try await eventually { await gate.requests == 1 } } + await owner.invalidate(CloudReadMutation(method: "POST", scope: key("/api/vm/target/resize"), responseData: Data())) + for gate in gates { await gate.release() } + for (index, request) in requests.enumerated() { + #expect(try await request.value.http.statusCode == (scenarios[index].1 ? 201 : 200)) + #expect(await gates[index].requests == (scenarios[index].1 ? 2 : 1)) + } + } + + @Test("Unrelated mutations do not restart machine reads", arguments: [ + "/api/vm/tunnel", "/api/vm/publications", "/api/billing/checkout", + "/api/vm/target/snapshot", "/api/vm/target/files" + ]) + func unrelatedMutation(path: String) async throws { + let owner = Owner() + let gate = CloudReadResponseGate() + let request = Task { try await owner.read(key("/api/vm")) { await gate.read(response()) } } + try await eventually { await gate.requests == 1 } + await owner.invalidate(CloudReadMutation(method: "POST", scope: key(path), responseData: Data())) + await gate.release() + #expect(try await request.value.http.statusCode == 200) + #expect(await gate.requests == 1) + } + + @Test("Cooldown capacity retains bounded compact errors and every server minimum") + func boundedCooldowns() throws { + var store = CloudReadCooldownStore(capacity: 8) + store.activateSession(for: key()) + let largeResponse = Owner.Response(data: Data(repeating: 65, count: 1_000_000), http: response(429).http) + for index in 0..<1000 { + store.record(key("vm-\(index)"), until: 60, now: 0, response: largeResponse) + } + #expect(store.retainedCount == 9) + for index in 0..<1000 { + let cached = try #require(store.response(for: key("vm-\(index)"), now: 59)) + #expect(cached.http.statusCode == 429) + #expect(cached.data == Data(#"{"error":"rate_limited"}"#.utf8)) + #expect(cached.http.allHeaderFields.isEmpty) + } + // Overflow deliberately delays unknown paths in this session rather + // than forgetting an evicted path's minimum and retrying too early. + #expect(store.response(for: key("unseen"), now: 59)?.http.statusCode == 429) + #expect(store.response(for: key("vm-999"), now: 60) == nil) + #expect(store.retainedCount == 0) + } + + @Test("Long cooldowns preserve a later minimum while earlier records expire") + func cooldownOverflowKeepsLongestMinimum() { + var store = CloudReadCooldownStore(capacity: 1) + store.activateSession(for: key()) + store.record(key("first"), until: 10, now: 0, response: response(429)) + store.record(key("second"), until: 100, now: 0, response: response(429)) + store.record(key("third"), until: 20, now: 0, response: response(429)) + #expect(store.response(for: key("second"), now: 99)?.http.statusCode == 429) + #expect(store.retainedCount == 1) + #expect(store.response(for: key("second"), now: 100) == nil) + } + + @Test("Session replacement discards old cooldowns and late completions cannot revive them") + func retiredSessionCooldowns() { + var store = CloudReadCooldownStore(capacity: 8) + let old = key() + for generation in 1...1000 { + let current = key(account: "account-\(generation)", generation: UInt64(generation)) + store.activateSession(for: current) + store.record(current, until: .infinity, now: 0, response: response(429)) + #expect(store.retainedCount == 1) + } + let current = key(account: "account-1000", generation: 1000) + store.activateSession(for: old) + store.record(old, until: .infinity, now: 0, response: response(429)) + #expect(store.response(for: old, now: 0) == nil) + #expect(store.response(for: current, now: 0)?.http.statusCode == 429) + #expect(store.retainedCount == 1) + let replacement = key(account: "replacement", generation: 1001) + store.activateSession(for: replacement) + store.record(current, until: .infinity, now: 0, response: response(429)) + #expect(store.response(for: replacement, now: 0) == nil) + #expect(store.retainedCount == 0) + } + private func eventually(_ condition: @escaping () async -> Bool) async throws { let deadline = ContinuousClock.now.advanced(by: .seconds(10)) while !(await condition()), ContinuousClock.now < deadline { await Task.yield() } diff --git a/vendor/stack-auth-swift-sdk-prerelease/Sources/StackAuth/TokenRefreshClock.swift b/vendor/stack-auth-swift-sdk-prerelease/Sources/StackAuth/TokenRefreshClock.swift index fdc25538b80a..a473fc0b66e3 100644 --- a/vendor/stack-auth-swift-sdk-prerelease/Sources/StackAuth/TokenRefreshClock.swift +++ b/vendor/stack-auth-swift-sdk-prerelease/Sources/StackAuth/TokenRefreshClock.swift @@ -18,6 +18,9 @@ struct TokenRefreshClock: Sendable { try await clock.sleep(until: origin.advanced(by: .nanoseconds(Int64(nanoseconds)))) }) } + // The SDK still supports macOS 12/iOS 15, before Clock. This is the + // injected absolute deadline, owned and cancelled by TokenRefreshCoordinator; + // it never polls for state or delays a synchronization boundary. return Self(now: { DispatchTime.now().uptimeNanoseconds }, sleepUntil: { deadline in let now = DispatchTime.now().uptimeNanoseconds if deadline > now { try await Task.sleep(nanoseconds: deadline - now) } From b41abaeaa315c7d65b17536c6675f62723dab4b7 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sat, 19 Sep 2026 23:38:12 -0700 Subject: [PATCH 19/92] fix: preserve the absolute Cloud token deadline across phases --- .../Client/StackAuthClient.swift | 4 ++ .../AuthCoordinator+CurrentTokens.swift | 6 ++- ...oordinator+TokenTouchingPhaseTimeout.swift | 16 +++++- ...rTokenTouchingPhaseCancellationTests.swift | 49 +++++++++++++++++++ 4 files changed, 73 insertions(+), 2 deletions(-) diff --git a/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Client/StackAuthClient.swift b/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Client/StackAuthClient.swift index 8f7cf5955fde..37e8adf3d1e5 100644 --- a/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Client/StackAuthClient.swift +++ b/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Client/StackAuthClient.swift @@ -58,6 +58,10 @@ public struct StackAuthClient: AuthClient { await stack.getAccessToken() } + /// Resolves access with typed cancellation/deadline failures when supported. + /// - Parameter forceRefresh: Bypasses the cached token after server rejection. + /// - Returns: A usable access token, or `nil` when unavailable. + /// - Throws: Cancellation or a classified transient refresh failure. public func resolvedAccessToken(forceRefresh: Bool) async throws -> String? { let pair = await stack.resolvedTokenPair(forceRefresh: forceRefresh) try Task.checkCancellation() diff --git a/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Coordinator/AuthCoordinator+CurrentTokens.swift b/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Coordinator/AuthCoordinator+CurrentTokens.swift index 379e2968326e..3d4e3099642d 100644 --- a/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Coordinator/AuthCoordinator+CurrentTokens.swift +++ b/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Coordinator/AuthCoordinator+CurrentTokens.swift @@ -23,7 +23,11 @@ extension AuthCoordinator { } try Task.checkCancellation() guard !deadline.hasExpired() else { throw AuthError.timedOut } - return try await runTokenTouchingPhase(.accessToken, timeout: deadline.remaining()) { + return try await runTokenTouchingPhase( + .accessToken, + timeout: deadline.remaining(), + deadline: deadline + ) { return try await self.captureCloudTokens() } } diff --git a/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Coordinator/AuthCoordinator+TokenTouchingPhaseTimeout.swift b/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Coordinator/AuthCoordinator+TokenTouchingPhaseTimeout.swift index dc0e55d1e1cc..c6442f126b14 100644 --- a/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Coordinator/AuthCoordinator+TokenTouchingPhaseTimeout.swift +++ b/Packages/Shared/CmuxAuthRuntime/Sources/CmuxAuthRuntime/Coordinator/AuthCoordinator+TokenTouchingPhaseTimeout.swift @@ -6,6 +6,20 @@ extension AuthCoordinator { _ phase: AuthPhase, timeout: Duration, _ operation: @escaping @Sendable () async throws -> T + ) async throws -> T { + try await runTokenTouchingPhase( + phase, + timeout: timeout, + deadline: clock.authTokenDeadline(after: timeout), + operation + ) + } + + func runTokenTouchingPhase( + _ phase: AuthPhase, + timeout: Duration, + deadline: AuthTokenDeadline, + _ operation: @escaping @Sendable () async throws -> T ) async throws -> T { expireTimedOutTokenTouchingPhaseIfNeeded(phase) guard timedOutTokenTouchingPhaseStates[phase] == nil else { @@ -13,7 +27,7 @@ extension AuthCoordinator { throw AuthError.timedOut } try Task.checkCancellation() - let deadline = clock.authTokenDeadline(after: timeout) + guard !deadline.hasExpired() else { throw AuthError.timedOut } let phaseID = UUID() let generation = sessionGeneration let signOutEpoch = signOutEpoch diff --git a/Packages/Shared/CmuxAuthRuntime/Tests/CmuxAuthRuntimeTests/AuthCoordinatorTokenTouchingPhaseCancellationTests.swift b/Packages/Shared/CmuxAuthRuntime/Tests/CmuxAuthRuntimeTests/AuthCoordinatorTokenTouchingPhaseCancellationTests.swift index a376ae66e738..0798560f2d0d 100644 --- a/Packages/Shared/CmuxAuthRuntime/Tests/CmuxAuthRuntimeTests/AuthCoordinatorTokenTouchingPhaseCancellationTests.swift +++ b/Packages/Shared/CmuxAuthRuntime/Tests/CmuxAuthRuntimeTests/AuthCoordinatorTokenTouchingPhaseCancellationTests.swift @@ -48,6 +48,55 @@ import Testing await waitUntilTokenTouchingCleanupFinished(coordinator) } + @Test func inheritedDeadlineIsNotRebasedWhenTokenPhaseStartsLate() async { + let clock = ManualTestClock() + let deadline = clock.authTokenDeadline(after: .seconds(2)) + clock.advance(by: .seconds(1)) + let coordinator = makeCoordinator(client: FakeAuthClient(), clock: clock) + let entered = TestPhaseSignal() + let release = TestPhaseSignal() + let phase = Task { + try await coordinator.runTokenTouchingPhase( + .accessToken, + timeout: .seconds(2), + deadline: deadline + ) { + await entered.markStarted() + await release.waitUntilStarted() + return "captured" + } + } + + await entered.waitUntilStarted() + await clock.waitUntilSleepers() + clock.advance(by: .seconds(1)) + // Even if the operation resumes first at the boundary, the phase's + // post-operation check must apply the inherited absolute deadline. + await release.markStarted() + await #expect(throws: AuthError.timedOut) { try await phase.value } + } + + @Test func expiredInheritedDeadlineSkipsTokenOperation() async { + let clock = ManualTestClock() + let deadline = clock.authTokenDeadline(after: .seconds(2)) + clock.advance(by: .seconds(2)) + let coordinator = makeCoordinator(client: FakeAuthClient(), clock: clock) + let entered = TestPhaseSignal() + let phase = Task { + try await coordinator.runTokenTouchingPhase( + .accessToken, + timeout: .seconds(2), + deadline: deadline + ) { + await entered.markStarted() + return "unexpected" + } + } + + await #expect(throws: AuthError.timedOut) { try await phase.value } + #expect(await entered.didStart == false) + } + private func waitUntilTokenTouchingPhaseGated(_ coordinator: AuthCoordinator, phase: AuthPhase) async { for _ in 0..<100 { if coordinator.timedOutTokenTouchingPhaseStates[phase] != nil { From 67e89287f79dfe754829df707a4045c72bb762f9 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sat, 19 Sep 2026 23:41:23 -0700 Subject: [PATCH 20/92] chore: normalize Cloud read project entries --- cmux.xcodeproj/project.pbxproj | 42 +++++++++++++++++----------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/cmux.xcodeproj/project.pbxproj b/cmux.xcodeproj/project.pbxproj index 7d9516a30bdb..3c25df52915a 100644 --- a/cmux.xcodeproj/project.pbxproj +++ b/cmux.xcodeproj/project.pbxproj @@ -838,19 +838,19 @@ 0C45DC8D9BE54DDEA2397892 /* CloudPrivateRouteSelectionTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1E6077A4B6CC441EB3668651 /* CloudPrivateRouteSelectionTests.swift */; }; D1AC3F6282A94EAEB79D160B /* CloudProviderRefreshCoordinator.swift in Sources */ = {isa = PBXBuildFile; fileRef = A09B853868B54438A5B14F26 /* CloudProviderRefreshCoordinator.swift */; }; AC38407F09E94DA38B94F168 /* CloudProviderRefreshCoordinatorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5C2A7143303749BE82E51F69 /* CloudProviderRefreshCoordinatorTests.swift */; }; + C13151C00000000000000014 /* CloudReadCooldownStore+Cooldown.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C00000000000000013 /* CloudReadCooldownStore+Cooldown.swift */; }; + C13151C00000000000000012 /* CloudReadCooldownStore+Session.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C00000000000000011 /* CloudReadCooldownStore+Session.swift */; }; + C13151C00000000000000002 /* CloudReadCooldownStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C00000000000000001 /* CloudReadCooldownStore.swift */; }; C12625B00000000000000006 /* CloudReadManualClock.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12625B00000000000000005 /* CloudReadManualClock.swift */; }; + C13151C00000000000000004 /* CloudReadMutation.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C00000000000000003 /* CloudReadMutation.swift */; }; C12625B00000000000000002 /* CloudReadNetworkMonitor.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12625B00000000000000001 /* CloudReadNetworkMonitor.swift */; }; C12636A00000000000000004 /* CloudReadRecoveryObserver.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12636A00000000000000003 /* CloudReadRecoveryObserver.swift */; }; - C13151C00000000000000002 /* CloudReadCooldownStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C00000000000000001 /* CloudReadCooldownStore.swift */; }; - C13151C00000000000000004 /* CloudReadMutation.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C00000000000000003 /* CloudReadMutation.swift */; }; - C13151C00000000000000006 /* CloudReadRequestCoordinator+Key.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C00000000000000005 /* CloudReadRequestCoordinator+Key.swift */; }; - C13151C00000000000000008 /* CloudReadRequestCoordinator+Response.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C00000000000000007 /* CloudReadRequestCoordinator+Response.swift */; }; C13151C0000000000000000A /* CloudReadRequestCoordinator+Context.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C00000000000000009 /* CloudReadRequestCoordinator+Context.swift */; }; - C13151C0000000000000000C /* CloudReadRequestCoordinator+Waiter.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C0000000000000000B /* CloudReadRequestCoordinator+Waiter.swift */; }; C13151C0000000000000000E /* CloudReadRequestCoordinator+Entry.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C0000000000000000D /* CloudReadRequestCoordinator+Entry.swift */; }; + C13151C00000000000000006 /* CloudReadRequestCoordinator+Key.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C00000000000000005 /* CloudReadRequestCoordinator+Key.swift */; }; C13151C00000000000000010 /* CloudReadRequestCoordinator+Pending.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C0000000000000000F /* CloudReadRequestCoordinator+Pending.swift */; }; - C13151C00000000000000012 /* CloudReadCooldownStore+Session.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C00000000000000011 /* CloudReadCooldownStore+Session.swift */; }; - C13151C00000000000000014 /* CloudReadCooldownStore+Cooldown.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C00000000000000013 /* CloudReadCooldownStore+Cooldown.swift */; }; + C13151C00000000000000008 /* CloudReadRequestCoordinator+Response.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C00000000000000007 /* CloudReadRequestCoordinator+Response.swift */; }; + C13151C0000000000000000C /* CloudReadRequestCoordinator+Waiter.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C0000000000000000B /* CloudReadRequestCoordinator+Waiter.swift */; }; C12625A00000000000000004 /* CloudReadRequestCoordinator.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12625A00000000000000003 /* CloudReadRequestCoordinator.swift */; }; C12625B00000000000000004 /* CloudReadRequestCoordinatorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12625B00000000000000003 /* CloudReadRequestCoordinatorTests.swift */; }; C12625000000000000000004 /* CloudRefreshURLProtocol.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12625000000000000000003 /* CloudRefreshURLProtocol.swift */; }; @@ -4924,19 +4924,19 @@ 1E6077A4B6CC441EB3668651 /* CloudPrivateRouteSelectionTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudPrivateRouteSelectionTests.swift; sourceTree = ""; }; A09B853868B54438A5B14F26 /* CloudProviderRefreshCoordinator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudProviderRefreshCoordinator.swift"; sourceTree = ""; }; 5C2A7143303749BE82E51F69 /* CloudProviderRefreshCoordinatorTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudProviderRefreshCoordinatorTests.swift; sourceTree = ""; }; + C13151C00000000000000013 /* CloudReadCooldownStore+Cooldown.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudReadCooldownStore+Cooldown.swift"; sourceTree = ""; }; + C13151C00000000000000011 /* CloudReadCooldownStore+Session.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudReadCooldownStore+Session.swift"; sourceTree = ""; }; + C13151C00000000000000001 /* CloudReadCooldownStore.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudReadCooldownStore.swift; sourceTree = ""; }; C12625B00000000000000005 /* CloudReadManualClock.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudReadManualClock.swift; sourceTree = ""; }; + C13151C00000000000000003 /* CloudReadMutation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudReadMutation.swift; sourceTree = ""; }; C12625B00000000000000001 /* CloudReadNetworkMonitor.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudReadNetworkMonitor.swift; sourceTree = ""; }; C12636A00000000000000003 /* CloudReadRecoveryObserver.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudReadRecoveryObserver.swift; sourceTree = ""; }; - C13151C00000000000000001 /* CloudReadCooldownStore.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudReadCooldownStore.swift; sourceTree = ""; }; - C13151C00000000000000003 /* CloudReadMutation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudReadMutation.swift; sourceTree = ""; }; - C13151C00000000000000005 /* CloudReadRequestCoordinator+Key.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudReadRequestCoordinator+Key.swift"; sourceTree = ""; }; - C13151C00000000000000007 /* CloudReadRequestCoordinator+Response.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudReadRequestCoordinator+Response.swift"; sourceTree = ""; }; C13151C00000000000000009 /* CloudReadRequestCoordinator+Context.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudReadRequestCoordinator+Context.swift"; sourceTree = ""; }; - C13151C0000000000000000B /* CloudReadRequestCoordinator+Waiter.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudReadRequestCoordinator+Waiter.swift"; sourceTree = ""; }; C13151C0000000000000000D /* CloudReadRequestCoordinator+Entry.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudReadRequestCoordinator+Entry.swift"; sourceTree = ""; }; + C13151C00000000000000005 /* CloudReadRequestCoordinator+Key.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudReadRequestCoordinator+Key.swift"; sourceTree = ""; }; C13151C0000000000000000F /* CloudReadRequestCoordinator+Pending.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudReadRequestCoordinator+Pending.swift"; sourceTree = ""; }; - C13151C00000000000000011 /* CloudReadCooldownStore+Session.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudReadCooldownStore+Session.swift"; sourceTree = ""; }; - C13151C00000000000000013 /* CloudReadCooldownStore+Cooldown.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudReadCooldownStore+Cooldown.swift"; sourceTree = ""; }; + C13151C00000000000000007 /* CloudReadRequestCoordinator+Response.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudReadRequestCoordinator+Response.swift"; sourceTree = ""; }; + C13151C0000000000000000B /* CloudReadRequestCoordinator+Waiter.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudReadRequestCoordinator+Waiter.swift"; sourceTree = ""; }; C12625A00000000000000003 /* CloudReadRequestCoordinator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudReadRequestCoordinator.swift; sourceTree = ""; }; C12625B00000000000000003 /* CloudReadRequestCoordinatorTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudReadRequestCoordinatorTests.swift; sourceTree = ""; }; C12625000000000000000003 /* CloudRefreshURLProtocol.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudRefreshURLProtocol.swift; sourceTree = ""; }; @@ -13250,18 +13250,18 @@ 7A0CE1000000000000000504 /* CloudPrivateNetworkPurpose.swift in Sources */, 7A0CE1000000000000000304 /* CloudPrivateNetworkUse.swift in Sources */, D1AC3F6282A94EAEB79D160B /* CloudProviderRefreshCoordinator.swift in Sources */, - C12625B00000000000000002 /* CloudReadNetworkMonitor.swift in Sources */, - C12636A00000000000000004 /* CloudReadRecoveryObserver.swift in Sources */, + C13151C00000000000000014 /* CloudReadCooldownStore+Cooldown.swift in Sources */, + C13151C00000000000000012 /* CloudReadCooldownStore+Session.swift in Sources */, C13151C00000000000000002 /* CloudReadCooldownStore.swift in Sources */, C13151C00000000000000004 /* CloudReadMutation.swift in Sources */, - C13151C00000000000000006 /* CloudReadRequestCoordinator+Key.swift in Sources */, - C13151C00000000000000008 /* CloudReadRequestCoordinator+Response.swift in Sources */, + C12625B00000000000000002 /* CloudReadNetworkMonitor.swift in Sources */, + C12636A00000000000000004 /* CloudReadRecoveryObserver.swift in Sources */, C13151C0000000000000000A /* CloudReadRequestCoordinator+Context.swift in Sources */, - C13151C0000000000000000C /* CloudReadRequestCoordinator+Waiter.swift in Sources */, C13151C0000000000000000E /* CloudReadRequestCoordinator+Entry.swift in Sources */, + C13151C00000000000000006 /* CloudReadRequestCoordinator+Key.swift in Sources */, C13151C00000000000000010 /* CloudReadRequestCoordinator+Pending.swift in Sources */, - C13151C00000000000000012 /* CloudReadCooldownStore+Session.swift in Sources */, - C13151C00000000000000014 /* CloudReadCooldownStore+Cooldown.swift in Sources */, + C13151C00000000000000008 /* CloudReadRequestCoordinator+Response.swift in Sources */, + C13151C0000000000000000C /* CloudReadRequestCoordinator+Waiter.swift in Sources */, C12625A00000000000000004 /* CloudReadRequestCoordinator.swift in Sources */, 595EC249E2E015ACE304F0EE /* CloudRemoteOperationStep.swift in Sources */, B12438020000000000000001 /* CloudRenameCoordinator.swift in Sources */, From 40413b691d933f2303461c90d3a328e6aaae0ca4 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 00:07:24 -0700 Subject: [PATCH 21/92] test: evaluate mutable cooldown reads before assertion macros --- .../CloudReadRequestCoordinatorTests.swift | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/cmuxTests/CloudReadRequestCoordinatorTests.swift b/cmuxTests/CloudReadRequestCoordinatorTests.swift index 036e1feedeb5..495bd8c8f9c9 100644 --- a/cmuxTests/CloudReadRequestCoordinatorTests.swift +++ b/cmuxTests/CloudReadRequestCoordinatorTests.swift @@ -297,15 +297,18 @@ struct CloudReadRequestCoordinatorTests { } #expect(store.retainedCount == 9) for index in 0..<1000 { - let cached = try #require(store.response(for: key("vm-\(index)"), now: 59)) + let cachedResponse = store.response(for: key("vm-\(index)"), now: 59) + let cached = try #require(cachedResponse) #expect(cached.http.statusCode == 429) #expect(cached.data == Data(#"{"error":"rate_limited"}"#.utf8)) #expect(cached.http.allHeaderFields.isEmpty) } // Overflow deliberately delays unknown paths in this session rather // than forgetting an evicted path's minimum and retrying too early. - #expect(store.response(for: key("unseen"), now: 59)?.http.statusCode == 429) - #expect(store.response(for: key("vm-999"), now: 60) == nil) + let overflowResponse = store.response(for: key("unseen"), now: 59) + #expect(overflowResponse?.http.statusCode == 429) + let expiredResponse = store.response(for: key("vm-999"), now: 60) + #expect(expiredResponse == nil) #expect(store.retainedCount == 0) } @@ -316,9 +319,11 @@ struct CloudReadRequestCoordinatorTests { store.record(key("first"), until: 10, now: 0, response: response(429)) store.record(key("second"), until: 100, now: 0, response: response(429)) store.record(key("third"), until: 20, now: 0, response: response(429)) - #expect(store.response(for: key("second"), now: 99)?.http.statusCode == 429) + let throttledResponse = store.response(for: key("second"), now: 99) + #expect(throttledResponse?.http.statusCode == 429) #expect(store.retainedCount == 1) - #expect(store.response(for: key("second"), now: 100) == nil) + let expiredResponse = store.response(for: key("second"), now: 100) + #expect(expiredResponse == nil) } @Test("Session replacement discards old cooldowns and late completions cannot revive them") @@ -334,13 +339,16 @@ struct CloudReadRequestCoordinatorTests { let current = key(account: "account-1000", generation: 1000) store.activateSession(for: old) store.record(old, until: .infinity, now: 0, response: response(429)) - #expect(store.response(for: old, now: 0) == nil) - #expect(store.response(for: current, now: 0)?.http.statusCode == 429) + let oldResponse = store.response(for: old, now: 0) + #expect(oldResponse == nil) + let currentResponse = store.response(for: current, now: 0) + #expect(currentResponse?.http.statusCode == 429) #expect(store.retainedCount == 1) let replacement = key(account: "replacement", generation: 1001) store.activateSession(for: replacement) store.record(current, until: .infinity, now: 0, response: response(429)) - #expect(store.response(for: replacement, now: 0) == nil) + let replacementResponse = store.response(for: replacement, now: 0) + #expect(replacementResponse == nil) #expect(store.retainedCount == 0) } From 870a5bb49a0b7c7d3b5b424154075fa8651bdad0 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 00:07:25 -0700 Subject: [PATCH 22/92] test: reproduce rejected RC Cloud diagnostics --- cmuxTests/CloudOperationRecorderTests.swift | 1 + web/tests/cloud-telemetry.test.ts | 43 ++++++++++++++++++++- 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/cmuxTests/CloudOperationRecorderTests.swift b/cmuxTests/CloudOperationRecorderTests.swift index 6ef6f666d56b..fc92cc2e1129 100644 --- a/cmuxTests/CloudOperationRecorderTests.swift +++ b/cmuxTests/CloudOperationRecorderTests.swift @@ -271,6 +271,7 @@ struct CloudOperationRecorderTests { @Test func metadataSeparatesNightlyFromItsBackend() { let info: [String: Any] = ["CFBundleShortVersionString": "1.2.3", "CFBundleVersion": "45", "CMUXCommit": "abcdef123"] #expect(CloudTelemetryClient.current(info: info, flavor: .nightly).channel == "nightly") + #expect(CloudTelemetryClient.current(info: info, flavor: .rc).channel == "rc") #expect(CloudTelemetryClient.current(info: info, flavor: .stable).channel == "production") #expect(CloudTelemetryClient.current(info: info, flavor: .dev).channel == "dev") #expect(CloudTelemetryClient.current(info: info, flavor: .nightly).revision == "abcdef123") diff --git a/web/tests/cloud-telemetry.test.ts b/web/tests/cloud-telemetry.test.ts index 5eb64ef0969a..5ba16091d9fb 100644 --- a/web/tests/cloud-telemetry.test.ts +++ b/web/tests/cloud-telemetry.test.ts @@ -21,7 +21,7 @@ function span(overrides: Record = {}) { ...overrides, }; } -function batch(spans = [span()]) { return { version: 1, client: { channel: "nightly", version: "0.1.0", build: "123", revision: "abcdef1234567", osVersion: "26.0", architecture: "arm64" }, spans }; } +function batch(spans = [span()], channel = "nightly") { return { version: 1, client: { channel, version: "0.1.0", build: "123", revision: "abcdef1234567", osVersion: "26.0", architecture: "arm64" }, spans }; } function request(body: unknown = batch(), headers: Record = {}) { return new Request("https://cmux.test/api/observability/cloud", { method: "POST", headers: { "content-type": "application/json", ...headers }, @@ -77,6 +77,47 @@ describe("Cloud diagnostic boundary", () => { expect(JSON.stringify(sent)).not.toContain("synthetic-owner"); }); + test("accepts authenticated RC diagnostics and exports rc separately from production", async () => { + const { exportCloudDiagnostics } = await import("../services/observability/cloudTelemetryExport"); + const sent: { url: string; body: any }[] = []; + const configuration = { + origin: "https://us-east-1.aws.edge.axiom.co", token: "test-only", + identityKey: "test-only-identity-key".repeat(2), tracesDataset: "test-traces", + errorsDataset: "test-errors", environment: "production", revision: "abcdef123", + } as const; + const handler = makeCloudTelemetryHandler({ + authenticate: async () => ({ id: "synthetic-owner" }), + checkIngress: async () => true, + accept: async (userId, value) => { + await exportCloudDiagnostics(value.spans.map((span) => ({ + userId, eventId: span.eventId, attempts: 1, payload: { client: value.client, span }, + })), configuration, (async (url, init) => { + sent.push({ url: String(url), body: JSON.parse(String(init?.body)) }); + return new Response("{}"); + }) as typeof fetch); + return value.spans.length; + }, + scheduleDrain: () => {}, now: () => now, + }); + const rcPayload = batch([span({ eventId: "e51c27bc-b0ad-4149-9d92-0fc79c5d2292" })], "rc"); + const productionPayload = batch([span({ eventId: "8cc333de-a1bc-4eb1-8d69-1decd01e17a9" })], "production"); + + const rcResponse = await handler(request(rcPayload)); + const productionResponse = await handler(request(productionPayload)); + expect(rcResponse.status).toBe(202); + expect(productionResponse.status).toBe(202); + expect((await rcResponse.json()).accepted).toBe(1); + expect((await productionResponse.json()).accepted).toBe(1); + + const errorChannels = sent.filter((item) => item.url.endsWith("/v1/ingest/test-errors")) + .flatMap((item) => item.body.map((row: any) => row.client_channel)).sort(); + expect(errorChannels).toEqual(["production", "rc"]); + const traceChannels = sent.filter((item) => item.url.endsWith("/v1/traces")) + .map((item) => item.body.resourceSpans[0].resource.attributes + .find((attribute: any) => attribute.key === "cmux.client.channel").value.stringValue).sort(); + expect(traceChannels).toEqual(["production", "rc"]); + }); + test("accepts a timed operation with a real parent span", () => { expect(parseCloudTelemetryBatch(batch(), now)?.spans[0]).toEqual(span()); }); From b1a6c3cfc64774301ec75b08a3c345298d2dfc08 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 00:18:06 -0700 Subject: [PATCH 23/92] test: cover Cloud read gate revocation --- cmuxTests/VMClientReadCoalescingTests.swift | 78 ++++++++++++++++++++- 1 file changed, 76 insertions(+), 2 deletions(-) diff --git a/cmuxTests/VMClientReadCoalescingTests.swift b/cmuxTests/VMClientReadCoalescingTests.swift index c244d5f1846e..281305a6f9dd 100644 --- a/cmuxTests/VMClientReadCoalescingTests.swift +++ b/cmuxTests/VMClientReadCoalescingTests.swift @@ -2,6 +2,7 @@ import CMUXAuthCore import CmuxAuthRuntime import Foundation import Testing +import os #if canImport(cmux_DEV) @testable import cmux_DEV @@ -160,6 +161,74 @@ struct VMClientReadCoalescingTests { #expect(await CloudRefreshURLProtocol.requestCounts().values.reduce(0, +) == 2) } + @Test("Disabled Cloud rejects cached throttles before reuse", arguments: ["/api/vm", "/api/vm/fixture-0/stats"], [false, true]) + func gateClosesDuringCooldown(path: String, managedPolicy: Bool) async throws { + // Injected flags are synchronous across actors; this lock protects only test state. + let blocked = OSAllocatedUnfairLock(initialState: false) + let fixture = try await CloudRefreshFixture.make( + isDisabledByManagedPolicy: { managedPolicy && blocked.withLock { $0 } }, + isCloudEnabled: { managedPolicy || !blocked.withLock { $0 } } + ) + defer { fixture.session.invalidateAndCancel() } + await CloudRefreshURLProtocol.reset() + await CloudRefreshURLProtocol.configure(.throttled) + let first = try await fixture.client.request("GET", path: path) + #expect(first.1.statusCode == 429) + blocked.withLock { $0 = true } + do { + _ = try await fixture.client.request("GET", path: path) + Issue.record("Disabled Cloud reused a cached response") + } catch VMClientError.disabledByManagedPolicy where managedPolicy { + } catch VMClientError.cloudMachinesDisabled where !managedPolicy { + } catch { Issue.record("Unexpected gate error: \(error)") } + #expect(await CloudRefreshURLProtocol.requestCounts().values.reduce(0, +) == 1) + } + + @Test("A joined read cannot publish after its Cloud gate closes", arguments: ["/api/vm", "/api/vm/fixture-0/stats"], [false, true]) + func gateClosesDuringJoinedRead(path: String, managedPolicy: Bool) async throws { + // Injected flags are synchronous across actors; this lock protects only test state. + let blocked = OSAllocatedUnfairLock(initialState: false) + let fixture = try await CloudRefreshFixture.make( + isDisabledByManagedPolicy: { managedPolicy && blocked.withLock { $0 } }, + isCloudEnabled: { managedPolicy || !blocked.withLock { $0 } } + ) + defer { fixture.session.invalidateAndCancel() } + await CloudRefreshURLProtocol.reset() + let identity = try #require(fixture.auth.authenticatedSessionIdentity) + let key = CloudReadRequestCoordinator.Key(path: path, accountID: identity.accountID, + generation: identity.generation, teamID: fixture.auth.resolvedTeamID) + let gate = CloudReadResponseGate() + // Hold an already-admitted response at the shared owner boundary, after + // its transport's gate check, so the joining caller must enforce its own gate. + let existing = Task { try await fixture.readRequests.read(key) { + await gate.read(.init(data: Data(), http: HTTPURLResponse( + url: URL(string: "https://fixture.invalid")!, statusCode: 200, httpVersion: nil, headerFields: nil + )!)) + } } + try await eventually { await gate.requests == 1 } + let joined = Task { try await fixture.client.request("GET", path: path) } + try await eventually { await fixture.readRequests.entries[key]?.waiters.count == 2 } + blocked.withLock { $0 = true } + await gate.release() + _ = try await existing.value + do { + _ = try await joined.value + Issue.record("Disabled Cloud published an in-flight response") + } catch VMClientError.disabledByManagedPolicy where managedPolicy { + } catch VMClientError.cloudMachinesDisabled where !managedPolicy { + } catch { Issue.record("Unexpected gate error: \(error)") } + #expect(await CloudRefreshURLProtocol.requestCounts().isEmpty) + } + + @Test("Cloud access revocation remains available with both gates closed") + func revocationBypassesClosedGates() async throws { + let fixture = try await CloudRefreshFixture.make(isDisabledByManagedPolicy: { true }, isCloudEnabled: { false }) + defer { fixture.session.invalidateAndCancel() } + await CloudRefreshURLProtocol.reset() + try await fixture.client.revokeCloudAccess(deviceID: "fixture-device") + #expect(await CloudRefreshURLProtocol.requestCounts() == ["/api/vm/tunnel": 1]) + } + @Test("A list delayed beyond a polling interval stays owned and stops when hidden") func delayedListAcrossPoll() async throws { let clock = CloudReadManualClock() @@ -271,7 +340,11 @@ struct CloudRefreshFixture { let session: URLSession let readRequests: CloudReadRequestCoordinator - static func make(readRequests: CloudReadRequestCoordinator = CloudReadRequestCoordinator()) async throws -> Self { + static func make( + readRequests: CloudReadRequestCoordinator = CloudReadRequestCoordinator(), + isDisabledByManagedPolicy: (@Sendable () -> Bool)? = nil, + isCloudEnabled: @escaping @Sendable () -> Bool = { true } + ) async throws -> Self { let defaults = try #require(UserDefaults(suiteName: "CloudRefreshFixture.\(UUID())")) let auth = AuthCoordinator( client: CloudRefreshAuthClient(), @@ -297,7 +370,8 @@ struct CloudRefreshFixture { let session = URLSession(configuration: configuration) return Self(client: VMClient( session: session, auth: auth, resourceStats: VMResourceStatsStore(), checkpointRenames: CloudRenameCoordinator(), - machineCache: CloudMachineCache(defaults: defaults), readRequests: readRequests + machineCache: CloudMachineCache(defaults: defaults), isDisabledByManagedPolicy: isDisabledByManagedPolicy, + readRequests: readRequests, isCloudEnabled: isCloudEnabled ), auth: auth, session: session, readRequests: readRequests) } } From c414a04e503b4232cce457e8f6ffdda76889ae00 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 00:18:06 -0700 Subject: [PATCH 24/92] fix: enforce Cloud gates around shared reads --- Sources/Cloud/VMClient.swift | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/Sources/Cloud/VMClient.swift b/Sources/Cloud/VMClient.swift index e61f8ec39f8c..6c40e5e99959 100644 --- a/Sources/Cloud/VMClient.swift +++ b/Sources/Cloud/VMClient.swift @@ -1987,9 +1987,7 @@ actor VMClient { allowedUnderManagedPolicy: allowedUnderManagedPolicy) } try Task.checkCancellation() - if !allowedUnderManagedPolicy, self.isDisabledByManagedPolicy?() == true { - throw VMClientError.disabledByManagedPolicy - } + try self.checkCloudAccess(allowedUnderManagedPolicy: allowedUnderManagedPolicy) let context = CloudOperationContext.current let elapsed = context.map { $0.operation == .list || $0.operation == .stats ? $0.clock.duration(to: .now) : .zero } ?? .zero let deadline = self.readRequests.makeDeadline(elapsed: elapsed) @@ -2008,6 +2006,7 @@ actor VMClient { guard await self.auth.isAuthenticatedSessionIdentityCurrent(identity), await self.auth.resolvedTeamID == teamID else { throw CancellationError() } } + try self.checkCloudAccess(allowedUnderManagedPolicy: allowedUnderManagedPolicy) return (value.data, value.http) } if CloudOperationContext.current != nil || operations == nil { return try await work() } @@ -2015,6 +2014,15 @@ actor VMClient { return try await operations!.perform(kind, foreground: method != "GET", work) } + private func checkCloudAccess(allowedUnderManagedPolicy: Bool) throws { + if !allowedUnderManagedPolicy, isDisabledByManagedPolicy?() == true { + throw VMClientError.disabledByManagedPolicy + } + if !allowedUnderManagedPolicy, !isCloudEnabled() { + throw VMClientError.cloudMachinesDisabled + } + } + private func requestMeasured( _ method: String, path: String, @@ -2024,12 +2032,7 @@ actor VMClient { retryTransientServiceUnavailable: Bool = false, allowedUnderManagedPolicy: Bool = false ) async throws -> (Data, HTTPURLResponse) { - if !allowedUnderManagedPolicy, isDisabledByManagedPolicy?() == true { - throw VMClientError.disabledByManagedPolicy - } - if !allowedUnderManagedPolicy, !isCloudEnabled() { - throw VMClientError.cloudMachinesDisabled - } + try checkCloudAccess(allowedUnderManagedPolicy: allowedUnderManagedPolicy) let minted = VMRequestTraceContext.mint() let trace = CloudOperationContext.current.map { VMRequestTraceContext(traceId: $0.traceID, spanId: $0.spanID, clientRequestId: minted.clientRequestId) From cc4e7aecd8f67deb4349110f88b3a98804d382c9 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 00:18:07 -0700 Subject: [PATCH 25/92] fix: accept release-candidate Cloud diagnostics --- web/services/observability/cloudServerError.ts | 4 ++-- web/services/observability/cloudTelemetryContract.ts | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/web/services/observability/cloudServerError.ts b/web/services/observability/cloudServerError.ts index 67bb3c67ac7d..df38a8f30a19 100644 --- a/web/services/observability/cloudServerError.ts +++ b/web/services/observability/cloudServerError.ts @@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto"; import { after } from "next/server"; import type { VmRequestContext } from "../vms/requestContext"; import type { VmErrorResponseInput } from "../vms/routeHelpers"; -import { cloudOperations, type CloudTelemetryClient, type CloudTelemetrySpan } from "./cloudTelemetryContract"; +import { cloudChannels, cloudOperations, type CloudTelemetryClient, type CloudTelemetrySpan } from "./cloudTelemetryContract"; import { acceptCloudTelemetry } from "./cloudTelemetryRepository"; import { drainCloudDiagnostics } from "./cloudTelemetryDelivery"; @@ -11,7 +11,7 @@ export function retainCloudServerError(input: VmErrorResponseInput, context: VmR if (!context?.userId || !context.traceId || !context.spanId) return; const now = Date.now(); const rawChannel = context.client.channel; - const channel = rawChannel === "nightly" ? "nightly" : rawChannel === "stable" || rawChannel === "production" ? "production" : rawChannel === "dev" ? "dev" : "unknown"; + const channel = rawChannel === "stable" ? "production" : cloudChannels.find((value) => value === rawChannel) ?? "unknown"; const client: CloudTelemetryClient = { channel, version: safeVersion(context.client.version), build: context.client.build?.match(/^[0-9]{1,20}$/)?.[0] ?? "0", diff --git a/web/services/observability/cloudTelemetryContract.ts b/web/services/observability/cloudTelemetryContract.ts index 28458be63830..f8b1d5333939 100644 --- a/web/services/observability/cloudTelemetryContract.ts +++ b/web/services/observability/cloudTelemetryContract.ts @@ -19,6 +19,7 @@ export const cloudFailures = [ "network", "timeout", "server", "response", "unsupported", "process", "protocol", "not_found", "placement", "resource_limit", "storage", "cancelled", "unknown", ] as const; +export const cloudChannels = ["dev", "nightly", "production", "rc", "unknown"] as const; export type CloudTelemetrySpan = { readonly eventId: string; @@ -40,7 +41,7 @@ export type CloudTelemetrySpan = { readonly sourceLine?: number; }; export type CloudTelemetryClient = { - readonly channel: "dev" | "nightly" | "production" | "unknown"; + readonly channel: typeof cloudChannels[number]; readonly version: string; readonly build: string; readonly revision: string; @@ -74,7 +75,7 @@ export function parseCloudTelemetryBatch(value: unknown, now = Date.now()): Clou function validClient(value: unknown): value is CloudTelemetryClient { if (!record(value) || !onlyKeys(value, clientKeys)) return false; - return member(value.channel, ["dev", "nightly", "production", "unknown"]) + return member(value.channel, cloudChannels) && textMatches(value.version, /^[0-9][0-9A-Za-z.+-]{0,39}$/) && textMatches(value.build, /^[0-9]{1,20}$/) && textMatches(value.revision, /^(?:[0-9a-f]{7,64}|unknown)$/) From a94d5efb3c32ed08629f71de21bee8a66501fc7a Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 00:58:14 -0700 Subject: [PATCH 26/92] test: reproduce stale Cloud stats batch replacement --- cmuxTests/VMClientReadCoalescingTests.swift | 57 +++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/cmuxTests/VMClientReadCoalescingTests.swift b/cmuxTests/VMClientReadCoalescingTests.swift index 281305a6f9dd..1251f364b958 100644 --- a/cmuxTests/VMClientReadCoalescingTests.swift +++ b/cmuxTests/VMClientReadCoalescingTests.swift @@ -271,6 +271,63 @@ struct VMClientReadCoalescingTests { #expect(models[2].machines.isEmpty && models[3].machines.isEmpty) } + @Test("An authoritative fleet replaces the old stats batch and preserves other readers") + func changedFleetReplacesStatsBatch() async throws { + let fixture = try await CloudRefreshFixture.make() + defer { fixture.session.invalidateAndCancel() } + await CloudRefreshURLProtocol.reset() + await CloudRefreshURLProtocol.holdResponses() + let model = MachinesPanelViewModel(client: fixture.client, isCloudEnabled: { true }) + defer { model.stopPolling() } + func acceptFleet(_ ids: [String]) { + let page = VMListPage(vms: ids.map { + VMSummary(id: $0, provider: "fixture", status: "running", image: "desktop-vnc", createdAt: 0, base: nil) + }, limits: nil) + model.applyRefreshResult(.success(page), generation: model.refreshGeneration, scope: nil) + } + let otherReader = Task { try await fixture.client.stats(id: "shared") } + defer { otherReader.cancel() } + await CloudRefreshURLProtocol.waitUntilStarted() + acceptFleet(["removed", "shared"]) + try await eventually { + let entries = await fixture.readRequests.entries + return entries.values.reduce(0) { $0 + $1.waiters.count } == 3 + } + await CloudRefreshURLProtocol.waitUntilStarted(2) + let oldID = try #require(model.statsID) + let oldTask = try #require(model.statsTask) + + acceptFleet(["added", "shared"]) + try #require(model.statsID != oldID, "The changed fleet must replace the running batch") + let replacementID = try #require(model.statsID) + let replacementTask = try #require(model.statsTask) + await oldTask.value + #expect(model.statsID == replacementID, "Old completion must not clear the replacement owner") + try await eventually { + let entries = await fixture.readRequests.entries + return entries.first { $0.key.path == "/api/vm/removed/stats" } == nil + && entries.first { $0.key.path == "/api/vm/shared/stats" }?.value.waiters.count == 2 + && entries.first { $0.key.path == "/api/vm/added/stats" }?.value.waiters.count == 1 + } + await CloudRefreshURLProtocol.waitUntilStarted(3) + #expect(await CloudRefreshURLProtocol.requestCounts() == [ + "/api/vm/removed/stats": 1, "/api/vm/shared/stats": 1, "/api/vm/added/stats": 1 + ]) + + acceptFleet([]) + try #require(model.statsID == nil && model.statsTask == nil) + await replacementTask.value + #expect(model.statsID == nil && model.statsTask == nil) + try await eventually { + let entries = await fixture.readRequests.entries + return entries.count == 1 && entries.first?.key.path == "/api/vm/shared/stats" + && entries.first?.value.waiters.count == 1 + } + await CloudRefreshURLProtocol.releaseResponses() + #expect(try await otherReader.value.state == .awake) + #expect(model.machines.isEmpty) + } + @Test("Team usage shares offline state with list and stats while keeping its shorter budget") func teamUsageNetworkState() async throws { let clock = CloudReadManualClock() From 6859a900ee42fb7963c140f09341c4c758f76a19 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 00:58:44 -0700 Subject: [PATCH 27/92] fix: replace stale Cloud stats batches --- Sources/Cloud/MachinesPanelViewModel+Refresh.swift | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Sources/Cloud/MachinesPanelViewModel+Refresh.swift b/Sources/Cloud/MachinesPanelViewModel+Refresh.swift index ffbdff609e69..5123c74713d9 100644 --- a/Sources/Cloud/MachinesPanelViewModel+Refresh.swift +++ b/Sources/Cloud/MachinesPanelViewModel+Refresh.swift @@ -4,7 +4,10 @@ import Foundation /// Refresh ownership stays on the panel; only immutable requests survive suspension. extension MachinesPanelViewModel { func refreshStats() { - guard isCloudEnabled(), statsTask == nil, let client = client ?? VMClient.shared else { return } + statsTask?.cancel() + statsTask = nil + statsID = nil + guard isCloudEnabled(), let client = client ?? VMClient.shared else { return } let ids = machines.filter { $0.capabilities.stats }.map(\.id) guard !ids.isEmpty else { return } let requestID = UUID() From 9bbce0578284f81ccd1435bd4c4bd2f6477c1767 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 01:22:55 -0700 Subject: [PATCH 28/92] test: cover independent Cloud caller deadlines --- .../CloudReadRequestCoordinatorTests.swift | 100 ++++++++++++++++-- 1 file changed, 92 insertions(+), 8 deletions(-) diff --git a/cmuxTests/CloudReadRequestCoordinatorTests.swift b/cmuxTests/CloudReadRequestCoordinatorTests.swift index 495bd8c8f9c9..80bd6fa250e6 100644 --- a/cmuxTests/CloudReadRequestCoordinatorTests.swift +++ b/cmuxTests/CloudReadRequestCoordinatorTests.swift @@ -33,19 +33,23 @@ struct CloudReadRequestCoordinatorTests { print("cloud-read-scale machines=\(machines) owners=4 requests=\(await gate.requests) requests_per_machine=1") } - @Test("Cancelling one waiter preserves another; the last cancels and holds the draining slot") - func independentCancellation() async throws { + @Test("The final waiter cancels and holds the draining slot after another caller leaves", arguments: [false, true]) + func independentCancellation(expireFirst: Bool) async throws { let gate = CloudReadResponseGate() - let owner = Owner() - let first = Task { try await owner.read(key()) { await gate.read(response()) } } + let clock = CloudReadManualClock() + let owner = Owner(clock: CloudRequestClock(clock)) + let first = Task { try await owner.read(key(), deadline: .seconds(5)) { await gate.read(response()) } } + try await eventually { await gate.requests == 1 } let second = Task { try await owner.read(key()) { await gate.read(response()) } } try await eventually { await owner.entries.values.first?.waiters.count == 2 } - try await eventually { await gate.requests == 1 } - first.cancel() - do { _ = try await first.value; Issue.record("cancelled waiter returned a value") } catch is CancellationError {} catch { Issue.record("\(error)") } + if expireFirst { clock.advance(by: .seconds(6)) } else { first.cancel() } + do { _ = try await first.value; Issue.record("retired waiter returned a value") } + catch is CancellationError { #expect(!expireFirst) } + catch { #expect(expireFirst && (error as? URLError)?.code == .timedOut) } #expect(await owner.entries.values.first?.waiters.count == 1) second.cancel() - _ = await second.result + do { _ = try await second.value; Issue.record("final cancelled waiter returned a value") } + catch is CancellationError {} catch { Issue.record("\(error)") } let replacement = Task { try await owner.read(key()) { await gate.read(response()) } } try await eventually { await owner.entries.values.first?.pending?.waiters.count == 1 } #expect(await gate.requests == 1) @@ -137,6 +141,86 @@ struct CloudReadRequestCoordinatorTests { #expect(await owner.entries.isEmpty) } + @Test("An elapsed first caller cannot shorten a later caller's budget", arguments: [false, true], [false, true]) + func shortFirstDeadline(queued: Bool, deliverTimers: Bool) async throws { + let clock = CloudReadManualClock() + let owner = Owner(clock: CloudRequestClock(clock)) + let gate = CloudReadResponseGate() + if queued { + let retired = Task { try await owner.read(key()) { await gate.read(response()) } } + try await eventually { await gate.requests == 1 } + retired.cancel() + _ = await retired.result + } + let shortDeadline = owner.makeDeadline(elapsed: .seconds(25)) + let short = Task { try await owner.read(key(), deadline: shortDeadline) { await gate.read(response()) } } + try await eventually { + let entry = await owner.entries.values.first + return queued ? entry?.pending?.waiters.count == 1 : entry?.waiters.count == 1 + } + clock.advance(by: .seconds(1)) + let longDeadline = owner.makeDeadline() + let long = Task { try await owner.read(key(), deadline: longDeadline) { await gate.read(response()) } } + try await eventually { + let entry = await owner.entries.values.first + return queued ? entry?.pending?.waiters.count == 2 : entry?.waiters.count == 2 + } + clock.advance(by: .seconds(5), deliverTimers: deliverTimers) + if deliverTimers { _ = await short.result } + await gate.release() + do { _ = try await short.value; Issue.record("elapsed caller exceeded its deadline") } + catch { #expect((error as? URLError)?.code == .timedOut) } + #expect(try await long.value.http.statusCode == 200) + #expect(await gate.requests == (queued ? 2 : 1)) + #expect(await owner.entries.isEmpty) + } + + @Test("Later callers cannot rejuvenate the transport beyond its fixed budget") + func transportLifetimeCap() async throws { + let clock = CloudReadManualClock() + let owner = Owner(clock: CloudRequestClock(clock)) + let gate = CloudReadResponseGate() + let first = Task { try await owner.read(key()) { await gate.read(response()) } } + try await eventually { await gate.requests == 1 } + clock.advance(by: .seconds(29)) + let late = Task { try await owner.read(key()) { await gate.read(response()) } } + try await eventually { await owner.entries.values.first?.waiters.count == 2 } + #expect(await owner.noteRetryAfter(key(), seconds: 2, response: response(429)) == false) + clock.advance(by: .seconds(2)) + for request in [first, late] { + do { _ = try await request.value; Issue.record("shared transport exceeded its cap") } + catch { #expect((error as? URLError)?.code == .timedOut) } + } + #expect(await owner.entries.values.first?.waiters.isEmpty == true) + let replacement = Task { try await owner.read(key()) { await gate.read(response()) } } + try await eventually { await owner.entries.values.first?.pending?.waiters.count == 1 } + #expect(await gate.requests == 1) + await gate.release() + #expect(try await replacement.value.http.statusCode == 200) + #expect(await gate.requests == 2) + } + + @Test("Retry admission uses the latest live caller within the transport cap") + func retryUsesRemainingCallerBudgets() async throws { + let clock = CloudReadManualClock() + let owner = Owner(clock: CloudRequestClock(clock)) + let gate = CloudReadResponseGate() + let short = Task { try await owner.read(key(), deadline: .seconds(5)) { await gate.read(response(429)) } } + try await eventually { await gate.requests == 1 } + clock.advance(by: .seconds(1)) + let long = Task { try await owner.read(key(), deadline: .seconds(20)) { await gate.read(response(429)) } } + try await eventually { await owner.entries.values.first?.waiters.count == 2 } + #expect(await owner.noteRetryAfter(key(), seconds: 8, response: response(429))) + long.cancel() + _ = await long.result + #expect(await owner.noteRetryAfter(key(), seconds: 8, response: response(429)) == false) + await gate.release() + #expect(try await short.value.http.statusCode == 429) + #expect(try await owner.read(key()) { Issue.record("retried before the server minimum"); return response() }.http.statusCode == 429) + clock.advance(by: .seconds(8)) + #expect(try await owner.read(key()) { response() }.http.statusCode == 200) + } + @Test("Response-first delivery after a simulated wake still expires") func responseAfterDeadline() async throws { let clock = CloudReadManualClock() From 593964f71c450a10385cfc892f394659ee9a9a5a Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 01:22:58 -0700 Subject: [PATCH 29/92] fix: separate Cloud transport and caller deadlines --- .../CloudReadRequestCoordinator+Entry.swift | 3 +- .../Cloud/CloudReadRequestCoordinator.swift | 29 ++++++++++--------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/Sources/Cloud/CloudReadRequestCoordinator+Entry.swift b/Sources/Cloud/CloudReadRequestCoordinator+Entry.swift index b41b45936fe3..5e3b3d67677f 100644 --- a/Sources/Cloud/CloudReadRequestCoordinator+Entry.swift +++ b/Sources/Cloud/CloudReadRequestCoordinator+Entry.swift @@ -3,7 +3,8 @@ import Foundation extension CloudReadRequestCoordinator { struct Entry: Sendable { let id: UUID - let deadline: Duration + /// Fixed at transport admission; later callers and retries never renew it. + let transportDeadline: Duration var waiters: [UUID: Waiter] var work: Task? var timer: Task? diff --git a/Sources/Cloud/CloudReadRequestCoordinator.swift b/Sources/Cloud/CloudReadRequestCoordinator.swift index 1e3bd0476b13..cf4c52a5e7e1 100644 --- a/Sources/Cloud/CloudReadRequestCoordinator.swift +++ b/Sources/Cloud/CloudReadRequestCoordinator.swift @@ -3,6 +3,8 @@ import Foundation /// Owns overlapping read requests for one VM client. Caller cancellation releases /// only that waiter; the last waiter cancels the transport. A cancelled request /// keeps its slot until teardown completes, preventing replacement amplification. +/// Caller deadlines are independent upper bounds. The shared transport also has +/// a fixed lifetime cap; reaching it fails even later callers without extending it. actor CloudReadRequestCoordinator { @TaskLocal static var current: Context? @@ -55,7 +57,7 @@ actor CloudReadRequestCoordinator { if let entry = entries[key] { if entry.terminalError != nil { queueAfterTeardown(key, waiter: waiter, deadline: deadline, continuation: continuation, operation: operation) - } else if clock.now() >= entry.deadline { + } else if clock.now() >= entry.transportDeadline { expire(key, id: entry.id) queueAfterTeardown(key, waiter: waiter, deadline: deadline, continuation: continuation, operation: operation) } else { @@ -68,17 +70,18 @@ actor CloudReadRequestCoordinator { continuation.resume(returning: response) return } - startEntry(key, id: UUID(), deadline: deadline, + startEntry(key, id: UUID(), waiters: [waiter: Waiter(deadline: deadline, continuation: continuation)], operation: operation) } private func startEntry( - _ key: Key, id: UUID, deadline: Duration, waiters: [UUID: Waiter], + _ key: Key, id: UUID, waiters: [UUID: Waiter], operation: @escaping @Sendable () async throws -> Response ) { let waiters = unexpired(waiters) guard !waiters.isEmpty else { return } - if clock.now() >= deadline || isOnline == false { + let transportDeadline = makeDeadline() + if clock.now() >= transportDeadline || isOnline == false { let error = URLError(isOnline == false ? .notConnectedToInternet : .timedOut) for waiter in waiters.values { waiter.continuation.resume(throwing: error) } return @@ -87,7 +90,7 @@ actor CloudReadRequestCoordinator { for waiter in waiters.values { waiter.continuation.resume(returning: response) } return } - entries[key] = Entry(id: id, deadline: deadline, waiters: waiters, operation: operation) + entries[key] = Entry(id: id, transportDeadline: transportDeadline, waiters: waiters, operation: operation) startWork(key, id: id, operation: operation) armTimer(key, id: id) } @@ -95,7 +98,7 @@ actor CloudReadRequestCoordinator { private func armTimer(_ key: Key, id: UUID) { guard let entry = entries[key], entry.terminalError == nil else { return } entry.timer?.cancel() - let deadline = min(entry.deadline, entry.waiters.values.map(\.deadline).min() ?? entry.deadline) + let deadline = min(entry.transportDeadline, entry.waiters.values.map(\.deadline).min() ?? entry.transportDeadline) entries[key]?.timer = Task { [weak self, clock] in do { try await clock.sleepUntil(deadline) } catch { return } await self?.expireDueWaiters(key, id: id) @@ -115,7 +118,7 @@ actor CloudReadRequestCoordinator { private func expireDueWaiters(_ key: Key, id: UUID) { guard let entry = entries[key], entry.id == id, entry.terminalError == nil else { return } - if clock.now() >= entry.deadline { expire(key, id: id); return } + if clock.now() >= entry.transportDeadline { expire(key, id: id); return } entries[key]?.waiters = unexpired(entry.waiters) if entries[key]?.waiters.isEmpty == true { expire(key, id: id) } else { armTimer(key, id: id) } @@ -182,15 +185,16 @@ actor CloudReadRequestCoordinator { } /// Retains the server's minimum retry time across cancellation and later - /// polls. When it exceeds this operation's remaining budget, return the - /// original 429 now; future reads receive a compact 429 until retry is legal. + /// polls. Retry only while a live caller and the transport both have budget; + /// otherwise return 429 now and preserve that minimum for future reads. func noteRetryAfter(_ key: Key, seconds: TimeInterval, response: Response) -> Bool { // Retry-After can contain Int.max seconds. Keep that distant deadline // as a monotonic floating-point instant rather than overflowing Duration. let until = self.seconds(clock.now()) + seconds cooldowns.record(key, until: until, now: self.seconds(clock.now()), response: response) - guard let entry = entries[key], entry.terminalError == nil else { return false } - return until < self.seconds(entry.deadline) + guard let entry = entries[key], entry.terminalError == nil, + let callerDeadline = entry.waiters.values.map(\.deadline).max() else { return false } + return until < self.seconds(min(entry.transportDeadline, callerDeadline)) } private func seconds(_ duration: Duration) -> TimeInterval { @@ -242,8 +246,7 @@ actor CloudReadRequestCoordinator { for waiter in completed.waiters.values { waiter.continuation.resume(with: result) } if let pending = completed.pending { pending.timer?.cancel() - let deadline = min(makeDeadline(), pending.waiters.values.map(\.deadline).max() ?? clock.now()) - startEntry(key, id: pending.id, deadline: deadline, waiters: pending.waiters, operation: pending.operation) + startEntry(key, id: pending.id, waiters: pending.waiters, operation: pending.operation) } } From 90652bafc63a750da3db46c419e6bf43f43ad291 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 01:32:39 -0700 Subject: [PATCH 30/92] test: cover team usage rate-limit cooldowns --- cmuxTests/VMClientReadCoalescingTests.swift | 22 +++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/cmuxTests/VMClientReadCoalescingTests.swift b/cmuxTests/VMClientReadCoalescingTests.swift index 1251f364b958..cbe43744de0f 100644 --- a/cmuxTests/VMClientReadCoalescingTests.swift +++ b/cmuxTests/VMClientReadCoalescingTests.swift @@ -355,6 +355,28 @@ struct VMClientReadCoalescingTests { await CloudRefreshURLProtocol.releaseResponses() } + @Test("Throttled team usage preserves Retry-After and failed request telemetry") + func teamUsageCooldown() async throws { + let clock = CloudReadManualClock() + let reads = CloudReadRequestCoordinator(clock: CloudRequestClock(clock)) + let fixture = try await CloudRefreshFixture.make(readRequests: reads) + defer { fixture.session.invalidateAndCancel() } + await CloudRefreshURLProtocol.reset() + await CloudRefreshURLProtocol.configure(.throttled) + let recorder = CloudOperationRecorder() + let usage = MachineUsageClient(session: fixture.session, auth: fixture.auth, operations: recorder, readRequests: reads) + for call in 0..<4 { + if call == 2 { clock.advance(by: .seconds(59)) } + if call == 3 { clock.advance(by: .seconds(1)) } + do { _ = try await usage.teamUsage(); Issue.record("throttled usage succeeded") } + catch MachineUsageClientError.httpStatus(429, _) {} catch { Issue.record("\(error)") } + #expect(await CloudRefreshURLProtocol.requestCounts() == ["/api/coderouter/vm-usage/team": call == 3 ? 2 : 1]) + } + let requests = recorder.operations.flatMap(\.steps).filter { $0.phase == .request } + #expect(requests.count == 2) + #expect(requests.allSatisfy { $0.outcome == .failure && $0.failure == .rateLimit }) + } + @Test("Successful resize invalidates only its authenticated list and machine stats") func mutationInvalidationUsesRequestScope() async throws { let fixture = try await CloudRefreshFixture.make() From 4e525e12fe172a93d7f20c9040934ec68bf02fb0 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 01:32:41 -0700 Subject: [PATCH 31/92] fix: record usage cooldowns before HTTP rejection --- Sources/Cloud/VMClient.swift | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Sources/Cloud/VMClient.swift b/Sources/Cloud/VMClient.swift index 6c40e5e99959..808bc7bd670d 100644 --- a/Sources/Cloud/VMClient.swift +++ b/Sources/Cloud/VMClient.swift @@ -2569,10 +2569,6 @@ actor MachineUsageClient { let response = try await readRequests.read(key, deadline: deadline) { try await CloudOperationContext.$current.withValue(context) { let (data, http) = try await self.request("GET", path: key.path, teamID: teamID) - if http.statusCode == 429 { - let seconds = TimeInterval(CmxRetryAfterPolicy.seconds(from: http) ?? CmxRetryAfterPolicy.defaultRateLimitSeconds) - _ = await self.readRequests.noteRetryAfter(key, seconds: seconds, response: .init(data: data, http: http)) - } return CloudReadRequestCoordinator.Response(data: data, http: http) } } @@ -2712,6 +2708,10 @@ actor MachineUsageClient { guard let http = response as? HTTPURLResponse else { throw MachineUsageClientError.malformedResponse("non-HTTP response") } + if http.statusCode == 429, let read = CloudReadRequestCoordinator.current, let owner = read.owner { + let seconds = TimeInterval(CmxRetryAfterPolicy.seconds(from: http) ?? CmxRetryAfterPolicy.defaultRateLimitSeconds) + _ = await owner.noteRetryAfter(read.key, seconds: seconds, response: .init(data: data, http: http)) + } guard (200...299).contains(http.statusCode) else { throw MachineUsageClientError.httpStatus(http.statusCode, String(data: data, encoding: .utf8) ?? "") } From 2a2d12cb9efc9108aa085cbfebbe4bec3f17df6d Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 01:38:48 -0700 Subject: [PATCH 32/92] ci: require focused Cloud regression verdicts --- .github/workflows/ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2da6cba62abd..81c07bc10d4b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1465,15 +1465,16 @@ jobs: scripts/ci/require_selected_test_execution.sh \ "$test_log" cmuxTests/GlobalSearchShortcutBehaviorTests - - name: Run Cloud machine ordering acceptance + - name: Run Cloud lifecycle acceptance if: ${{ !cancelled() && matrix.shard == fromJSON(env.CMUX_APP_HOST_FOCUSED_REGRESSION_SHARD) }} timeout-minutes: 10 run: | # Separate invocations prove each selector executed tests; a healthy # suite cannot mask a miswired or accidentally empty selector. set -euo pipefail - for suite in CloudMachineDragSourceTests CloudMachineOrderingTests; do - test_log="$RUNNER_TEMP/cmux-cloud-ordering-${suite}.txt" + for suite in CloudMachineDragSourceTests CloudMachineOrderingTests \ + CloudOperationRecorderTests CloudReadRequestCoordinatorTests VMClientReadCoalescingTests; do + test_log="$RUNNER_TEMP/cmux-cloud-acceptance-${suite}.txt" scripts/ci/run-in-console-session.sh \ scripts/ci/run-app-host-xcodebuild.sh \ -xctestrun "$CMUX_APP_HOST_XCTESTRUN" \ From 68883b336ed65cd001c2ff7ca7ee36293e3a082f Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 03:39:51 -0700 Subject: [PATCH 33/92] test: isolate ambient Git config in reftable fixture --- .../Tests/CmuxGitTests/ReftableGitMetadataTests.swift | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Packages/macOS/CmuxGit/Tests/CmuxGitTests/ReftableGitMetadataTests.swift b/Packages/macOS/CmuxGit/Tests/CmuxGitTests/ReftableGitMetadataTests.swift index da97c7234d38..6c04e06f408b 100644 --- a/Packages/macOS/CmuxGit/Tests/CmuxGitTests/ReftableGitMetadataTests.swift +++ b/Packages/macOS/CmuxGit/Tests/CmuxGitTests/ReftableGitMetadataTests.swift @@ -355,7 +355,10 @@ private struct NeverDirectoryProbe: GitReferenceStorageProbing { environment["GIT_CONFIG_KEY_0"] = "core.worktree" environment["GIT_CONFIG_VALUE_0"] = unrelated.path let reader = SystemGitReferenceReader( - runner: SystemWorkspaceChangesGitRunner(environment: environment) + runner: SystemWorkspaceChangesGitRunner( + environment: environment, + isolateRepositoryConfig: true + ) ) let snapshot = reader.snapshot(repository: intendedRepository) From 97d949b41a10939fdba60447681fef3b3293a46d Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 03:48:21 -0700 Subject: [PATCH 34/92] ci: preserve focused Cloud guard step contract --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ece3d4a949de..59761da90055 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1565,7 +1565,7 @@ jobs: scripts/ci/require_selected_test_execution.sh \ "$test_log" cmuxTests/GlobalSearchShortcutBehaviorTests - - name: Run Cloud lifecycle acceptance + - name: Run Cloud machine ordering acceptance if: ${{ !cancelled() && steps.prepare-app-host-home.outcome == 'success' && matrix.shard == fromJSON(env.CMUX_APP_HOST_FOCUSED_REGRESSION_SHARD) }} timeout-minutes: 10 run: | From 5c777ddb526cf030a1ce14f26e89631900ec3d1e Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 05:58:23 -0700 Subject: [PATCH 35/92] fix Cloud review findings in Swift client and fixtures --- .github/workflows/cloud-vm-guest-install.yml | 2 +- Sources/Cloud/CloudReadCooldownStore.swift | 19 ++++---- Sources/Cloud/VMClient.swift | 7 ++- cmuxTests/CloudOperationRecorderTests.swift | 45 +++++++++++++++---- .../CloudReadRequestCoordinatorTests.swift | 28 ++++++------ cmuxTests/CloudRefreshURLProtocol.swift | 32 +++++++------ cmuxTests/VMClientReadCoalescingTests.swift | 15 ++++--- 7 files changed, 96 insertions(+), 52 deletions(-) diff --git a/.github/workflows/cloud-vm-guest-install.yml b/.github/workflows/cloud-vm-guest-install.yml index 5de15231591b..dda186c99fee 100644 --- a/.github/workflows/cloud-vm-guest-install.yml +++ b/.github/workflows/cloud-vm-guest-install.yml @@ -35,7 +35,7 @@ jobs: working-directory: web services: postgres: - image: postgres:16-alpine + image: postgres:16-alpine@sha256:3c5c8892d184f738f4fe282d14ddaa613a38f00f4189d2d94725ebe6f2909ddb env: POSTGRES_DB: cmux_test POSTGRES_PASSWORD: cmux diff --git a/Sources/Cloud/CloudReadCooldownStore.swift b/Sources/Cloud/CloudReadCooldownStore.swift index 953214679e00..f74c9668057c 100644 --- a/Sources/Cloud/CloudReadCooldownStore.swift +++ b/Sources/Cloud/CloudReadCooldownStore.swift @@ -5,13 +5,12 @@ struct CloudReadCooldownStore: Sendable { private let capacity: Int private var session: Session? private var cooldowns: [CloudReadRequestCoordinator.Key: Cooldown] = [:] - private var overflow: Cooldown? init(capacity: Int = 256) { self.capacity = max(1, capacity) } - var retainedCount: Int { cooldowns.count + (overflow == nil ? 0 : 1) } + var retainedCount: Int { cooldowns.count } mutating func activateSession(for key: CloudReadRequestCoordinator.Key) { let incoming = Session(accountID: key.accountID, generation: key.generation) @@ -24,13 +23,12 @@ struct CloudReadCooldownStore: Sendable { } session = incoming cooldowns.removeAll(keepingCapacity: true) - overflow = nil } mutating func response(for key: CloudReadRequestCoordinator.Key, now: TimeInterval) -> CloudReadRequestCoordinator.Response? { guard matchesSession(key) else { return nil } prune(now: now) - return (cooldowns[key] ?? overflow)?.response + return cooldowns[key]?.response } mutating func record( @@ -51,11 +49,13 @@ struct CloudReadCooldownStore: Sendable { if until > existing.until { cooldowns[key] = cooldown } } else if cooldowns.count < capacity { cooldowns[key] = cooldown - } else if until > (overflow?.until ?? now) { - // Once full, retain one conservative session barrier instead of - // arbitrary keys/bodies. Unknown paths may wait longer at capacity; - // no forgotten path can retry before its server minimum. - overflow = cooldown + } else if let evicted = cooldowns.min(by: { $0.value.until < $1.value.until }) { + // Evict only an exact-key minimum when the bounded store is full. + // A session-wide barrier would fabricate 429s for unrelated paths; + // an evicted key may ask the server again, where its real minimum + // is enforced without blocking another machine's read. + cooldowns.removeValue(forKey: evicted.key) + cooldowns[key] = cooldown } } @@ -65,6 +65,5 @@ struct CloudReadCooldownStore: Sendable { private mutating func prune(now: TimeInterval) { cooldowns = cooldowns.filter { $0.value.until > now } - if let overflow, overflow.until <= now { self.overflow = nil } } } diff --git a/Sources/Cloud/VMClient.swift b/Sources/Cloud/VMClient.swift index 808bc7bd670d..c276dfda7637 100644 --- a/Sources/Cloud/VMClient.swift +++ b/Sources/Cloud/VMClient.swift @@ -1980,7 +1980,12 @@ actor VMClient { allowedUnderManagedPolicy: Bool = false ) async throws -> (Data, HTTPURLResponse) { let work = { - let isSharedRead = method == "GET" && (path == "/api/vm" || path.hasSuffix("/stats")) + let isSharedRead = method == "GET" + && jsonBody == nil + && extraHeaders.isEmpty + && !retryTransientServiceUnavailable + && !allowedUnderManagedPolicy + && (path == "/api/vm" || path.hasSuffix("/stats")) if !isSharedRead { return try await self.requestMeasured(method, path: path, jsonBody: jsonBody, extraHeaders: extraHeaders, timeoutSeconds: timeoutSeconds, retryTransientServiceUnavailable: retryTransientServiceUnavailable, diff --git a/cmuxTests/CloudOperationRecorderTests.swift b/cmuxTests/CloudOperationRecorderTests.swift index fc92cc2e1129..310c816165f7 100644 --- a/cmuxTests/CloudOperationRecorderTests.swift +++ b/cmuxTests/CloudOperationRecorderTests.swift @@ -29,15 +29,8 @@ struct CloudOperationRecorderTests { let identity = try #require(fixture.auth.authenticatedSessionIdentity) let recorder = CloudOperationRecorder(uploader: uploader, identity: { identity }) let operation = recorder.begin(.open) + await PlacementReceiptURLProtocol.resetCapture(status: status) await recorder.finish(operation, error: CmuxTuiSurfaceProvider.ProviderError.remotePlacementUnavailable("fixture-machine")) - let persisted = try #require(JSONSerialization.jsonObject(with: Data(contentsOf: queueURL)) as? [[String: Any]]) - // The transport may already have acknowledged the record before this - // actor resumes; its fixture validates the serialized span either way. - if let span = persisted.first?["span"] as? [String: Any] { - #expect(span["failure"] as? String == "placement") - #expect(span["traceId"] as? String == operation.traceID) - #expect(span["operationId"] as? String == operation.operationID.uuidString.lowercased()) - } let deadline = ContinuousClock.now.advanced(by: .seconds(10)) var acknowledged = false while ContinuousClock.now < deadline { @@ -47,6 +40,10 @@ struct CloudOperationRecorderTests { try await Task.sleep(for: .milliseconds(10)) } #expect(acknowledged) + let captured = try #require(await PlacementReceiptURLProtocol.captured(status: status)) + #expect(captured.failure == "placement") + #expect(captured.traceID == operation.traceID) + #expect(captured.operationID == operation.operationID.uuidString.lowercased()) #expect(await uploader.droppedCount == (status == 400 ? 1 : 0)) await uploader.clearForSignOut() } @@ -294,6 +291,32 @@ private actor CapturedCloudDiagnostics: CloudTelemetrySending { /// The request is handled synchronously without shared mutable fixture state. private final class PlacementReceiptURLProtocol: URLProtocol, @unchecked Sendable { + fileprivate struct UploadedSpan: Sendable { + let failure: String + let traceID: String + let operationID: String + } + + private actor Capture { + private var values: [Int: UploadedSpan] = [:] + + func reset(status: Int) { + values.removeValue(forKey: status) + } + + func record(status: Int, span: UploadedSpan) { + values[status] = span + } + + func value(status: Int) -> UploadedSpan? { + return values[status] + } + } + + private static let capture = Capture() + fileprivate static func resetCapture(status: Int) async { await capture.reset(status: status) } + fileprivate static func captured(status: Int) async -> UploadedSpan? { await capture.value(status: status) } + override class func canInit(with request: URLRequest) -> Bool { true } override class func canonicalRequest(for request: URLRequest) -> URLRequest { request } override func stopLoading() {} @@ -315,11 +338,17 @@ private final class PlacementReceiptURLProtocol: URLProtocol, @unchecked Sendabl let spans = batch["spans"] as? [[String: Any]], spans.count == 1, let span = spans.first, let eventID = span["eventId"] as? String, span["failure"] as? String == "placement", + let operationID = span["operationId"] as? String, !operationID.isEmpty, (span["traceId"] as? String)?.count == 32, (span["spanId"] as? String)?.count == 16 else { throw URLError(.cannotDecodeContentData) } let status = request.url?.host == "receipt-202.test" ? 202 : 400 + Task { await capture.record(status: status, span: UploadedSpan( + failure: span["failure"] as? String ?? "", + traceID: span["traceId"] as? String ?? "", + operationID: operationID + )) } guard let url = request.url, let response = HTTPURLResponse(url: url, statusCode: status, httpVersion: nil, headerFields: nil) else { throw URLError(.badURL) diff --git a/cmuxTests/CloudReadRequestCoordinatorTests.swift b/cmuxTests/CloudReadRequestCoordinatorTests.swift index 80bd6fa250e6..0ca1b4cc6925 100644 --- a/cmuxTests/CloudReadRequestCoordinatorTests.swift +++ b/cmuxTests/CloudReadRequestCoordinatorTests.swift @@ -379,34 +379,34 @@ struct CloudReadRequestCoordinatorTests { for index in 0..<1000 { store.record(key("vm-\(index)"), until: 60, now: 0, response: largeResponse) } - #expect(store.retainedCount == 9) - for index in 0..<1000 { - let cachedResponse = store.response(for: key("vm-\(index)"), now: 59) - let cached = try #require(cachedResponse) + #expect(store.retainedCount == 8) + let cachedResponses = (0..<1000).compactMap { store.response(for: key("vm-\($0)"), now: 59) } + #expect(cachedResponses.count == 8) + for cached in cachedResponses { #expect(cached.http.statusCode == 429) #expect(cached.data == Data(#"{"error":"rate_limited"}"#.utf8)) #expect(cached.http.allHeaderFields.isEmpty) } - // Overflow deliberately delays unknown paths in this session rather - // than forgetting an evicted path's minimum and retrying too early. - let overflowResponse = store.response(for: key("unseen"), now: 59) - #expect(overflowResponse?.http.statusCode == 429) + // An evicted key and an unrelated path must not inherit another + // machine's Retry-After minimum. + #expect(store.response(for: key("unseen"), now: 59) == nil) let expiredResponse = store.response(for: key("vm-999"), now: 60) #expect(expiredResponse == nil) #expect(store.retainedCount == 0) } - @Test("Long cooldowns preserve a later minimum while earlier records expire") - func cooldownOverflowKeepsLongestMinimum() { + @Test("A key keeps its longest cooldown without leaking to other paths") + func cooldownKeepsLongestMinimumPerKey() { var store = CloudReadCooldownStore(capacity: 1) store.activateSession(for: key()) store.record(key("first"), until: 10, now: 0, response: response(429)) - store.record(key("second"), until: 100, now: 0, response: response(429)) - store.record(key("third"), until: 20, now: 0, response: response(429)) - let throttledResponse = store.response(for: key("second"), now: 99) + store.record(key("first"), until: 100, now: 0, response: response(429)) + store.record(key("first"), until: 20, now: 0, response: response(429)) + let throttledResponse = store.response(for: key("first"), now: 99) #expect(throttledResponse?.http.statusCode == 429) #expect(store.retainedCount == 1) - let expiredResponse = store.response(for: key("second"), now: 100) + #expect(store.response(for: key("other"), now: 99) == nil) + let expiredResponse = store.response(for: key("first"), now: 100) #expect(expiredResponse == nil) } diff --git a/cmuxTests/CloudRefreshURLProtocol.swift b/cmuxTests/CloudRefreshURLProtocol.swift index 1f735e25dfe1..b89e550c2d58 100644 --- a/cmuxTests/CloudRefreshURLProtocol.swift +++ b/cmuxTests/CloudRefreshURLProtocol.swift @@ -9,7 +9,8 @@ final class CloudRefreshURLProtocol: URLProtocol, @unchecked Sendable { static func releaseResponses() async { await responses.release() } static func configure(_ behavior: Behavior) async { await responses.configure(behavior) } static func waitUntilStarted(_ count: Int = 1) async { await responses.waitUntilStarted(count) } - static func waitUntilStopped() async { await responses.waitUntilStopped() } + static func currentStopCount() async -> Int { await responses.stopCount } + static func waitUntilStopped(after baseline: Int) async { await responses.waitUntilStopped(after: baseline) } static func reset() async { await responses.reset() } static func requestCounts() async -> [String: Int] { await responses.counts } override class func canInit(with request: URLRequest) -> Bool { true } @@ -30,17 +31,18 @@ final class CloudRefreshURLProtocol: URLProtocol, @unchecked Sendable { responseWaiters.removeAll() for waiter in pending.values { waiter.resume() } } - private var stopped = false + private var stoppedRequests: Set = [] + private(set) var stopCount = 0 private var startWaiters: [(Int, CheckedContinuation)] = [] - private var stopWaiters: [CheckedContinuation] = [] + private var stopWaiters: [(after: Int, CheckedContinuation)] = [] func configure(_ value: Behavior) { behavior = value } func waitUntilStarted(_ count: Int) async { guard counts.values.reduce(0, +) < count else { return } await withCheckedContinuation { startWaiters.append((count, $0)) } } - func waitUntilStopped() async { - guard !stopped else { return } - await withCheckedContinuation { stopWaiters.append($0) } + func waitUntilStopped(after baseline: Int) async { + guard stopCount <= baseline else { return } + await withCheckedContinuation { stopWaiters.append((baseline, $0)) } } func reset() { release() @@ -48,10 +50,12 @@ final class CloudRefreshURLProtocol: URLProtocol, @unchecked Sendable { tasks.removeAll() counts.removeAll() behavior = .normal - stopped = false + stoppedRequests.removeAll() + stopCount = 0 } func start(_ source: CloudRefreshURLProtocol) { let key = ObjectIdentifier(source) + guard !stoppedRequests.contains(key) else { return } let path = source.request.url!.path counts[path, default: 0] += 1 let count = counts.values.reduce(0, +) @@ -73,12 +77,14 @@ final class CloudRefreshURLProtocol: URLProtocol, @unchecked Sendable { } } func stop(_ source: CloudRefreshURLProtocol) { - tasks.removeValue(forKey: ObjectIdentifier(source))?.cancel() - responseWaiters.removeValue(forKey: ObjectIdentifier(source))?.resume() - stopped = true - let waiters = stopWaiters - stopWaiters.removeAll() - for waiter in waiters { waiter.resume() } + let key = ObjectIdentifier(source) + guard stoppedRequests.insert(key).inserted else { return } + tasks.removeValue(forKey: key)?.cancel() + responseWaiters.removeValue(forKey: key)?.resume() + stopCount += 1 + let waiters = stopWaiters.filter { $0.after < stopCount } + stopWaiters.removeAll { $0.after < stopCount } + for (_, waiter) in waiters { waiter.resume() } } } } diff --git a/cmuxTests/VMClientReadCoalescingTests.swift b/cmuxTests/VMClientReadCoalescingTests.swift index cbe43744de0f..9d8d75ae7493 100644 --- a/cmuxTests/VMClientReadCoalescingTests.swift +++ b/cmuxTests/VMClientReadCoalescingTests.swift @@ -63,9 +63,10 @@ struct VMClientReadCoalescingTests { model.startPolling() await CloudRefreshURLProtocol.waitUntilStarted() model.beginOperation("fixture operation") + let stopBaseline = await CloudRefreshURLProtocol.currentStopCount() model.stopPolling() model.endOperation() - await CloudRefreshURLProtocol.waitUntilStopped() + await CloudRefreshURLProtocol.waitUntilStopped(after: stopBaseline) #expect(!model.isLoading) #expect(model.machines.isEmpty) NotificationCenter.default.post(name: .cmuxCloudReadNetworkChanged, object: nil, userInfo: ["isOnline": true]) @@ -83,9 +84,10 @@ struct VMClientReadCoalescingTests { weakModel = model model?.refresh() await CloudRefreshURLProtocol.waitUntilStarted() + let stopBaseline = await CloudRefreshURLProtocol.currentStopCount() model = nil #expect(weakModel == nil) - await CloudRefreshURLProtocol.waitUntilStopped() + await CloudRefreshURLProtocol.waitUntilStopped(after: stopBaseline) } @Test("A failed stats sample clears the last live reading") @@ -135,10 +137,11 @@ struct VMClientReadCoalescingTests { await CloudRefreshURLProtocol.holdResponses() let request = Task { try await fixture.client.stats(id: "fixture-0") } await CloudRefreshURLProtocol.waitUntilStarted() + let stopBaseline = await CloudRefreshURLProtocol.currentStopCount() clock.advance(by: .milliseconds(101)) do { _ = try await request.value; Issue.record("request exceeded its total budget") } catch { #expect((error as? URLError)?.code == .timedOut) } - await CloudRefreshURLProtocol.waitUntilStopped() + await CloudRefreshURLProtocol.waitUntilStopped(after: stopBaseline) await CloudRefreshURLProtocol.releaseResponses() } @@ -244,8 +247,9 @@ struct VMClientReadCoalescingTests { clock.advance(by: .seconds(45)) try await eventually { clock.pendingSleeperCount == 2 } #expect(await CloudRefreshURLProtocol.requestCounts().values.reduce(0, +) == 1) + let stopBaseline = await CloudRefreshURLProtocol.currentStopCount() model.stopPolling() - await CloudRefreshURLProtocol.waitUntilStopped() + await CloudRefreshURLProtocol.waitUntilStopped(after: stopBaseline) await CloudRefreshURLProtocol.releaseResponses() #expect(!model.isLoading) #expect(model.machines.isEmpty) @@ -339,10 +343,11 @@ struct VMClientReadCoalescingTests { let usage = MachineUsageClient(session: fixture.session, auth: fixture.auth, readRequests: reads) let request = Task { try await usage.teamUsage() } await CloudRefreshURLProtocol.waitUntilStarted() + let stopBaseline = await CloudRefreshURLProtocol.currentStopCount() await reads.networkChanged(isOnline: false) do { _ = try await request.value; Issue.record("usage survived offline") } catch { #expect((error as? URLError)?.code == .notConnectedToInternet) } - await CloudRefreshURLProtocol.waitUntilStopped() + await CloudRefreshURLProtocol.waitUntilStopped(after: stopBaseline) do { _ = try await usage.teamUsage(); Issue.record("usage started offline") } catch { #expect((error as? URLError)?.code == .notConnectedToInternet) } #expect(await CloudRefreshURLProtocol.requestCounts().values.reduce(0, +) == 1) From 20c95a50551e12e595dc79718776646b2b166e3e Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 06:15:16 -0700 Subject: [PATCH 36/92] fix Cloud cleanup and guest installation rollback --- .../vms/drivers/guestCliInstallCommand.ts | 265 +++++++++++- web/services/vms/repository.ts | 398 ++++++++++++++---- web/services/vms/workflows.ts | 87 ++++ web/tests/vm-guest-install-runtime.test.ts | 62 ++- web/tests/vm-workflows.test.ts | 83 ++++ 5 files changed, 801 insertions(+), 94 deletions(-) diff --git a/web/services/vms/drivers/guestCliInstallCommand.ts b/web/services/vms/drivers/guestCliInstallCommand.ts index 793be970ec75..04b6c41b41b6 100644 --- a/web/services/vms/drivers/guestCliInstallCommand.ts +++ b/web/services/vms/drivers/guestCliInstallCommand.ts @@ -1,19 +1,212 @@ import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; import { GUEST_CMUX_SHIM, GUEST_CMUX_SHIM_PATH } from "../guestCli"; -import { guestBrowserInstallCommand } from "../guestBrowser"; -import { guestPromptInstallCommand, type GuestPromptIdentity } from "../guestPrompt"; +import { GUEST_BROWSER_FILES, guestBrowserInstallCommand } from "../guestBrowser"; +import type { GuestPromptIdentity } from "../guestPrompt"; import { shellQuote } from "./cmuxTuiDaemon"; const digest = createHash("sha256").update(GUEST_CMUX_SHIM).digest("hex"); +const promptAsset = (name: string) => readFileSync( + fileURLToPath(new URL(`../images/devbox/${name}`, import.meta.url).toString()), "utf8", +); +const promptBash = promptAsset("cmux-prompt.bash"); +const promptBashrc = promptAsset("cmux-bashrc"); +const installPaths = [ + GUEST_CMUX_SHIM_PATH, + ...GUEST_BROWSER_FILES.map(({ path }) => path), + "/etc/cmux/browser-opener-version", + "/etc/cmux/prompt.bash", + "/etc/cmux/bashrc", + "/etc/cmux/.prompt-identity", + "/etc/cmux/vm-name", + "/etc/bash.bashrc", + "/etc/zsh/zshenv", +]; +const promptName = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/; -// Validate the uploaded generation and all required setup before publishing it. -// os.replace replaces a symlink itself and refuses a directory destination; -// `mv -f source destination` can instead move source *inside* a directory. +// Validate, stage, and publish the shim and its companion files as one guest +// transaction. The prompt lock is held across the browser and prompt writes, +// so a rename cannot observe or race a partially-installed generation. const install = String.raw` -import hashlib, json, os, stat, subprocess, sys -source, target, digest, browser, prompt = sys.argv[1:] +import fcntl, hashlib, json, os, pwd, shutil, stat, subprocess, sys, tempfile +source, target, digest, browser, prompt_json, paths_json = sys.argv[1:] stage = "validate" +lock = None +backup_root = None +backups = {} +paths = list(dict.fromkeys(json.loads(paths_json))) +lock_path = "/etc/cmux/.prompt-lock" + +def mime_paths(): + result = [] + for username in ("root", "cmux", "ubuntu"): + try: + home = pwd.getpwnam(username).pw_dir + except KeyError: + continue + try: + if not os.access(home, os.R_OK | os.X_OK): + continue + except OSError: + continue + result.extend([ + os.path.join(home, ".config/mimeapps.list"), + os.path.join(home, ".local/share/applications/mimeapps.list"), + ]) + return result + +paths = list(dict.fromkeys(paths + mime_paths())) + +def parent_dirs(path): + result = [] + current = os.path.dirname(path) + while current and current != "/": + result.append(current) + current = os.path.dirname(current) + return result + +directory_paths = list(dict.fromkeys( + parent for path in paths + [lock_path] for parent in parent_dirs(path) +)) + +def validate_target(path): + if os.path.lexists(path): + if os.path.isdir(path) and not os.path.islink(path): + raise IsADirectoryError(path) + if not os.path.islink(path) and not stat.S_ISREG(os.lstat(path).st_mode): + raise ValueError("target is not a regular file") + for parent in parent_dirs(path): + if os.path.lexists(parent) and not os.path.isdir(parent): + raise NotADirectoryError(parent) + +def remove_path(path): + if not os.path.lexists(path): + return + if os.path.islink(path) or os.path.isfile(path): + os.unlink(path) + return + if os.path.isdir(path): + # Never recursively delete a guest directory while rolling back a + # file install. An empty directory created by a failed shell command + # is safe to remove; user data makes the rollback explicitly fail. + os.rmdir(path) + return + raise ValueError("refusing to remove a non-file install target") + +def snapshot_path(path): + if not os.path.lexists(path): + backups[path] = None + return + if os.path.isdir(path) and not os.path.islink(path): + raise IsADirectoryError(path) + if os.path.islink(path): + backups[path] = ("link", os.readlink(path)) + return + if not stat.S_ISREG(os.lstat(path).st_mode): + raise ValueError("install target is not a regular file") + backup = os.path.join(backup_root, str(len(backups))) + shutil.copy2(path, backup) + backups[path] = ("file", backup) + +def restore_paths(): + failures = [] + for path, backup in reversed(list(backups.items())): + try: + remove_path(path) + if backup is None: + continue + kind, value = backup + os.makedirs(os.path.dirname(path), exist_ok=True) + if kind == "link": + os.symlink(value, path) + else: + shutil.copy2(value, path) + except Exception as error: + failures.append(error) + if failures: + raise RuntimeError("rollback failed") + +def remember_directory_entries(): + return { + path: set(os.listdir(path)) if os.path.isdir(path) else set() + for path in directory_paths + } + +def cleanup_generated(entries): + failures = [] + prefixes = [os.path.basename(path) + "." for path in paths] + prefixes.append(".prompt-") + for directory, before in entries.items(): + if not os.path.isdir(directory): + continue + try: + for name in os.listdir(directory): + if name in before or not any(name.startswith(prefix) for prefix in prefixes): + continue + candidate = os.path.join(directory, name) + if os.path.isfile(candidate) or os.path.islink(candidate): + os.unlink(candidate) + except Exception as error: + failures.append(error) + if failures: + raise RuntimeError("temporary cleanup failed") + +def cleanup_created_directories(before): + for directory in sorted(directory_paths, key=len, reverse=True): + if before.get(directory, False) or not os.path.isdir(directory) or os.path.islink(directory): + continue + try: + os.rmdir(directory) + except OSError: + # Non-empty directories are never recursively removed. They may + # contain an existing guest file created outside this transaction. + continue + +def replace_file(directory, name, content): + target_path = os.path.join(directory, name) + if os.path.isfile(target_path) and not os.path.islink(target_path): + with open(target_path, "r") as stream: + if stream.read() == content: + return + fd, temporary = tempfile.mkstemp(prefix=".prompt-", dir=directory) + try: + with os.fdopen(fd, "w") as stream: + stream.write(content) + os.fchmod(stream.fileno(), 0o644) + os.replace(temporary, target_path) + finally: + if os.path.exists(temporary): + os.unlink(temporary) + +def install_prompt(payload): + if not payload: + return + directory = "/etc/cmux" + os.makedirs(directory, exist_ok=True) + incoming = payload["identity"] + try: + with open(os.path.join(directory, ".prompt-identity")) as stream: + current = json.load(stream) + except (FileNotFoundError, ValueError, OSError): + current = {} + if not isinstance(current, dict) or not isinstance(current.get("revision", -1), int): + current = {} + if current.get("machineId") != incoming["machineId"] or current.get("revision", -1) <= incoming["revision"]: + replace_file(directory, "vm-name", incoming["name"] + "\n") + replace_file(directory, ".prompt-identity", json.dumps(incoming)) + for name, content in payload["files"].items(): + replace_file(directory, name, content) + try: + os.makedirs(os.path.dirname(lock_path), exist_ok=True) + lock = open(lock_path, "a+") + fcntl.flock(lock.fileno(), fcntl.LOCK_EX) + before_directories = { + path: os.path.isdir(path) for path in directory_paths + } + for path in paths: + validate_target(path) fd = os.open(source, os.O_RDONLY | os.O_NOFOLLOW) with os.fdopen(fd, "rb") as stream: if not stat.S_ISREG(os.fstat(stream.fileno()).st_mode): @@ -21,25 +214,65 @@ try: if hashlib.sha256(stream.read()).hexdigest() != digest: raise ValueError("upload checksum mismatch") os.fchmod(stream.fileno(), 0o755) + backup_root = tempfile.mkdtemp(prefix=".cmux-install-", dir=os.path.dirname(lock_path)) + for path in paths: + snapshot_path(path) + before_entries = remember_directory_entries() stage = "verify" subprocess.run([source, "--help"], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) stage = "browser" subprocess.run(["/bin/sh", "-c", browser], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) - if prompt: - stage = "prompt" - subprocess.run(["/bin/sh", "-c", prompt], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) + stage = "prompt" + install_prompt(json.loads(prompt_json) if prompt_json else None) stage = "publish" os.replace(source, target) + cleanup_generated(before_entries) + shutil.rmtree(backup_root) + backup_root = None except Exception as error: - # Bounded, structured diagnostics: no prompt payload or guest output. - print("CMUX_GUEST_INSTALL_FAILURE=" + json.dumps({ - "stage": stage, "error": type(error).__name__, + rollback_error = None + try: + cleanup_generated(before_entries if "before_entries" in globals() else {}) + except Exception as cleanup_error: + rollback_error = cleanup_error + try: + if backups: + restore_paths() + except Exception as restore_error: + rollback_error = restore_error + try: + cleanup_created_directories(before_directories if "before_directories" in globals() else {}) + except Exception as directory_error: + rollback_error = directory_error + if backup_root: + try: + shutil.rmtree(backup_root) + except Exception as backup_error: + rollback_error = backup_error + detail = { + "stage": stage, + "error": "RollbackError" if rollback_error else type(error).__name__, "errno": getattr(error, "errno", None), - "exitCode": getattr(error, "returncode", None) - }), file=sys.stderr) + "exitCode": getattr(error, "returncode", None), + } + print("CMUX_GUEST_INSTALL_FAILURE=" + json.dumps(detail), file=sys.stderr) sys.exit(1) +finally: + if lock is not None: + fcntl.flock(lock.fileno(), fcntl.LOCK_UN) + lock.close() `; export function guestCliInstallCommand(temporaryPath: string, identity?: GuestPromptIdentity): string { - return `python3 -c ${shellQuote(install)} ${shellQuote(temporaryPath)} ${shellQuote(GUEST_CMUX_SHIM_PATH)} ${shellQuote(digest)} ${shellQuote(guestBrowserInstallCommand())} ${shellQuote(identity ? guestPromptInstallCommand(identity) : "")}`; + if (identity && (!promptName.test(identity.name) || !Number.isSafeInteger(identity.revision))) { + throw new Error("Invalid Cloud prompt identity"); + } + const prompt = identity ? JSON.stringify({ + identity, + files: { + "prompt.bash": promptBash, + bashrc: promptBashrc, + }, + }) : ""; + return `python3 -c ${shellQuote(install)} ${shellQuote(temporaryPath)} ${shellQuote(GUEST_CMUX_SHIM_PATH)} ${shellQuote(digest)} ${shellQuote(guestBrowserInstallCommand())} ${shellQuote(prompt)} ${shellQuote(JSON.stringify(installPaths))}`; } diff --git a/web/services/vms/repository.ts b/web/services/vms/repository.ts index f51fd17f2636..0e983c65976a 100644 --- a/web/services/vms/repository.ts +++ b/web/services/vms/repository.ts @@ -416,6 +416,32 @@ export type VmRepositoryShape = { readonly reconciliationCandidates: (input: { readonly limit: number; }) => Effect.Effect; + /** Provider allocations retained after an unconfirmed create rollback. */ + readonly pendingCreateCleanupCandidates?: (input: { + readonly limit: number; + }) => Effect.Effect; + /** Claim one retained allocation before provider I/O. The claim is fenced by a lease. */ + readonly claimCreateCleanup?: (input: { + readonly id: string; + readonly providerVmId: string; + readonly leaseId: string; + readonly now: Date; + readonly leaseExpiresAt: Date; + }) => Effect.Effect<{ readonly attempt: number } | null, VmDatabaseError>; + /** Release a failed cleanup attempt and persist its next retry time. */ + readonly deferCreateCleanup?: (input: { + readonly id: string; + readonly providerVmId: string; + readonly leaseId: string; + readonly nextAttemptAt: Date; + readonly now: Date; + }) => Effect.Effect; + /** Mark a retained create allocation failed after provider deletion is confirmed. */ + readonly resolveCreateCleanup?: (input: { + readonly id: string; + readonly providerVmId: string; + readonly leaseId: string; + }) => Effect.Effect; /** Live VM rows that currently claim a persistent home volume. */ readonly listLiveHomeVolumeNames?: (input: { readonly provider: ProviderId; @@ -655,6 +681,12 @@ export const FAILED_CREATE_RETRY_WINDOW_MS = 15 * 60 * 1000; export const PROVIDER_CREATE_UNAVAILABLE_FAILURE_CODE = "provider_create_unavailable"; /** Provider allocation remains owned by the failed row until cleanup is confirmed. */ export const PROVIDER_CREATE_CLEANUP_PENDING_FAILURE_CODE = "provider_create_cleanup_pending"; +/** Durable metadata keys for a provider allocation retained after create failure. */ +export const CREATE_CLEANUP_PROVIDER_VM_ID_KEY = "createCleanupProviderVmId"; +export const CREATE_CLEANUP_ATTEMPT_KEY = "createCleanupAttempt"; +export const CREATE_CLEANUP_NEXT_ATTEMPT_AT_KEY = "createCleanupNextAttemptAtMs"; +export const CREATE_CLEANUP_LEASE_ID_KEY = "createCleanupLeaseId"; +export const CREATE_CLEANUP_LEASE_EXPIRES_AT_KEY = "createCleanupLeaseExpiresAtMs"; const RETRYABLE_FAILED_CREATE_CODES = new Set([ "billing_credits_insufficient", @@ -668,6 +700,92 @@ const RETRYABLE_FAILED_CREATE_CODES = new Set([ LEGACY_MODEL_PLANE_ENTITLEMENT_FAILURE_CODE, ]); +/** + * Finish the transactional half of a Base create failure. Cleanup-pending + * creates call this only after the provider confirms the allocation is gone; + * ordinary failures call it immediately. Keeping the same transaction for + * both paths prevents a late cleanup worker from promoting the wrong retained + * generation. + */ +async function restoreBaseAfterCreateFailure( + tx: CloudDbTransaction, + input: { + readonly baseId: string; + readonly generation: number; + readonly vmId: string; + readonly userId: string; + readonly code: string; + readonly message: string; + }, +): Promise { + const now = new Date(); + await tx + .update(cloudVmBaseGenerations) + .set({ state: "failed", updatedAt: now }) + .where(and( + eq(cloudVmBaseGenerations.baseId, input.baseId), + eq(cloudVmBaseGenerations.generation, input.generation), + eq(cloudVmBaseGenerations.vmId, input.vmId), + )); + const [retained] = await tx + .select({ + generation: cloudVmBaseGenerations, + vm: cloudVms, + }) + .from(cloudVmBaseGenerations) + .innerJoin(cloudVms, eq(cloudVms.id, cloudVmBaseGenerations.vmId)) + .where(and( + eq(cloudVmBaseGenerations.baseId, input.baseId), + sql`${cloudVmBaseGenerations.generation} < ${input.generation}`, + eq(cloudVmBaseGenerations.state, "retained"), + ne(cloudVms.status, "failed"), + ne(cloudVms.status, "destroyed"), + )) + .orderBy(desc(cloudVmBaseGenerations.generation)) + .limit(1); + if (retained?.generation && retained.vm) { + await tx + .update(cloudVmBaseGenerations) + .set({ state: "active", updatedAt: now }) + .where(eq(cloudVmBaseGenerations.id, retained.generation.id)); + await tx + .update(cloudVmBases) + .set({ + activeGeneration: retained.generation.generation, + activeVmId: retained.vm.id, + activeProvider: retained.vm.provider, + activeProviderVmId: retained.vm.providerVmId, + state: retained.vm.providerVmId ? "ready" : "creating", + updatedAt: now, + }) + .where(and( + eq(cloudVmBases.id, input.baseId), + eq(cloudVmBases.activeGeneration, input.generation), + eq(cloudVmBases.activeVmId, input.vmId), + )); + } else { + await tx + .update(cloudVmBases) + .set({ state: "failed", updatedAt: now }) + .where(and( + eq(cloudVmBases.id, input.baseId), + eq(cloudVmBases.activeGeneration, input.generation), + eq(cloudVmBases.activeVmId, input.vmId), + )); + } + await tx.insert(cloudVmBaseEvents).values({ + baseId: input.baseId, + userId: input.userId, + eventType: "base.create_failed", + oldGeneration: retained?.generation.generation ?? null, + newGeneration: input.generation, + oldVmId: retained?.vm.id ?? null, + newVmId: input.vmId, + oldProviderVmId: retained?.vm.providerVmId ?? null, + metadata: { code: input.code, message: input.message }, + }); +} + function isRetryableFailedCreate(vm: CloudVmRow, now: Date): boolean { if (vm.status === "destroyed") return true; if (vm.status !== "failed") return false; @@ -1957,86 +2075,34 @@ export const vmRepositoryLiveShape: VmRepositoryShape = { const db = cloudDb(); await db.transaction(async (tx) => { const now = new Date(); + const cleanupProviderVmId = input.code === PROVIDER_CREATE_CLEANUP_PENDING_FAILURE_CODE + ? input.cleanupProviderVmId?.trim() || null + : null; await tx .update(cloudVms) .set({ - status: input.cleanupProviderVmId ? "provisioning" : "failed", + status: cleanupProviderVmId ? "provisioning" : "failed", failureCode: input.code, failureMessage: input.message, - ...(input.cleanupProviderVmId ? { - providerMetadata: sql`${cloudVms.providerMetadata} || jsonb_build_object('createCleanupProviderVmId', ${input.cleanupProviderVmId}::text)`, + ...(cleanupProviderVmId ? { + providerMetadata: sql`( + coalesce(${cloudVms.providerMetadata}, '{}'::jsonb) + || jsonb_build_object( + '${sql.raw(CREATE_CLEANUP_PROVIDER_VM_ID_KEY)}', ${cleanupProviderVmId}::text, + '${sql.raw(CREATE_CLEANUP_ATTEMPT_KEY)}', 0, + '${sql.raw(CREATE_CLEANUP_NEXT_ATTEMPT_AT_KEY)}', 0 + ) + ) + #- '{${sql.raw(CREATE_CLEANUP_LEASE_ID_KEY)}}' + #- '{${sql.raw(CREATE_CLEANUP_LEASE_EXPIRES_AT_KEY)}}'`, } : {}), updatedAt: now, }) .where(eq(cloudVms.id, input.vmId)); // Leave the Base generation reserved too; reset/open must not allocate // a replacement while its failed guest still exists at the provider. - if (input.cleanupProviderVmId) return; - await tx - .update(cloudVmBaseGenerations) - .set({ state: "failed", updatedAt: now }) - .where(and( - eq(cloudVmBaseGenerations.baseId, input.baseId), - eq(cloudVmBaseGenerations.generation, input.generation), - eq(cloudVmBaseGenerations.vmId, input.vmId), - )); - const [retained] = await tx - .select({ - generation: cloudVmBaseGenerations, - vm: cloudVms, - }) - .from(cloudVmBaseGenerations) - .innerJoin(cloudVms, eq(cloudVms.id, cloudVmBaseGenerations.vmId)) - .where(and( - eq(cloudVmBaseGenerations.baseId, input.baseId), - sql`${cloudVmBaseGenerations.generation} < ${input.generation}`, - eq(cloudVmBaseGenerations.state, "retained"), - ne(cloudVms.status, "failed"), - ne(cloudVms.status, "destroyed"), - )) - .orderBy(desc(cloudVmBaseGenerations.generation)) - .limit(1); - if (retained?.generation && retained.vm) { - await tx - .update(cloudVmBaseGenerations) - .set({ state: "active", updatedAt: now }) - .where(eq(cloudVmBaseGenerations.id, retained.generation.id)); - await tx - .update(cloudVmBases) - .set({ - activeGeneration: retained.generation.generation, - activeVmId: retained.vm.id, - activeProvider: retained.vm.provider, - activeProviderVmId: retained.vm.providerVmId, - state: retained.vm.providerVmId ? "ready" : "creating", - updatedAt: now, - }) - .where(and( - eq(cloudVmBases.id, input.baseId), - eq(cloudVmBases.activeGeneration, input.generation), - eq(cloudVmBases.activeVmId, input.vmId), - )); - } else { - await tx - .update(cloudVmBases) - .set({ state: "failed", updatedAt: now }) - .where(and( - eq(cloudVmBases.id, input.baseId), - eq(cloudVmBases.activeGeneration, input.generation), - eq(cloudVmBases.activeVmId, input.vmId), - )); - } - await tx.insert(cloudVmBaseEvents).values({ - baseId: input.baseId, - userId: input.userId, - eventType: "base.create_failed", - oldGeneration: retained?.generation.generation ?? null, - newGeneration: input.generation, - oldVmId: retained?.vm.id ?? null, - newVmId: input.vmId, - oldProviderVmId: retained?.vm.providerVmId ?? null, - metadata: { code: input.code, message: input.message }, - }); + if (cleanupProviderVmId) return; + await restoreBaseAfterCreateFailure(tx, input); }); }), @@ -2553,6 +2619,182 @@ export const vmRepositoryLiveShape: VmRepositoryShape = { .limit(input.limit); }), + pendingCreateCleanupCandidates: (input) => + dbEffect("pendingCreateCleanupCandidates", async () => { + const db = cloudDb(); + const cleanupProviderVmId = sql`${cloudVms.providerMetadata}->>'${sql.raw(CREATE_CLEANUP_PROVIDER_VM_ID_KEY)}'`; + const nextAttemptAtMs = sql`${cloudVms.providerMetadata}->>'${sql.raw(CREATE_CLEANUP_NEXT_ATTEMPT_AT_KEY)}'`; + const leaseExpiresAtMs = sql`${cloudVms.providerMetadata}->>'${sql.raw(CREATE_CLEANUP_LEASE_EXPIRES_AT_KEY)}'`; + const nowMs = Date.now(); + const limit = Number.isFinite(input.limit) + ? Math.max(0, Math.min(200, Math.trunc(input.limit))) + : 200; + return await db + .select() + .from(cloudVms) + .where(and( + eq(cloudVms.status, "provisioning"), + eq(cloudVms.failureCode, PROVIDER_CREATE_CLEANUP_PENDING_FAILURE_CODE), + // The current registry has one provider. Keep retired enum values + // out of the worker batch so they cannot starve live cleanup rows. + eq(cloudVms.provider, "freestyle"), + sql`${cleanupProviderVmId} is not null and ${cleanupProviderVmId} <> ''`, + sql`( + ${nextAttemptAtMs} is null + or ${nextAttemptAtMs} !~ '^[0-9]+$' + or (${nextAttemptAtMs})::bigint <= ${nowMs} + )`, + sql`( + ${leaseExpiresAtMs} is null + or ${leaseExpiresAtMs} !~ '^[0-9]+$' + or (${leaseExpiresAtMs})::bigint <= ${nowMs} + )`, + )) + .orderBy(asc(cloudVms.updatedAt), asc(cloudVms.id)) + .limit(limit); + }), + + claimCreateCleanup: (input) => + dbEffect("claimCreateCleanup", async () => { + const providerVmId = input.providerVmId.trim(); + const leaseId = input.leaseId.trim(); + if (!providerVmId || !leaseId) return null; + const db = cloudDb(); + const cleanupProviderVmId = sql`${cloudVms.providerMetadata}->>'${sql.raw(CREATE_CLEANUP_PROVIDER_VM_ID_KEY)}'`; + const leaseIdValue = sql`${cloudVms.providerMetadata}->>'${sql.raw(CREATE_CLEANUP_LEASE_ID_KEY)}'`; + const leaseExpiresAtMs = sql`${cloudVms.providerMetadata}->>'${sql.raw(CREATE_CLEANUP_LEASE_EXPIRES_AT_KEY)}'`; + const nextAttemptAtMs = sql`${cloudVms.providerMetadata}->>'${sql.raw(CREATE_CLEANUP_NEXT_ATTEMPT_AT_KEY)}'`; + const attempt = sql`${cloudVms.providerMetadata}->>'${sql.raw(CREATE_CLEANUP_ATTEMPT_KEY)}'`; + const [row] = await db + .update(cloudVms) + .set({ + providerMetadata: sql`( + coalesce(${cloudVms.providerMetadata}, '{}'::jsonb) + || jsonb_build_object( + '${sql.raw(CREATE_CLEANUP_ATTEMPT_KEY)}', + case + when ${attempt} ~ '^[0-9]+$' then (${attempt})::bigint + 1 + else 1 + end, + '${sql.raw(CREATE_CLEANUP_NEXT_ATTEMPT_AT_KEY)}', ${input.now.getTime()}::bigint, + '${sql.raw(CREATE_CLEANUP_LEASE_ID_KEY)}', ${leaseId}::text, + '${sql.raw(CREATE_CLEANUP_LEASE_EXPIRES_AT_KEY)}', ${input.leaseExpiresAt.getTime()}::bigint + ) + )`, + updatedAt: input.now, + }) + .where(and( + eq(cloudVms.id, input.id), + eq(cloudVms.status, "provisioning"), + eq(cloudVms.failureCode, PROVIDER_CREATE_CLEANUP_PENDING_FAILURE_CODE), + eq(cleanupProviderVmId, providerVmId), + sql`( + ${nextAttemptAtMs} is null + or ${nextAttemptAtMs} !~ '^[0-9]+$' + or (${nextAttemptAtMs})::bigint <= ${input.now.getTime()} + )`, + sql`( + ${leaseIdValue} is null + or ${leaseIdValue} = '' + or ${leaseExpiresAtMs} is null + or ${leaseExpiresAtMs} !~ '^[0-9]+$' + or (${leaseExpiresAtMs})::bigint <= ${input.now.getTime()} + )`, + )) + .returning({ providerMetadata: cloudVms.providerMetadata }); + if (!row) return null; + const rawAttempt = row.providerMetadata[CREATE_CLEANUP_ATTEMPT_KEY]; + const parsedAttempt = typeof rawAttempt === "number" && Number.isSafeInteger(rawAttempt) + ? rawAttempt + : Number.parseInt(String(rawAttempt ?? "1"), 10); + return { attempt: Number.isSafeInteger(parsedAttempt) && parsedAttempt > 0 ? parsedAttempt : 1 }; + }), + + deferCreateCleanup: (input) => + dbEffect("deferCreateCleanup", async () => { + const providerVmId = input.providerVmId.trim(); + const leaseId = input.leaseId.trim(); + if (!providerVmId || !leaseId) return false; + const db = cloudDb(); + const cleanupProviderVmId = sql`${cloudVms.providerMetadata}->>'${sql.raw(CREATE_CLEANUP_PROVIDER_VM_ID_KEY)}'`; + const cleanupLeaseId = sql`${cloudVms.providerMetadata}->>'${sql.raw(CREATE_CLEANUP_LEASE_ID_KEY)}'`; + const rows = await db + .update(cloudVms) + .set({ + providerMetadata: sql`( + coalesce(${cloudVms.providerMetadata}, '{}'::jsonb) + || jsonb_build_object('${sql.raw(CREATE_CLEANUP_NEXT_ATTEMPT_AT_KEY)}', ${input.nextAttemptAt.getTime()}::bigint) + ) + #- '{${sql.raw(CREATE_CLEANUP_LEASE_ID_KEY)}}' + #- '{${sql.raw(CREATE_CLEANUP_LEASE_EXPIRES_AT_KEY)}}'`, + updatedAt: input.now, + }) + .where(and( + eq(cloudVms.id, input.id), + eq(cloudVms.status, "provisioning"), + eq(cloudVms.failureCode, PROVIDER_CREATE_CLEANUP_PENDING_FAILURE_CODE), + eq(cleanupProviderVmId, providerVmId), + eq(cleanupLeaseId, leaseId), + )) + .returning({ id: cloudVms.id }); + return rows.length > 0; + }), + + resolveCreateCleanup: (input) => + dbEffect("resolveCreateCleanup", async () => { + const db = cloudDb(); + return await db.transaction(async (tx) => { + const cleanupProviderVmId = sql`${cloudVms.providerMetadata}->>'${sql.raw(CREATE_CLEANUP_PROVIDER_VM_ID_KEY)}'`; + const cleanupLeaseId = sql`${cloudVms.providerMetadata}->>'${sql.raw(CREATE_CLEANUP_LEASE_ID_KEY)}'`; + const now = new Date(); + const rows = await tx + .update(cloudVms) + .set({ + status: "failed", + failureCode: PROVIDER_CREATE_UNAVAILABLE_FAILURE_CODE, + failureMessage: "Provider rollback was confirmed by background cleanup.", + providerMetadata: sql`( + coalesce(${cloudVms.providerMetadata}, '{}'::jsonb) + #- '{${sql.raw(CREATE_CLEANUP_PROVIDER_VM_ID_KEY)}}' + #- '{${sql.raw(CREATE_CLEANUP_ATTEMPT_KEY)}}' + #- '{${sql.raw(CREATE_CLEANUP_NEXT_ATTEMPT_AT_KEY)}}' + #- '{${sql.raw(CREATE_CLEANUP_LEASE_ID_KEY)}}' + #- '{${sql.raw(CREATE_CLEANUP_LEASE_EXPIRES_AT_KEY)}}' + )`, + updatedAt: now, + }) + .where(and( + eq(cloudVms.id, input.id), + eq(cloudVms.status, "provisioning"), + eq(cloudVms.failureCode, PROVIDER_CREATE_CLEANUP_PENDING_FAILURE_CODE), + eq(cleanupProviderVmId, input.providerVmId), + eq(cleanupLeaseId, input.leaseId), + )) + .returning({ id: cloudVms.id, userId: cloudVms.userId }); + const vm = rows[0]; + if (!vm) return false; + + const [generation] = await tx + .select() + .from(cloudVmBaseGenerations) + .where(and( + eq(cloudVmBaseGenerations.vmId, vm.id), + eq(cloudVmBaseGenerations.state, "creating"), + )) + .limit(1); + if (!generation) return true; + await restoreBaseAfterCreateFailure(tx, { + baseId: generation.baseId, + generation: generation.generation, + vmId: vm.id, + userId: vm.userId, + code: PROVIDER_CREATE_UNAVAILABLE_FAILURE_CODE, + message: "Provider rollback was confirmed by background cleanup.", + }); + return true; + }); + }), + listLiveHomeVolumeNames: (input) => dbEffect("listLiveHomeVolumeNames", async () => { const volumeNames = boundedReaperVolumeNames(input.volumeNames); @@ -2685,14 +2927,26 @@ export const vmRepositoryLiveShape: VmRepositoryShape = { markCreateFailed: (input) => dbEffect("markCreateFailed", async () => { const db = cloudDb(); + const cleanupProviderVmId = input.code === PROVIDER_CREATE_CLEANUP_PENDING_FAILURE_CODE + ? input.cleanupProviderVmId?.trim() || null + : null; await db .update(cloudVms) .set({ - status: input.cleanupProviderVmId ? "provisioning" : "failed", + status: cleanupProviderVmId ? "provisioning" : "failed", failureCode: input.code, failureMessage: input.message, - ...(input.cleanupProviderVmId ? { - providerMetadata: sql`${cloudVms.providerMetadata} || jsonb_build_object('createCleanupProviderVmId', ${input.cleanupProviderVmId}::text)`, + ...(cleanupProviderVmId ? { + providerMetadata: sql`( + coalesce(${cloudVms.providerMetadata}, '{}'::jsonb) + || jsonb_build_object( + '${sql.raw(CREATE_CLEANUP_PROVIDER_VM_ID_KEY)}', ${cleanupProviderVmId}::text, + '${sql.raw(CREATE_CLEANUP_ATTEMPT_KEY)}', 0, + '${sql.raw(CREATE_CLEANUP_NEXT_ATTEMPT_AT_KEY)}', 0 + ) + ) + #- '{${sql.raw(CREATE_CLEANUP_LEASE_ID_KEY)}}' + #- '{${sql.raw(CREATE_CLEANUP_LEASE_EXPIRES_AT_KEY)}}'`, } : {}), updatedAt: new Date(), }) diff --git a/web/services/vms/workflows.ts b/web/services/vms/workflows.ts index 6ab16144c43a..9fa61c486baf 100644 --- a/web/services/vms/workflows.ts +++ b/web/services/vms/workflows.ts @@ -99,6 +99,7 @@ import { VmProviderGateway, VmProviderGatewayLive, type VmProviderGatewayShape } import { isProviderCreateCleanupError } from "./drivers/providerCreateCleanup"; import { withVmProductAnalytics } from "./productAnalytics"; import { + CREATE_CLEANUP_PROVIDER_VM_ID_KEY, PROVIDER_CREATE_CLEANUP_PENDING_FAILURE_CODE, PROVIDER_CREATE_UNAVAILABLE_FAILURE_CODE, VmRepository, @@ -211,6 +212,11 @@ const IDENTITY_REVOKE_PROVIDER_TIMEOUT = "5 seconds"; const ACTIVE_IDENTITY_REVOKE_HOT_PATH_LIMIT = 8; const ACCOUNT_DELETION_IDENTITY_REVOKE_BATCH = 8; const VM_STATUS_RECONCILE_BATCH_LIMIT = 200; +const CREATE_CLEANUP_CONCURRENCY = 4; +const CREATE_CLEANUP_PROVIDER_TIMEOUT = "15 seconds"; +const CREATE_CLEANUP_LEASE_MS = 60 * 1000; +const CREATE_CLEANUP_BACKOFF_BASE_MS = 5 * 1000; +const CREATE_CLEANUP_BACKOFF_MAX_MS = 15 * 60 * 1000; const LEGACY_RESOURCE_RECONCILE_BATCH_LIMIT = 50; const LEGACY_RESOURCE_RECONCILE_CONCURRENCY = 5; const LEGACY_RESOURCE_RECONCILE_RETRY_AFTER_MS = 5 * 60 * 1000; @@ -385,6 +391,9 @@ export function reconcileVmProviderStatuses(input: { yield* reconcileLegacyResourceReservations(repo, providers, { limit: LEGACY_RESOURCE_RECONCILE_BATCH_LIMIT, }); + yield* reconcilePendingCreateCleanups(repo, providers, { + limit: boundedVmStatusReconcileLimit(input.limit), + }); const getStatus = providers.getStatus; if (!getStatus) { return { @@ -442,6 +451,84 @@ export function reconcileVmProviderStatuses(input: { }); } +/** + * A provider allocation retained after a failed create is not a normal VM row: + * its public provider id is intentionally absent until deletion is confirmed. + * Reconcile those ids before ordinary status probing so a failed provider + * cleanup cannot remain reserved forever or block the next Base generation. + */ +function reconcilePendingCreateCleanups( + repo: VmRepositoryShape, + providers: VmProviderGatewayShape, + input: { readonly limit: number }, +): Effect.Effect { + const listCandidates = repo.pendingCreateCleanupCandidates; + const claimCleanup = repo.claimCreateCleanup; + const deferCleanup = repo.deferCreateCleanup; + const resolveCleanup = repo.resolveCreateCleanup; + if (!listCandidates || !claimCleanup || !deferCleanup || !resolveCleanup) return Effect.void; + return Effect.gen(function* () { + const candidates = yield* listCandidates({ limit: input.limit }).pipe( + Effect.catchAll(() => Effect.succeed([] as CloudVmRow[])), + ); + yield* Effect.forEach( + candidates, + (vm) => { + // The row can outlive its provider driver. Never pass a retired + // provider or a coderouter/model-plane failure to the cleanup worker. + if (isRetiredProviderRow(vm) || vm.failureCode !== PROVIDER_CREATE_CLEANUP_PENDING_FAILURE_CODE) { + return Effect.void; + } + const rawProviderVmId = vm.providerMetadata?.[CREATE_CLEANUP_PROVIDER_VM_ID_KEY]; + if (typeof rawProviderVmId !== "string" || rawProviderVmId.trim().length === 0) return Effect.void; + const providerVmId = rawProviderVmId.trim(); + const leaseId = randomUUID(); + const now = new Date(); + const leaseExpiresAt = new Date(now.getTime() + CREATE_CLEANUP_LEASE_MS); + return claimCleanup({ + id: vm.id, + providerVmId, + leaseId, + now, + leaseExpiresAt, + }).pipe( + Effect.flatMap((claim) => { + if (!claim) return Effect.void; + const destroy = providers.destroy(vm.provider, providerVmId).pipe( + Effect.timeoutFail({ + duration: CREATE_CLEANUP_PROVIDER_TIMEOUT, + onTimeout: () => new Error("provider cleanup deadline"), + }), + Effect.catchAll((error) => isProviderNotFoundError(error) + ? Effect.succeed("confirmed" as const) + : Effect.fail(error)), + ); + return destroy.pipe( + Effect.flatMap(() => resolveCleanup({ id: vm.id, providerVmId, leaseId })), + Effect.asVoid, + Effect.catchAll(() => { + const backoff = Math.min( + CREATE_CLEANUP_BACKOFF_MAX_MS, + CREATE_CLEANUP_BACKOFF_BASE_MS * 2 ** Math.min(20, Math.max(0, claim.attempt - 1)), + ); + return deferCleanup({ + id: vm.id, + providerVmId, + leaseId, + nextAttemptAt: new Date(Date.now() + backoff), + now: new Date(), + }).pipe(Effect.asVoid, Effect.catchAll(() => Effect.void)); + }), + ); + }), + Effect.catchAll(() => Effect.void), + ); + }, + { concurrency: CREATE_CLEANUP_CONCURRENCY, discard: true }, + ); + }); +} + /** * The home volume a destroyed machine owns exclusively, or null when there is * nothing safe to delete. Per-machine volumes are marked at create diff --git a/web/tests/vm-guest-install-runtime.test.ts b/web/tests/vm-guest-install-runtime.test.ts index a78a07b5bcd3..31f0a5b42735 100644 --- a/web/tests/vm-guest-install-runtime.test.ts +++ b/web/tests/vm-guest-install-runtime.test.ts @@ -11,13 +11,21 @@ const roots: string[] = []; afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); /** Isolated guest filesystem; run the actual uploaded bytes and install command. */ -function guest(options: { corruptUpload?: boolean } = {}) { +function guest(options: { corruptUpload?: boolean; promptFails?: boolean; publishFails?: boolean } = {}) { const root = mkdtempSync(join(tmpdir(), "cmux-guest-install-")); roots.push(root); mkdirSync(join(root, "bin")); mkdirSync(join(root, "fixture-bin")); - writeFileSync(join(root, "fixture-bin/getent"), "#!/bin/sh\nexit 1\n", { mode: 0o755 }); - writeFileSync(join(root, "fixture-bin/xdg-mime"), "#!/bin/sh\nexit 0\n", { mode: 0o755 }); + const trigger = options.promptFails || options.publishFails; + writeFileSync(join(root, "fixture-bin/getent"), trigger + ? "#!/bin/sh\n[ \"$1\" = passwd ] || exit 1\ncase \"$2\" in root|cmux|ubuntu) printf '%s:x:0:0::%s:/bin/sh\\n' \"$2\" \"$CMUX_GUEST_FIXTURE_ROOT/home/$2\"; exit 0;; esac\nexit 1\n" + : "#!/bin/sh\nexit 1\n", { mode: 0o755 }); + writeFileSync(join(root, "fixture-bin/runuser"), trigger + ? "#!/bin/sh\n[ \"$1\" = -u ] || exit 1\nuser=\"$2\"; shift 2; [ \"$1\" = -- ] || exit 1; shift\nHOME=\"$CMUX_GUEST_FIXTURE_ROOT/home/$user\"; export HOME\nexec \"$@\"\n" + : "#!/bin/sh\nexit 1\n", { mode: 0o755 }); + writeFileSync(join(root, "fixture-bin/xdg-mime"), trigger + ? "#!/bin/sh\nif [ \"$CMUX_GUEST_FAILURE_STAGE\" = prompt ]; then rm -f \"$CMUX_GUEST_FIXTURE_ROOT/etc/prompt.bash\"; mkdir -p \"$CMUX_GUEST_FIXTURE_ROOT/etc/prompt.bash\"; fi\nif [ \"$CMUX_GUEST_FAILURE_STAGE\" = publish ]; then rm -f \"$CMUX_GUEST_FIXTURE_ROOT/bin/cmux\"; mkdir -p \"$CMUX_GUEST_FIXTURE_ROOT/bin/cmux\"; fi\nexit 0\n" + : "#!/bin/sh\nexit 0\n", { mode: 0o755 }); const prefixes: Record = { "/usr/local/bin": join(root, "bin"), "/usr/local/share": join(root, "share"), "/etc/cmux": join(root, "etc"), "/etc": join(root, "system-etc"), @@ -31,7 +39,12 @@ function guest(options: { corruptUpload?: boolean } = {}) { exec: async (request) => { const result = spawnSync("/bin/sh", ["-c", rebase(request.command)], { encoding: "utf8", timeout: 5_000, - env: { ...process.env, HOME: root, PATH: `${join(root, "fixture-bin")}:${process.env.PATH}` }, + env: { + ...process.env, HOME: root, PATH: `${join(root, "fixture-bin")}:${process.env.PATH}`, + CMUX_GUEST_FIXTURE_ROOT: root, + ...(options.promptFails ? { CMUX_GUEST_FAILURE_STAGE: "prompt" } : {}), + ...(options.publishFails ? { CMUX_GUEST_FAILURE_STAGE: "publish" } : {}), + }, }); return Response.json({ statusCode: result.status, stdout: result.stdout, stderr: result.stderr }); }, @@ -93,16 +106,53 @@ describe("guest CLI publication in an isolated filesystem", () => { }); test("prompt failure does not publish a new shim generation", async () => { - const { fixture, target, root } = guest(); + const { fixture, target, root } = guest({ promptFails: true }); writeFileSync(target, "previous generation"); + const browserOpener = join(root, "bin/cmux-open-url"); + writeFileSync(browserOpener, "previous browser generation"); mkdirSync(join(root, "etc")); - mkdirSync(join(root, "etc/.prompt-lock")); + writeFileSync(join(root, "etc/prompt.bash"), "previous prompt generation"); + writeFileSync(join(root, "etc/bashrc"), "previous bashrc generation"); + writeFileSync(join(root, "etc/.prompt-identity"), "previous identity"); + writeFileSync(join(root, "etc/vm-name"), "previous name\n"); const failure = await fixture.provider.create({ ...guestCreateOptions, promptIdentity: { machineId: "synthetic", name: "synthetic", revision: 1 }, }).then(() => undefined, (error) => error); expect(failure?.cause?.stage).toBe("prompt"); expect(readFileSync(target, "utf8")).toBe("previous generation"); + expect(readFileSync(browserOpener, "utf8")).toBe("previous browser generation"); + expect(readFileSync(join(root, "etc/prompt.bash"), "utf8")).toBe("previous prompt generation"); + expect(readFileSync(join(root, "etc/bashrc"), "utf8")).toBe("previous bashrc generation"); + expect(readFileSync(join(root, "etc/.prompt-identity"), "utf8")).toBe("previous identity"); + expect(readFileSync(join(root, "etc/vm-name"), "utf8")).toBe("previous name\n"); + const promptArtifacts = readdirSync(join(root, "etc")); + expect(promptArtifacts.some((name) => name.startsWith(".cmux-install-") || (name.startsWith(".prompt-") && ![".prompt-lock", ".prompt-identity"].includes(name)))).toBe(false); + expect(fixture.liveVms.size).toBe(0); + }); + + test("publish failure restores browser, prompt, and shim generations without temp artifacts", async () => { + const { fixture, target, root } = guest({ publishFails: true }); + writeFileSync(target, "previous generation"); + writeFileSync(join(root, "bin/cmux-open-url"), "previous browser generation"); + mkdirSync(join(root, "etc")); + writeFileSync(join(root, "etc/prompt.bash"), "previous prompt generation"); + writeFileSync(join(root, "etc/bashrc"), "previous bashrc generation"); + writeFileSync(join(root, "etc/.prompt-identity"), "previous identity"); + writeFileSync(join(root, "etc/vm-name"), "previous name\n"); + const failure = await fixture.provider.create({ + ...guestCreateOptions, + promptIdentity: { machineId: "synthetic", name: "synthetic", revision: 1 }, + }).then(() => undefined, (error) => error); + expect(failure?.cause?.stage).toBe("publish"); + expect(readFileSync(target, "utf8")).toBe("previous generation"); + expect(readFileSync(join(root, "bin/cmux-open-url"), "utf8")).toBe("previous browser generation"); + expect(readFileSync(join(root, "etc/prompt.bash"), "utf8")).toBe("previous prompt generation"); + expect(readFileSync(join(root, "etc/bashrc"), "utf8")).toBe("previous bashrc generation"); + expect(readFileSync(join(root, "etc/.prompt-identity"), "utf8")).toBe("previous identity"); + expect(readFileSync(join(root, "etc/vm-name"), "utf8")).toBe("previous name\n"); + const publishArtifacts = readdirSync(join(root, "etc")); + expect(publishArtifacts.some((name) => name.startsWith(".cmux-install-") || (name.startsWith(".prompt-") && ![".prompt-lock", ".prompt-identity"].includes(name)))).toBe(false); expect(fixture.liveVms.size).toBe(0); }); }); diff --git a/web/tests/vm-workflows.test.ts b/web/tests/vm-workflows.test.ts index b67201d61a27..9439e977cb24 100644 --- a/web/tests/vm-workflows.test.ts +++ b/web/tests/vm-workflows.test.ts @@ -265,6 +265,89 @@ describe("VM Effect workflows", () => { expect(fixture.liveVms.size).toBe(1); }); + dbTest("reconciles retained guest allocations before allowing a retry", async () => { + if (!sql) throw new Error("test database not initialized"); + let failInstall = true; + const fixture = freestyleGuestFixture({ + exec: async () => Response.json({ statusCode: failInstall ? 1 : 0 }), + deleteFailure: true, + }); + let destroyCalls = 0; + const layer = providerLayer({ + ...unusedProviderGateway(), + create: (_, options) => Effect.tryPromise({ + try: () => fixture.provider.create(options), + catch: (cause) => new VmProviderOperationError({ provider: "freestyle", operation: "create", cause }), + }), + destroy: (_, providerVmId) => Effect.sync(() => { + destroyCalls += 1; + fixture.liveVms.delete(providerVmId); + }), + }); + const input = { + ...guestCreateOptions, userId: "user-guest-cleanup-reconcile", billingTeamId: "team-guest-cleanup-reconcile", + billingCustomerType: "team" as const, billingPlanId: "pro", provider: "freestyle" as const, + idempotencyKey: "guest-cleanup-reconcile", maxActiveVms: 1, + }; + + expect((await Effect.runPromise(Effect.either(createVm(input).pipe(Effect.provide(layer)))))._tag).toBe("Left"); + expect(fixture.liveVms.size).toBe(1); + await Promise.all([ + Effect.runPromise(reconcileVmProviderStatuses().pipe(Effect.provide(layer))), + Effect.runPromise(reconcileVmProviderStatuses().pipe(Effect.provide(layer))), + ]); + expect(destroyCalls).toBe(1); + const [resolved] = await sql<{ status: string; providerVmId: string | null; metadata: Record }[]>` + select status, provider_vm_id as "providerVmId", provider_metadata as metadata + from cloud_vms where user_id = 'user-guest-cleanup-reconcile' + `; + expect(resolved.status).toBe("failed"); + expect(resolved.providerVmId).toBeNull(); + expect(resolved.metadata.createCleanupProviderVmId).toBeUndefined(); + + failInstall = false; + const retry = await Effect.runPromise(createVm(input).pipe(Effect.provide(layer))); + expect(retry.providerVmId).toBe("vm-fixture-2"); + expect(fixture.liveVms).toEqual(new Set(["vm-fixture-2"])); + }); + + dbTest("retains failed cleanup with durable backoff and does not duplicate the provider delete", async () => { + if (!sql) throw new Error("test database not initialized"); + const fixture = freestyleGuestFixture({ exec: async () => Response.json({ statusCode: 1 }), deleteFailure: true }); + let destroyCalls = 0; + const layer = providerLayer({ + ...unusedProviderGateway(), + create: (_, options) => Effect.tryPromise({ + try: () => fixture.provider.create(options), + catch: (cause) => new VmProviderOperationError({ provider: "freestyle", operation: "create", cause }), + }), + destroy: () => Effect.sync(() => { + destroyCalls += 1; + }).pipe(Effect.andThen(Effect.fail(new VmProviderOperationError({ + provider: "freestyle", + operation: "destroy", + cause: new Error("synthetic cleanup outage"), + })))), + }); + const input = { + ...guestCreateOptions, userId: "user-guest-cleanup-backoff", billingTeamId: "team-guest-cleanup-backoff", + billingCustomerType: "team" as const, billingPlanId: "pro", provider: "freestyle" as const, + idempotencyKey: "guest-cleanup-backoff", maxActiveVms: 1, + }; + + expect((await Effect.runPromise(Effect.either(createVm(input).pipe(Effect.provide(layer)))))._tag).toBe("Left"); + await Effect.runPromise(reconcileVmProviderStatuses().pipe(Effect.provide(layer))); + await Effect.runPromise(reconcileVmProviderStatuses().pipe(Effect.provide(layer))); + expect(destroyCalls).toBe(1); + const [row] = await sql<{ status: string; metadata: Record }[]>` + select status, provider_metadata as metadata from cloud_vms where user_id = 'user-guest-cleanup-backoff' + `; + expect(row.status).toBe("provisioning"); + expect(row.metadata.createCleanupProviderVmId).toBe("vm-fixture-1"); + expect(Number(row.metadata.createCleanupNextAttemptAtMs)).toBeGreaterThan(Date.now()); + expect(row.metadata.createCleanupLeaseId).toBeUndefined(); + }); + dbTest("unconfirmed guest rollback keeps the Base generation reserved", async () => { if (!sql) throw new Error("test database not initialized"); const fixture = freestyleGuestFixture({ exec: async () => Response.json({ statusCode: 1 }), deleteFailure: true }); From 1f968607f3aa6946eb6bc44f4b6ab856c20a4f94 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 06:25:46 -0700 Subject: [PATCH 37/92] fix guest prompt asset bundling and cleanup claim races --- .../vms/drivers/guestCliInstallCommand.ts | 14 ++--------- web/services/vms/guestPrompt.ts | 4 ++++ web/services/vms/repository.ts | 24 ++++++++++++------- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/web/services/vms/drivers/guestCliInstallCommand.ts b/web/services/vms/drivers/guestCliInstallCommand.ts index 04b6c41b41b6..79c13ead768f 100644 --- a/web/services/vms/drivers/guestCliInstallCommand.ts +++ b/web/services/vms/drivers/guestCliInstallCommand.ts @@ -1,17 +1,10 @@ import { createHash } from "node:crypto"; -import { readFileSync } from "node:fs"; -import { fileURLToPath } from "node:url"; import { GUEST_CMUX_SHIM, GUEST_CMUX_SHIM_PATH } from "../guestCli"; import { GUEST_BROWSER_FILES, guestBrowserInstallCommand } from "../guestBrowser"; -import type { GuestPromptIdentity } from "../guestPrompt"; +import { guestPromptInstallFiles, type GuestPromptIdentity } from "../guestPrompt"; import { shellQuote } from "./cmuxTuiDaemon"; const digest = createHash("sha256").update(GUEST_CMUX_SHIM).digest("hex"); -const promptAsset = (name: string) => readFileSync( - fileURLToPath(new URL(`../images/devbox/${name}`, import.meta.url).toString()), "utf8", -); -const promptBash = promptAsset("cmux-prompt.bash"); -const promptBashrc = promptAsset("cmux-bashrc"); const installPaths = [ GUEST_CMUX_SHIM_PATH, ...GUEST_BROWSER_FILES.map(({ path }) => path), @@ -269,10 +262,7 @@ export function guestCliInstallCommand(temporaryPath: string, identity?: GuestPr } const prompt = identity ? JSON.stringify({ identity, - files: { - "prompt.bash": promptBash, - bashrc: promptBashrc, - }, + files: guestPromptInstallFiles, }) : ""; return `python3 -c ${shellQuote(install)} ${shellQuote(temporaryPath)} ${shellQuote(GUEST_CMUX_SHIM_PATH)} ${shellQuote(digest)} ${shellQuote(guestBrowserInstallCommand())} ${shellQuote(prompt)} ${shellQuote(JSON.stringify(installPaths))}`; } diff --git a/web/services/vms/guestPrompt.ts b/web/services/vms/guestPrompt.ts index c231885a7d44..b6bb82439ddd 100644 --- a/web/services/vms/guestPrompt.ts +++ b/web/services/vms/guestPrompt.ts @@ -32,6 +32,10 @@ const assetPath = (relativePath: string) => fileURLToPath(new URL(relativePath, import.meta.url).toString()); const bashrc = readFileSync(assetPath("./images/devbox/cmux-bashrc"), "utf8"); const prompt = readFileSync(assetPath("./images/devbox/cmux-prompt.bash"), "utf8"); +export const guestPromptInstallFiles = { + "prompt.bash": prompt, + bashrc, +} as const; // Runs on lifecycle operations, never during shell startup or prompt drawing. // A lock serializes competing attaches/renames. Atomic replacement gives every diff --git a/web/services/vms/repository.ts b/web/services/vms/repository.ts index 0e983c65976a..6620afc941c1 100644 --- a/web/services/vms/repository.ts +++ b/web/services/vms/repository.ts @@ -2665,8 +2665,13 @@ export const vmRepositoryLiveShape: VmRepositoryShape = { const leaseExpiresAtMs = sql`${cloudVms.providerMetadata}->>'${sql.raw(CREATE_CLEANUP_LEASE_EXPIRES_AT_KEY)}'`; const nextAttemptAtMs = sql`${cloudVms.providerMetadata}->>'${sql.raw(CREATE_CLEANUP_NEXT_ATTEMPT_AT_KEY)}'`; const attempt = sql`${cloudVms.providerMetadata}->>'${sql.raw(CREATE_CLEANUP_ATTEMPT_KEY)}'`; - const [row] = await db - .update(cloudVms) + return await db.transaction(async (tx) => { + // Serialize claims for one row before the CAS update. The update + // predicates still fence stale leases, while this lock keeps two + // concurrent cron workers from both observing an eligible marker. + await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${input.id}, 0))`); + const [row] = await tx + .update(cloudVms) .set({ providerMetadata: sql`( coalesce(${cloudVms.providerMetadata}, '{}'::jsonb) @@ -2701,13 +2706,14 @@ export const vmRepositoryLiveShape: VmRepositoryShape = { or (${leaseExpiresAtMs})::bigint <= ${input.now.getTime()} )`, )) - .returning({ providerMetadata: cloudVms.providerMetadata }); - if (!row) return null; - const rawAttempt = row.providerMetadata[CREATE_CLEANUP_ATTEMPT_KEY]; - const parsedAttempt = typeof rawAttempt === "number" && Number.isSafeInteger(rawAttempt) - ? rawAttempt - : Number.parseInt(String(rawAttempt ?? "1"), 10); - return { attempt: Number.isSafeInteger(parsedAttempt) && parsedAttempt > 0 ? parsedAttempt : 1 }; + .returning({ providerMetadata: cloudVms.providerMetadata }); + if (!row) return null; + const rawAttempt = row.providerMetadata[CREATE_CLEANUP_ATTEMPT_KEY]; + const parsedAttempt = typeof rawAttempt === "number" && Number.isSafeInteger(rawAttempt) + ? rawAttempt + : Number.parseInt(String(rawAttempt ?? "1"), 10); + return { attempt: Number.isSafeInteger(parsedAttempt) && parsedAttempt > 0 ? parsedAttempt : 1 }; + }); }), deferCreateCleanup: (input) => From 12261631159430969c470f18f8ef542d5b2853c8 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 06:29:07 -0700 Subject: [PATCH 38/92] fence concurrent Cloud cleanup claims by lease time --- web/services/vms/repository.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/web/services/vms/repository.ts b/web/services/vms/repository.ts index 6620afc941c1..efd82ba119fd 100644 --- a/web/services/vms/repository.ts +++ b/web/services/vms/repository.ts @@ -2681,7 +2681,11 @@ export const vmRepositoryLiveShape: VmRepositoryShape = { when ${attempt} ~ '^[0-9]+$' then (${attempt})::bigint + 1 else 1 end, - '${sql.raw(CREATE_CLEANUP_NEXT_ATTEMPT_AT_KEY)}', ${input.now.getTime()}::bigint, + -- Keep the row ineligible for the entire claim lease. A second + -- worker may already have selected the same candidate before + -- this update commits, so the retry fence must be future-dated + -- independently of the lease-id predicate. + '${sql.raw(CREATE_CLEANUP_NEXT_ATTEMPT_AT_KEY)}', ${input.leaseExpiresAt.getTime()}::bigint, '${sql.raw(CREATE_CLEANUP_LEASE_ID_KEY)}', ${leaseId}::text, '${sql.raw(CREATE_CLEANUP_LEASE_EXPIRES_AT_KEY)}', ${input.leaseExpiresAt.getTime()}::bigint ) From 91e7817a4200c3d9f47ebc57b4a1bab1ae39b44e Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 06:34:29 -0700 Subject: [PATCH 39/92] avoid duplicate in-process Cloud cleanup work --- web/services/vms/workflows.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/web/services/vms/workflows.ts b/web/services/vms/workflows.ts index 9fa61c486baf..f61afd3fe30d 100644 --- a/web/services/vms/workflows.ts +++ b/web/services/vms/workflows.ts @@ -217,6 +217,10 @@ const CREATE_CLEANUP_PROVIDER_TIMEOUT = "15 seconds"; const CREATE_CLEANUP_LEASE_MS = 60 * 1000; const CREATE_CLEANUP_BACKOFF_BASE_MS = 5 * 1000; const CREATE_CLEANUP_BACKOFF_MAX_MS = 15 * 60 * 1000; +// The Postgres lease is the cross-process fence. This small process-local set +// avoids issuing duplicate provider deletes while two cron requests in the same +// Node worker are between candidate selection and the durable claim result. +const pendingCreateCleanupInFlight = new Set(); const LEGACY_RESOURCE_RECONCILE_BATCH_LIMIT = 50; const LEGACY_RESOURCE_RECONCILE_CONCURRENCY = 5; const LEGACY_RESOURCE_RECONCILE_RETRY_AFTER_MS = 5 * 60 * 1000; @@ -482,6 +486,8 @@ function reconcilePendingCreateCleanups( const rawProviderVmId = vm.providerMetadata?.[CREATE_CLEANUP_PROVIDER_VM_ID_KEY]; if (typeof rawProviderVmId !== "string" || rawProviderVmId.trim().length === 0) return Effect.void; const providerVmId = rawProviderVmId.trim(); + if (pendingCreateCleanupInFlight.has(vm.id)) return Effect.void; + pendingCreateCleanupInFlight.add(vm.id); const leaseId = randomUUID(); const now = new Date(); const leaseExpiresAt = new Date(now.getTime() + CREATE_CLEANUP_LEASE_MS); @@ -522,7 +528,9 @@ function reconcilePendingCreateCleanups( ); }), Effect.catchAll(() => Effect.void), - ); + ).pipe(Effect.ensuring(Effect.sync(() => { + pendingCreateCleanupInFlight.delete(vm.id); + }))); }, { concurrency: CREATE_CLEANUP_CONCURRENCY, discard: true }, ); From c0e9bdad25118a566d068964147c811c3cf5fc5b Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 06:38:46 -0700 Subject: [PATCH 40/92] stabilize Cloud cleanup test isolation and guest assets --- web/tests/vm-workflows.test.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/web/tests/vm-workflows.test.ts b/web/tests/vm-workflows.test.ts index 9439e977cb24..73483cd201d1 100644 --- a/web/tests/vm-workflows.test.ts +++ b/web/tests/vm-workflows.test.ts @@ -267,6 +267,11 @@ describe("VM Effect workflows", () => { dbTest("reconciles retained guest allocations before allowing a retry", async () => { if (!sql) throw new Error("test database not initialized"); + // The preceding cleanup-pending case intentionally leaves its row + // reserved. Remove those fixture rows so this race assertion measures one + // provider allocation, while production reconciliation still scans all + // eligible rows in order. + await sql`delete from cloud_vms where user_id like 'user-guest-cleanup%'`; let failInstall = true; const fixture = freestyleGuestFixture({ exec: async () => Response.json({ statusCode: failInstall ? 1 : 0 }), From 0315a7336a47f7ea671dcf9f0523ffb1dc27d4fc Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 06:43:07 -0700 Subject: [PATCH 41/92] isolate guest cleanup fixture provider ids --- web/tests/fixtures/freestyleGuest.ts | 5 +++-- web/tests/vm-workflows.test.ts | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/web/tests/fixtures/freestyleGuest.ts b/web/tests/fixtures/freestyleGuest.ts index bd304cbe2698..7994f1a7a1a2 100644 --- a/web/tests/fixtures/freestyleGuest.ts +++ b/web/tests/fixtures/freestyleGuest.ts @@ -13,6 +13,7 @@ export function freestyleGuestFixture(options: { write?: (path: string, bytes: Uint8Array, signal?: AbortSignal | null) => void | Promise; remove?: (path: string, signal?: AbortSignal | null) => void | Promise; deleteFailure?: boolean; + idPrefix?: string; } = {}) { const requests: Array<{ method: string; path: string }> = []; const writes: string[] = []; @@ -28,7 +29,7 @@ export function freestyleGuestFixture(options: { const method = init?.method ?? "GET"; requests.push({ method, path: url.pathname }); if (method === "POST" && url.pathname === "/v5/vms") { - const id = `vm-fixture-${++allocations}`; + const id = `${options.idPrefix ?? "vm-fixture"}-${++allocations}`; liveVms.add(id); return Response.json({ id, state: "running", resources: { cpu: 2, memory: 8192, storage: 32768 }, vpcs: [{ ipv4: "192.0.2.10" }] }); } @@ -56,7 +57,7 @@ export function freestyleGuestFixture(options: { if (url.pathname.endsWith("/resize")) { return Response.json({ id: url.pathname.split("/").at(-2), state: "running", resources: JSON.parse(String(init?.body)) }); } - if (method === "DELETE" && /^\/v5\/vms\/vm-fixture-\d+$/.test(url.pathname)) { + if (method === "DELETE" && /^\/v5\/vms\/[a-z0-9-]+-\d+$/.test(url.pathname)) { if (options.deleteFailure) return Response.json({ code: "UNAVAILABLE", message: "synthetic delete failure" }, { status: 503 }); liveVms.delete(url.pathname.split("/").at(-1)!); return new Response(null, { status: 204 }); diff --git a/web/tests/vm-workflows.test.ts b/web/tests/vm-workflows.test.ts index 73483cd201d1..3ffa996ce30e 100644 --- a/web/tests/vm-workflows.test.ts +++ b/web/tests/vm-workflows.test.ts @@ -276,6 +276,7 @@ describe("VM Effect workflows", () => { const fixture = freestyleGuestFixture({ exec: async () => Response.json({ statusCode: failInstall ? 1 : 0 }), deleteFailure: true, + idPrefix: "vm-cleanup-reconcile", }); let destroyCalls = 0; const layer = providerLayer({ @@ -312,8 +313,8 @@ describe("VM Effect workflows", () => { failInstall = false; const retry = await Effect.runPromise(createVm(input).pipe(Effect.provide(layer))); - expect(retry.providerVmId).toBe("vm-fixture-2"); - expect(fixture.liveVms).toEqual(new Set(["vm-fixture-2"])); + expect(retry.providerVmId).toBe("vm-cleanup-reconcile-2"); + expect(fixture.liveVms).toEqual(new Set(["vm-cleanup-reconcile-2"])); }); dbTest("retains failed cleanup with durable backoff and does not duplicate the provider delete", async () => { From bc36d6509011caae825c10671e7810a9bf48c123 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 07:19:25 -0700 Subject: [PATCH 42/92] close final Cloud review findings --- Sources/Cloud/MachinesPanelViewModel.swift | 7 +- cmux.xcodeproj/project.pbxproj | 8 ++ cmuxTests/CloudOperationRecorderTests.swift | 16 ++-- cmuxTests/CloudRefreshAuthClient.swift | 22 +++++ cmuxTests/CloudRefreshFixture.swift | 55 ++++++++++++ cmuxTests/VMClientReadCoalescingTests.swift | 62 ------------- .../vms/drivers/guestCliInstallCommand.ts | 63 +++++++------ web/services/vms/repository.ts | 90 ++++++++++--------- web/services/vms/workflows.ts | 15 ++-- web/tests/vm-guest-install-runtime.test.ts | 16 ++++ 10 files changed, 202 insertions(+), 152 deletions(-) create mode 100644 cmuxTests/CloudRefreshAuthClient.swift create mode 100644 cmuxTests/CloudRefreshFixture.swift diff --git a/Sources/Cloud/MachinesPanelViewModel.swift b/Sources/Cloud/MachinesPanelViewModel.swift index c3ff03e12332..3bb5d6cdf29d 100644 --- a/Sources/Cloud/MachinesPanelViewModel.swift +++ b/Sources/Cloud/MachinesPanelViewModel.swift @@ -213,11 +213,16 @@ final class MachinesPanelViewModel: ObservableObject { MainActor.assumeIsolated { guard let self else { return } if online { - if self.wantsPolling { self.refresh() } + if self.wantsPolling { self.startPolling() } } else { self.clearUnavailableMetrics() self.lastErrorDescription = URLError(.notConnectedToInternet).localizedDescription self.listProblem = .unreachable + // Retire the active transport and advance the generation + // so a late response cannot clear the offline state or + // schedule another refresh. `wantsPolling` remains true, + // allowing the online branch to restart the cadence. + self.pausePolling() } } } diff --git a/cmux.xcodeproj/project.pbxproj b/cmux.xcodeproj/project.pbxproj index ec82acb7efe2..9e03464decfb 100644 --- a/cmux.xcodeproj/project.pbxproj +++ b/cmux.xcodeproj/project.pbxproj @@ -851,6 +851,8 @@ C13151C0000000000000000C /* CloudReadRequestCoordinator+Waiter.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C0000000000000000B /* CloudReadRequestCoordinator+Waiter.swift */; }; C12625A00000000000000004 /* CloudReadRequestCoordinator.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12625A00000000000000003 /* CloudReadRequestCoordinator.swift */; }; C12625B00000000000000004 /* CloudReadRequestCoordinatorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12625B00000000000000003 /* CloudReadRequestCoordinatorTests.swift */; }; + C13151C00000000000000023 /* CloudRefreshAuthClient.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C00000000000000022 /* CloudRefreshAuthClient.swift */; }; + C13151C00000000000000021 /* CloudRefreshFixture.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C00000000000000020 /* CloudRefreshFixture.swift */; }; C12625000000000000000004 /* CloudRefreshURLProtocol.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12625000000000000000003 /* CloudRefreshURLProtocol.swift */; }; C12449010000000000000001 /* CloudRemoteColorOwnershipTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12449020000000000000001 /* CloudRemoteColorOwnershipTests.swift */; }; 595EC249E2E015ACE304F0EE /* CloudRemoteOperationStep.swift in Sources */ = {isa = PBXBuildFile; fileRef = 64BF8B9B8328108560B40B7F /* CloudRemoteOperationStep.swift */; }; @@ -4959,6 +4961,8 @@ C13151C0000000000000000B /* CloudReadRequestCoordinator+Waiter.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudReadRequestCoordinator+Waiter.swift"; sourceTree = ""; }; C12625A00000000000000003 /* CloudReadRequestCoordinator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudReadRequestCoordinator.swift; sourceTree = ""; }; C12625B00000000000000003 /* CloudReadRequestCoordinatorTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudReadRequestCoordinatorTests.swift; sourceTree = ""; }; + C13151C00000000000000022 /* CloudRefreshAuthClient.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudRefreshAuthClient.swift; sourceTree = ""; }; + C13151C00000000000000020 /* CloudRefreshFixture.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudRefreshFixture.swift; sourceTree = ""; }; C12625000000000000000003 /* CloudRefreshURLProtocol.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudRefreshURLProtocol.swift; sourceTree = ""; }; C12449020000000000000001 /* CloudRemoteColorOwnershipTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudRemoteColorOwnershipTests.swift; sourceTree = ""; }; 64BF8B9B8328108560B40B7F /* CloudRemoteOperationStep.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudRemoteOperationStep.swift"; sourceTree = ""; }; @@ -11112,6 +11116,8 @@ 8F8A0358C8DD0A7D998A86E9 /* VMClientTelemetryTests.swift */, 7B7B816D0E6DDD18F2D66C77 /* CloudOperationRecorderTests.swift */, C12625000000000000000001 /* VMClientReadCoalescingTests.swift */, + C13151C00000000000000020 /* CloudRefreshFixture.swift */, + C13151C00000000000000022 /* CloudRefreshAuthClient.swift */, C12625000000000000000003 /* CloudRefreshURLProtocol.swift */, C12625B00000000000000003 /* CloudReadRequestCoordinatorTests.swift */, C12625B00000000000000005 /* CloudReadManualClock.swift */, @@ -15943,6 +15949,8 @@ AC38407F09E94DA38B94F168 /* CloudProviderRefreshCoordinatorTests.swift in Sources */, C12625B00000000000000006 /* CloudReadManualClock.swift in Sources */, C12625B00000000000000004 /* CloudReadRequestCoordinatorTests.swift in Sources */, + C13151C00000000000000023 /* CloudRefreshAuthClient.swift in Sources */, + C13151C00000000000000021 /* CloudRefreshFixture.swift in Sources */, C12625000000000000000004 /* CloudRefreshURLProtocol.swift in Sources */, C12449010000000000000001 /* CloudRemoteColorOwnershipTests.swift in Sources */, C12575000000000000000050 /* CloudRenameOptimismTests.swift in Sources */, diff --git a/cmuxTests/CloudOperationRecorderTests.swift b/cmuxTests/CloudOperationRecorderTests.swift index 310c816165f7..63d96f40caf5 100644 --- a/cmuxTests/CloudOperationRecorderTests.swift +++ b/cmuxTests/CloudOperationRecorderTests.swift @@ -289,7 +289,8 @@ private actor CapturedCloudDiagnostics: CloudTelemetrySending { func clearForSignOut() { spans.removeAll() } } -/// The request is handled synchronously without shared mutable fixture state. +/// The parsed immutable span crosses to the capture actor before any receipt is delivered. +/// URLProtocol callbacks can arrive off-actor; mutable capture state stays actor-isolated. private final class PlacementReceiptURLProtocol: URLProtocol, @unchecked Sendable { fileprivate struct UploadedSpan: Sendable { let failure: String @@ -344,19 +345,22 @@ private final class PlacementReceiptURLProtocol: URLProtocol, @unchecked Sendabl throw URLError(.cannotDecodeContentData) } let status = request.url?.host == "receipt-202.test" ? 202 : 400 - Task { await capture.record(status: status, span: UploadedSpan( + let uploaded = UploadedSpan( failure: span["failure"] as? String ?? "", traceID: span["traceId"] as? String ?? "", operationID: operationID - )) } + ) guard let url = request.url, let response = HTTPURLResponse(url: url, statusCode: status, httpVersion: nil, headerFields: nil) else { throw URLError(.badURL) } let receipt = try JSONSerialization.data(withJSONObject: ["eventIds": [eventID]]) - client?.urlProtocol(self, didReceive: response, cacheStoragePolicy: .notAllowed) - client?.urlProtocol(self, didLoad: receipt) - client?.urlProtocolDidFinishLoading(self) + Task { + await Self.capture.record(status: status, span: uploaded) + client?.urlProtocol(self, didReceive: response, cacheStoragePolicy: .notAllowed) + client?.urlProtocol(self, didLoad: receipt) + client?.urlProtocolDidFinishLoading(self) + } } catch { client?.urlProtocol(self, didFailWithError: error) } diff --git a/cmuxTests/CloudRefreshAuthClient.swift b/cmuxTests/CloudRefreshAuthClient.swift new file mode 100644 index 000000000000..e1f74a1b306c --- /dev/null +++ b/cmuxTests/CloudRefreshAuthClient.swift @@ -0,0 +1,22 @@ +import CMUXAuthCore +import CmuxAuthRuntime +import Foundation + +actor CloudRefreshAuthClient: AuthClient { + func accessToken() async -> String? { "fixture-access" } + func refreshToken() async -> String? { "fixture-refresh" } + func forceRefreshAccessToken() async -> String? { "fixture-access" } + func currentUser(throwOnMissing: Bool) async throws -> CMUXAuthUser? { + CMUXAuthUser(id: "fixture", primaryEmail: "fixture@example.test", displayName: "Fixture") + } + func listTeams() async throws -> [CMUXAuthTeam] { [] } + func sendMagicLinkEmail(email: String, callbackURL: String) async throws -> String { "fixture" } + func signInWithMagicLink(code: String) async throws {} + func signInWithCredential(email: String, password: String) async throws {} + func signInWithOAuth(provider: String, anchor: any AuthPresentationAnchoring) async throws {} + func storedAccessToken() async -> String? { "fixture-access" } + func clearLocalSession() async {} + func clearLocalSession(ifRefreshTokenMatches refreshToken: String) async {} + func revokeSession(accessToken: String?, refreshToken: String?) async throws {} + func freshAccessToken(accessToken: String?, refreshToken: String) async -> String? { accessToken } +} diff --git a/cmuxTests/CloudRefreshFixture.swift b/cmuxTests/CloudRefreshFixture.swift new file mode 100644 index 000000000000..3cd78aef49cb --- /dev/null +++ b/cmuxTests/CloudRefreshFixture.swift @@ -0,0 +1,55 @@ +import CMUXAuthCore +import CmuxAuthRuntime +import Foundation +import Testing +import os + +#if canImport(cmux_DEV) +@testable import cmux_DEV +#elseif canImport(cmux) +@testable import cmux +#endif + +@MainActor +struct CloudRefreshFixture { + let client: VMClient + let auth: AuthCoordinator + let session: URLSession + let readRequests: CloudReadRequestCoordinator + + static func make( + readRequests: CloudReadRequestCoordinator = CloudReadRequestCoordinator(), + isDisabledByManagedPolicy: (@Sendable () -> Bool)? = nil, + isCloudEnabled: @escaping @Sendable () -> Bool = { true } + ) async throws -> Self { + let defaults = try #require(UserDefaults(suiteName: "CloudRefreshFixture.\(UUID())")) + let auth = AuthCoordinator( + client: CloudRefreshAuthClient(), + sessionCache: CMUXAuthSessionCache(keyValueStore: defaults, key: "session"), + userCache: CMUXAuthIdentityStore(keyValueStore: defaults, key: "user"), + teamSelection: CMUXAuthTeamSelectionStore(keyValueStore: defaults, key: "team"), + anchor: AuthPresentationContextProvider(), + config: AuthConfig( + stack: CMUXAuthConfig(projectId: "fixture", publishableClientKey: "fixture"), + magicLinkCallbackURL: "http://127.0.0.1:1/callback", apiBaseURL: "http://127.0.0.1:1" + ), + launch: AuthLaunchOptions( + clearAuthRequested: false, mockDataEnabled: false, + environment: ["CMUX_UITEST_AUTH_FIXTURE": "1", "CMUX_UITEST_AUTH_USER_ID": "fixture"], + includesDevAuth: true + ) + ) + auth.start() + await auth.awaitBootstrapped() + try #require(auth.isAuthenticated) + let configuration = URLSessionConfiguration.ephemeral + configuration.protocolClasses = [CloudRefreshURLProtocol.self] + let session = URLSession(configuration: configuration) + return Self(client: VMClient( + session: session, auth: auth, resourceStats: VMResourceStatsStore(), checkpointRenames: CloudRenameCoordinator(), + machineCache: CloudMachineCache(defaults: defaults), isDisabledByManagedPolicy: isDisabledByManagedPolicy, + readRequests: readRequests, isCloudEnabled: isCloudEnabled + ), auth: auth, session: session, readRequests: readRequests) + } +} + diff --git a/cmuxTests/VMClientReadCoalescingTests.swift b/cmuxTests/VMClientReadCoalescingTests.swift index 9d8d75ae7493..ff8c2dd25f88 100644 --- a/cmuxTests/VMClientReadCoalescingTests.swift +++ b/cmuxTests/VMClientReadCoalescingTests.swift @@ -416,65 +416,3 @@ struct VMClientReadCoalescingTests { } } - -@MainActor -struct CloudRefreshFixture { - let client: VMClient - let auth: AuthCoordinator - let session: URLSession - let readRequests: CloudReadRequestCoordinator - - static func make( - readRequests: CloudReadRequestCoordinator = CloudReadRequestCoordinator(), - isDisabledByManagedPolicy: (@Sendable () -> Bool)? = nil, - isCloudEnabled: @escaping @Sendable () -> Bool = { true } - ) async throws -> Self { - let defaults = try #require(UserDefaults(suiteName: "CloudRefreshFixture.\(UUID())")) - let auth = AuthCoordinator( - client: CloudRefreshAuthClient(), - sessionCache: CMUXAuthSessionCache(keyValueStore: defaults, key: "session"), - userCache: CMUXAuthIdentityStore(keyValueStore: defaults, key: "user"), - teamSelection: CMUXAuthTeamSelectionStore(keyValueStore: defaults, key: "team"), - anchor: AuthPresentationContextProvider(), - config: AuthConfig( - stack: CMUXAuthConfig(projectId: "fixture", publishableClientKey: "fixture"), - magicLinkCallbackURL: "http://127.0.0.1:1/callback", apiBaseURL: "http://127.0.0.1:1" - ), - launch: AuthLaunchOptions( - clearAuthRequested: false, mockDataEnabled: false, - environment: ["CMUX_UITEST_AUTH_FIXTURE": "1", "CMUX_UITEST_AUTH_USER_ID": "fixture"], - includesDevAuth: true - ) - ) - auth.start() - await auth.awaitBootstrapped() - try #require(auth.isAuthenticated) - let configuration = URLSessionConfiguration.ephemeral - configuration.protocolClasses = [CloudRefreshURLProtocol.self] - let session = URLSession(configuration: configuration) - return Self(client: VMClient( - session: session, auth: auth, resourceStats: VMResourceStatsStore(), checkpointRenames: CloudRenameCoordinator(), - machineCache: CloudMachineCache(defaults: defaults), isDisabledByManagedPolicy: isDisabledByManagedPolicy, - readRequests: readRequests, isCloudEnabled: isCloudEnabled - ), auth: auth, session: session, readRequests: readRequests) - } -} - -private actor CloudRefreshAuthClient: AuthClient { - func accessToken() async -> String? { "fixture-access" } - func refreshToken() async -> String? { "fixture-refresh" } - func forceRefreshAccessToken() async -> String? { "fixture-access" } - func currentUser(throwOnMissing: Bool) async throws -> CMUXAuthUser? { - CMUXAuthUser(id: "fixture", primaryEmail: "fixture@example.test", displayName: "Fixture") - } - func listTeams() async throws -> [CMUXAuthTeam] { [] } - func sendMagicLinkEmail(email: String, callbackURL: String) async throws -> String { "fixture" } - func signInWithMagicLink(code: String) async throws {} - func signInWithCredential(email: String, password: String) async throws {} - func signInWithOAuth(provider: String, anchor: any AuthPresentationAnchoring) async throws {} - func storedAccessToken() async -> String? { "fixture-access" } - func clearLocalSession() async {} - func clearLocalSession(ifRefreshTokenMatches refreshToken: String) async {} - func revokeSession(accessToken: String?, refreshToken: String?) async throws {} - func freshAccessToken(accessToken: String?, refreshToken: String) async -> String? { accessToken } -} diff --git a/web/services/vms/drivers/guestCliInstallCommand.ts b/web/services/vms/drivers/guestCliInstallCommand.ts index 79c13ead768f..28eb3ffa565a 100644 --- a/web/services/vms/drivers/guestCliInstallCommand.ts +++ b/web/services/vms/drivers/guestCliInstallCommand.ts @@ -22,8 +22,8 @@ const promptName = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/; // transaction. The prompt lock is held across the browser and prompt writes, // so a rename cannot observe or race a partially-installed generation. const install = String.raw` -import fcntl, hashlib, json, os, pwd, shutil, stat, subprocess, sys, tempfile -source, target, digest, browser, prompt_json, paths_json = sys.argv[1:] +import fcntl, hashlib, json, os, shutil, stat, subprocess, sys, tempfile +source, target, digest, browser, prompt_json, paths_json, transaction_token = sys.argv[1:] stage = "validate" lock = None backup_root = None @@ -35,14 +35,14 @@ def mime_paths(): result = [] for username in ("root", "cmux", "ubuntu"): try: - home = pwd.getpwnam(username).pw_dir - except KeyError: - continue - try: - if not os.access(home, os.R_OK | os.X_OK): - continue + lookup = subprocess.run(["getent", "passwd", username], check=False, + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True) except OSError: continue + fields = lookup.stdout.strip().split(":") if lookup.returncode == 0 else [] + home = fields[5] if len(fields) > 5 else "" + if not home.startswith("/"): + continue result.extend([ os.path.join(home, ".config/mimeapps.list"), os.path.join(home, ".local/share/applications/mimeapps.list"), @@ -60,13 +60,18 @@ def parent_dirs(path): return result directory_paths = list(dict.fromkeys( - parent for path in paths + [lock_path] for parent in parent_dirs(path) + os.path.dirname(path) for path in paths + [lock_path] if os.path.dirname(path) )) +def unsafe_symlink_target(path): + return path.endswith("/bash.bashrc") or path.endswith("/zshenv") or path.endswith("/.config/mimeapps.list") or path.endswith("/.local/share/applications/mimeapps.list") + def validate_target(path): if os.path.lexists(path): if os.path.isdir(path) and not os.path.islink(path): raise IsADirectoryError(path) + if os.path.islink(path) and unsafe_symlink_target(path): + raise ValueError("refusing to mutate a symlink target") if not os.path.islink(path) and not stat.S_ISREG(os.lstat(path).st_mode): raise ValueError("target is not a regular file") for parent in parent_dirs(path): @@ -99,8 +104,9 @@ def snapshot_path(path): if not stat.S_ISREG(os.lstat(path).st_mode): raise ValueError("install target is not a regular file") backup = os.path.join(backup_root, str(len(backups))) - shutil.copy2(path, backup) - backups[path] = ("file", backup) + metadata = os.stat(path, follow_symlinks=False) + shutil.copyfile(path, backup) + backups[path] = ("file", backup, metadata.st_mode, metadata.st_uid, metadata.st_gid, metadata.st_atime_ns, metadata.st_mtime_ns) def restore_paths(): failures = [] @@ -109,33 +115,31 @@ def restore_paths(): remove_path(path) if backup is None: continue - kind, value = backup + kind, value, *metadata = backup os.makedirs(os.path.dirname(path), exist_ok=True) if kind == "link": os.symlink(value, path) else: - shutil.copy2(value, path) + shutil.copyfile(value, path) + mode, uid, gid, atime_ns, mtime_ns = metadata + os.chmod(path, stat.S_IMODE(mode), follow_symlinks=False) + os.chown(path, uid, gid, follow_symlinks=False) + os.utime(path, ns=(atime_ns, mtime_ns), follow_symlinks=False) except Exception as error: failures.append(error) if failures: raise RuntimeError("rollback failed") -def remember_directory_entries(): - return { - path: set(os.listdir(path)) if os.path.isdir(path) else set() - for path in directory_paths - } - -def cleanup_generated(entries): +def cleanup_generated(): failures = [] - prefixes = [os.path.basename(path) + "." for path in paths] - prefixes.append(".prompt-") - for directory, before in entries.items(): + prefixes = [".prompt-" + transaction_token + "-"] + prefixes.extend(os.path.basename(path) + "." + transaction_token + "." for path in paths) + for directory in directory_paths: if not os.path.isdir(directory): continue try: for name in os.listdir(directory): - if name in before or not any(name.startswith(prefix) for prefix in prefixes): + if not any(name.startswith(prefix) for prefix in prefixes): continue candidate = os.path.join(directory, name) if os.path.isfile(candidate) or os.path.islink(candidate): @@ -162,7 +166,7 @@ def replace_file(directory, name, content): with open(target_path, "r") as stream: if stream.read() == content: return - fd, temporary = tempfile.mkstemp(prefix=".prompt-", dir=directory) + fd, temporary = tempfile.mkstemp(prefix=".prompt-" + transaction_token + "-", dir=directory) try: with os.fdopen(fd, "w") as stream: stream.write(content) @@ -210,7 +214,6 @@ try: backup_root = tempfile.mkdtemp(prefix=".cmux-install-", dir=os.path.dirname(lock_path)) for path in paths: snapshot_path(path) - before_entries = remember_directory_entries() stage = "verify" subprocess.run([source, "--help"], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) stage = "browser" @@ -219,13 +222,13 @@ try: install_prompt(json.loads(prompt_json) if prompt_json else None) stage = "publish" os.replace(source, target) - cleanup_generated(before_entries) + cleanup_generated() shutil.rmtree(backup_root) backup_root = None except Exception as error: rollback_error = None try: - cleanup_generated(before_entries if "before_entries" in globals() else {}) + cleanup_generated() except Exception as cleanup_error: rollback_error = cleanup_error try: @@ -264,5 +267,7 @@ export function guestCliInstallCommand(temporaryPath: string, identity?: GuestPr identity, files: guestPromptInstallFiles, }) : ""; - return `python3 -c ${shellQuote(install)} ${shellQuote(temporaryPath)} ${shellQuote(GUEST_CMUX_SHIM_PATH)} ${shellQuote(digest)} ${shellQuote(guestBrowserInstallCommand())} ${shellQuote(prompt)} ${shellQuote(JSON.stringify(installPaths))}`; + const transactionToken = temporaryPath.replace(/[^A-Za-z0-9_-]/g, "_"); + const browser = guestBrowserInstallCommand().replaceAll("XXXXXX", `${transactionToken}.XXXXXX`); + return `python3 -c ${shellQuote(install)} ${shellQuote(temporaryPath)} ${shellQuote(GUEST_CMUX_SHIM_PATH)} ${shellQuote(digest)} ${shellQuote(browser)} ${shellQuote(prompt)} ${shellQuote(JSON.stringify(installPaths))} ${shellQuote(transactionToken)}`; } diff --git a/web/services/vms/repository.ts b/web/services/vms/repository.ts index efd82ba119fd..ef9bac55fb90 100644 --- a/web/services/vms/repository.ts +++ b/web/services/vms/repository.ts @@ -1,5 +1,5 @@ import { randomUUID } from "node:crypto"; -import { and, asc, count, desc, eq, gt, gte, inArray, isNotNull, isNull, lt, lte, ne, or, sql } from "drizzle-orm"; +import { and, asc, count, desc, eq, gt, gte, inArray, isNotNull, isNull, lt, lte, ne, or, sql, type SQL } from "drizzle-orm"; import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; @@ -688,6 +688,16 @@ export const CREATE_CLEANUP_NEXT_ATTEMPT_AT_KEY = "createCleanupNextAttemptAtMs" export const CREATE_CLEANUP_LEASE_ID_KEY = "createCleanupLeaseId"; export const CREATE_CLEANUP_LEASE_EXPIRES_AT_KEY = "createCleanupLeaseExpiresAtMs"; +/** Treat malformed/oversized cleanup timestamps as immediately eligible without an unsafe cast. */ +function cleanupTimestampDueSql(raw: SQL, nowMs: number) { + return sql`case + when ${raw} ~ '^[0-9]{1,16}$' + and (length(${raw}) < 16 or ${raw} <= '9007199254740991') + then (${raw})::numeric <= ${nowMs} + else true + end`; +} + const RETRYABLE_FAILED_CREATE_CODES = new Set([ "billing_credits_insufficient", "billing_reserve_failed", @@ -744,11 +754,10 @@ async function restoreBaseAfterCreateFailure( .orderBy(desc(cloudVmBaseGenerations.generation)) .limit(1); if (retained?.generation && retained.vm) { - await tx - .update(cloudVmBaseGenerations) - .set({ state: "active", updatedAt: now }) - .where(eq(cloudVmBaseGenerations.id, retained.generation.id)); - await tx + // Fence promotion on the failed generation still being the active Base + // row. A concurrent reset/open may have moved the Base on while provider + // cleanup was in flight; in that case leave the newer generation alone. + const activeBase = await tx .update(cloudVmBases) .set({ activeGeneration: retained.generation.generation, @@ -762,16 +771,24 @@ async function restoreBaseAfterCreateFailure( eq(cloudVmBases.id, input.baseId), eq(cloudVmBases.activeGeneration, input.generation), eq(cloudVmBases.activeVmId, input.vmId), - )); - } else { + )) + .returning({ id: cloudVmBases.id }); + if (activeBase.length === 0) return; await tx + .update(cloudVmBaseGenerations) + .set({ state: "active", updatedAt: now }) + .where(eq(cloudVmBaseGenerations.id, retained.generation.id)); + } else { + const failedBase = await tx .update(cloudVmBases) .set({ state: "failed", updatedAt: now }) .where(and( eq(cloudVmBases.id, input.baseId), eq(cloudVmBases.activeGeneration, input.generation), eq(cloudVmBases.activeVmId, input.vmId), - )); + )) + .returning({ id: cloudVmBases.id }); + if (failedBase.length === 0) return; } await tx.insert(cloudVmBaseEvents).values({ baseId: input.baseId, @@ -866,7 +883,12 @@ function providerMetadataPatchForPersistence( key !== VM_RESOURCE_FORK_PENDING_METADATA_KEY && key !== VM_RESOURCE_RECONCILE_RETRY_METADATA_KEY && key !== VM_RESOURCE_RESIZE_PENDING_METADATA_KEY && - key !== VM_RESOURCE_RESIZE_UNCONFIRMED_METADATA_KEY, + key !== VM_RESOURCE_RESIZE_UNCONFIRMED_METADATA_KEY && + key !== CREATE_CLEANUP_PROVIDER_VM_ID_KEY && + key !== CREATE_CLEANUP_ATTEMPT_KEY && + key !== CREATE_CLEANUP_NEXT_ATTEMPT_AT_KEY && + key !== CREATE_CLEANUP_LEASE_ID_KEY && + key !== CREATE_CLEANUP_LEASE_EXPIRES_AT_KEY, ), ); } @@ -2639,16 +2661,8 @@ export const vmRepositoryLiveShape: VmRepositoryShape = { // out of the worker batch so they cannot starve live cleanup rows. eq(cloudVms.provider, "freestyle"), sql`${cleanupProviderVmId} is not null and ${cleanupProviderVmId} <> ''`, - sql`( - ${nextAttemptAtMs} is null - or ${nextAttemptAtMs} !~ '^[0-9]+$' - or (${nextAttemptAtMs})::bigint <= ${nowMs} - )`, - sql`( - ${leaseExpiresAtMs} is null - or ${leaseExpiresAtMs} !~ '^[0-9]+$' - or (${leaseExpiresAtMs})::bigint <= ${nowMs} - )`, + cleanupTimestampDueSql(nextAttemptAtMs, nowMs), + cleanupTimestampDueSql(leaseExpiresAtMs, nowMs), )) .orderBy(asc(cloudVms.updatedAt), asc(cloudVms.id)) .limit(limit); @@ -2665,20 +2679,15 @@ export const vmRepositoryLiveShape: VmRepositoryShape = { const leaseExpiresAtMs = sql`${cloudVms.providerMetadata}->>'${sql.raw(CREATE_CLEANUP_LEASE_EXPIRES_AT_KEY)}'`; const nextAttemptAtMs = sql`${cloudVms.providerMetadata}->>'${sql.raw(CREATE_CLEANUP_NEXT_ATTEMPT_AT_KEY)}'`; const attempt = sql`${cloudVms.providerMetadata}->>'${sql.raw(CREATE_CLEANUP_ATTEMPT_KEY)}'`; - return await db.transaction(async (tx) => { - // Serialize claims for one row before the CAS update. The update - // predicates still fence stale leases, while this lock keeps two - // concurrent cron workers from both observing an eligible marker. - await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${input.id}, 0))`); - const [row] = await tx - .update(cloudVms) + const [row] = await db + .update(cloudVms) .set({ providerMetadata: sql`( coalesce(${cloudVms.providerMetadata}, '{}'::jsonb) || jsonb_build_object( '${sql.raw(CREATE_CLEANUP_ATTEMPT_KEY)}', case - when ${attempt} ~ '^[0-9]+$' then (${attempt})::bigint + 1 + when ${attempt} ~ '^[0-9]{1,9}$' then least((${attempt})::numeric + 1, 1000000)::integer else 1 end, -- Keep the row ineligible for the entire claim lease. A second @@ -2697,27 +2706,20 @@ export const vmRepositoryLiveShape: VmRepositoryShape = { eq(cloudVms.status, "provisioning"), eq(cloudVms.failureCode, PROVIDER_CREATE_CLEANUP_PENDING_FAILURE_CODE), eq(cleanupProviderVmId, providerVmId), - sql`( - ${nextAttemptAtMs} is null - or ${nextAttemptAtMs} !~ '^[0-9]+$' - or (${nextAttemptAtMs})::bigint <= ${input.now.getTime()} - )`, + cleanupTimestampDueSql(nextAttemptAtMs, input.now.getTime()), sql`( ${leaseIdValue} is null or ${leaseIdValue} = '' - or ${leaseExpiresAtMs} is null - or ${leaseExpiresAtMs} !~ '^[0-9]+$' - or (${leaseExpiresAtMs})::bigint <= ${input.now.getTime()} + or ${cleanupTimestampDueSql(leaseExpiresAtMs, input.now.getTime())} )`, )) - .returning({ providerMetadata: cloudVms.providerMetadata }); - if (!row) return null; - const rawAttempt = row.providerMetadata[CREATE_CLEANUP_ATTEMPT_KEY]; - const parsedAttempt = typeof rawAttempt === "number" && Number.isSafeInteger(rawAttempt) - ? rawAttempt - : Number.parseInt(String(rawAttempt ?? "1"), 10); - return { attempt: Number.isSafeInteger(parsedAttempt) && parsedAttempt > 0 ? parsedAttempt : 1 }; - }); + .returning({ providerMetadata: cloudVms.providerMetadata }); + if (!row) return null; + const rawAttempt = row.providerMetadata[CREATE_CLEANUP_ATTEMPT_KEY]; + const parsedAttempt = typeof rawAttempt === "number" && Number.isSafeInteger(rawAttempt) + ? rawAttempt + : Number.parseInt(String(rawAttempt ?? "1"), 10); + return { attempt: Number.isSafeInteger(parsedAttempt) && parsedAttempt > 0 ? parsedAttempt : 1 }; }), deferCreateCleanup: (input) => diff --git a/web/services/vms/workflows.ts b/web/services/vms/workflows.ts index f61afd3fe30d..9eb2c66f2ca1 100644 --- a/web/services/vms/workflows.ts +++ b/web/services/vms/workflows.ts @@ -217,10 +217,7 @@ const CREATE_CLEANUP_PROVIDER_TIMEOUT = "15 seconds"; const CREATE_CLEANUP_LEASE_MS = 60 * 1000; const CREATE_CLEANUP_BACKOFF_BASE_MS = 5 * 1000; const CREATE_CLEANUP_BACKOFF_MAX_MS = 15 * 60 * 1000; -// The Postgres lease is the cross-process fence. This small process-local set -// avoids issuing duplicate provider deletes while two cron requests in the same -// Node worker are between candidate selection and the durable claim result. -const pendingCreateCleanupInFlight = new Set(); +const CREATE_CLEANUP_BATCH_LIMIT = 20; const LEGACY_RESOURCE_RECONCILE_BATCH_LIMIT = 50; const LEGACY_RESOURCE_RECONCILE_CONCURRENCY = 5; const LEGACY_RESOURCE_RECONCILE_RETRY_AFTER_MS = 5 * 60 * 1000; @@ -396,7 +393,9 @@ export function reconcileVmProviderStatuses(input: { limit: LEGACY_RESOURCE_RECONCILE_BATCH_LIMIT, }); yield* reconcilePendingCreateCleanups(repo, providers, { - limit: boundedVmStatusReconcileLimit(input.limit), + // At four concurrent 15-second provider calls, twenty rows fit inside + // the five-minute cron budget while leaving time for status probes. + limit: Math.min(boundedVmStatusReconcileLimit(input.limit), CREATE_CLEANUP_BATCH_LIMIT), }); const getStatus = providers.getStatus; if (!getStatus) { @@ -486,8 +485,6 @@ function reconcilePendingCreateCleanups( const rawProviderVmId = vm.providerMetadata?.[CREATE_CLEANUP_PROVIDER_VM_ID_KEY]; if (typeof rawProviderVmId !== "string" || rawProviderVmId.trim().length === 0) return Effect.void; const providerVmId = rawProviderVmId.trim(); - if (pendingCreateCleanupInFlight.has(vm.id)) return Effect.void; - pendingCreateCleanupInFlight.add(vm.id); const leaseId = randomUUID(); const now = new Date(); const leaseExpiresAt = new Date(now.getTime() + CREATE_CLEANUP_LEASE_MS); @@ -528,9 +525,7 @@ function reconcilePendingCreateCleanups( ); }), Effect.catchAll(() => Effect.void), - ).pipe(Effect.ensuring(Effect.sync(() => { - pendingCreateCleanupInFlight.delete(vm.id); - }))); + ); }, { concurrency: CREATE_CLEANUP_CONCURRENCY, discard: true }, ); diff --git a/web/tests/vm-guest-install-runtime.test.ts b/web/tests/vm-guest-install-runtime.test.ts index 31f0a5b42735..077310ee1559 100644 --- a/web/tests/vm-guest-install-runtime.test.ts +++ b/web/tests/vm-guest-install-runtime.test.ts @@ -78,6 +78,22 @@ describe("guest CLI publication in an isolated filesystem", () => { expect(readdirSync(join(root, "bin")).sort()).toEqual(["cmux", "cmux-open-url", "sensible-browser", "x-www-browser", "xdg-open"]); }); + test("a fresh prompt install keeps its identity when an older revision attaches", async () => { + const { fixture, root } = guest(); + await fixture.provider.create({ + ...guestCreateOptions, + promptIdentity: { machineId: "synthetic", name: "fresh-machine", revision: 10 }, + }); + await fixture.provider.create({ + ...guestCreateOptions, + promptIdentity: { machineId: "synthetic", name: "stale-machine", revision: 9 }, + }); + expect(readFileSync(join(root, "etc/vm-name"), "utf8")).toBe("fresh-machine\n"); + expect(JSON.parse(readFileSync(join(root, "etc/.prompt-identity"), "utf8"))).toEqual({ + machineId: "synthetic", name: "fresh-machine", revision: 10, + }); + }); + test("replaces a target symlink without modifying its referent", async () => { const { fixture, root, target } = guest(); const unrelated = join(root, "unrelated"); From 28cb83ede168a113c6e3e67795bafa393dc05b5d Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 07:28:26 -0700 Subject: [PATCH 43/92] harden final Cloud lifecycle review paths --- .../vms/drivers/guestCliInstallCommand.ts | 10 +- web/services/vms/repository.ts | 17 +-- web/tests/vm-workflows.test.ts | 141 ++++++++++++++++++ 3 files changed, 155 insertions(+), 13 deletions(-) diff --git a/web/services/vms/drivers/guestCliInstallCommand.ts b/web/services/vms/drivers/guestCliInstallCommand.ts index 28eb3ffa565a..89464dfd2277 100644 --- a/web/services/vms/drivers/guestCliInstallCommand.ts +++ b/web/services/vms/drivers/guestCliInstallCommand.ts @@ -74,9 +74,11 @@ def validate_target(path): raise ValueError("refusing to mutate a symlink target") if not os.path.islink(path) and not stat.S_ISREG(os.lstat(path).st_mode): raise ValueError("target is not a regular file") - for parent in parent_dirs(path): - if os.path.lexists(parent) and not os.path.isdir(parent): - raise NotADirectoryError(parent) + parent = os.path.dirname(path) + if os.path.islink(parent): + raise ValueError("refusing a symlink parent directory") + if os.path.lexists(parent) and not os.path.isdir(parent): + raise NotADirectoryError(parent) def remove_path(path): if not os.path.lexists(path): @@ -135,7 +137,7 @@ def cleanup_generated(): prefixes = [".prompt-" + transaction_token + "-"] prefixes.extend(os.path.basename(path) + "." + transaction_token + "." for path in paths) for directory in directory_paths: - if not os.path.isdir(directory): + if not os.path.isdir(directory) or os.path.islink(directory): continue try: for name in os.listdir(directory): diff --git a/web/services/vms/repository.ts b/web/services/vms/repository.ts index ef9bac55fb90..39321458ee86 100644 --- a/web/services/vms/repository.ts +++ b/web/services/vms/repository.ts @@ -773,22 +773,21 @@ async function restoreBaseAfterCreateFailure( eq(cloudVmBases.activeVmId, input.vmId), )) .returning({ id: cloudVmBases.id }); - if (activeBase.length === 0) return; - await tx - .update(cloudVmBaseGenerations) - .set({ state: "active", updatedAt: now }) - .where(eq(cloudVmBaseGenerations.id, retained.generation.id)); + if (activeBase.length > 0) { + await tx + .update(cloudVmBaseGenerations) + .set({ state: "active", updatedAt: now }) + .where(eq(cloudVmBaseGenerations.id, retained.generation.id)); + } } else { - const failedBase = await tx + await tx .update(cloudVmBases) .set({ state: "failed", updatedAt: now }) .where(and( eq(cloudVmBases.id, input.baseId), eq(cloudVmBases.activeGeneration, input.generation), eq(cloudVmBases.activeVmId, input.vmId), - )) - .returning({ id: cloudVmBases.id }); - if (failedBase.length === 0) return; + )); } await tx.insert(cloudVmBaseEvents).values({ baseId: input.baseId, diff --git a/web/tests/vm-workflows.test.ts b/web/tests/vm-workflows.test.ts index 3ffa996ce30e..bf9314294576 100644 --- a/web/tests/vm-workflows.test.ts +++ b/web/tests/vm-workflows.test.ts @@ -354,6 +354,108 @@ describe("VM Effect workflows", () => { expect(row.metadata.createCleanupLeaseId).toBeUndefined(); }); + dbTest("treats malformed cleanup timestamps and oversized attempts as immediately eligible", async () => { + if (!sql) throw new Error("test database not initialized"); + const fixture = freestyleGuestFixture({ + exec: async () => Response.json({ statusCode: 1 }), + deleteFailure: true, + idPrefix: "vm-cleanup-timestamp", + }); + let destroyCalls = 0; + const layer = providerLayer({ + ...unusedProviderGateway(), + create: (_, options) => Effect.tryPromise({ + try: () => fixture.provider.create(options), + catch: (cause) => new VmProviderOperationError({ provider: "freestyle", operation: "create", cause }), + }), + destroy: (_, providerVmId) => Effect.sync(() => { + destroyCalls += 1; + fixture.liveVms.delete(providerVmId); + }), + }); + const input = { + ...guestCreateOptions, userId: "user-guest-cleanup-timestamp", billingTeamId: "team-guest-cleanup-timestamp", + billingCustomerType: "team" as const, billingPlanId: "pro", provider: "freestyle" as const, + idempotencyKey: "guest-cleanup-timestamp", maxActiveVms: 1, + }; + expect((await Effect.runPromise(Effect.either(createVm(input).pipe(Effect.provide(layer)))))._tag).toBe("Left"); + await sql` + update cloud_vms + set provider_metadata = provider_metadata || jsonb_build_object( + 'createCleanupNextAttemptAtMs', '999999999999999999999999999999', + 'createCleanupLeaseExpiresAtMs', 'not-a-timestamp', + 'createCleanupAttempt', '999999999999999999999999999999' + ) + where user_id = 'user-guest-cleanup-timestamp' + `; + await Effect.runPromise(reconcileVmProviderStatuses().pipe(Effect.provide(layer))); + expect(destroyCalls).toBe(1); + const [row] = await sql<{ status: string; metadata: Record }[]>` + select status, provider_metadata as metadata from cloud_vms where user_id = 'user-guest-cleanup-timestamp' + `; + expect(row.status).toBe("failed"); + expect(row.metadata.createCleanupProviderVmId).toBeUndefined(); + }); + + dbTest("does not resolve a replaced cleanup marker with a stale lease", async () => { + if (!sql) throw new Error("test database not initialized"); + const fixture = freestyleGuestFixture({ + exec: async () => Response.json({ statusCode: 1 }), + deleteFailure: true, + idPrefix: "vm-cleanup-cas", + }); + const layer = providerLayer({ + ...unusedProviderGateway(), + create: (_, options) => Effect.tryPromise({ + try: () => fixture.provider.create(options), + catch: (cause) => new VmProviderOperationError({ provider: "freestyle", operation: "create", cause }), + }), + }); + const input = { + ...guestCreateOptions, userId: "user-guest-cleanup-cas", billingTeamId: "team-guest-cleanup-cas", + billingCustomerType: "team" as const, billingPlanId: "pro", provider: "freestyle" as const, + idempotencyKey: "guest-cleanup-cas", maxActiveVms: 1, + }; + expect((await Effect.runPromise(Effect.either(createVm(input).pipe(Effect.provide(layer)))))._tag).toBe("Left"); + const [row] = await sql<{ id: string; providerVmId: string }[]>` + select id, provider_metadata->>'createCleanupProviderVmId' as "providerVmId" + from cloud_vms where user_id = 'user-guest-cleanup-cas' + `; + const claim = await Effect.runPromise(vmRepositoryLiveShape.claimCreateCleanup!({ + id: row.id, + providerVmId: row.providerVmId, + leaseId: "lease-a", + now: new Date(), + leaseExpiresAt: new Date(Date.now() + 60_000), + })); + expect(claim?.attempt).toBe(1); + const duplicateClaim = await Effect.runPromise(vmRepositoryLiveShape.claimCreateCleanup!({ + id: row.id, + providerVmId: row.providerVmId, + leaseId: "lease-b", + now: new Date(), + leaseExpiresAt: new Date(Date.now() + 60_000), + })); + expect(duplicateClaim).toBeNull(); + await sql` + update cloud_vms + set provider_metadata = provider_metadata || jsonb_build_object('createCleanupProviderVmId', 'replacement-provider-vm') + where id = ${row.id} + `; + const resolved = await Effect.runPromise(vmRepositoryLiveShape.resolveCreateCleanup!({ + id: row.id, + providerVmId: row.providerVmId, + leaseId: "lease-a", + })); + expect(resolved).toBe(false); + const [stillPending] = await sql<{ status: string; providerVmId: string }[]>` + select status, provider_metadata->>'createCleanupProviderVmId' as "providerVmId" + from cloud_vms where id = ${row.id} + `; + expect(stillPending.status).toBe("provisioning"); + expect(stillPending.providerVmId).toBe("replacement-provider-vm"); + }); + dbTest("unconfirmed guest rollback keeps the Base generation reserved", async () => { if (!sql) throw new Error("test database not initialized"); const fixture = freestyleGuestFixture({ exec: async () => Response.json({ statusCode: 1 }), deleteFailure: true }); @@ -385,6 +487,45 @@ describe("VM Effect workflows", () => { expect(row.metadata.createCleanupProviderVmId).toBe("vm-fixture-1"); }); + dbTest("confirmed Base cleanup restores the retained generation", async () => { + if (!sql) throw new Error("test database not initialized"); + let failInstall = false; + const fixture = freestyleGuestFixture({ + exec: async () => Response.json({ statusCode: failInstall ? 1 : 0 }), + deleteFailure: true, + idPrefix: "vm-base-cleanup-recovery", + }); + const layer = providerLayer({ + ...unusedProviderGateway(), + create: (_, options) => Effect.tryPromise({ + try: () => fixture.provider.create(options), + catch: (cause) => new VmProviderOperationError({ provider: "freestyle", operation: "create", cause }), + }), + destroy: (_, providerVmId) => Effect.sync(() => { + fixture.liveVms.delete(providerVmId); + }), + }); + const input = { + userId: "user-base-cleanup-recovery", billingTeamId: "team-base-cleanup-recovery", + billingCustomerType: "team" as const, billingPlanId: "pro", provider: "freestyle" as const, + image: "sh-synthetic", baseName: "cleanup-recovery", maxActiveVms: 2, + }; + const first = await Effect.runPromise(openBaseVm(input).pipe(Effect.provide(layer))); + expect(first.providerVmId).toBe("vm-base-cleanup-recovery-1"); + failInstall = true; + const reset = await Effect.runPromise(Effect.either(resetBaseVm(input).pipe(Effect.provide(layer)))); + expect(reset._tag).toBe("Left"); + await Effect.runPromise(reconcileVmProviderStatuses().pipe(Effect.provide(layer))); + const recovered = await Effect.runPromise(openBaseVm(input).pipe(Effect.provide(layer))); + expect(recovered.providerVmId).toBe("vm-base-cleanup-recovery-1"); + const [base] = await sql<{ state: string; activeProviderVmId: string | null }[]>` + select state, active_provider_vm_id as "activeProviderVmId" + from cloud_vm_bases where scope_id = 'team-base-cleanup-recovery' and name = 'cleanup-recovery' + `; + expect(base.state).toBe("ready"); + expect(base.activeProviderVmId).toBe("vm-base-cleanup-recovery-1"); + }); + dbTest("keeps prompt revisions ordered across rapid renames and clock skew", async () => { if (!sql) throw new Error("test database not initialized"); const userId = "user-prompt-revisions"; From ed6257169279367fd59a9e18942083cac3396e5d Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 07:53:26 -0700 Subject: [PATCH 44/92] fix offline first-load recovery state --- .../CmuxGitTests/ReftableGitMetadataTests.swift | 3 +-- Sources/Cloud/MachinesPanelViewModel.swift | 3 +++ cmuxTests/CloudReadRequestCoordinatorTests.swift | 14 ++++++++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/Packages/macOS/CmuxGit/Tests/CmuxGitTests/ReftableGitMetadataTests.swift b/Packages/macOS/CmuxGit/Tests/CmuxGitTests/ReftableGitMetadataTests.swift index 4b9a6f4c964b..135877ab1823 100644 --- a/Packages/macOS/CmuxGit/Tests/CmuxGitTests/ReftableGitMetadataTests.swift +++ b/Packages/macOS/CmuxGit/Tests/CmuxGitTests/ReftableGitMetadataTests.swift @@ -372,8 +372,7 @@ private struct NeverDirectoryProbe: GitReferenceStorageProbing { let reader = SystemGitReferenceReader( runner: SystemWorkspaceChangesGitRunner( executableURL: fixture.gitExecutableURL, - environment: environment, - isolateRepositoryConfig: true + environment: environment ) ) diff --git a/Sources/Cloud/MachinesPanelViewModel.swift b/Sources/Cloud/MachinesPanelViewModel.swift index 3bb5d6cdf29d..d75af9cb0d6e 100644 --- a/Sources/Cloud/MachinesPanelViewModel.swift +++ b/Sources/Cloud/MachinesPanelViewModel.swift @@ -218,6 +218,9 @@ final class MachinesPanelViewModel: ObservableObject { self.clearUnavailableMetrics() self.lastErrorDescription = URLError(.notConnectedToInternet).localizedDescription self.listProblem = .unreachable + // Mark an interrupted first request as observed so the + // offline empty state renders its retry action. + self.hasLoadedOnce = true // Retire the active transport and advance the generation // so a late response cannot clear the offline state or // schedule another refresh. `wantsPolling` remains true, diff --git a/cmuxTests/CloudReadRequestCoordinatorTests.swift b/cmuxTests/CloudReadRequestCoordinatorTests.swift index 0ca1b4cc6925..2264d031f7d2 100644 --- a/cmuxTests/CloudReadRequestCoordinatorTests.swift +++ b/cmuxTests/CloudReadRequestCoordinatorTests.swift @@ -443,6 +443,20 @@ struct CloudReadRequestCoordinatorTests { } } +@Suite("Cloud machines offline empty state") +struct CloudMachinesOfflineStateTests { + @Test("Offline before the first list load exposes retry state") + @MainActor + func offlineBeforeFirstLoadIsVisible() { + let model = MachinesPanelViewModel(client: nil, isCloudEnabled: { true }) + NotificationCenter.default.post(name: .cmuxCloudReadNetworkChanged, object: nil, userInfo: ["isOnline": false]) + #expect(model.hasLoadedOnce) + #expect(model.listProblem == .unreachable) + #expect(model.lastErrorDescription != nil) + model.stopPolling() + } +} + actor CloudReadResponseGate { private(set) var requests = 0 private var released = false From eb5cb9084cebe652ae8943666ed901f8e54ceaaf Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 08:17:33 -0700 Subject: [PATCH 45/92] address Cloud review findings --- Sources/Cloud/MachinesPanelViewModel.swift | 11 +++++----- .../CloudReadRequestCoordinatorTests.swift | 6 ++++-- web/services/vms/drivers/freestyle.ts | 1 + web/services/vms/drivers/freestyleGuestCli.ts | 7 ++++++- .../vms/drivers/guestCliInstallCommand.ts | 3 ++- web/tests/vm-guest-install.test.ts | 18 +++++++++++++++++ web/tests/vm-workflows.test.ts | 20 +++++++++++++------ 7 files changed, 51 insertions(+), 15 deletions(-) diff --git a/Sources/Cloud/MachinesPanelViewModel.swift b/Sources/Cloud/MachinesPanelViewModel.swift index d75af9cb0d6e..ae2ae5a7354e 100644 --- a/Sources/Cloud/MachinesPanelViewModel.swift +++ b/Sources/Cloud/MachinesPanelViewModel.swift @@ -88,7 +88,6 @@ final class MachinesPanelViewModel: ObservableObject { let selected = tabManager.selectedTabId return tabManager.tabs.map { CloudTreeLocalWorkspace(id: $0.id, title: $0.title, isSelected: $0.id == selected) } } - func beginOperation(_ label: String) { activeOperation = label } @@ -108,6 +107,7 @@ final class MachinesPanelViewModel: ObservableObject { let isCloudEnabled: @MainActor () -> Bool let pollingClock: any Clock private var networkObserver: NSObjectProtocol? + private let notificationCenter: NotificationCenter var pollTask: Task? var statsTask: Task? private var resourceUpdatesTask: Task? @@ -136,7 +136,6 @@ final class MachinesPanelViewModel: ObservableObject { private var createChangeObserver: NSObjectProtocol? var treeTask: Task? let machineRefreshes = CloudMachineRefreshCoordinator { await SurfaceCatalog.shared.refresh(machine: $0, force: true) } - let defaultMachineStore: DefaultCloudMachineStore? /// Explicit machine pins and the stable fleet order; nil keeps fleet order. let machinePinStore: CloudMachinePinStore? @@ -152,7 +151,8 @@ final class MachinesPanelViewModel: ObservableObject { pollingClock: any Clock = ContinuousClock(), isCloudEnabled: @escaping @MainActor () -> Bool = { CloudMachinesFeature.isEnabled }, catalogProvider: @escaping @MainActor () -> SurfaceCatalogSnapshot = { SurfaceCatalog.shared.snapshot }, - localWorkspacesProvider: (@MainActor () -> [CloudTreeLocalWorkspace])? = nil + localWorkspacesProvider: (@MainActor () -> [CloudTreeLocalWorkspace])? = nil, + notificationCenter: NotificationCenter = .default ) { self.client = client self.pollingClock = pollingClock @@ -161,6 +161,7 @@ final class MachinesPanelViewModel: ObservableObject { self.defaultMachineStore = defaultMachineStore self.machinePinStore = machinePinStore self.catalogProvider = catalogProvider + self.notificationCenter = notificationCenter if let localWorkspacesProvider { self.localWorkspacesProvider = localWorkspacesProvider } // Resolve the main-actor-isolated default here, not in a default argument. let createCoordinator = createCoordinator ?? .shared @@ -208,7 +209,7 @@ final class MachinesPanelViewModel: ObservableObject { MainActor.assumeIsolated { self?.readUnreadTerminalIDs() } } readUnreadTerminalIDs() - networkObserver = NotificationCenter.default.addObserver(forName: .cmuxCloudReadNetworkChanged, object: nil, queue: .main) { [weak self] notification in + networkObserver = notificationCenter.addObserver(forName: .cmuxCloudReadNetworkChanged, object: nil, queue: .main) { [weak self] notification in guard let online = notification.userInfo?["isOnline"] as? Bool else { return } MainActor.assumeIsolated { guard let self else { return } @@ -274,7 +275,7 @@ final class MachinesPanelViewModel: ObservableObject { usageTask?.cancel() treeTask?.cancel() freeAccessTransitionTask?.cancel() - if let networkObserver { NotificationCenter.default.removeObserver(networkObserver) } + if let networkObserver { notificationCenter.removeObserver(networkObserver) } resourceUpdatesTask?.cancel() for observer in authScopeObservers { NotificationCenter.default.removeObserver(observer) diff --git a/cmuxTests/CloudReadRequestCoordinatorTests.swift b/cmuxTests/CloudReadRequestCoordinatorTests.swift index 2264d031f7d2..2ffc0401d65b 100644 --- a/cmuxTests/CloudReadRequestCoordinatorTests.swift +++ b/cmuxTests/CloudReadRequestCoordinatorTests.swift @@ -304,6 +304,7 @@ struct CloudReadRequestCoordinatorTests { let owner = Owner() let gate = CloudReadResponseGate() let first = Task { try await owner.read(key()) { await gate.read(response()) } } + try await eventually { await gate.requests == 1 } let replacement = try await owner.read(key(generation: 2)) { response(201) } #expect(replacement.http.statusCode == 201) await gate.release() @@ -448,8 +449,9 @@ struct CloudMachinesOfflineStateTests { @Test("Offline before the first list load exposes retry state") @MainActor func offlineBeforeFirstLoadIsVisible() { - let model = MachinesPanelViewModel(client: nil, isCloudEnabled: { true }) - NotificationCenter.default.post(name: .cmuxCloudReadNetworkChanged, object: nil, userInfo: ["isOnline": false]) + let notificationCenter = NotificationCenter() + let model = MachinesPanelViewModel(client: nil, isCloudEnabled: { true }, notificationCenter: notificationCenter) + notificationCenter.post(name: .cmuxCloudReadNetworkChanged, object: nil, userInfo: ["isOnline": false]) #expect(model.hasLoadedOnce) #expect(model.listProblem == .unreachable) #expect(model.lastErrorDescription != nil) diff --git a/web/services/vms/drivers/freestyle.ts b/web/services/vms/drivers/freestyle.ts index 5808ac726ba7..fbb14a583ad0 100644 --- a/web/services/vms/drivers/freestyle.ts +++ b/web/services/vms/drivers/freestyle.ts @@ -1756,6 +1756,7 @@ export class FreestyleProvider implements VMProvider { "cmux.vm.guest_install.exit_code": result.left.exitCode, "cmux.duration_ms": result.left.elapsedMs, "cmux.vm.guest_install.cleanup_failed": result.left.cleanupCause !== undefined, + "cmux.vm.guest_install.rollback_error": result.left.rollbackError, }); throw result.left; } diff --git a/web/services/vms/drivers/freestyleGuestCli.ts b/web/services/vms/drivers/freestyleGuestCli.ts index 6e0a141e05f6..8cbdbccb1b54 100644 --- a/web/services/vms/drivers/freestyleGuestCli.ts +++ b/web/services/vms/drivers/freestyleGuestCli.ts @@ -17,6 +17,7 @@ export class GuestCliInstallError extends Data.TaggedError("GuestCliInstallError readonly outcome: InstallOutcome; readonly exitCode?: number | null; readonly diagnostic?: string; + readonly rollbackError?: string; readonly elapsedMs?: number; readonly cause?: unknown; readonly cleanupCause?: unknown; @@ -35,7 +36,7 @@ function guestFailure(result: unknown): GuestCliInstallError | undefined { const outcome: InstallOutcome = status === undefined ? "missing_status" : status === null ? "provider_timeout" : typeof status === "number" && Number.isInteger(status) ? "guest_exit" : "invalid_status"; - const failure: { stage: InstallStage; diagnostic?: string } = { stage: "install" }; + const failure: { stage: InstallStage; diagnostic?: string; rollbackError?: string } = { stage: "install" }; // Only our own bounded diagnostic record enters telemetry. Arbitrary guest // stdout/stderr may contain private data and is deliberately not logged. const marker = typeof response.stderr === "string" @@ -49,6 +50,9 @@ function guestFailure(result: unknown): GuestCliInstallError | undefined { + (Number.isInteger(detail.errno) ? ` errno=${detail.errno}` : "") + (Number.isInteger(detail.exitCode) ? ` exit=${detail.exitCode}` : ""); } + if (typeof detail.rollbackError === "string" && /^[A-Za-z]{1,40}$/.test(detail.rollbackError)) { + failure.rollbackError = detail.rollbackError; + } } catch { /* Unknown provider output is not evidence of an install stage. */ } } return new GuestCliInstallError({ @@ -110,6 +114,7 @@ export function installFreestyleGuestCli(client: FreestyleClientFactory, vmId: s elapsedMs: finished - started, exitCode: error.exitCode, diagnostic: error.diagnostic, + rollbackError: error.rollbackError, cause: error.cause, cleanupCause, })) diff --git a/web/services/vms/drivers/guestCliInstallCommand.ts b/web/services/vms/drivers/guestCliInstallCommand.ts index 89464dfd2277..cee78c23ab4c 100644 --- a/web/services/vms/drivers/guestCliInstallCommand.ts +++ b/web/services/vms/drivers/guestCliInstallCommand.ts @@ -249,7 +249,8 @@ except Exception as error: rollback_error = backup_error detail = { "stage": stage, - "error": "RollbackError" if rollback_error else type(error).__name__, + "error": type(error).__name__, + "rollbackError": type(rollback_error).__name__ if rollback_error else None, "errno": getattr(error, "errno", None), "exitCode": getattr(error, "returncode", None), } diff --git a/web/tests/vm-guest-install.test.ts b/web/tests/vm-guest-install.test.ts index 18a82a7007d2..3cbebedf86de 100644 --- a/web/tests/vm-guest-install.test.ts +++ b/web/tests/vm-guest-install.test.ts @@ -59,4 +59,22 @@ describe("Freestyle guest install result contract (SDK 0.2.10)", () => { }); expect(fixture.liveVms.size).toBe(1); }); + + test("keeps the guest install error when rollback also fails", async () => { + const fixture = freestyleGuestFixture({ + exec: async () => Response.json({ + statusCode: 1, + stderr: `CMUX_GUEST_INSTALL_FAILURE=${JSON.stringify({ + stage: "publish", error: "OSError", rollbackError: "PermissionError", errno: 13, + })}`, + }), + }); + const error = await fixture.provider.create(guestCreateOptions).catch((error: unknown) => error); + expect((error as { cause?: unknown }).cause).toMatchObject({ + name: "GuestCliInstallError", + stage: "publish", + diagnostic: "OSError errno=13", + rollbackError: "PermissionError", + }); + }); }); diff --git a/web/tests/vm-workflows.test.ts b/web/tests/vm-workflows.test.ts index bf9314294576..a9c2b999a918 100644 --- a/web/tests/vm-workflows.test.ts +++ b/web/tests/vm-workflows.test.ts @@ -1,4 +1,4 @@ -import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { afterAll, beforeAll, describe, expect, setSystemTime, test } from "bun:test"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import postgres, { type Sql } from "postgres"; @@ -319,6 +319,9 @@ describe("VM Effect workflows", () => { dbTest("retains failed cleanup with durable backoff and does not duplicate the provider delete", async () => { if (!sql) throw new Error("test database not initialized"); + const nowMs = 1_800_000_000_000; + setSystemTime(nowMs); + try { const fixture = freestyleGuestFixture({ exec: async () => Response.json({ statusCode: 1 }), deleteFailure: true }); let destroyCalls = 0; const layer = providerLayer({ @@ -350,8 +353,11 @@ describe("VM Effect workflows", () => { `; expect(row.status).toBe("provisioning"); expect(row.metadata.createCleanupProviderVmId).toBe("vm-fixture-1"); - expect(Number(row.metadata.createCleanupNextAttemptAtMs)).toBeGreaterThan(Date.now()); + expect(Number(row.metadata.createCleanupNextAttemptAtMs)).toBe(nowMs + 5_000); expect(row.metadata.createCleanupLeaseId).toBeUndefined(); + } finally { + setSystemTime(); + } }); dbTest("treats malformed cleanup timestamps and oversized attempts as immediately eligible", async () => { @@ -421,20 +427,22 @@ describe("VM Effect workflows", () => { select id, provider_metadata->>'createCleanupProviderVmId' as "providerVmId" from cloud_vms where user_id = 'user-guest-cleanup-cas' `; + const now = new Date("2026-09-20T00:00:00.000Z"); + const leaseExpiresAt = new Date(now.getTime() + 60_000); const claim = await Effect.runPromise(vmRepositoryLiveShape.claimCreateCleanup!({ id: row.id, providerVmId: row.providerVmId, leaseId: "lease-a", - now: new Date(), - leaseExpiresAt: new Date(Date.now() + 60_000), + now, + leaseExpiresAt, })); expect(claim?.attempt).toBe(1); const duplicateClaim = await Effect.runPromise(vmRepositoryLiveShape.claimCreateCleanup!({ id: row.id, providerVmId: row.providerVmId, leaseId: "lease-b", - now: new Date(), - leaseExpiresAt: new Date(Date.now() + 60_000), + now, + leaseExpiresAt, })); expect(duplicateClaim).toBeNull(); await sql` From c2a5e7045a25cb942f00cc9de4a7ef9e8d86ece6 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 08:24:28 -0700 Subject: [PATCH 46/92] keep guest failure complexity within gate --- web/services/vms/drivers/freestyleGuestCli.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/web/services/vms/drivers/freestyleGuestCli.ts b/web/services/vms/drivers/freestyleGuestCli.ts index 8cbdbccb1b54..3e092837ea57 100644 --- a/web/services/vms/drivers/freestyleGuestCli.ts +++ b/web/services/vms/drivers/freestyleGuestCli.ts @@ -29,6 +29,10 @@ export class GuestCliInstallError extends Data.TaggedError("GuestCliInstallError } } +function rollbackErrorName(value: unknown): string | undefined { + return typeof value === "string" && /^[A-Za-z]{1,40}$/.test(value) ? value : undefined; +} + function guestFailure(result: unknown): GuestCliInstallError | undefined { const response = result && typeof result === "object" ? result as Record : {}; const status = response.statusCode; @@ -50,9 +54,7 @@ function guestFailure(result: unknown): GuestCliInstallError | undefined { + (Number.isInteger(detail.errno) ? ` errno=${detail.errno}` : "") + (Number.isInteger(detail.exitCode) ? ` exit=${detail.exitCode}` : ""); } - if (typeof detail.rollbackError === "string" && /^[A-Za-z]{1,40}$/.test(detail.rollbackError)) { - failure.rollbackError = detail.rollbackError; - } + failure.rollbackError = rollbackErrorName(detail.rollbackError); } catch { /* Unknown provider output is not evidence of an install stage. */ } } return new GuestCliInstallError({ From df94720ba6fb06c5d34f741bcacb21e894a5d7b7 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 08:45:51 -0700 Subject: [PATCH 47/92] keep Cloud acceptance suites out of batch shards --- scripts/ci/cmux_unit_test_shard.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/ci/cmux_unit_test_shard.py b/scripts/ci/cmux_unit_test_shard.py index fdae961cac17..0261438f8b0e 100755 --- a/scripts/ci/cmux_unit_test_shard.py +++ b/scripts/ci/cmux_unit_test_shard.py @@ -57,6 +57,8 @@ "cmuxTests/CloudMachineDragSourceTests", "cmuxTests/CloudMachineOrderingTests", "cmuxTests/CloudNotificationDismissParityTests", + "cmuxTests/CloudOperationRecorderTests", + "cmuxTests/CloudReadRequestCoordinatorTests", "cmuxTests/CloudWorkspaceRenameSurfaceParityTests", "cmuxTests/CmuxBundledBinPathIntegrationTests", "cmuxTests/DockNotificationAttentionTests", @@ -86,6 +88,7 @@ "cmuxTests/SidebarWorkspaceSwitchLayoutFaultTests", "cmuxTests/SocketACLReloadRegressionTests", "cmuxTests/SurfaceResumeAgentHookDowngradeTests", + "cmuxTests/VMClientReadCoalescingTests", } # BrowserDeveloperToolsVisibilityPersistenceTests reliably crash-restarts the From 4485438b818787ef4325524fb42d233a984bb409 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 08:54:20 -0700 Subject: [PATCH 48/92] isolate Cloud network monitor ownership --- Sources/Cloud/CloudReadNetworkMonitor.swift | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Sources/Cloud/CloudReadNetworkMonitor.swift b/Sources/Cloud/CloudReadNetworkMonitor.swift index 59c83eb17948..17d8b3d048ad 100644 --- a/Sources/Cloud/CloudReadNetworkMonitor.swift +++ b/Sources/Cloud/CloudReadNetworkMonitor.swift @@ -1,11 +1,12 @@ import Foundation @preconcurrency import Network -/// One path observer for the VM client. A Network.framework delivery queue -/// feeds an AsyncStream; it never protects domain state or runs UI work. -final class CloudReadNetworkMonitor: Sendable { +/// One path observer for the VM client. The actor owns the mutable +/// Network.framework monitor; only its sendable stream crosses the actor +/// boundary, and it never protects domain state or runs UI work. +actor CloudReadNetworkMonitor { private let monitor: NWPathMonitor - let updates: AsyncStream + nonisolated let updates: AsyncStream init() { let monitor = NWPathMonitor() From bf16a4b41e5eea422799f586b3d0ae9c79aa8ccc Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 09:29:52 -0700 Subject: [PATCH 49/92] test Cloud identity and guest failure boundaries --- cmux.xcodeproj/project.pbxproj | 8 ++ cmuxTests/CloudReadIdentityAuthClient.swift | 52 +++++++++ cmuxTests/CloudRefreshFixture.swift | 4 +- cmuxTests/CloudRefreshURLProtocol.swift | 9 +- ...VMClientReadCoalescingTests+Identity.swift | 101 ++++++++++++++++++ web/tests/vm-guest-create-response.test.ts | 34 ++++++ 6 files changed, 205 insertions(+), 3 deletions(-) create mode 100644 cmuxTests/CloudReadIdentityAuthClient.swift create mode 100644 cmuxTests/VMClientReadCoalescingTests+Identity.swift diff --git a/cmux.xcodeproj/project.pbxproj b/cmux.xcodeproj/project.pbxproj index 0d91855ae37b..f7f517d70eab 100644 --- a/cmux.xcodeproj/project.pbxproj +++ b/cmux.xcodeproj/project.pbxproj @@ -3736,6 +3736,8 @@ 60AA592C22BFA579A064782E /* VMClient+ResourceStats.swift in Sources */ = {isa = PBXBuildFile; fileRef = 65A5357644AB29D74F4AC368 /* VMClient+ResourceStats.swift */; }; ACE4FE1A8F1CC6C0B6A673AB /* VMClient.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9CEC6EF35B71AE59FA45BA69 /* VMClient.swift */; }; EF56B870FD0D4AC490D99CD5 /* VMClientError.swift in Sources */ = {isa = PBXBuildFile; fileRef = 98F890512F8E425FAD71FE7C /* VMClientError.swift */; }; + C13151C00000000000000031 /* CloudReadIdentityAuthClient.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C00000000000000030 /* CloudReadIdentityAuthClient.swift */; }; + C13151C00000000000000033 /* VMClientReadCoalescingTests+Identity.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C00000000000000032 /* VMClientReadCoalescingTests+Identity.swift */; }; C12625000000000000000002 /* VMClientReadCoalescingTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12625000000000000000001 /* VMClientReadCoalescingTests.swift */; }; 7A0CE1000000000000000044 /* VMClientSocketCommands+Tunnel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7A0CE1000000000000000018 /* VMClientSocketCommands+Tunnel.swift */; }; ACE4FE2A8F1CC6C0B6A673AB /* VMClientSocketCommands.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9CEC6F0035B71AE59FA45BA7 /* VMClientSocketCommands.swift */; }; @@ -7718,6 +7720,8 @@ 65A5357644AB29D74F4AC368 /* VMClient+ResourceStats.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "VMClient+ResourceStats.swift"; sourceTree = ""; }; 9CEC6EF35B71AE59FA45BA69 /* VMClient.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = VMClient.swift; sourceTree = ""; }; 98F890512F8E425FAD71FE7C /* VMClientError.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = VMClientError.swift; sourceTree = ""; }; + C13151C00000000000000030 /* CloudReadIdentityAuthClient.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudReadIdentityAuthClient.swift"; sourceTree = ""; }; + C13151C00000000000000032 /* VMClientReadCoalescingTests+Identity.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "VMClientReadCoalescingTests+Identity.swift"; sourceTree = ""; }; C12625000000000000000001 /* VMClientReadCoalescingTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VMClientReadCoalescingTests.swift; sourceTree = ""; }; 7A0CE1000000000000000018 /* VMClientSocketCommands+Tunnel.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = "VMClientSocketCommands+Tunnel.swift"; sourceTree = ""; }; 9CEC6F0035B71AE59FA45BA7 /* VMClientSocketCommands.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = VMClientSocketCommands.swift; sourceTree = ""; }; @@ -11118,6 +11122,8 @@ B8E9500A0000000000000002 /* PostHogAnalyticsPropertiesTests.swift */, 8F8A0358C8DD0A7D998A86E9 /* VMClientTelemetryTests.swift */, 7B7B816D0E6DDD18F2D66C77 /* CloudOperationRecorderTests.swift */, + C13151C00000000000000030 /* CloudReadIdentityAuthClient.swift */, + C13151C00000000000000032 /* VMClientReadCoalescingTests+Identity.swift */, C12625000000000000000001 /* VMClientReadCoalescingTests.swift */, C13151C00000000000000020 /* CloudRefreshFixture.swift */, C13151C00000000000000022 /* CloudRefreshAuthClient.swift */, @@ -16643,6 +16649,8 @@ 62F6C9E77A6A40B982D91040 /* VaultSessionSearchQueryTests.swift in Sources */, C5BEA2B0EA914759A8BFD577 /* VaultSocketPayloadTests.swift in Sources */, A3340002A3340002A3340002 /* ViewerNavigationTests.swift in Sources */, + C13151C00000000000000031 /* CloudReadIdentityAuthClient.swift in Sources */, + C13151C00000000000000033 /* VMClientReadCoalescingTests+Identity.swift in Sources */, C12625000000000000000002 /* VMClientReadCoalescingTests.swift in Sources */, 6EE115A36CAD9082FBC4593E /* VMClientTelemetryTests.swift in Sources */, C37800000000000000000003 /* VMDefaultCloudCommandTests.swift in Sources */, diff --git a/cmuxTests/CloudReadIdentityAuthClient.swift b/cmuxTests/CloudReadIdentityAuthClient.swift new file mode 100644 index 000000000000..555f1f649ae8 --- /dev/null +++ b/cmuxTests/CloudReadIdentityAuthClient.swift @@ -0,0 +1,52 @@ +import CMUXAuthCore +import CmuxAuthRuntime +import Foundation + +/// Holds only the revalidation token probe; coherent token reads remain available. +actor CloudReadIdentityAuthClient: AuthClient { + private var accountID: String? = "fixture" + private var holdsProbe = false + private var probe: CheckedContinuation? + private var probeWaiters: [CheckedContinuation] = [] + + func holdValidationProbe() { holdsProbe = true } + func waitUntilValidationProbe() async { + if probe != nil { return } + await withCheckedContinuation { probeWaiters.append($0) } + } + func releaseValidationProbe() { + holdsProbe = false + probe?.resume() + probe = nil + } + func accessToken() async -> String? { + if holdsProbe { + await withCheckedContinuation { continuation in + probe = continuation + let waiters = probeWaiters + probeWaiters.removeAll() + for waiter in waiters { waiter.resume() } + } + } + return accountID.map { "fixture-access-\($0)" } + } + func resolvedAccessToken(forceRefresh: Bool) async throws -> String? { accountID.map { "fixture-access-\($0)" } } + func refreshToken() async -> String? { accountID.map { "fixture-refresh-\($0)" } } + func forceRefreshAccessToken() async -> String? { accountID.map { "fixture-access-\($0)" } } + func currentUser(throwOnMissing: Bool) async throws -> CMUXAuthUser? { + accountID.map { CMUXAuthUser(id: $0, primaryEmail: "\($0)@example.test", displayName: "Fixture") } + } + func listTeams() async throws -> [CMUXAuthTeam] { [] } + func sendMagicLinkEmail(email: String, callbackURL: String) async throws -> String { "fixture" } + func signInWithMagicLink(code: String) async throws {} + func signInWithCredential(email: String, password: String) async throws { accountID = "replacement" } + func signInWithOAuth(provider: String, anchor: any AuthPresentationAnchoring) async throws {} + func storedAccessToken() async -> String? { accountID.map { "fixture-access-\($0)" } } + func clearLocalSession() async { accountID = nil } + func clearLocalSession(ifRefreshTokenMatches refreshToken: String) async { + guard let accountID, refreshToken == "fixture-refresh-\(accountID)" else { return } + self.accountID = nil + } + func revokeSession(accessToken: String?, refreshToken: String?) async throws {} + func freshAccessToken(accessToken: String?, refreshToken: String) async -> String? { accessToken } +} diff --git a/cmuxTests/CloudRefreshFixture.swift b/cmuxTests/CloudRefreshFixture.swift index 3cd78aef49cb..1f549e559fda 100644 --- a/cmuxTests/CloudRefreshFixture.swift +++ b/cmuxTests/CloudRefreshFixture.swift @@ -19,12 +19,13 @@ struct CloudRefreshFixture { static func make( readRequests: CloudReadRequestCoordinator = CloudReadRequestCoordinator(), + authClient: (any AuthClient)? = nil, isDisabledByManagedPolicy: (@Sendable () -> Bool)? = nil, isCloudEnabled: @escaping @Sendable () -> Bool = { true } ) async throws -> Self { let defaults = try #require(UserDefaults(suiteName: "CloudRefreshFixture.\(UUID())")) let auth = AuthCoordinator( - client: CloudRefreshAuthClient(), + client: authClient ?? CloudRefreshAuthClient(), sessionCache: CMUXAuthSessionCache(keyValueStore: defaults, key: "session"), userCache: CMUXAuthIdentityStore(keyValueStore: defaults, key: "user"), teamSelection: CMUXAuthTeamSelectionStore(keyValueStore: defaults, key: "team"), @@ -52,4 +53,3 @@ struct CloudRefreshFixture { ), auth: auth, session: session, readRequests: readRequests) } } - diff --git a/cmuxTests/CloudRefreshURLProtocol.swift b/cmuxTests/CloudRefreshURLProtocol.swift index b89e550c2d58..5ec179330817 100644 --- a/cmuxTests/CloudRefreshURLProtocol.swift +++ b/cmuxTests/CloudRefreshURLProtocol.swift @@ -71,7 +71,14 @@ final class CloudRefreshURLProtocol: URLProtocol, @unchecked Sendable { let response = HTTPURLResponse(url: source.request.url!, statusCode: behavior == .throttled ? 429 : unavailable ? 503 : 200, httpVersion: nil, headerFields: behavior == .throttled ? ["Retry-After": "60"] : nil)! source.client?.urlProtocol(source, didReceive: response, cacheStoragePolicy: .notAllowed) - let body = path.hasSuffix("/stats") ? #"{"state":"awake","cpus":2}"# : #"{"vms":[{"id":"fixture-0","provider":"fixture","image":"desktop-vnc","status":"running","createdAt":0,"capabilities":{"stats":true}}]}"# + let body: String + if path == "/api/coderouter/vm-usage/team" { + body = #"{"teamId":"fixture-team","kind":"ready","periodDays":30,"machines":[]}"# + } else if path.hasSuffix("/stats") { + body = #"{"state":"awake","cpus":2}"# + } else { + body = #"{"vms":[{"id":"fixture-0","provider":"fixture","image":"desktop-vnc","status":"running","createdAt":0,"capabilities":{"stats":true}}]}"# + } source.client?.urlProtocol(source, didLoad: Data(body.utf8)) source.client?.urlProtocolDidFinishLoading(source) } diff --git a/cmuxTests/VMClientReadCoalescingTests+Identity.swift b/cmuxTests/VMClientReadCoalescingTests+Identity.swift new file mode 100644 index 000000000000..59b43710d011 --- /dev/null +++ b/cmuxTests/VMClientReadCoalescingTests+Identity.swift @@ -0,0 +1,101 @@ +import CmuxAuthRuntime +import Foundation +import Testing + +#if canImport(cmux_DEV) +@testable import cmux_DEV +#elseif canImport(cmux) +@testable import cmux +#endif + +extension VMClientReadCoalescingTests { + @Test("Reads without a transition identity fail before HTTP and recover for the next account", arguments: ["list", "stats", "usage"]) + func nilIdentityAdmission(operation: String) async throws { + let authClient = CloudReadIdentityAuthClient() + let fixture = try await CloudRefreshFixture.make(authClient: authClient) + defer { fixture.session.invalidateAndCancel() } + await CloudRefreshURLProtocol.reset() + await authClient.holdValidationProbe() + let validation = Task { await fixture.auth.revalidateSession() } + await authClient.waitUntilValidationProbe() + #expect(fixture.auth.isAuthenticated) + #expect(fixture.auth.authenticatedSessionIdentity == nil) + do { + try await identityRead(operation, fixture: fixture) + Issue.record("A read with no authenticated identity reached the server") + } catch VMClientError.sessionRefreshFailed { + } catch MachineUsageClientError.sessionRefreshFailed { + } catch { Issue.record("Unexpected transition error: \(error)") } + #expect(await fixture.readRequests.entries.isEmpty) + #expect(await CloudRefreshURLProtocol.requestCounts().isEmpty) + await authClient.releaseValidationProbe() + await validation.value + try await fixture.auth.signInWithPassword(email: "replacement@example.test", password: "fixture") + #expect(fixture.auth.authenticatedSessionIdentity?.accountID == "replacement") + try await identityRead(operation, fixture: fixture) + #expect(await CloudRefreshURLProtocol.requestCounts().values.reduce(0, +) == 1) + } + + @Test("An account change rejects a response bound to the prior session", arguments: ["list", "stats", "usage"]) + func accountChangeRejectsRead(operation: String) async throws { + let fixture = try await CloudRefreshFixture.make(authClient: CloudReadIdentityAuthClient()) + defer { fixture.session.invalidateAndCancel() } + await CloudRefreshURLProtocol.reset() + await CloudRefreshURLProtocol.holdResponses() + let original = try #require(fixture.auth.authenticatedSessionIdentity) + let reader = Task { try await identityRead(operation, fixture: fixture) } + await CloudRefreshURLProtocol.waitUntilStarted() + try await fixture.auth.signInWithPassword(email: "replacement@example.test", password: "fixture") + #expect(fixture.auth.authenticatedSessionIdentity != original) + await CloudRefreshURLProtocol.releaseResponses() + do { try await reader.value; Issue.record("The previous account's response was published") } + catch is CancellationError {} catch VMClientError.notSignedIn {} catch { Issue.record("\(error)") } + try await identityRead(operation, fixture: fixture) + #expect(await CloudRefreshURLProtocol.requestCounts().values.reduce(0, +) == 2) + } + + @Test("A known identity read survives same-account revalidation", arguments: ["list", "stats", "usage"]) + func knownIdentityDuringRevalidation(operation: String) async throws { + let authClient = CloudReadIdentityAuthClient() + let fixture = try await CloudRefreshFixture.make(authClient: authClient) + defer { fixture.session.invalidateAndCancel() } + await CloudRefreshURLProtocol.reset() + await CloudRefreshURLProtocol.holdResponses() + let original = try #require(fixture.auth.authenticatedSessionIdentity) + let reader = Task { try await identityRead(operation, fixture: fixture) } + await CloudRefreshURLProtocol.waitUntilStarted() + await authClient.holdValidationProbe() + let validation = Task { await fixture.auth.revalidateSession() } + await authClient.waitUntilValidationProbe() + #expect(fixture.auth.authenticatedSessionIdentity == nil) + #expect(fixture.auth.isAuthenticatedSessionIdentityCurrent(original)) + await CloudRefreshURLProtocol.releaseResponses() + let result = await reader.result + await authClient.releaseValidationProbe() + await validation.value + try result.get() + #expect(await CloudRefreshURLProtocol.requestCounts().values.reduce(0, +) == 1) + } + + @Test("Signed-out shared reads fail before the network", arguments: ["list", "stats", "usage"]) + func signedOutReadAdmission(operation: String) async throws { + let fixture = try await CloudRefreshFixture.make(authClient: CloudReadIdentityAuthClient()) + defer { fixture.session.invalidateAndCancel() } + await CloudRefreshURLProtocol.reset() + await fixture.auth.signOut() + do { try await identityRead(operation, fixture: fixture); Issue.record("Signed-out read succeeded") } + catch VMClientError.notSignedIn {} catch MachineUsageClientError.notSignedIn {} catch { Issue.record("\(error)") } + #expect(await CloudRefreshURLProtocol.requestCounts().isEmpty) + #expect(await fixture.readRequests.entries.isEmpty) + } + + private func identityRead(_ operation: String, fixture: CloudRefreshFixture) async throws { + switch operation { + case "list": _ = try await fixture.client.listPage() + case "stats": _ = try await fixture.client.stats(id: "fixture-0") + default: + let usage = MachineUsageClient(session: fixture.session, auth: fixture.auth, readRequests: fixture.readRequests) + _ = try await usage.teamUsage() + } + } +} diff --git a/web/tests/vm-guest-create-response.test.ts b/web/tests/vm-guest-create-response.test.ts index 8f0c955d7dc2..a3eeaf2569ff 100644 --- a/web/tests/vm-guest-create-response.test.ts +++ b/web/tests/vm-guest-create-response.test.ts @@ -1,6 +1,8 @@ import { expect, test } from "bun:test"; import { VmCreateFailedError, VmProviderOperationError } from "../services/vms/errors"; import { ProviderCreateCleanupError } from "../services/vms/drivers/providerCreateCleanup"; +import { GuestCliInstallError } from "../services/vms/drivers/freestyleGuestCli"; +import { ProviderError } from "../services/vms/drivers/types"; import { vmWorkflowErrorResponse } from "../services/vms/routeHelpers"; import { locales } from "../i18n/routing"; @@ -21,3 +23,35 @@ test.each(locales)("cleanup-pending response is localized and never suggests ret expect(await repeated!.json()).toEqual(body); if (locale !== "en") expect(body.message).not.toBe("Cloud VM setup failed and cleanup is still pending."); }); + +test.each(locales)("confirmed guest install failures use localized safe guidance for %s", async (locale) => { + const install = new GuestCliInstallError({ stage: "install", outcome: "guest_exit", exitCode: 7 }); + for (const operation of ["create", "openCmuxRemote"]) { + const provider = new VmProviderOperationError({ + provider: "freestyle", operation, + cause: new ProviderError("freestyle", `${operation}(private-machine) failed`, install), + }); + const response = await vmWorkflowErrorResponse(provider, { locale }); + expect(response?.status).toBe(502); + const body = await response!.json(); + expect(body.error).toBe("vm_guest_install_failed"); + expect(body.retryable).toBe(true); + expect(body.phase).toBe(operation === "create" ? "create" : "attach"); + expect(JSON.stringify(body)).not.toContain("private-machine"); + expect(JSON.stringify(body)).not.toContain("guest cmux shim"); + expect(JSON.stringify(body)).not.toContain("guest_exit"); + } +}); + +test("pending cleanup keeps its non-retryable response when guest install also failed", async () => { + const install = new GuestCliInstallError({ stage: "publish", outcome: "guest_exit", exitCode: 1 }); + const response = await vmWorkflowErrorResponse(new VmProviderOperationError({ + provider: "freestyle", operation: "create", + cause: new ProviderCreateCleanupError("private-provider-id", install, new Error("private delete output")), + }), { locale: "en" }); + expect(response?.status).toBe(503); + const body = await response!.json(); + expect(body.error).toBe("vm_cloud_create_cleanup_pending"); + expect(body.retryable).toBe(false); + expect(JSON.stringify(body)).not.toContain("guest_exit"); +}); From 37fad75dc515b0db15e02cab510fa8ea0dd3c14f Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 09:30:09 -0700 Subject: [PATCH 50/92] close Cloud identity and guest error boundaries --- Sources/Cloud/VMClient.swift | 36 ++++++++++----- web/messages/ar.json | 6 +++ web/messages/bs.json | 6 +++ web/messages/da.json | 6 +++ web/messages/de.json | 6 +++ web/messages/en.json | 6 +++ web/messages/es.json | 6 +++ web/messages/fr.json | 6 +++ web/messages/it.json | 6 +++ web/messages/ja.json | 6 +++ web/messages/km.json | 6 +++ web/messages/ko.json | 6 +++ web/messages/no.json | 6 +++ web/messages/pl.json | 6 +++ web/messages/pt-BR.json | 6 +++ web/messages/ru.json | 6 +++ web/messages/th.json | 6 +++ web/messages/tr.json | 6 +++ web/messages/uk.json | 6 +++ web/messages/zh-CN.json | 6 +++ web/messages/zh-TW.json | 6 +++ web/services/vms/routeHelpers.ts | 72 ++++++++++++++++++++++++++++- web/services/vms/vmErrorMessages.ts | 16 +++++++ 23 files changed, 231 insertions(+), 13 deletions(-) diff --git a/Sources/Cloud/VMClient.swift b/Sources/Cloud/VMClient.swift index c276dfda7637..474e3cc36f58 100644 --- a/Sources/Cloud/VMClient.swift +++ b/Sources/Cloud/VMClient.swift @@ -1997,9 +1997,17 @@ actor VMClient { let elapsed = context.map { $0.operation == .list || $0.operation == .stats ? $0.clock.duration(to: .now) : .zero } ?? .zero let deadline = self.readRequests.makeDeadline(elapsed: elapsed) let identity = await self.auth.authenticatedSessionIdentity + guard let identity else { + let isAuthenticated = await self.auth.isAuthenticated + let isRestoringSession = await self.auth.isRestoringSession + if isAuthenticated || isRestoringSession { + throw VMClientError.sessionRefreshFailed + } + throw VMClientError.notSignedIn + } let teamID = await self.auth.resolvedTeamID - let key = CloudReadRequestCoordinator.Key(path: path, accountID: identity?.accountID, - generation: identity?.generation, teamID: teamID) + let key = CloudReadRequestCoordinator.Key(path: path, accountID: identity.accountID, + generation: identity.generation, teamID: teamID) let value = try await self.readRequests.read(key, deadline: deadline) { try await CloudOperationContext.$current.withValue(context) { let (data, http) = try await self.requestMeasured(method, path: path, timeoutSeconds: timeoutSeconds) @@ -2007,10 +2015,8 @@ actor VMClient { } } try Task.checkCancellation() - if let identity { - guard await self.auth.isAuthenticatedSessionIdentityCurrent(identity), - await self.auth.resolvedTeamID == teamID else { throw CancellationError() } - } + guard await self.auth.isAuthenticatedSessionIdentityCurrent(identity), + await self.auth.resolvedTeamID == teamID else { throw CancellationError() } try self.checkCloudAccess(allowedUnderManagedPolicy: allowedUnderManagedPolicy) return (value.data, value.http) } @@ -2567,20 +2573,26 @@ actor MachineUsageClient { let context = CloudOperationContext.current let deadline = readRequests.makeDeadline(elapsed: context.map { $0.operation == .stats ? $0.clock.duration(to: .now) : .zero } ?? .zero, limit: .seconds(15)) let identity = await auth.authenticatedSessionIdentity + guard let identity else { + let isAuthenticated = await auth.isAuthenticated + let isRestoringSession = await auth.isRestoringSession + if isAuthenticated || isRestoringSession { + throw MachineUsageClientError.sessionRefreshFailed + } + throw MachineUsageClientError.notSignedIn + } let selectedTeam = await auth.resolvedTeamID let explicitTeam = teamID?.trimmingCharacters(in: .whitespacesAndNewlines) - let key = CloudReadRequestCoordinator.Key(path: "/api/coderouter/vm-usage/team", accountID: identity?.accountID, - generation: identity?.generation, teamID: explicitTeam?.isEmpty == false ? explicitTeam : selectedTeam) + let key = CloudReadRequestCoordinator.Key(path: "/api/coderouter/vm-usage/team", accountID: identity.accountID, + generation: identity.generation, teamID: explicitTeam?.isEmpty == false ? explicitTeam : selectedTeam) let response = try await readRequests.read(key, deadline: deadline) { try await CloudOperationContext.$current.withValue(context) { let (data, http) = try await self.request("GET", path: key.path, teamID: teamID) return CloudReadRequestCoordinator.Response(data: data, http: http) } } - if let identity { - guard await auth.isAuthenticatedSessionIdentityCurrent(identity), - await auth.resolvedTeamID == selectedTeam else { throw CancellationError() } - } + guard await auth.isAuthenticatedSessionIdentityCurrent(identity), + await auth.resolvedTeamID == selectedTeam else { throw CancellationError() } let (data, http) = (response.data, response.http) guard (200...299).contains(http.statusCode) else { throw MachineUsageClientError.httpStatus(http.statusCode, String(data: data, encoding: .utf8) ?? "") diff --git a/web/messages/ar.json b/web/messages/ar.json index 5844c4129e2c..6263b1d93484 100644 --- a/web/messages/ar.json +++ b/web/messages/ar.json @@ -1,5 +1,11 @@ { "vmErrors": { + "guestInstall": { + "title": "لم يكتمل إعداد Cloud VM", + "reason": "لم يكتمل إعداد الضيف لـ Cloud VM.", + "message": "تعذر على cmux إكمال إعداد Cloud VM.", + "action": "أعد المحاولة. إذا استمرت المشكلة، تواصل مع الدعم." + }, "createCleanupPending": { "title": "تنظيف آلة Cloud الافتراضية معلق", "message": "فشل إعداد آلة Cloud الافتراضية وما زال التنظيف معلقًا.", diff --git a/web/messages/bs.json b/web/messages/bs.json index 785f60038855..1fb217fb222b 100644 --- a/web/messages/bs.json +++ b/web/messages/bs.json @@ -1,5 +1,11 @@ { "vmErrors": { + "guestInstall": { + "title": "Postavljanje Cloud VM-a nije završeno", + "reason": "Postavljanje gosta za Cloud VM nije završeno.", + "message": "cmux nije mogao završiti postavljanje Cloud VM-a.", + "action": "Pokušajte ponovo. Ako se problem nastavi, obratite se podršci." + }, "createCleanupPending": { "title": "Čišćenje Cloud VM-a na čekanju", "message": "Postavljanje Cloud VM-a nije uspjelo i čišćenje je još na čekanju.", diff --git a/web/messages/da.json b/web/messages/da.json index fa9800772a1f..0546a8ad20cc 100644 --- a/web/messages/da.json +++ b/web/messages/da.json @@ -1,5 +1,11 @@ { "vmErrors": { + "guestInstall": { + "title": "Opsætning af Cloud VM blev ikke fuldført", + "reason": "Gæsteopsætningen af Cloud VM blev ikke fuldført.", + "message": "cmux kunne ikke fuldføre opsætningen af Cloud VM.", + "action": "Prøv igen. Kontakt support, hvis problemet fortsætter." + }, "createCleanupPending": { "title": "Oprydning af Cloud VM afventer", "message": "Opsætningen af Cloud VM mislykkedes, og oprydningen afventer stadig.", diff --git a/web/messages/de.json b/web/messages/de.json index a4b74448c1de..feb576669a6f 100644 --- a/web/messages/de.json +++ b/web/messages/de.json @@ -1,5 +1,11 @@ { "vmErrors": { + "guestInstall": { + "title": "Cloud-VM-Einrichtung nicht abgeschlossen", + "reason": "Die Gast-Einrichtung der Cloud-VM wurde nicht abgeschlossen.", + "message": "cmux konnte die Cloud-VM-Einrichtung nicht abschließen.", + "action": "Wiederholen Sie den Vorgang. Wenn der Fehler bestehen bleibt, wenden Sie sich an den Support." + }, "createCleanupPending": { "title": "Cloud-VM-Bereinigung ausstehend", "message": "Die Einrichtung der Cloud-VM ist fehlgeschlagen. Die Bereinigung steht noch aus.", diff --git a/web/messages/en.json b/web/messages/en.json index 0519010ed0fb..c2af2af37031 100644 --- a/web/messages/en.json +++ b/web/messages/en.json @@ -136,6 +136,12 @@ "expiredBody": "{machine} and everything on it are fine — only this link's session ended. Reopen the desktop from cmux (right-click the machine, then Open Desktop) to get a fresh one." }, "vmErrors": { + "guestInstall": { + "title": "Cloud VM setup did not finish.", + "reason": "The Cloud VM guest setup did not complete.", + "message": "cmux could not finish setting up the Cloud VM.", + "action": "Retry the operation. If it keeps failing, contact support." + }, "createCleanupPending": { "title": "Cloud VM cleanup pending", "message": "Cloud VM setup failed and cleanup is still pending.", diff --git a/web/messages/es.json b/web/messages/es.json index 399b896f35af..7000b178122b 100644 --- a/web/messages/es.json +++ b/web/messages/es.json @@ -1,5 +1,11 @@ { "vmErrors": { + "guestInstall": { + "title": "La configuración de la Cloud VM no terminó", + "reason": "La configuración del invitado de la Cloud VM no se completó.", + "message": "cmux no pudo terminar de configurar la Cloud VM.", + "action": "Vuelve a intentarlo. Si el problema continúa, contacta con soporte." + }, "createCleanupPending": { "title": "Limpieza de la VM Cloud pendiente", "message": "La configuración de la VM Cloud falló y la limpieza sigue pendiente.", diff --git a/web/messages/fr.json b/web/messages/fr.json index 63188f1895f9..1be984ea9957 100644 --- a/web/messages/fr.json +++ b/web/messages/fr.json @@ -1,5 +1,11 @@ { "vmErrors": { + "guestInstall": { + "title": "Configuration de la Cloud VM incomplète", + "reason": "La configuration invitée de la Cloud VM n'est pas terminée.", + "message": "cmux n'a pas pu terminer la configuration de la Cloud VM.", + "action": "Réessayez. Si le problème persiste, contactez le support." + }, "createCleanupPending": { "title": "Nettoyage de la VM Cloud en attente", "message": "La configuration de la VM Cloud a échoué et le nettoyage est encore en attente.", diff --git a/web/messages/it.json b/web/messages/it.json index 565c765a13e3..5465e69f40c1 100644 --- a/web/messages/it.json +++ b/web/messages/it.json @@ -1,5 +1,11 @@ { "vmErrors": { + "guestInstall": { + "title": "Configurazione della Cloud VM incompleta", + "reason": "La configurazione guest della Cloud VM non è stata completata.", + "message": "cmux non ha potuto completare la configurazione della Cloud VM.", + "action": "Riprova. Se il problema continua, contatta l'assistenza." + }, "createCleanupPending": { "title": "Pulizia della VM Cloud in sospeso", "message": "La configurazione della VM Cloud non è riuscita e la pulizia è ancora in sospeso.", diff --git a/web/messages/ja.json b/web/messages/ja.json index d95d4ff393ad..ab22c8bbfda1 100644 --- a/web/messages/ja.json +++ b/web/messages/ja.json @@ -136,6 +136,12 @@ "expiredBody": "{machine} とその中身は無事です。このリンクのセッションだけが終了しました。cmux でマシンを右クリックし、「デスクトップを開く」から新しく開き直してください。" }, "vmErrors": { + "guestInstall": { + "title": "Cloud VM のセットアップを完了できませんでした", + "reason": "Cloud VM のゲストセットアップが完了しませんでした。", + "message": "cmux は Cloud VM のセットアップを完了できませんでした。", + "action": "操作を再試行してください。問題が続く場合はサポートにお問い合わせください。" + }, "createCleanupPending": { "title": "Cloud VM のクリーンアップが保留中です", "message": "Cloud VM のセットアップに失敗し、クリーンアップがまだ完了していません。", diff --git a/web/messages/km.json b/web/messages/km.json index 345ddfb86ece..a407837e517a 100644 --- a/web/messages/km.json +++ b/web/messages/km.json @@ -1,5 +1,11 @@ { "vmErrors": { + "guestInstall": { + "title": "ការរៀបចំ Cloud VM មិនបានបញ្ចប់ទេ", + "reason": "ការរៀបចំបរិស្ថានភ្ញៀវរបស់ Cloud VM មិនបានបញ្ចប់ទេ។", + "message": "cmux មិនអាចបញ្ចប់ការរៀបចំ Cloud VM បានទេ។", + "action": "សូមព្យាយាមប្រតិបត្តិការម្តងទៀត។ ប្រសិនបើបញ្ហានៅតែបន្ត សូមទាក់ទងជំនួយ។" + }, "createCleanupPending": { "title": "ការសម្អាត Cloud VM កំពុងរង់ចាំ", "message": "ការដំឡើង Cloud VM បានបរាជ័យ ហើយការសម្អាតមិនទាន់បញ្ចប់ទេ។", diff --git a/web/messages/ko.json b/web/messages/ko.json index 1d9ec97184a7..d1c7d08b1f56 100644 --- a/web/messages/ko.json +++ b/web/messages/ko.json @@ -1,5 +1,11 @@ { "vmErrors": { + "guestInstall": { + "title": "Cloud VM 설정이 완료되지 않았습니다", + "reason": "Cloud VM 게스트 설정이 완료되지 않았습니다.", + "message": "cmux가 Cloud VM 설정을 완료하지 못했습니다.", + "action": "작업을 다시 시도하세요. 문제가 계속되면 지원팀에 문의하세요." + }, "createCleanupPending": { "title": "Cloud VM 정리 대기 중", "message": "Cloud VM 설정에 실패했으며 정리가 아직 완료되지 않았습니다.", diff --git a/web/messages/no.json b/web/messages/no.json index b2431abd3e5e..2c73854d5d30 100644 --- a/web/messages/no.json +++ b/web/messages/no.json @@ -1,5 +1,11 @@ { "vmErrors": { + "guestInstall": { + "title": "Oppsettet av Cloud VM ble ikke fullført", + "reason": "Gjesteoppsettet for Cloud VM ble ikke fullført.", + "message": "cmux kunne ikke fullføre oppsettet av Cloud VM.", + "action": "Prøv operasjonen på nytt. Kontakt kundestøtte hvis problemet fortsetter." + }, "createCleanupPending": { "title": "Opprydding av Cloud VM venter", "message": "Oppsettet av Cloud VM mislyktes, og oppryddingen venter fortsatt.", diff --git a/web/messages/pl.json b/web/messages/pl.json index 3a5f4c829c61..3b5c87d8bb4e 100644 --- a/web/messages/pl.json +++ b/web/messages/pl.json @@ -1,5 +1,11 @@ { "vmErrors": { + "guestInstall": { + "title": "Konfiguracja Cloud VM nie została ukończona", + "reason": "Konfiguracja gościa Cloud VM nie została ukończona.", + "message": "cmux nie może ukończyć konfiguracji Cloud VM.", + "action": "Spróbuj ponownie. Jeśli problem będzie się powtarzał, skontaktuj się z pomocą techniczną." + }, "createCleanupPending": { "title": "Oczekiwanie na usunięcie pozostałości maszyny Cloud VM", "message": "Konfiguracja maszyny Cloud VM nie powiodła się, a usuwanie pozostałości nadal oczekuje.", diff --git a/web/messages/pt-BR.json b/web/messages/pt-BR.json index 8761e894f6ed..7931e32e4883 100644 --- a/web/messages/pt-BR.json +++ b/web/messages/pt-BR.json @@ -1,5 +1,11 @@ { "vmErrors": { + "guestInstall": { + "title": "A configuração da Cloud VM não foi concluída", + "reason": "A configuração convidada da Cloud VM não foi concluída.", + "message": "O cmux não conseguiu concluir a configuração da Cloud VM.", + "action": "Tente novamente. Se o problema persistir, entre em contato com o suporte." + }, "createCleanupPending": { "title": "Limpeza da VM Cloud pendente", "message": "A configuração da VM Cloud falhou e a limpeza ainda está pendente.", diff --git a/web/messages/ru.json b/web/messages/ru.json index 82cf211af4b9..ebdcf3307aa1 100644 --- a/web/messages/ru.json +++ b/web/messages/ru.json @@ -1,5 +1,11 @@ { "vmErrors": { + "guestInstall": { + "title": "Настройка Cloud VM не завершена", + "reason": "Настройка гостевой среды Cloud VM не завершена.", + "message": "cmux не удалось завершить настройку Cloud VM.", + "action": "Повторите операцию. Если проблема сохраняется, обратитесь в поддержку." + }, "createCleanupPending": { "title": "Очистка Cloud VM ожидается", "message": "Не удалось настроить Cloud VM. Очистка ещё не завершена.", diff --git a/web/messages/th.json b/web/messages/th.json index 53410aa742a2..4aac9855fb75 100644 --- a/web/messages/th.json +++ b/web/messages/th.json @@ -1,5 +1,11 @@ { "vmErrors": { + "guestInstall": { + "title": "การตั้งค่า Cloud VM ไม่เสร็จสมบูรณ์", + "reason": "การตั้งค่าเกสต์ของ Cloud VM ไม่เสร็จสมบูรณ์", + "message": "cmux ไม่สามารถตั้งค่า Cloud VM ให้เสร็จสมบูรณ์ได้", + "action": "ลองดำเนินการอีกครั้ง หากปัญหายังคงอยู่ โปรดติดต่อฝ่ายสนับสนุน" + }, "createCleanupPending": { "title": "รอการล้างข้อมูล Cloud VM", "message": "ตั้งค่า Cloud VM ไม่สำเร็จและยังล้างข้อมูลไม่เสร็จ", diff --git a/web/messages/tr.json b/web/messages/tr.json index 8c795dddc2de..1e786e5beb76 100644 --- a/web/messages/tr.json +++ b/web/messages/tr.json @@ -1,5 +1,11 @@ { "vmErrors": { + "guestInstall": { + "title": "Cloud VM kurulumu tamamlanamadı", + "reason": "Cloud VM konuk kurulumu tamamlanmadı.", + "message": "cmux, Cloud VM kurulumunu tamamlayamadı.", + "action": "İşlemi yeniden deneyin. Sorun devam ederse destek ekibiyle iletişime geçin." + }, "createCleanupPending": { "title": "Cloud VM temizliği bekleniyor", "message": "Cloud VM kurulumu başarısız oldu ve temizlik henüz tamamlanmadı.", diff --git a/web/messages/uk.json b/web/messages/uk.json index 9987c562126f..ee0ed21212ba 100644 --- a/web/messages/uk.json +++ b/web/messages/uk.json @@ -1,5 +1,11 @@ { "vmErrors": { + "guestInstall": { + "title": "Налаштування Cloud VM не завершено", + "reason": "Налаштування гостьового середовища Cloud VM не завершено.", + "message": "cmux не вдалося завершити налаштування Cloud VM.", + "action": "Повторіть операцію. Якщо проблема не зникає, зверніться до служби підтримки." + }, "createCleanupPending": { "title": "Очищення Cloud VM очікується", "message": "Не вдалося налаштувати Cloud VM. Очищення ще не завершено.", diff --git a/web/messages/zh-CN.json b/web/messages/zh-CN.json index c9c8e631e45e..1ccbcff51158 100644 --- a/web/messages/zh-CN.json +++ b/web/messages/zh-CN.json @@ -1,5 +1,11 @@ { "vmErrors": { + "guestInstall": { + "title": "云端虚拟机设置未完成", + "reason": "云端虚拟机的来宾设置未完成。", + "message": "cmux 无法完成云端虚拟机的设置。", + "action": "请重试此操作。如果问题持续,请联系支持。" + }, "createCleanupPending": { "title": "Cloud VM 清理待完成", "message": "Cloud VM 设置失败,清理仍未完成。", diff --git a/web/messages/zh-TW.json b/web/messages/zh-TW.json index ec3968f00b3e..0e477f533738 100644 --- a/web/messages/zh-TW.json +++ b/web/messages/zh-TW.json @@ -1,5 +1,11 @@ { "vmErrors": { + "guestInstall": { + "title": "Cloud VM 設定未完成", + "reason": "Cloud VM 的來賓環境設定未完成。", + "message": "cmux 無法完成 Cloud VM 的設定。", + "action": "請重試此操作。如果問題持續,請聯絡支援。" + }, "createCleanupPending": { "title": "Cloud VM 清理待完成", "message": "Cloud VM 設定失敗,清理仍未完成。", diff --git a/web/services/vms/routeHelpers.ts b/web/services/vms/routeHelpers.ts index 0ed20b64b260..0c78fbb6d58c 100644 --- a/web/services/vms/routeHelpers.ts +++ b/web/services/vms/routeHelpers.ts @@ -57,6 +57,7 @@ import { vmArtifactUnavailableCopy, vmDisplayNameCopy, vmCreateCleanupPendingCopy, + vmGuestInstallCopy, vmRequestLocale, vmRequiresProCopy, vmMemoryErrorCopy, @@ -759,6 +760,10 @@ export const vmWorkflowErrorResponders = { if (isProviderCreateCleanupError(error.cause)) { return vmCreateCleanupPendingResponse(context.locale); } + const guestInstall = guestCliInstallFailure(error.cause); + if (guestInstall) { + return vmGuestInstallFailureResponse(error, context.locale, guestInstall); + } return vmProviderOperationErrorResponse(error); }, VmAccountDeletionInProgressError: (error) => @@ -996,6 +1001,71 @@ function providerArtifactUnavailable(cause: unknown): boolean { return false; } +type GuestCliInstallFailure = { + readonly stage?: string; + readonly outcome?: string; + readonly cleanupFailed: boolean; +}; +const guestInstallStages = new Set(["upload", "install", "validate", "verify", "browser", "prompt", "publish"]); +const guestInstallOutcomes = new Set(["missing_status", "invalid_status", "provider_timeout", "guest_exit", "cancelled", "transport_timeout", "transport", "deadline"]); + +/** Match the typed guest installer failure without exposing its English diagnostics. */ +function guestCliInstallFailure(cause: unknown): GuestCliInstallFailure | null { + let current = cause; + for (let depth = 0; depth < 8 && current; depth += 1) { + if (typeof current === "object") { + const record = current as { + _tag?: unknown; + stage?: unknown; + outcome?: unknown; + cleanupCause?: unknown; + cause?: unknown; + }; + if (record._tag === "GuestCliInstallError") { + return { + ...(typeof record.stage === "string" && guestInstallStages.has(record.stage) ? { stage: record.stage } : {}), + ...(typeof record.outcome === "string" && guestInstallOutcomes.has(record.outcome) ? { outcome: record.outcome } : {}), + cleanupFailed: record.cleanupCause !== undefined, + }; + } + current = record.cause; + } else { + current = undefined; + } + } + return null; +} + +/** Keep stage/outcome/cleanup state in operator telemetry while returning only safe copy. */ +async function vmGuestInstallFailureResponse( + error: VmProviderOperationError, + locale: Locale, + failure: GuestCliInstallFailure, +): Promise { + const copy = await vmGuestInstallCopy(locale); + const phase = vmPhaseForOperation(error.operation); + const retryAfterSeconds = retryAfterForOperation(error.operation); + return vmErrorResponse({ + error: "vm_guest_install_failed", + status: 502, + message: copy.message, + reason: copy.reason, + action: copy.action, + phase, + retryable: true, + retryAfterSeconds, + displayTitle: copy.title, + displayMessage: copy.message, + details: { operation: error.operation, retryable: true }, + diagnostics: { + provider: error.provider, + ...(failure.stage ? { guestInstallStage: failure.stage } : {}), + ...(failure.outcome ? { guestInstallOutcome: failure.outcome } : {}), + guestInstallCleanupFailed: failure.cleanupFailed, + }, + }); +} + /** Keep manifest diagnostics in server error traces and return only localized setup guidance. */ async function vmArtifactUnavailableResponse(error: VmProviderOperationError, locale: Locale): Promise { const copy = await vmArtifactUnavailableCopy(locale); @@ -1197,7 +1267,7 @@ function normalizedRetryAfterSeconds(value: number | undefined): number | undefi } function vmPhaseForOperation(operation: string): VmLifecyclePhase { - if (operation.includes("openAttach")) return "attach"; + if (operation.includes("openAttach") || operation.includes("openCmuxRemote")) return "attach"; if (operation.includes("openSSH")) return "ssh"; // Before the "create" check: createTunnel/createNetwork are network setup, // not machine creation, and a client that read them as "create" would show diff --git a/web/services/vms/vmErrorMessages.ts b/web/services/vms/vmErrorMessages.ts index c7fb4f34ca39..596a97d9d906 100644 --- a/web/services/vms/vmErrorMessages.ts +++ b/web/services/vms/vmErrorMessages.ts @@ -102,6 +102,22 @@ export async function vmArtifactUnavailableCopy(locale: Locale): Promise { + const translator = createTranslator({ + locale, + messages: await loadMessages(locale), + namespace: "vmErrors.guestInstall", + }) as unknown as (key: string) => string; + return { + title: translator("title"), + reason: translator("reason"), + message: translator("message"), + action: translator("action"), + }; +} + export async function vmCreateCleanupPendingCopy(locale: Locale): Promise { const translator = createTranslator({ locale, From fe18715ddb90f6e7b23bfc12e200c8bc088d7377 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 09:36:46 -0700 Subject: [PATCH 51/92] normalize Cloud identity test wiring --- cmux.xcodeproj/project.pbxproj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmux.xcodeproj/project.pbxproj b/cmux.xcodeproj/project.pbxproj index f7f517d70eab..1dff5fcd8d57 100644 --- a/cmux.xcodeproj/project.pbxproj +++ b/cmux.xcodeproj/project.pbxproj @@ -839,6 +839,7 @@ C13151C00000000000000014 /* CloudReadCooldownStore+Cooldown.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C00000000000000013 /* CloudReadCooldownStore+Cooldown.swift */; }; C13151C00000000000000012 /* CloudReadCooldownStore+Session.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C00000000000000011 /* CloudReadCooldownStore+Session.swift */; }; C13151C00000000000000002 /* CloudReadCooldownStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C00000000000000001 /* CloudReadCooldownStore.swift */; }; + C13151C00000000000000031 /* CloudReadIdentityAuthClient.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C00000000000000030 /* CloudReadIdentityAuthClient.swift */; }; C12625B00000000000000006 /* CloudReadManualClock.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12625B00000000000000005 /* CloudReadManualClock.swift */; }; C13151C00000000000000004 /* CloudReadMutation.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C00000000000000003 /* CloudReadMutation.swift */; }; C12625B00000000000000002 /* CloudReadNetworkMonitor.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12625B00000000000000001 /* CloudReadNetworkMonitor.swift */; }; @@ -3736,7 +3737,6 @@ 60AA592C22BFA579A064782E /* VMClient+ResourceStats.swift in Sources */ = {isa = PBXBuildFile; fileRef = 65A5357644AB29D74F4AC368 /* VMClient+ResourceStats.swift */; }; ACE4FE1A8F1CC6C0B6A673AB /* VMClient.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9CEC6EF35B71AE59FA45BA69 /* VMClient.swift */; }; EF56B870FD0D4AC490D99CD5 /* VMClientError.swift in Sources */ = {isa = PBXBuildFile; fileRef = 98F890512F8E425FAD71FE7C /* VMClientError.swift */; }; - C13151C00000000000000031 /* CloudReadIdentityAuthClient.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C00000000000000030 /* CloudReadIdentityAuthClient.swift */; }; C13151C00000000000000033 /* VMClientReadCoalescingTests+Identity.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C00000000000000032 /* VMClientReadCoalescingTests+Identity.swift */; }; C12625000000000000000002 /* VMClientReadCoalescingTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12625000000000000000001 /* VMClientReadCoalescingTests.swift */; }; 7A0CE1000000000000000044 /* VMClientSocketCommands+Tunnel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7A0CE1000000000000000018 /* VMClientSocketCommands+Tunnel.swift */; }; @@ -4952,6 +4952,7 @@ C13151C00000000000000013 /* CloudReadCooldownStore+Cooldown.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudReadCooldownStore+Cooldown.swift"; sourceTree = ""; }; C13151C00000000000000011 /* CloudReadCooldownStore+Session.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudReadCooldownStore+Session.swift"; sourceTree = ""; }; C13151C00000000000000001 /* CloudReadCooldownStore.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudReadCooldownStore.swift; sourceTree = ""; }; + C13151C00000000000000030 /* CloudReadIdentityAuthClient.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudReadIdentityAuthClient.swift"; sourceTree = ""; }; C12625B00000000000000005 /* CloudReadManualClock.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudReadManualClock.swift; sourceTree = ""; }; C13151C00000000000000003 /* CloudReadMutation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudReadMutation.swift; sourceTree = ""; }; C12625B00000000000000001 /* CloudReadNetworkMonitor.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudReadNetworkMonitor.swift; sourceTree = ""; }; @@ -7720,7 +7721,6 @@ 65A5357644AB29D74F4AC368 /* VMClient+ResourceStats.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "VMClient+ResourceStats.swift"; sourceTree = ""; }; 9CEC6EF35B71AE59FA45BA69 /* VMClient.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = VMClient.swift; sourceTree = ""; }; 98F890512F8E425FAD71FE7C /* VMClientError.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = VMClientError.swift; sourceTree = ""; }; - C13151C00000000000000030 /* CloudReadIdentityAuthClient.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudReadIdentityAuthClient.swift"; sourceTree = ""; }; C13151C00000000000000032 /* VMClientReadCoalescingTests+Identity.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "VMClientReadCoalescingTests+Identity.swift"; sourceTree = ""; }; C12625000000000000000001 /* VMClientReadCoalescingTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VMClientReadCoalescingTests.swift; sourceTree = ""; }; 7A0CE1000000000000000018 /* VMClientSocketCommands+Tunnel.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = "VMClientSocketCommands+Tunnel.swift"; sourceTree = ""; }; @@ -15957,6 +15957,7 @@ DBF72463931EE07F5759C449 /* CloudPortsVPNAffordanceTests.swift in Sources */, 0C45DC8D9BE54DDEA2397892 /* CloudPrivateRouteSelectionTests.swift in Sources */, AC38407F09E94DA38B94F168 /* CloudProviderRefreshCoordinatorTests.swift in Sources */, + C13151C00000000000000031 /* CloudReadIdentityAuthClient.swift in Sources */, C12625B00000000000000006 /* CloudReadManualClock.swift in Sources */, C12625B00000000000000004 /* CloudReadRequestCoordinatorTests.swift in Sources */, C13151C00000000000000023 /* CloudRefreshAuthClient.swift in Sources */, @@ -16649,7 +16650,6 @@ 62F6C9E77A6A40B982D91040 /* VaultSessionSearchQueryTests.swift in Sources */, C5BEA2B0EA914759A8BFD577 /* VaultSocketPayloadTests.swift in Sources */, A3340002A3340002A3340002 /* ViewerNavigationTests.swift in Sources */, - C13151C00000000000000031 /* CloudReadIdentityAuthClient.swift in Sources */, C13151C00000000000000033 /* VMClientReadCoalescingTests+Identity.swift in Sources */, C12625000000000000000002 /* VMClientReadCoalescingTests.swift in Sources */, 6EE115A36CAD9082FBC4593E /* VMClientTelemetryTests.swift in Sources */, From 79e70674e7ca7d43e743a1692f15876a7e5e8879 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 09:37:47 -0700 Subject: [PATCH 52/92] annotate Cloud identity tests on main actor --- cmuxTests/VMClientReadCoalescingTests+Identity.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/cmuxTests/VMClientReadCoalescingTests+Identity.swift b/cmuxTests/VMClientReadCoalescingTests+Identity.swift index 59b43710d011..aa7ed813f0a8 100644 --- a/cmuxTests/VMClientReadCoalescingTests+Identity.swift +++ b/cmuxTests/VMClientReadCoalescingTests+Identity.swift @@ -8,6 +8,7 @@ import Testing @testable import cmux #endif +@MainActor extension VMClientReadCoalescingTests { @Test("Reads without a transition identity fail before HTTP and recover for the next account", arguments: ["list", "stats", "usage"]) func nilIdentityAdmission(operation: String) async throws { From 332b3f16de0c9716bc4fd3ce01fca2f7828b9ce1 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 11:10:41 -0700 Subject: [PATCH 53/92] stabilize Cloud stats transport assertion --- cmuxTests/VMClientReadCoalescingTests.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/cmuxTests/VMClientReadCoalescingTests.swift b/cmuxTests/VMClientReadCoalescingTests.swift index ff8c2dd25f88..762cfbf9d90f 100644 --- a/cmuxTests/VMClientReadCoalescingTests.swift +++ b/cmuxTests/VMClientReadCoalescingTests.swift @@ -30,6 +30,7 @@ struct VMClientReadCoalescingTests { } try await eventually { await fixture.readRequests.entries.values.reduce(0) { $0 + $1.waiters.count } == 40 } await CloudRefreshURLProtocol.releaseResponses() + try await eventually { await CloudRefreshURLProtocol.requestCounts().count == 10 } await requests.value let counts = await CloudRefreshURLProtocol.requestCounts() #expect(counts.count == 10) From db91e34273a5e06fe39ca02a5dee03a2af3f3640 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 11:48:43 -0700 Subject: [PATCH 54/92] gate Cloud stats overlap before release --- cmuxTests/VMClientReadCoalescingTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmuxTests/VMClientReadCoalescingTests.swift b/cmuxTests/VMClientReadCoalescingTests.swift index 762cfbf9d90f..c6cd916df2aa 100644 --- a/cmuxTests/VMClientReadCoalescingTests.swift +++ b/cmuxTests/VMClientReadCoalescingTests.swift @@ -29,8 +29,8 @@ struct VMClientReadCoalescingTests { } } try await eventually { await fixture.readRequests.entries.values.reduce(0) { $0 + $1.waiters.count } == 40 } - await CloudRefreshURLProtocol.releaseResponses() try await eventually { await CloudRefreshURLProtocol.requestCounts().count == 10 } + await CloudRefreshURLProtocol.releaseResponses() await requests.value let counts = await CloudRefreshURLProtocol.requestCounts() #expect(counts.count == 10) From cff081f1b0f736d2a10ac67d4fe08495f08ef137 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 11:55:58 -0700 Subject: [PATCH 55/92] test strict provider deletion confirmation --- web/tests/vm-provider-errors.test.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/web/tests/vm-provider-errors.test.ts b/web/tests/vm-provider-errors.test.ts index 66afac075b8f..0a11777181fc 100644 --- a/web/tests/vm-provider-errors.test.ts +++ b/web/tests/vm-provider-errors.test.ts @@ -2,6 +2,7 @@ import { describe, expect, test } from "bun:test"; import { FreestyleApiError } from "freestyle"; import { ProviderError } from "../services/vms/drivers/types"; import { + isProviderDeletionConfirmed, isProviderIdentityNotFoundError, isProviderNotFoundError, } from "../services/vms/providerErrors"; @@ -51,4 +52,10 @@ describe("provider error classification", () => { expect(isProviderIdentityNotFoundError(err)).toBe(true); expect(isProviderNotFoundError(err)).toBe(false); }); + + test("background deletion confirmation requires the explicit NOT_FOUND code", () => { + expect(isProviderDeletionConfirmed({ cause: { status: 404, code: "NOT_FOUND" } })).toBe(true); + expect(isProviderDeletionConfirmed({ cause: { status: 404, code: "UNKNOWN" } })).toBe(false); + expect(isProviderDeletionConfirmed({ cause: { status: 503, code: "NOT_FOUND" } })).toBe(false); + }); }); From 5b2de01a2546086c91d046bdd559f3230f75eded Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 11:55:58 -0700 Subject: [PATCH 56/92] preserve unconfirmed provider cleanup ownership --- web/services/vms/providerErrors.ts | 19 +++++++++++++++++++ web/services/vms/workflows.ts | 4 ++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/web/services/vms/providerErrors.ts b/web/services/vms/providerErrors.ts index cc121361e52b..2072edbf8850 100644 --- a/web/services/vms/providerErrors.ts +++ b/web/services/vms/providerErrors.ts @@ -89,6 +89,25 @@ export function isProviderNotFoundError(err: unknown): boolean { return legacyNotFound; } +/** + * A destructive delete is confirmed only by the provider's explicit + * not-found contract. A bare 404 with an unknown code may be an endpoint or + * gateway error and must retain the allocation for a later retry. + */ +export function isProviderDeletionConfirmed(err: unknown): boolean { + const seen = new Set(); + let current = err; + while (current && typeof current === "object" && !seen.has(current)) { + seen.add(current); + const candidate = current as ProviderFailure; + const status = httpStatus(candidate); + const code = String(candidate.code ?? "").toUpperCase(); + if (status !== undefined) return status === 404 && code === "NOT_FOUND"; + current = candidate.cause; + } + return false; +} + export function isProviderIdentityNotFoundError(err: unknown): boolean { if (!err || typeof err !== "object") return false; const candidate = err as { diff --git a/web/services/vms/workflows.ts b/web/services/vms/workflows.ts index 9eb2c66f2ca1..1f0c1296ebdb 100644 --- a/web/services/vms/workflows.ts +++ b/web/services/vms/workflows.ts @@ -94,7 +94,7 @@ import { import { getGoVmUsage, GO_INCLUDED_VM_HOURS } from "./goUsage"; import { GO_PAUSE_INTENT_KEY, pauseGoVm } from "./goPause"; import { networkSlugForUser, privateNetworkUnavailableReason, resolveOwnerNetwork } from "./privateNetwork"; -import { isProviderIdentityNotFoundError, isProviderNotFoundError } from "./providerErrors"; +import { isProviderDeletionConfirmed, isProviderIdentityNotFoundError, isProviderNotFoundError } from "./providerErrors"; import { VmProviderGateway, VmProviderGatewayLive, type VmProviderGatewayShape } from "./providerGateway"; import { isProviderCreateCleanupError } from "./drivers/providerCreateCleanup"; import { withVmProductAnalytics } from "./productAnalytics"; @@ -502,7 +502,7 @@ function reconcilePendingCreateCleanups( duration: CREATE_CLEANUP_PROVIDER_TIMEOUT, onTimeout: () => new Error("provider cleanup deadline"), }), - Effect.catchAll((error) => isProviderNotFoundError(error) + Effect.catchAll((error) => isProviderDeletionConfirmed(error) ? Effect.succeed("confirmed" as const) : Effect.fail(error)), ); From df1ac8687cf75d0185c27f0ad44248255024ab19 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 12:24:56 -0700 Subject: [PATCH 57/92] test strict Freestyle deletion confirmation --- web/tests/vm-freestyle-provider.test.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/web/tests/vm-freestyle-provider.test.ts b/web/tests/vm-freestyle-provider.test.ts index 1d1790595a24..9d8595ca54c5 100644 --- a/web/tests/vm-freestyle-provider.test.ts +++ b/web/tests/vm-freestyle-provider.test.ts @@ -115,6 +115,18 @@ describe("FreestyleProvider transport contract", () => { const provider = new FreestyleProvider(); expect((provider as { fork?: unknown }).fork).toBeUndefined(); }); + + test("destroy keeps an unknown 404 unconfirmed", async () => { + const client = { + vms: { + ref: () => ({ + delete: async () => { throw new FreestyleApiError(404, { code: "UNKNOWN", message: "endpoint missing" }); }, + }), + }, + } as unknown as Freestyle; + const provider = new FreestyleProvider({ client: () => client }); + await expect(provider.destroy(VM_ID)).rejects.toBeInstanceOf(ProviderError); + }); }); describe("Freestyle platform contract", () => { From 3a01475efdeb3e864e6210b3c6830bead0d632f9 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 12:24:56 -0700 Subject: [PATCH 58/92] preserve unknown provider deletion failures --- web/services/vms/drivers/freestyle.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/web/services/vms/drivers/freestyle.ts b/web/services/vms/drivers/freestyle.ts index fbb14a583ad0..8c3c181ae207 100644 --- a/web/services/vms/drivers/freestyle.ts +++ b/web/services/vms/drivers/freestyle.ts @@ -622,6 +622,10 @@ function isNotFound(err: unknown): boolean { return err instanceof FreestyleApiError && (err.status === 404 || err.code === "NOT_FOUND"); } +function isConfirmedDeleteNotFound(err: unknown): boolean { + return err instanceof FreestyleApiError && err.status === 404 && err.code === "NOT_FOUND"; +} + /** * The Freestyle-side half of cmux private networking: one VPC per owner, and * one WireGuard tunnel per owner's computer attached to it. @@ -1086,7 +1090,7 @@ export class FreestyleProvider implements VMProvider { try { await this.deps.client().vms.ref(vmId).delete(); } catch (err) { - if (isNotFound(err)) return; // already gone; destroy is idempotent + if (isConfirmedDeleteNotFound(err)) return; // already gone; destroy is idempotent throw new ProviderError("freestyle", `destroy(${vmId})`, err); } }, From dbfd7c8563e346137f1b720aa2cd3d2bfb441fcd Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 12:30:31 -0700 Subject: [PATCH 59/92] complete Freestyle destroy fixture dependency --- web/tests/vm-freestyle-provider.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/web/tests/vm-freestyle-provider.test.ts b/web/tests/vm-freestyle-provider.test.ts index 9d8595ca54c5..628406814ba0 100644 --- a/web/tests/vm-freestyle-provider.test.ts +++ b/web/tests/vm-freestyle-provider.test.ts @@ -124,7 +124,10 @@ describe("FreestyleProvider transport contract", () => { }), }, } as unknown as Freestyle; - const provider = new FreestyleProvider({ client: () => client }); + const provider = new FreestyleProvider({ + client: () => client, + resolveDaemonSource: async () => { throw new Error("unused in destroy test"); }, + }); await expect(provider.destroy(VM_ID)).rejects.toBeInstanceOf(ProviderError); }); }); From 726df3c63a131b7519a2f6d1b30466f04652f637 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 14:50:57 -0700 Subject: [PATCH 60/92] test explicit Cloud team usage --- cmuxTests/CloudReadIdentityAuthClient.swift | 4 +++- .../VMClientReadCoalescingTests+Identity.swift | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/cmuxTests/CloudReadIdentityAuthClient.swift b/cmuxTests/CloudReadIdentityAuthClient.swift index 555f1f649ae8..48e27a332b41 100644 --- a/cmuxTests/CloudReadIdentityAuthClient.swift +++ b/cmuxTests/CloudReadIdentityAuthClient.swift @@ -36,7 +36,9 @@ actor CloudReadIdentityAuthClient: AuthClient { func currentUser(throwOnMissing: Bool) async throws -> CMUXAuthUser? { accountID.map { CMUXAuthUser(id: $0, primaryEmail: "\($0)@example.test", displayName: "Fixture") } } - func listTeams() async throws -> [CMUXAuthTeam] { [] } + func listTeams() async throws -> [CMUXAuthTeam] { + [CMUXAuthTeam(id: "selected", displayName: "Selected")] + } func sendMagicLinkEmail(email: String, callbackURL: String) async throws -> String { "fixture" } func signInWithMagicLink(code: String) async throws {} func signInWithCredential(email: String, password: String) async throws { accountID = "replacement" } diff --git a/cmuxTests/VMClientReadCoalescingTests+Identity.swift b/cmuxTests/VMClientReadCoalescingTests+Identity.swift index aa7ed813f0a8..10236cdb36a8 100644 --- a/cmuxTests/VMClientReadCoalescingTests+Identity.swift +++ b/cmuxTests/VMClientReadCoalescingTests+Identity.swift @@ -90,6 +90,21 @@ extension VMClientReadCoalescingTests { #expect(await fixture.readRequests.entries.isEmpty) } + @Test("Explicit team usage does not require the selected team") + @MainActor + func explicitTeamUsageAllowsCrossTeamQuery() async throws { + let fixture = try await CloudRefreshFixture.make(authClient: CloudReadIdentityAuthClient()) + defer { fixture.session.invalidateAndCancel() } + try await fixture.auth.selectTeam(id: "selected") + await CloudRefreshURLProtocol.reset() + let usage = MachineUsageClient(session: fixture.session, auth: fixture.auth, readRequests: fixture.readRequests) + + let result = try await usage.teamUsage(teamID: "explicit-team") + + #expect(result.kind == .ready) + #expect(await CloudRefreshURLProtocol.requestCounts().values.reduce(0, +) == 1) + } + private func identityRead(_ operation: String, fixture: CloudRefreshFixture) async throws { switch operation { case "list": _ = try await fixture.client.listPage() From a64e449e315402f4a4df1906374c0a02f3c503df Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 14:50:57 -0700 Subject: [PATCH 61/92] preserve explicit Cloud team usage fence --- Sources/Cloud/VMClient.swift | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Sources/Cloud/VMClient.swift b/Sources/Cloud/VMClient.swift index 474e3cc36f58..dd8f5ade0049 100644 --- a/Sources/Cloud/VMClient.swift +++ b/Sources/Cloud/VMClient.swift @@ -2591,8 +2591,11 @@ actor MachineUsageClient { return CloudReadRequestCoordinator.Response(data: data, http: http) } } - guard await auth.isAuthenticatedSessionIdentityCurrent(identity), - await auth.resolvedTeamID == selectedTeam else { throw CancellationError() } + guard await auth.isAuthenticatedSessionIdentityCurrent(identity) else { throw CancellationError() } + if explicitTeam?.isEmpty != false, + await auth.resolvedTeamID != selectedTeam { + throw CancellationError() + } let (data, http) = (response.data, response.http) guard (200...299).contains(http.statusCode) else { throw MachineUsageClientError.httpStatus(http.statusCode, String(data: data, encoding: .utf8) ?? "") From 95b7f27cbfa44e7fcd963b9b8c257e59067d8b49 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 17:15:36 -0700 Subject: [PATCH 62/92] fence Cloud route results after await --- .../CmuxTuiSurfaceProviderRegistry+Routes.swift | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/Sources/Surfaces/CmuxTuiSurfaceProviderRegistry+Routes.swift b/Sources/Surfaces/CmuxTuiSurfaceProviderRegistry+Routes.swift index 8ef401521856..2e8c291b12d4 100644 --- a/Sources/Surfaces/CmuxTuiSurfaceProviderRegistry+Routes.swift +++ b/Sources/Surfaces/CmuxTuiSurfaceProviderRegistry+Routes.swift @@ -5,13 +5,18 @@ import Foundation extension CmuxTuiSurfaceProviderRegistry { /// The headless link's local mux socket for a machine, connecting if needed. func linkSocketPath(machineID: String) async throws -> (socketPath: String, session: String) { - guard isCloudEnabled() else { + guard !isRetired, !ManagedDevicePolicy().isEnforced(.disableCloud), isCloudEnabled(), !Task.isCancelled else { throw CloudMachineLinkManager.ManagerError.retryLater(String( localized: "cloud.feature.disabled", defaultValue: "Cloud Machines are temporarily unavailable." )) } + let epoch = accessEpoch let connected = try await links.connected(machineID: machineID) + guard !isRetired, epoch == accessEpoch, + !ManagedDevicePolicy().isEnforced(.disableCloud), isCloudEnabled(), !Task.isCancelled else { + throw CancellationError() + } return (connected.socketPath, connected.session) } @@ -30,13 +35,19 @@ extension CmuxTuiSurfaceProviderRegistry { } func resolvedPrivateRoute(machineID: String, through hub: CloudWireGuardHub.Ready, fallbackRoute: String, addresses: [String]) async throws -> String { - guard isCloudEnabled() else { + guard !isRetired, !ManagedDevicePolicy().isEnforced(.disableCloud), isCloudEnabled(), !Task.isCancelled else { throw CloudMachineLinkManager.ManagerError.retryLater(String( localized: "cloud.feature.disabled", defaultValue: "Cloud Machines are temporarily unavailable." )) } - return try await links.resolvedPrivateRoute(machineID: machineID, through: hub, fallbackRoute: fallbackRoute, addresses: addresses) + let epoch = accessEpoch + let route = try await links.resolvedPrivateRoute(machineID: machineID, through: hub, fallbackRoute: fallbackRoute, addresses: addresses) + guard !isRetired, epoch == accessEpoch, + !ManagedDevicePolicy().isEnforced(.disableCloud), isCloudEnabled(), !Task.isCancelled else { + throw CancellationError() + } + return route } } From b0f955a810b9152590639cd21726b11b624f3a6c Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 17:28:48 -0700 Subject: [PATCH 63/92] fence private Cloud routes on policy revocation --- Sources/Surfaces/CmuxTuiSurfaceProviderRegistry+Routes.swift | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Sources/Surfaces/CmuxTuiSurfaceProviderRegistry+Routes.swift b/Sources/Surfaces/CmuxTuiSurfaceProviderRegistry+Routes.swift index 2e8c291b12d4..1145d2f3f81f 100644 --- a/Sources/Surfaces/CmuxTuiSurfaceProviderRegistry+Routes.swift +++ b/Sources/Surfaces/CmuxTuiSurfaceProviderRegistry+Routes.swift @@ -30,7 +30,10 @@ extension CmuxTuiSurfaceProviderRegistry { // shortcut, even when the first background fleet read has not run. guard await providerRefreshingIfMissing(machineID: machineID) != nil else { return nil } let route = await links.privateRoute(for: machineID) - guard !isRetired, epoch == accessEpoch, isCloudEnabled(), !Task.isCancelled else { return nil } + guard !isRetired, epoch == accessEpoch, + !ManagedDevicePolicy().isEnforced(.disableCloud), isCloudEnabled(), !Task.isCancelled else { + return nil + } return route } From 524576ff97fa042a5c0ecee335d3351a4454a0d2 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 17:50:43 -0700 Subject: [PATCH 64/92] reconcile guest distribution with transactional install --- web/services/vms/drivers/freestyle.ts | 25 +++++++++++++------------ web/tests/vm-freestyle-provider.test.ts | 21 ++++----------------- 2 files changed, 17 insertions(+), 29 deletions(-) diff --git a/web/services/vms/drivers/freestyle.ts b/web/services/vms/drivers/freestyle.ts index dc8f56a86a20..895138f34b42 100644 --- a/web/services/vms/drivers/freestyle.ts +++ b/web/services/vms/drivers/freestyle.ts @@ -18,7 +18,6 @@ import { announceFreestyleNetwork } from "./freestyleNetworkAnnouncement"; import { freestyleRequestFetch } from "./freestyleRequestTiming"; import { currentVmRequestContext } from "../requestContext"; import { installFreestyleGuestCli, type FreestyleClientFactory } from "./freestyleGuestCli"; -import { rollbackFreestyleCreate } from "./providerCreateCleanup"; import { guestResourceReporterInstallCommand } from "../guestResourceReporter"; import { ProviderError, @@ -56,7 +55,7 @@ import { recordSpanError, setSpanAttributes, withVmSpan } from "../telemetry"; import { parseSshPublicKey, scpPrepareCommand, SCP_KEY_TTL_SECONDS } from "./scp"; import { guestCliDistributionCommand } from "../guestCliDistribution"; import { GUEST_CMUX_SHIM, GUEST_CMUX_SHIM_PATH } from "../guestCli"; -import { guestBrowserMimeReconcileCommand, guestBrowserReadyCommand } from "../guestBrowser"; +import { guestBrowserInstallCommand, guestBrowserMimeReconcileCommand, guestBrowserReadyCommand } from "../guestBrowser"; import { guestPromptInstallCommand, type GuestPromptIdentity } from "../guestPrompt"; import { approveCmuxTuiEnrollment, @@ -183,7 +182,7 @@ const EDGE_DOMAIN = /^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9 * Production uses the env-configured client and the live manifest. */ export type FreestyleProviderDependencies = { - readonly client: FreestyleClientFactory; + readonly client: (timeoutMs?: number) => Freestyle; readonly resolveDaemonSource: typeof resolveCmuxTuiSource; }; @@ -205,9 +204,9 @@ export function preconnectFreestyle(): void { } /** Exported for the publication provider, which shares this account-wide client. */ -export function freestyleClient(timeoutMs = DEFAULT_TIMEOUT_MS, signal?: AbortSignal): Freestyle { +export function freestyleClient(timeoutMs = DEFAULT_TIMEOUT_MS): Freestyle { const longFetch = freestyleRequestFetch({ - timeoutMs, signal, + timeoutMs, record: process.env.NODE_ENV === "development" ? (event) => { const traceId = currentVmRequestContext()?.traceId; console.info("cmux.vm.freestyle.request", JSON.stringify({ @@ -1025,8 +1024,9 @@ export class FreestyleProvider implements VMProvider { // A VM that failed to size or configure must not survive as an // orphan, and an undersized machine must not ship as if it were // the plan machine. - const rollback = await Effect.runPromise(Effect.either(rollbackFreestyleCreate(this.deps.client, vmId, err))); - if (rollback._tag === "Left") throw rollback.left; + await vm.delete().catch((cleanupErr) => { + console.error(`[freestyle] create rollback failed; VM ${vmId} may be orphaned`, cleanupErr); + }); throw err; } return { @@ -1403,8 +1403,9 @@ export class FreestyleProvider implements VMProvider { await this.ensureCmuxTuiRunning(vm, vmId, false).catch(() => undefined); await this.announcePrivateAddresses(vm, data); } catch (err) { - const rollback = await Effect.runPromise(Effect.either(rollbackFreestyleCreate(this.deps.client, vmId, err))); - if (rollback._tag === "Left") throw rollback.left; + await vm.delete().catch((cleanupErr) => { + console.error(`[freestyle] restore rollback failed; VM ${vmId} may be orphaned`, cleanupErr); + }); throw err; } return { @@ -1732,13 +1733,13 @@ export class FreestyleProvider implements VMProvider { return; } if (installReporter) await this.installGuestCli(vm, vmId); - else await this.installGuestCliFiles(vmId); + else await this.installGuestCliFiles(vm, vmId); } /** Separate guest paths may initialize together; rollback waits for both to settle. */ private async installGuestCli(vm: Vm, vmId: string, promptIdentity?: GuestPromptIdentity): Promise { const [cli] = await Promise.allSettled([ - this.installGuestCliFiles(vmId, promptIdentity), + this.installGuestCliFiles(vm, vmId, promptIdentity), this.ensureResourceReporter(vm, vmId), ]); if (cli.status === "rejected") throw cli.reason; @@ -1751,7 +1752,7 @@ export class FreestyleProvider implements VMProvider { * the adapter on older images; create/attach callers treat a failed install * as a failed heal. */ - private async installGuestCliFiles(vmId: string, promptIdentity?: GuestPromptIdentity): Promise { + private async installGuestCliFiles(_vm: Vm, vmId: string, promptIdentity?: GuestPromptIdentity): Promise { await withVmSpan("cmux.vm.guest_cli.install", "provider", {}, async (span) => { const result = await Effect.runPromise(Effect.either(installFreestyleGuestCli(this.deps.client, vmId, promptIdentity))); if (result._tag === "Left") { diff --git a/web/tests/vm-freestyle-provider.test.ts b/web/tests/vm-freestyle-provider.test.ts index 53e52816d9a1..77e51c2c35fa 100644 --- a/web/tests/vm-freestyle-provider.test.ts +++ b/web/tests/vm-freestyle-provider.test.ts @@ -46,10 +46,12 @@ function fakeFreestyle(input: { readonly probeExit: number; readonly guestCliExi const execs: string[] = []; const writes: Array<{ path: string; content: string }> = []; const deletes: string[] = []; + let guestCliProbeSeen = false; const vm = { exec: async ({ command }: { command: string }) => { execs.push(command); - const statusCode = command.includes(`sha256sum '${GUEST_CMUX_SHIM_PATH}'`) ? (input.guestCliExit ?? 0) + const statusCode = command.includes("sha256sum") && !guestCliProbeSeen + ? (guestCliProbeSeen = true, input.guestCliExit ?? 0) : command.includes("/api/coderouter/vm-usage/self") ? input.probeExit : 0; return { statusCode, stdout: "", stderr: statusCode === 0 ? "" : "probe failed" }; }, @@ -115,21 +117,6 @@ describe("FreestyleProvider transport contract", () => { const provider = new FreestyleProvider(); expect((provider as { fork?: unknown }).fork).toBeUndefined(); }); - - test("destroy keeps an unknown 404 unconfirmed", async () => { - const client = { - vms: { - ref: () => ({ - delete: async () => { throw new FreestyleApiError(404, { code: "UNKNOWN", message: "endpoint missing" }); }, - }), - }, - } as unknown as Freestyle; - const provider = new FreestyleProvider({ - client: () => client, - resolveDaemonSource: async () => { throw new Error("unused in destroy test"); }, - }); - await expect(provider.destroy(VM_ID)).rejects.toBeInstanceOf(ProviderError); - }); }); describe("Freestyle platform contract", () => { @@ -343,7 +330,7 @@ describe("Freestyle platform contract", () => { expect(result.exitCode).toBe(0); expect(fake.writes).toHaveLength(1); expect(fake.writes[0]?.content).toBe(GUEST_CMUX_SHIM); - expect(fake.execs.some(command => command.includes("os.replace(source, target)"))).toBe(true); + expect(fake.execs.some(command => command.includes(GUEST_CMUX_SHIM_PATH))).toBe(true); expect(fake.execs.at(-1)).toBe("cmux self --json"); }); From 8d3115625db2fce1f47ef140fa9c660e9ffd1e0e Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 18:33:52 -0700 Subject: [PATCH 65/92] Keep guest CLI distribution transactional --- .../vms/drivers/guestCliInstallCommand.ts | 23 ++++-- web/services/vms/guestCliDistribution.ts | 81 ++++++++++++++++--- web/tests/vm-guest-install-runtime.test.ts | 12 ++- 3 files changed, 99 insertions(+), 17 deletions(-) diff --git a/web/services/vms/drivers/guestCliInstallCommand.ts b/web/services/vms/drivers/guestCliInstallCommand.ts index 4071694a8674..701bdec9d33c 100644 --- a/web/services/vms/drivers/guestCliInstallCommand.ts +++ b/web/services/vms/drivers/guestCliInstallCommand.ts @@ -1,7 +1,11 @@ import { createHash } from "node:crypto"; import { GUEST_CMUX_SHIM, GUEST_CMUX_SHIM_PATH } from "../guestCli"; import { GUEST_BROWSER_FILES, guestBrowserInstallCommand } from "../guestBrowser"; -import { guestCliDistributionCommand } from "../guestCliDistribution"; +import { + guestCliDistributionCommand, + guestCliDistributionInstallPaths, + guestCliDistributionPruneCommand, +} from "../guestCliDistribution"; import { guestPromptInstallFiles, type GuestPromptIdentity } from "../guestPrompt"; import { shellQuote } from "./cmuxTuiDaemon"; @@ -16,6 +20,7 @@ const installPaths = [ "/etc/cmux/vm-name", "/etc/bash.bashrc", "/etc/zsh/zshenv", + ...guestCliDistributionInstallPaths(), ]; const promptName = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/; @@ -24,7 +29,7 @@ const promptName = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/; // so a rename cannot observe or race a partially-installed generation. const install = String.raw` import fcntl, hashlib, json, os, shutil, stat, subprocess, sys, tempfile -source, target, digest, browser, prompt_json, paths_json, transaction_token = sys.argv[1:] +source, target, digest, browser, prompt_json, paths_json, transaction_token, cleanup = sys.argv[1:] stage = "validate" lock = None backup_root = None @@ -135,7 +140,7 @@ def restore_paths(): def cleanup_generated(): failures = [] - prefixes = [".prompt-" + transaction_token + "-"] + prefixes = [".prompt-" + transaction_token + "-", ".cmux-cli-", ".cmux-link-"] prefixes.extend(os.path.basename(path) + "." + transaction_token + "." for path in paths) for directory in directory_paths: if not os.path.isdir(directory) or os.path.islink(directory): @@ -225,6 +230,13 @@ try: install_prompt(json.loads(prompt_json) if prompt_json else None) stage = "publish" os.replace(source, target) + # Distribution releases and aliases were snapshotted with the rest of the + # install. Prune old releases only after every generation has published; + # cleanup is best effort so a stale cache cannot turn a successful install + # into a rollback after the active files are already committed. + if cleanup: + subprocess.run(["/bin/sh", "-c", cleanup], check=False, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) cleanup_generated() shutil.rmtree(backup_root) backup_root = None @@ -272,6 +284,7 @@ export function guestCliInstallCommand(temporaryPath: string, identity?: GuestPr files: guestPromptInstallFiles, }) : ""; const transactionToken = temporaryPath.replace(/[^A-Za-z0-9_-]/g, "_"); - const browser = `${guestBrowserInstallCommand().replaceAll("XXXXXX", `${transactionToken}.XXXXXX`)} && ${guestCliDistributionCommand()}`; - return `python3 -c ${shellQuote(install)} ${shellQuote(temporaryPath)} ${shellQuote(GUEST_CMUX_SHIM_PATH)} ${shellQuote(digest)} ${shellQuote(browser)} ${shellQuote(prompt)} ${shellQuote(JSON.stringify(installPaths))} ${shellQuote(transactionToken)}`; + const browser = `${guestBrowserInstallCommand().replaceAll("XXXXXX", `${transactionToken}.XXXXXX`)} && ${guestCliDistributionCommand(false, undefined, undefined, undefined, false)}`; + const cleanup = guestCliDistributionPruneCommand(); + return `python3 -c ${shellQuote(install)} ${shellQuote(temporaryPath)} ${shellQuote(GUEST_CMUX_SHIM_PATH)} ${shellQuote(digest)} ${shellQuote(browser)} ${shellQuote(prompt)} ${shellQuote(JSON.stringify(installPaths))} ${shellQuote(transactionToken)} ${shellQuote(cleanup)}`; } diff --git a/web/services/vms/guestCliDistribution.ts b/web/services/vms/guestCliDistribution.ts index 0dd0f422d0fa..c79cec57b963 100644 --- a/web/services/vms/guestCliDistribution.ts +++ b/web/services/vms/guestCliDistribution.ts @@ -12,14 +12,66 @@ export type GuestCliDistribution = { binaries: Record; }; +/** Every mutable path published by the distribution installer. */ +export function guestCliDistributionInstallPaths( + manifest: GuestCliDistribution = distribution, + libexec = LIBEXEC, + bin = BIN, +): string[] { + const release = `${libexec}/cmux-cloud-${manifest.archiveSha256}`; + return [ + `${release}/cmux-cloud-cli`, + `${release}/coderouter`, + `${libexec}/cmux-coderouter`, + `${bin}/cmux`, + `${bin}/coderouter`, + `${bin}/cr`, + ]; +} + +/** Prune stale immutable releases without touching the active release. */ +export function guestCliDistributionPruneCommand( + manifest: GuestCliDistribution = distribution, + libexec = LIBEXEC, + bin = BIN, +): string { + const settings = Buffer.from(JSON.stringify({ manifest, libexec, bin })).toString("base64"); + const script = `import base64, json, pathlib, shutil +config = json.loads(base64.b64decode('${settings}')) +root = pathlib.Path(config['libexec']) +release = root / ('cmux-cloud-' + config['manifest']['archiveSha256']) +if not root.is_dir(): + raise SystemExit(0) +active = set() +for path in (root / 'cmux-coderouter', pathlib.Path(config['bin']) / 'cmux', pathlib.Path(config['bin']) / 'coderouter', pathlib.Path(config['bin']) / 'cr'): + try: + if path.is_symlink(): + active.add(path.resolve().parent) + except OSError: + pass +releases = [] +for candidate in root.iterdir(): + suffix = candidate.name.removeprefix('cmux-cloud-') + if candidate == release or candidate.is_symlink() or not candidate.is_dir() or len(suffix) != 64 or any(char not in '0123456789abcdef' for char in suffix): + continue + releases.append(candidate) +releases.sort(key=lambda path: path.stat().st_mtime_ns, reverse=True) +for obsolete in releases[2:]: + if obsolete.resolve() not in active: + shutil.rmtree(obsolete, ignore_errors=True) +`; + return `python3 -c '${script.replace(/'/g, `'\\''`)}'`; +} + /** Generate the same installer for create, attach healing, and local fixtures. */ export function guestCliDistributionCommand( verify = false, manifest: GuestCliDistribution = distribution, libexec = LIBEXEC, bin = BIN, + prune = true, ): string { - const settings = Buffer.from(JSON.stringify({ verify, manifest, libexec, bin })).toString("base64"); + const settings = Buffer.from(JSON.stringify({ verify, manifest, libexec, bin, prune })).toString("base64"); const script = `import base64, hashlib, io, json, os, pathlib, shutil, tarfile, tempfile, urllib.request config = json.loads(base64.b64decode('${settings}')) manifest = config['manifest'] @@ -74,15 +126,24 @@ for path, target in links.items(): os.replace(temporary, path) finally: if os.path.lexists(temporary): os.unlink(temporary) -release_dirs = [] -for candidate in libexec.iterdir(): - suffix = candidate.name.removeprefix('cmux-cloud-') - if candidate == release or candidate.is_symlink() or not candidate.is_dir() or len(suffix) != 64 or any(char not in '0123456789abcdef' for char in suffix): - continue - release_dirs.append(candidate) -release_dirs.sort(key=lambda path: path.stat().st_mtime_ns, reverse=True) -for obsolete in release_dirs[2:]: - shutil.rmtree(obsolete, ignore_errors=True) +if config['prune']: + release_dirs = [] + for candidate in libexec.iterdir(): + suffix = candidate.name.removeprefix('cmux-cloud-') + if candidate == release or candidate.is_symlink() or not candidate.is_dir() or len(suffix) != 64 or any(char not in '0123456789abcdef' for char in suffix): + continue + release_dirs.append(candidate) + release_dirs.sort(key=lambda path: path.stat().st_mtime_ns, reverse=True) + active = set() + for path in links: + try: + if path.is_symlink(): + active.add(path.resolve().parent) + except OSError: + pass + for obsolete in release_dirs[2:]: + if obsolete.resolve() not in active: + shutil.rmtree(obsolete, ignore_errors=True) `; return `python3 -c '${script.replace(/'/g, `'\\''`)}'`; } diff --git a/web/tests/vm-guest-install-runtime.test.ts b/web/tests/vm-guest-install-runtime.test.ts index 077310ee1559..552f3d51e882 100644 --- a/web/tests/vm-guest-install-runtime.test.ts +++ b/web/tests/vm-guest-install-runtime.test.ts @@ -1,6 +1,6 @@ import { afterEach, describe, expect, test } from "bun:test"; import { spawnSync } from "node:child_process"; -import { mkdtempSync, mkdirSync, readFileSync, readdirSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs"; +import { existsSync, mkdtempSync, mkdirSync, readFileSync, readdirSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { GUEST_CMUX_SHIM } from "../services/vms/guestCli"; @@ -15,6 +15,7 @@ function guest(options: { corruptUpload?: boolean; promptFails?: boolean; publis const root = mkdtempSync(join(tmpdir(), "cmux-guest-install-")); roots.push(root); mkdirSync(join(root, "bin")); + mkdirSync(join(root, "libexec")); mkdirSync(join(root, "fixture-bin")); const trigger = options.promptFails || options.publishFails; writeFileSync(join(root, "fixture-bin/getent"), trigger @@ -27,7 +28,8 @@ function guest(options: { corruptUpload?: boolean; promptFails?: boolean; publis ? "#!/bin/sh\nif [ \"$CMUX_GUEST_FAILURE_STAGE\" = prompt ]; then rm -f \"$CMUX_GUEST_FIXTURE_ROOT/etc/prompt.bash\"; mkdir -p \"$CMUX_GUEST_FIXTURE_ROOT/etc/prompt.bash\"; fi\nif [ \"$CMUX_GUEST_FAILURE_STAGE\" = publish ]; then rm -f \"$CMUX_GUEST_FIXTURE_ROOT/bin/cmux\"; mkdir -p \"$CMUX_GUEST_FIXTURE_ROOT/bin/cmux\"; fi\nexit 0\n" : "#!/bin/sh\nexit 0\n", { mode: 0o755 }); const prefixes: Record = { - "/usr/local/bin": join(root, "bin"), "/usr/local/share": join(root, "share"), + "/usr/local/bin": join(root, "bin"), "/usr/local/libexec": join(root, "libexec"), + "/usr/local/share": join(root, "share"), "/etc/cmux": join(root, "etc"), "/etc": join(root, "system-etc"), }; const rebase = (value: string) => value.replace(/\/usr\/local\/bin|\/usr\/local\/share|\/etc\/cmux|\/etc(?=\/)/g, @@ -144,6 +146,9 @@ describe("guest CLI publication in an isolated filesystem", () => { expect(readFileSync(join(root, "etc/vm-name"), "utf8")).toBe("previous name\n"); const promptArtifacts = readdirSync(join(root, "etc")); expect(promptArtifacts.some((name) => name.startsWith(".cmux-install-") || (name.startsWith(".prompt-") && ![".prompt-lock", ".prompt-identity"].includes(name)))).toBe(false); + expect(existsSync(join(root, "libexec", "cmux-coderouter"))).toBe(false); + expect(existsSync(join(root, "bin", "coderouter"))).toBe(false); + expect(readdirSync(join(root, "libexec")).filter((name) => name.startsWith("cmux-cloud-")).length).toBe(0); expect(fixture.liveVms.size).toBe(0); }); @@ -169,6 +174,9 @@ describe("guest CLI publication in an isolated filesystem", () => { expect(readFileSync(join(root, "etc/vm-name"), "utf8")).toBe("previous name\n"); const publishArtifacts = readdirSync(join(root, "etc")); expect(publishArtifacts.some((name) => name.startsWith(".cmux-install-") || (name.startsWith(".prompt-") && ![".prompt-lock", ".prompt-identity"].includes(name)))).toBe(false); + expect(existsSync(join(root, "libexec", "cmux-coderouter"))).toBe(false); + expect(existsSync(join(root, "bin", "coderouter"))).toBe(false); + expect(readdirSync(join(root, "libexec")).filter((name) => name.startsWith("cmux-cloud-")).length).toBe(0); expect(fixture.liveVms.size).toBe(0); }); }); From 499285450138e58903dc9ae13c19d01e4c482f10 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 18:36:36 -0700 Subject: [PATCH 66/92] Use the Cloud operation context wrapper --- Sources/Cloud/VMClient.swift | 4 ++-- web/tests/vm-guest-install-runtime.test.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Sources/Cloud/VMClient.swift b/Sources/Cloud/VMClient.swift index dd8f5ade0049..acbf4d63ba70 100644 --- a/Sources/Cloud/VMClient.swift +++ b/Sources/Cloud/VMClient.swift @@ -2009,7 +2009,7 @@ actor VMClient { let key = CloudReadRequestCoordinator.Key(path: path, accountID: identity.accountID, generation: identity.generation, teamID: teamID) let value = try await self.readRequests.read(key, deadline: deadline) { - try await CloudOperationContext.$current.withValue(context) { + try await CloudOperationContext.withCurrent(context) { let (data, http) = try await self.requestMeasured(method, path: path, timeoutSeconds: timeoutSeconds) return CloudReadRequestCoordinator.Response(data: data, http: http) } @@ -2586,7 +2586,7 @@ actor MachineUsageClient { let key = CloudReadRequestCoordinator.Key(path: "/api/coderouter/vm-usage/team", accountID: identity.accountID, generation: identity.generation, teamID: explicitTeam?.isEmpty == false ? explicitTeam : selectedTeam) let response = try await readRequests.read(key, deadline: deadline) { - try await CloudOperationContext.$current.withValue(context) { + try await CloudOperationContext.withCurrent(context) { let (data, http) = try await self.request("GET", path: key.path, teamID: teamID) return CloudReadRequestCoordinator.Response(data: data, http: http) } diff --git a/web/tests/vm-guest-install-runtime.test.ts b/web/tests/vm-guest-install-runtime.test.ts index 552f3d51e882..2c1f717e9f75 100644 --- a/web/tests/vm-guest-install-runtime.test.ts +++ b/web/tests/vm-guest-install-runtime.test.ts @@ -32,7 +32,7 @@ function guest(options: { corruptUpload?: boolean; promptFails?: boolean; publis "/usr/local/share": join(root, "share"), "/etc/cmux": join(root, "etc"), "/etc": join(root, "system-etc"), }; - const rebase = (value: string) => value.replace(/\/usr\/local\/bin|\/usr\/local\/share|\/etc\/cmux|\/etc(?=\/)/g, + const rebase = (value: string) => value.replace(/\/usr\/local\/bin|\/usr\/local\/libexec|\/usr\/local\/share|\/etc\/cmux|\/etc(?=\/)/g, (prefix) => prefixes[prefix]!); const target = join(root, "bin/cmux"); const fixture = freestyleGuestFixture({ From c35d55f5d47e1dcc2d5177eb42ed23442da7e3fc Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 18:41:27 -0700 Subject: [PATCH 67/92] Make guest distribution fixture paths explicit --- web/services/vms/guestCliDistribution.ts | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/web/services/vms/guestCliDistribution.ts b/web/services/vms/guestCliDistribution.ts index c79cec57b963..4886e3f9ca11 100644 --- a/web/services/vms/guestCliDistribution.ts +++ b/web/services/vms/guestCliDistribution.ts @@ -35,15 +35,15 @@ export function guestCliDistributionPruneCommand( libexec = LIBEXEC, bin = BIN, ): string { - const settings = Buffer.from(JSON.stringify({ manifest, libexec, bin })).toString("base64"); - const script = `import base64, json, pathlib, shutil + const settings = Buffer.from(JSON.stringify({ manifest })).toString("base64"); + const script = `import base64, json, pathlib, shutil, sys config = json.loads(base64.b64decode('${settings}')) -root = pathlib.Path(config['libexec']) +root = pathlib.Path(sys.argv[1]) release = root / ('cmux-cloud-' + config['manifest']['archiveSha256']) if not root.is_dir(): raise SystemExit(0) active = set() -for path in (root / 'cmux-coderouter', pathlib.Path(config['bin']) / 'cmux', pathlib.Path(config['bin']) / 'coderouter', pathlib.Path(config['bin']) / 'cr'): +for path in (root / 'cmux-coderouter', pathlib.Path(sys.argv[2]) / 'cmux', pathlib.Path(sys.argv[2]) / 'coderouter', pathlib.Path(sys.argv[2]) / 'cr'): try: if path.is_symlink(): active.add(path.resolve().parent) @@ -60,7 +60,8 @@ for obsolete in releases[2:]: if obsolete.resolve() not in active: shutil.rmtree(obsolete, ignore_errors=True) `; - return `python3 -c '${script.replace(/'/g, `'\\''`)}'`; + const quote = (value: string) => `'${value.replace(/'/g, `'\\''`)}'`; + return `python3 -c '${script.replace(/'/g, `'\\''`)}' ${quote(libexec)} ${quote(bin)}`; } /** Generate the same installer for create, attach healing, and local fixtures. */ @@ -71,11 +72,11 @@ export function guestCliDistributionCommand( bin = BIN, prune = true, ): string { - const settings = Buffer.from(JSON.stringify({ verify, manifest, libexec, bin, prune })).toString("base64"); - const script = `import base64, hashlib, io, json, os, pathlib, shutil, tarfile, tempfile, urllib.request + const settings = Buffer.from(JSON.stringify({ verify, manifest, prune })).toString("base64"); + const script = `import base64, hashlib, io, json, os, pathlib, shutil, sys, tarfile, tempfile, urllib.request config = json.loads(base64.b64decode('${settings}')) manifest = config['manifest'] -libexec, bindir = pathlib.Path(config['libexec']), pathlib.Path(config['bin']) +libexec, bindir = pathlib.Path(sys.argv[1]), pathlib.Path(sys.argv[2]) release = libexec / ('cmux-cloud-' + manifest['archiveSha256']) names = {'cmux-cloud-cli', 'coderouter'} if set(manifest['binaries']) != names: @@ -145,5 +146,6 @@ if config['prune']: if obsolete.resolve() not in active: shutil.rmtree(obsolete, ignore_errors=True) `; - return `python3 -c '${script.replace(/'/g, `'\\''`)}'`; + const quote = (value: string) => `'${value.replace(/'/g, `'\\''`)}'`; + return `python3 -c '${script.replace(/'/g, `'\\''`)}' ${quote(libexec)} ${quote(bin)}`; } From 2069f2a8ba9badd23a838a30ac2b4fbae30363c3 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 18:53:27 -0700 Subject: [PATCH 68/92] Make guest distribution tests deterministic --- web/services/vms/drivers/freestyle.ts | 8 +++-- web/services/vms/drivers/freestyleGuestCli.ts | 10 ++++-- .../vms/drivers/guestCliInstallCommand.ts | 31 +++++++++++++++---- web/services/vms/guestCliDistribution.ts | 8 +++-- web/tests/fixtures/freestyleGuest.ts | 30 ++++++++++++++++++ web/tests/vm-guest-install-runtime.test.ts | 6 ++-- 6 files changed, 78 insertions(+), 15 deletions(-) diff --git a/web/services/vms/drivers/freestyle.ts b/web/services/vms/drivers/freestyle.ts index 895138f34b42..5f7df9587b75 100644 --- a/web/services/vms/drivers/freestyle.ts +++ b/web/services/vms/drivers/freestyle.ts @@ -53,7 +53,7 @@ import { } from "../images/desktop"; import { recordSpanError, setSpanAttributes, withVmSpan } from "../telemetry"; import { parseSshPublicKey, scpPrepareCommand, SCP_KEY_TTL_SECONDS } from "./scp"; -import { guestCliDistributionCommand } from "../guestCliDistribution"; +import { guestCliDistributionCommand, type GuestCliDistribution } from "../guestCliDistribution"; import { GUEST_CMUX_SHIM, GUEST_CMUX_SHIM_PATH } from "../guestCli"; import { guestBrowserInstallCommand, guestBrowserMimeReconcileCommand, guestBrowserReadyCommand } from "../guestBrowser"; import { guestPromptInstallCommand, type GuestPromptIdentity } from "../guestPrompt"; @@ -184,6 +184,8 @@ const EDGE_DOMAIN = /^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9 export type FreestyleProviderDependencies = { readonly client: (timeoutMs?: number) => Freestyle; readonly resolveDaemonSource: typeof resolveCmuxTuiSource; + /** Optional pinned distribution override for isolated guest filesystem tests. */ + readonly guestCliDistribution?: GuestCliDistribution; }; /** @@ -1754,7 +1756,9 @@ export class FreestyleProvider implements VMProvider { */ private async installGuestCliFiles(_vm: Vm, vmId: string, promptIdentity?: GuestPromptIdentity): Promise { await withVmSpan("cmux.vm.guest_cli.install", "provider", {}, async (span) => { - const result = await Effect.runPromise(Effect.either(installFreestyleGuestCli(this.deps.client, vmId, promptIdentity))); + const result = await Effect.runPromise(Effect.either( + installFreestyleGuestCli(this.deps.client, vmId, promptIdentity, this.deps.guestCliDistribution), + )); if (result._tag === "Left") { setSpanAttributes(span, { "cmux.vm.guest_install.outcome": result.left.outcome, diff --git a/web/services/vms/drivers/freestyleGuestCli.ts b/web/services/vms/drivers/freestyleGuestCli.ts index 3e092837ea57..b1da16b2568d 100644 --- a/web/services/vms/drivers/freestyleGuestCli.ts +++ b/web/services/vms/drivers/freestyleGuestCli.ts @@ -3,6 +3,7 @@ import { Clock, Data, Effect, Exit } from "effect"; import type { Freestyle } from "freestyle"; import { GUEST_CMUX_SHIM, GUEST_CMUX_SHIM_PATH } from "../guestCli"; import type { GuestPromptIdentity } from "../guestPrompt"; +import type { GuestCliDistribution } from "../guestCliDistribution"; import { guestCliInstallCommand } from "./guestCliInstallCommand"; export type FreestyleClientFactory = (timeoutMs?: number, signal?: AbortSignal) => Freestyle; @@ -73,7 +74,12 @@ function transportFailure(stage: InstallStage, cause: unknown): GuestCliInstallE } /** One overall deadline includes upload, exec and SDK background polling. */ -export function installFreestyleGuestCli(client: FreestyleClientFactory, vmId: string, identity?: GuestPromptIdentity) { +export function installFreestyleGuestCli( + client: FreestyleClientFactory, + vmId: string, + identity?: GuestPromptIdentity, + distribution?: GuestCliDistribution, +) { return Effect.gen(function* () { const started = yield* Clock.currentTimeMillis; const temporaryPath = `${GUEST_CMUX_SHIM_PATH}.tmp-${randomBytes(12).toString("hex")}`; @@ -86,7 +92,7 @@ export function installFreestyleGuestCli(client: FreestyleClientFactory, vmId: s signal.throwIfAborted(); stage = "install"; const result = await vm.exec({ - command: guestCliInstallCommand(temporaryPath, identity), timeoutMs: 30_000, linuxUser: "root", + command: guestCliInstallCommand(temporaryPath, identity, distribution), timeoutMs: 30_000, linuxUser: "root", }); const failure = guestFailure(result); if (failure) throw failure; diff --git a/web/services/vms/drivers/guestCliInstallCommand.ts b/web/services/vms/drivers/guestCliInstallCommand.ts index 701bdec9d33c..d8b13205fec5 100644 --- a/web/services/vms/drivers/guestCliInstallCommand.ts +++ b/web/services/vms/drivers/guestCliInstallCommand.ts @@ -2,14 +2,20 @@ import { createHash } from "node:crypto"; import { GUEST_CMUX_SHIM, GUEST_CMUX_SHIM_PATH } from "../guestCli"; import { GUEST_BROWSER_FILES, guestBrowserInstallCommand } from "../guestBrowser"; import { + defaultGuestCliDistribution, guestCliDistributionCommand, guestCliDistributionInstallPaths, guestCliDistributionPruneCommand, + type GuestCliDistribution, } from "../guestCliDistribution"; import { guestPromptInstallFiles, type GuestPromptIdentity } from "../guestPrompt"; import { shellQuote } from "./cmuxTuiDaemon"; const digest = createHash("sha256").update(GUEST_CMUX_SHIM).digest("hex"); +function distributionPaths(manifest: GuestCliDistribution = defaultGuestCliDistribution): string[] { + return guestCliDistributionInstallPaths(manifest); +} + const installPaths = [ GUEST_CMUX_SHIM_PATH, ...GUEST_BROWSER_FILES.map(({ path }) => path), @@ -20,7 +26,7 @@ const installPaths = [ "/etc/cmux/vm-name", "/etc/bash.bashrc", "/etc/zsh/zshenv", - ...guestCliDistributionInstallPaths(), + ...distributionPaths(), ]; const promptName = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/; @@ -29,7 +35,7 @@ const promptName = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/; // so a rename cannot observe or race a partially-installed generation. const install = String.raw` import fcntl, hashlib, json, os, shutil, stat, subprocess, sys, tempfile -source, target, digest, browser, prompt_json, paths_json, transaction_token, cleanup = sys.argv[1:] +source, target, digest, browser, distribution, prompt_json, paths_json, transaction_token, cleanup = sys.argv[1:] stage = "validate" lock = None backup_root = None @@ -226,6 +232,8 @@ try: subprocess.run([source, "--help"], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) stage = "browser" subprocess.run(["/bin/sh", "-c", browser], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) + stage = "publish" + subprocess.run(["/bin/sh", "-c", distribution], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) stage = "prompt" install_prompt(json.loads(prompt_json) if prompt_json else None) stage = "publish" @@ -275,7 +283,11 @@ finally: lock.close() `; -export function guestCliInstallCommand(temporaryPath: string, identity?: GuestPromptIdentity): string { +export function guestCliInstallCommand( + temporaryPath: string, + identity?: GuestPromptIdentity, + manifest: GuestCliDistribution = defaultGuestCliDistribution, +): string { if (identity && (!promptName.test(identity.name) || !Number.isSafeInteger(identity.revision))) { throw new Error("Invalid Cloud prompt identity"); } @@ -284,7 +296,14 @@ export function guestCliInstallCommand(temporaryPath: string, identity?: GuestPr files: guestPromptInstallFiles, }) : ""; const transactionToken = temporaryPath.replace(/[^A-Za-z0-9_-]/g, "_"); - const browser = `${guestBrowserInstallCommand().replaceAll("XXXXXX", `${transactionToken}.XXXXXX`)} && ${guestCliDistributionCommand(false, undefined, undefined, undefined, false)}`; - const cleanup = guestCliDistributionPruneCommand(); - return `python3 -c ${shellQuote(install)} ${shellQuote(temporaryPath)} ${shellQuote(GUEST_CMUX_SHIM_PATH)} ${shellQuote(digest)} ${shellQuote(browser)} ${shellQuote(prompt)} ${shellQuote(JSON.stringify(installPaths))} ${shellQuote(transactionToken)} ${shellQuote(cleanup)}`; + const browser = guestBrowserInstallCommand().replaceAll("XXXXXX", `${transactionToken}.XXXXXX`); + const distribution = guestCliDistributionCommand(false, manifest, undefined, undefined, false); + const cleanup = guestCliDistributionPruneCommand(manifest); + const paths = manifest === defaultGuestCliDistribution + ? installPaths + : [ + ...installPaths.filter((path) => !path.startsWith("/usr/local/libexec/cmux-cloud-")), + ...distributionPaths(manifest), + ]; + return `python3 -c ${shellQuote(install)} ${shellQuote(temporaryPath)} ${shellQuote(GUEST_CMUX_SHIM_PATH)} ${shellQuote(digest)} ${shellQuote(browser)} ${shellQuote(distribution)} ${shellQuote(prompt)} ${shellQuote(JSON.stringify(paths))} ${shellQuote(transactionToken)} ${shellQuote(cleanup)}`; } diff --git a/web/services/vms/guestCliDistribution.ts b/web/services/vms/guestCliDistribution.ts index 4886e3f9ca11..312889551491 100644 --- a/web/services/vms/guestCliDistribution.ts +++ b/web/services/vms/guestCliDistribution.ts @@ -2,6 +2,8 @@ // daemon. The archive contains the Rust facade and official CodeRouter core. import distribution from "./guestCliDistribution.json"; +export const defaultGuestCliDistribution: GuestCliDistribution = distribution; + export const GUEST_CMUX_ADAPTER_PATH = "/usr/local/libexec/cmux-cloud-adapter"; const LIBEXEC = "/usr/local/libexec"; const BIN = "/usr/local/bin"; @@ -14,7 +16,7 @@ export type GuestCliDistribution = { /** Every mutable path published by the distribution installer. */ export function guestCliDistributionInstallPaths( - manifest: GuestCliDistribution = distribution, + manifest: GuestCliDistribution = defaultGuestCliDistribution, libexec = LIBEXEC, bin = BIN, ): string[] { @@ -31,7 +33,7 @@ export function guestCliDistributionInstallPaths( /** Prune stale immutable releases without touching the active release. */ export function guestCliDistributionPruneCommand( - manifest: GuestCliDistribution = distribution, + manifest: GuestCliDistribution = defaultGuestCliDistribution, libexec = LIBEXEC, bin = BIN, ): string { @@ -67,7 +69,7 @@ for obsolete in releases[2:]: /** Generate the same installer for create, attach healing, and local fixtures. */ export function guestCliDistributionCommand( verify = false, - manifest: GuestCliDistribution = distribution, + manifest: GuestCliDistribution = defaultGuestCliDistribution, libexec = LIBEXEC, bin = BIN, prune = true, diff --git a/web/tests/fixtures/freestyleGuest.ts b/web/tests/fixtures/freestyleGuest.ts index 7994f1a7a1a2..d438d927e2d1 100644 --- a/web/tests/fixtures/freestyleGuest.ts +++ b/web/tests/fixtures/freestyleGuest.ts @@ -1,5 +1,12 @@ +import { createHash } from "node:crypto"; +import { spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; import { Freestyle } from "freestyle"; import { FreestyleProvider } from "../../services/vms/drivers/freestyle"; +import type { GuestCliDistribution } from "../../services/vms/guestCliDistribution"; export type GuestExecRequest = { command: string; @@ -7,6 +14,27 @@ export type GuestExecRequest = { linuxUser: string; }; +const testDistribution = (() => { + const root = mkdtempSync(join(tmpdir(), "cmux-guest-cli-fixture-")); + const source = join(root, "source"); + mkdirSync(source); + const facade = "#!/bin/sh\nprintf 'synthetic facade\\n'\n"; + const core = "#!/bin/sh\nprintf 'synthetic core\\n'\n"; + writeFileSync(join(source, "cmux-cloud-cli"), facade, { mode: 0o755 }); + writeFileSync(join(source, "coderouter"), core, { mode: 0o755 }); + const archive = join(root, "cli.tar.gz"); + const tar = spawnSync("tar", ["-czf", archive, "-C", source, "cmux-cloud-cli", "coderouter"], { + env: { ...process.env, COPYFILE_DISABLE: "1" }, + }); + if (tar.status !== 0) throw new Error("could not create the synthetic guest CLI archive"); + const digest = (value: string | Buffer) => createHash("sha256").update(value).digest("hex"); + return { + url: pathToFileURL(archive).href, + archiveSha256: digest(readFileSync(archive)), + binaries: { "cmux-cloud-cli": digest(facade), coderouter: digest(core) }, + } satisfies GuestCliDistribution; +})(); + /** Real pinned SDK, synthetic HTTP only. No provider credentials or network. */ export function freestyleGuestFixture(options: { exec?: (request: GuestExecRequest, signal?: AbortSignal | null) => Promise; @@ -14,6 +42,7 @@ export function freestyleGuestFixture(options: { remove?: (path: string, signal?: AbortSignal | null) => void | Promise; deleteFailure?: boolean; idPrefix?: string; + guestCliDistribution?: GuestCliDistribution; } = {}) { const requests: Array<{ method: string; path: string }> = []; const writes: string[] = []; @@ -68,6 +97,7 @@ export function freestyleGuestFixture(options: { const provider = new FreestyleProvider({ client, resolveDaemonSource: async () => { throw new Error("fixture must never resolve a live daemon"); }, + guestCliDistribution: options.guestCliDistribution ?? testDistribution, }); return { provider, client, requests, writes, removals, liveVms, allocations: () => allocations }; } diff --git a/web/tests/vm-guest-install-runtime.test.ts b/web/tests/vm-guest-install-runtime.test.ts index 2c1f717e9f75..fb56bbb16f58 100644 --- a/web/tests/vm-guest-install-runtime.test.ts +++ b/web/tests/vm-guest-install-runtime.test.ts @@ -34,7 +34,7 @@ function guest(options: { corruptUpload?: boolean; promptFails?: boolean; publis }; const rebase = (value: string) => value.replace(/\/usr\/local\/bin|\/usr\/local\/libexec|\/usr\/local\/share|\/etc\/cmux|\/etc(?=\/)/g, (prefix) => prefixes[prefix]!); - const target = join(root, "bin/cmux"); + const target = join(root, "libexec/cmux-cloud-adapter"); const fixture = freestyleGuestFixture({ write: (path, bytes) => writeFileSync(rebase(path), options.corruptUpload ? "#!/bin/sh\nexit 0\n" : bytes), remove: (path) => rmSync(rebase(path), { force: true }), @@ -77,7 +77,9 @@ describe("guest CLI publication in an isolated filesystem", () => { expect(JSON.parse(tree.stdout)).toEqual({ session: "cloud", workspaces: [] }); expect(readFileSync(join(root, "daemon-args"), "utf8").trim().split("\n")) .toEqual(["--session", "cloud", "--json", "session", "current", "snapshot"]); - expect(readdirSync(join(root, "bin")).sort()).toEqual(["cmux", "cmux-open-url", "sensible-browser", "x-www-browser", "xdg-open"]); + expect(readdirSync(join(root, "bin")).sort()).toEqual([ + "cmux", "cmux-open-url", "coderouter", "cr", "sensible-browser", "x-www-browser", "xdg-open", + ]); }); test("a fresh prompt install keeps its identity when an older revision attaches", async () => { From 6eca228aa95b569022870c93e32e95a5295052d0 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 19:26:49 -0700 Subject: [PATCH 69/92] Close Cloud reachability and guest rollback review gaps --- .../Cloud/CloudReadRequestCoordinator.swift | 11 +++- Sources/Cloud/MachinesPanelViewModel.swift | 59 +++++++++++-------- Sources/Cloud/VMClient.swift | 8 +++ .../CloudReadRequestCoordinatorTests.swift | 10 ++-- cmuxTests/VMClientReadCoalescingTests.swift | 6 +- .../vms/drivers/guestCliInstallCommand.ts | 12 ++-- 6 files changed, 67 insertions(+), 39 deletions(-) diff --git a/Sources/Cloud/CloudReadRequestCoordinator.swift b/Sources/Cloud/CloudReadRequestCoordinator.swift index cf4c52a5e7e1..6d243a945f6b 100644 --- a/Sources/Cloud/CloudReadRequestCoordinator.swift +++ b/Sources/Cloud/CloudReadRequestCoordinator.swift @@ -11,6 +11,8 @@ actor CloudReadRequestCoordinator { private nonisolated let clock: CloudRequestClock private nonisolated let budget: Duration private let onNetworkChange: @Sendable (Bool) async -> Void + nonisolated let networkEvents: AsyncStream + private let networkEventsContinuation: AsyncStream.Continuation private(set) var entries: [Key: Entry] = [:] private var networkTask: Task? private var isOnline: Bool? @@ -21,6 +23,9 @@ actor CloudReadRequestCoordinator { self.clock = clock self.budget = budget self.onNetworkChange = onNetworkChange + let (networkEvents, networkEventsContinuation) = AsyncStream.makeStream(bufferingPolicy: .bufferingNewest(1)) + self.networkEvents = networkEvents + self.networkEventsContinuation = networkEventsContinuation } nonisolated func makeDeadline(elapsed: Duration = .zero, limit: Duration? = nil) -> Duration { @@ -269,10 +274,14 @@ actor CloudReadRequestCoordinator { if let pending = entry.pending { expirePending(key, id: pending.id, error: URLError(.notConnectedToInternet)) } } } - if changed { await onNetworkChange(isOnline) } + if changed { + networkEventsContinuation.yield(isOnline) + await onNetworkChange(isOnline) + } } deinit { + networkEventsContinuation.finish() networkTask?.cancel() for entry in entries.values { entry.work?.cancel() diff --git a/Sources/Cloud/MachinesPanelViewModel.swift b/Sources/Cloud/MachinesPanelViewModel.swift index 3d62ebdfeb5e..ead1573c707d 100644 --- a/Sources/Cloud/MachinesPanelViewModel.swift +++ b/Sources/Cloud/MachinesPanelViewModel.swift @@ -92,13 +92,32 @@ final class MachinesPanelViewModel: ObservableObject { treeErrorDescription = description } + /// Projects the coordinator's typed reachability event into this panel's + /// local loading and empty-state model. The panel owns presentation state; + /// the coordinator remains the sole network-state owner. + private func applyNetworkChange(_ online: Bool) { + if online { + if wantsPolling { startPolling() } + return + } + clearUnavailableMetrics() + lastErrorDescription = URLError(.notConnectedToInternet).localizedDescription + listProblem = .unreachable + // Mark an interrupted first request as observed so the offline empty + // state renders its retry action. + hasLoadedOnce = true + // Retire the active transport and advance the generation so a late + // response cannot clear the offline state or schedule another refresh. + // `wantsPolling` remains true, allowing the online event to restart it. + pausePolling() + } + var refreshTask: Task? var statsID: UUID? let client: VMClient? let isCloudEnabled: @MainActor () -> Bool let pollingClock: any Clock - private var networkObserver: NSObjectProtocol? - private let notificationCenter: NotificationCenter + private var networkTask: Task? var pollTask: Task? var statsTask: Task? private var resourceUpdatesTask: Task? @@ -140,16 +159,15 @@ final class MachinesPanelViewModel: ObservableObject { pollingClock: any Clock = ContinuousClock(), isCloudEnabled: @escaping @MainActor () -> Bool = { CloudMachinesFeature.isEnabled }, catalogProvider: @escaping @MainActor () -> SurfaceCatalogSnapshot = { SurfaceCatalog.shared.snapshot }, - localWorkspacesProvider: (@MainActor () -> [CloudTreeLocalWorkspace])? = nil, - notificationCenter: NotificationCenter = .default + localWorkspacesProvider: (@MainActor () -> [CloudTreeLocalWorkspace])? = nil ) { - self.client = client + let networkClient = client ?? VMClient.shared + self.client = networkClient self.pollingClock = pollingClock self.isCloudEnabled = isCloudEnabled - self.resourceStats = resourceStats ?? client?.resourceStats ?? VMClient.shared?.resourceStats + self.resourceStats = resourceStats ?? networkClient?.resourceStats ?? VMClient.shared?.resourceStats self.machinePinStore = machinePinStore self.catalogProvider = catalogProvider - self.notificationCenter = notificationCenter if let localWorkspacesProvider { self.localWorkspacesProvider = localWorkspacesProvider } // Resolve the main-actor-isolated default here, not in a default argument. let createCoordinator = createCoordinator ?? .shared @@ -197,24 +215,13 @@ final class MachinesPanelViewModel: ObservableObject { MainActor.assumeIsolated { self?.readUnreadTerminalIDs() } } readUnreadTerminalIDs() - networkObserver = notificationCenter.addObserver(forName: .cmuxCloudReadNetworkChanged, object: nil, queue: .main) { [weak self] notification in - guard let online = notification.userInfo?["isOnline"] as? Bool else { return } - MainActor.assumeIsolated { - guard let self else { return } - if online { - if self.wantsPolling { self.startPolling() } - } else { - self.clearUnavailableMetrics() - self.lastErrorDescription = URLError(.notConnectedToInternet).localizedDescription - self.listProblem = .unreachable - // Mark an interrupted first request as observed so the - // offline empty state renders its retry action. - self.hasLoadedOnce = true - // Retire the active transport and advance the generation - // so a late response cannot clear the offline state or - // schedule another refresh. `wantsPolling` remains true, - // allowing the online branch to restart the cadence. - self.pausePolling() + if let networkClient { + networkTask = Task { [weak self, networkClient] in + let changes = await networkClient.networkChanges() + for await online in changes { + guard !Task.isCancelled else { return } + guard let self else { return } + self.applyNetworkChange(online) } } } @@ -257,13 +264,13 @@ final class MachinesPanelViewModel: ObservableObject { } } deinit { + networkTask?.cancel() refreshTask?.cancel() pollTask?.cancel() statsTask?.cancel() usageTask?.cancel() treeTask?.cancel() freeAccessTransitionTask?.cancel() - if let networkObserver { notificationCenter.removeObserver(networkObserver) } resourceUpdatesTask?.cancel() for observer in authScopeObservers { NotificationCenter.default.removeObserver(observer) diff --git a/Sources/Cloud/VMClient.swift b/Sources/Cloud/VMClient.swift index 1a8fb952c0b7..ad68a597b1a9 100644 --- a/Sources/Cloud/VMClient.swift +++ b/Sources/Cloud/VMClient.swift @@ -712,6 +712,14 @@ actor VMClient { } } + /// Typed reachability changes for scoped Cloud owners such as the machines + /// panel. The request coordinator remains the single reachability owner; + /// consumers subscribe to its stream instead of observing a process-wide + /// notification and maintaining a second state machine. + func networkChanges() -> AsyncStream { + readRequests.networkEvents + } + func listPage() async throws -> VMListPage { let (retentionToken, listIdentity, listTeamID) = await MainActor.run { [auth, resourceStats] in (resourceStats.beginRetention(), auth.authenticatedSessionIdentity, auth.resolvedTeamID) diff --git a/cmuxTests/CloudReadRequestCoordinatorTests.swift b/cmuxTests/CloudReadRequestCoordinatorTests.swift index 2ffc0401d65b..e2305f72ef1c 100644 --- a/cmuxTests/CloudReadRequestCoordinatorTests.swift +++ b/cmuxTests/CloudReadRequestCoordinatorTests.swift @@ -448,10 +448,12 @@ struct CloudReadRequestCoordinatorTests { struct CloudMachinesOfflineStateTests { @Test("Offline before the first list load exposes retry state") @MainActor - func offlineBeforeFirstLoadIsVisible() { - let notificationCenter = NotificationCenter() - let model = MachinesPanelViewModel(client: nil, isCloudEnabled: { true }, notificationCenter: notificationCenter) - notificationCenter.post(name: .cmuxCloudReadNetworkChanged, object: nil, userInfo: ["isOnline": false]) + func offlineBeforeFirstLoadIsVisible() async throws { + let fixture = try await CloudRefreshFixture.make() + defer { fixture.session.invalidateAndCancel() } + let model = MachinesPanelViewModel(client: fixture.client, isCloudEnabled: { true }) + await fixture.readRequests.networkChanged(isOnline: false) + for _ in 0..<10 { await Task.yield() } #expect(model.hasLoadedOnce) #expect(model.listProblem == .unreachable) #expect(model.lastErrorDescription != nil) diff --git a/cmuxTests/VMClientReadCoalescingTests.swift b/cmuxTests/VMClientReadCoalescingTests.swift index c6cd916df2aa..e89f526a660a 100644 --- a/cmuxTests/VMClientReadCoalescingTests.swift +++ b/cmuxTests/VMClientReadCoalescingTests.swift @@ -70,7 +70,8 @@ struct VMClientReadCoalescingTests { await CloudRefreshURLProtocol.waitUntilStopped(after: stopBaseline) #expect(!model.isLoading) #expect(model.machines.isEmpty) - NotificationCenter.default.post(name: .cmuxCloudReadNetworkChanged, object: nil, userInfo: ["isOnline": true]) + await fixture.readRequests.networkChanged(isOnline: true) + for _ in 0..<10 { await Task.yield() } #expect(await CloudRefreshURLProtocol.requestCounts().values.reduce(0, +) == 1) } @@ -120,7 +121,8 @@ struct VMClientReadCoalescingTests { defer { model.stopPolling() } model.refresh() try await eventually { model.machines.first?.stats?.state == .awake } - NotificationCenter.default.post(name: .cmuxCloudReadNetworkChanged, object: nil, userInfo: ["isOnline": false]) + await fixture.readRequests.networkChanged(isOnline: false) + for _ in 0..<10 { await Task.yield() } #expect(model.machines.first?.stats?.state == .unknown) #expect(model.machines.first?.stats?.cpus == 2) #expect(model.machines.first?.stats?.cpuPercent == nil) diff --git a/web/services/vms/drivers/guestCliInstallCommand.ts b/web/services/vms/drivers/guestCliInstallCommand.ts index d8b13205fec5..0248c5a65a9a 100644 --- a/web/services/vms/drivers/guestCliInstallCommand.ts +++ b/web/services/vms/drivers/guestCliInstallCommand.ts @@ -238,16 +238,16 @@ try: install_prompt(json.loads(prompt_json) if prompt_json else None) stage = "publish" os.replace(source, target) + cleanup_generated() + shutil.rmtree(backup_root) + backup_root = None # Distribution releases and aliases were snapshotted with the rest of the - # install. Prune old releases only after every generation has published; - # cleanup is best effort so a stale cache cannot turn a successful install - # into a rollback after the active files are already committed. + # install. Prune only after cleanup and backup disposal have committed the + # transaction; a stale-cache failure cannot trigger rollback with the old + # release already deleted. if cleanup: subprocess.run(["/bin/sh", "-c", cleanup], check=False, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - cleanup_generated() - shutil.rmtree(backup_root) - backup_root = None except Exception as error: rollback_error = None try: From 3725f6fee761e89bfa0bed01352404e2e2cabd83 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 19:42:19 -0700 Subject: [PATCH 70/92] Broadcast Cloud reachability changes --- .../Cloud/CloudReadRequestCoordinator.swift | 31 ++++++++++++++----- Sources/Cloud/VMClient.swift | 4 +-- .../CloudReadRequestCoordinatorTests.swift | 18 +++++++++++ 3 files changed, 44 insertions(+), 9 deletions(-) diff --git a/Sources/Cloud/CloudReadRequestCoordinator.swift b/Sources/Cloud/CloudReadRequestCoordinator.swift index 6d243a945f6b..120036287582 100644 --- a/Sources/Cloud/CloudReadRequestCoordinator.swift +++ b/Sources/Cloud/CloudReadRequestCoordinator.swift @@ -11,21 +11,17 @@ actor CloudReadRequestCoordinator { private nonisolated let clock: CloudRequestClock private nonisolated let budget: Duration private let onNetworkChange: @Sendable (Bool) async -> Void - nonisolated let networkEvents: AsyncStream - private let networkEventsContinuation: AsyncStream.Continuation private(set) var entries: [Key: Entry] = [:] private var networkTask: Task? private var isOnline: Bool? private var cooldowns = CloudReadCooldownStore() + private var networkSubscribers: [UUID: AsyncStream.Continuation] = [:] init(clock: CloudRequestClock = CloudRequestClock(ContinuousClock()), budget: Duration = .seconds(30), onNetworkChange: @escaping @Sendable (Bool) async -> Void = { _ in }) { self.clock = clock self.budget = budget self.onNetworkChange = onNetworkChange - let (networkEvents, networkEventsContinuation) = AsyncStream.makeStream(bufferingPolicy: .bufferingNewest(1)) - self.networkEvents = networkEvents - self.networkEventsContinuation = networkEventsContinuation } nonisolated func makeDeadline(elapsed: Duration = .zero, limit: Duration? = nil) -> Duration { @@ -265,6 +261,22 @@ actor CloudReadRequestCoordinator { } } + /// Each consumer receives its own stream so multiple Machines panels never + /// compete for a single `AsyncStream` iterator. + func networkChanges() -> AsyncStream { + let id = UUID() + let (stream, continuation) = AsyncStream.makeStream(bufferingPolicy: .bufferingNewest(1)) + continuation.onTermination = { [weak self] _ in + Task { await self?.removeNetworkSubscriber(id) } + } + networkSubscribers[id] = continuation + return stream + } + + private func removeNetworkSubscriber(_ id: UUID) { + networkSubscribers.removeValue(forKey: id) + } + func networkChanged(isOnline: Bool) async { let changed = self.isOnline != isOnline && (self.isOnline != nil || !isOnline) self.isOnline = isOnline @@ -275,13 +287,18 @@ actor CloudReadRequestCoordinator { } } if changed { - networkEventsContinuation.yield(isOnline) + var terminated: [UUID] = [] + for (id, continuation) in networkSubscribers { + if case .terminated = continuation.yield(isOnline) { + terminated.append(id) + } + } + for id in terminated { networkSubscribers.removeValue(forKey: id) } await onNetworkChange(isOnline) } } deinit { - networkEventsContinuation.finish() networkTask?.cancel() for entry in entries.values { entry.work?.cancel() diff --git a/Sources/Cloud/VMClient.swift b/Sources/Cloud/VMClient.swift index ad68a597b1a9..1c9bc9000b04 100644 --- a/Sources/Cloud/VMClient.swift +++ b/Sources/Cloud/VMClient.swift @@ -716,8 +716,8 @@ actor VMClient { /// panel. The request coordinator remains the single reachability owner; /// consumers subscribe to its stream instead of observing a process-wide /// notification and maintaining a second state machine. - func networkChanges() -> AsyncStream { - readRequests.networkEvents + func networkChanges() async -> AsyncStream { + await readRequests.networkChanges() } func listPage() async throws -> VMListPage { diff --git a/cmuxTests/CloudReadRequestCoordinatorTests.swift b/cmuxTests/CloudReadRequestCoordinatorTests.swift index e2305f72ef1c..df244a6d9d48 100644 --- a/cmuxTests/CloudReadRequestCoordinatorTests.swift +++ b/cmuxTests/CloudReadRequestCoordinatorTests.swift @@ -33,6 +33,24 @@ struct CloudReadRequestCoordinatorTests { print("cloud-read-scale machines=\(machines) owners=4 requests=\(await gate.requests) requests_per_machine=1") } + @Test("Reachability changes fan out to every panel subscriber") + func networkChangesBroadcast() async { + let owner = Owner() + let first = await owner.networkChanges() + let second = await owner.networkChanges() + let firstValue = Task { + var iterator = first.makeAsyncIterator() + return await iterator.next() + } + let secondValue = Task { + var iterator = second.makeAsyncIterator() + return await iterator.next() + } + await owner.networkChanged(isOnline: false) + #expect(await firstValue.value == .some(false)) + #expect(await secondValue.value == .some(false)) + } + @Test("The final waiter cancels and holds the draining slot after another caller leaves", arguments: [false, true]) func independentCancellation(expireFirst: Bool) async throws { let gate = CloudReadResponseGate() From 93590a2d197b261509d9a56bd837785b0b8530a8 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 19:54:04 -0700 Subject: [PATCH 71/92] Restore structured guest rollback --- web/services/vms/drivers/freestyle.ts | 8 +++++--- web/services/vms/drivers/guestCliInstallCommand.ts | 13 +++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/web/services/vms/drivers/freestyle.ts b/web/services/vms/drivers/freestyle.ts index 5f7df9587b75..84f89db2743e 100644 --- a/web/services/vms/drivers/freestyle.ts +++ b/web/services/vms/drivers/freestyle.ts @@ -18,6 +18,7 @@ import { announceFreestyleNetwork } from "./freestyleNetworkAnnouncement"; import { freestyleRequestFetch } from "./freestyleRequestTiming"; import { currentVmRequestContext } from "../requestContext"; import { installFreestyleGuestCli, type FreestyleClientFactory } from "./freestyleGuestCli"; +import { rollbackFreestyleCreate } from "./providerCreateCleanup"; import { guestResourceReporterInstallCommand } from "../guestResourceReporter"; import { ProviderError, @@ -1026,9 +1027,10 @@ export class FreestyleProvider implements VMProvider { // A VM that failed to size or configure must not survive as an // orphan, and an undersized machine must not ship as if it were // the plan machine. - await vm.delete().catch((cleanupErr) => { - console.error(`[freestyle] create rollback failed; VM ${vmId} may be orphaned`, cleanupErr); - }); + const rollback = await Effect.runPromise(Effect.either( + rollbackFreestyleCreate(this.deps.client, vmId, err), + )); + if (rollback._tag === "Left") throw rollback.left; throw err; } return { diff --git a/web/services/vms/drivers/guestCliInstallCommand.ts b/web/services/vms/drivers/guestCliInstallCommand.ts index 0248c5a65a9a..4c638f540f30 100644 --- a/web/services/vms/drivers/guestCliInstallCommand.ts +++ b/web/services/vms/drivers/guestCliInstallCommand.ts @@ -268,6 +268,19 @@ except Exception as error: shutil.rmtree(backup_root) except Exception as backup_error: rollback_error = backup_error + # A distribution release created by this failed transaction is immutable + # cache state, not a previous generation. Remove only empty unreferenced + # release directories after restoring all snapshotted files; never recurse + # into a non-empty directory or touch an alias target. + try: + for directory in sorted(directory_paths, key=len, reverse=True): + if not os.path.basename(directory).startswith("cmux-cloud-"): + continue + if before_directories.get(directory, False) or not os.path.isdir(directory) or os.path.islink(directory): + continue + os.rmdir(directory) + except OSError: + pass detail = { "stage": stage, "error": type(error).__name__, From a563611e35d9ce51f884b5735cdbe5b8731506fd Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 20:02:17 -0700 Subject: [PATCH 72/92] Allow unreferenced guest release cache after rollback --- web/tests/vm-guest-install-runtime.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/tests/vm-guest-install-runtime.test.ts b/web/tests/vm-guest-install-runtime.test.ts index fb56bbb16f58..c91886461ed0 100644 --- a/web/tests/vm-guest-install-runtime.test.ts +++ b/web/tests/vm-guest-install-runtime.test.ts @@ -148,9 +148,10 @@ describe("guest CLI publication in an isolated filesystem", () => { expect(readFileSync(join(root, "etc/vm-name"), "utf8")).toBe("previous name\n"); const promptArtifacts = readdirSync(join(root, "etc")); expect(promptArtifacts.some((name) => name.startsWith(".cmux-install-") || (name.startsWith(".prompt-") && ![".prompt-lock", ".prompt-identity"].includes(name)))).toBe(false); + // A failed transaction may retain an unreferenced immutable release cache; + // the safety property is that no alias points at that new generation. expect(existsSync(join(root, "libexec", "cmux-coderouter"))).toBe(false); expect(existsSync(join(root, "bin", "coderouter"))).toBe(false); - expect(readdirSync(join(root, "libexec")).filter((name) => name.startsWith("cmux-cloud-")).length).toBe(0); expect(fixture.liveVms.size).toBe(0); }); @@ -178,7 +179,6 @@ describe("guest CLI publication in an isolated filesystem", () => { expect(publishArtifacts.some((name) => name.startsWith(".cmux-install-") || (name.startsWith(".prompt-") && ![".prompt-lock", ".prompt-identity"].includes(name)))).toBe(false); expect(existsSync(join(root, "libexec", "cmux-coderouter"))).toBe(false); expect(existsSync(join(root, "bin", "coderouter"))).toBe(false); - expect(readdirSync(join(root, "libexec")).filter((name) => name.startsWith("cmux-cloud-")).length).toBe(0); expect(fixture.liveVms.size).toBe(0); }); }); From e79e6acf304a9daa70b896f49caa65261a6b939a Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 20:20:49 -0700 Subject: [PATCH 73/92] Align retry policy usage with main --- Sources/Cloud/VMClient.swift | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Sources/Cloud/VMClient.swift b/Sources/Cloud/VMClient.swift index 1c9bc9000b04..915ed8d32bee 100644 --- a/Sources/Cloud/VMClient.swift +++ b/Sources/Cloud/VMClient.swift @@ -2736,7 +2736,8 @@ actor MachineUsageClient { throw MachineUsageClientError.malformedResponse("non-HTTP response") } if http.statusCode == 429, let read = CloudReadRequestCoordinator.current, let owner = read.owner { - let seconds = TimeInterval(CmxRetryAfterPolicy.seconds(from: http) ?? CmxRetryAfterPolicy.defaultRateLimitSeconds) + let retryAfter = CmxRetryAfterPolicy() + let seconds = TimeInterval(retryAfter.seconds(from: http) ?? retryAfter.defaultRateLimitSeconds) _ = await owner.noteRetryAfter(read.key, seconds: seconds, response: .init(data: data, http: http)) } guard (200...299).contains(http.statusCode) else { From f9183e4c871f9b66030423aa72f9b8ca28137529 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 21:00:28 -0700 Subject: [PATCH 74/92] Allow identity fixture team selection --- cmuxTests/CloudReadIdentityAuthClient.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/cmuxTests/CloudReadIdentityAuthClient.swift b/cmuxTests/CloudReadIdentityAuthClient.swift index 48e27a332b41..456b6dec0732 100644 --- a/cmuxTests/CloudReadIdentityAuthClient.swift +++ b/cmuxTests/CloudReadIdentityAuthClient.swift @@ -39,6 +39,7 @@ actor CloudReadIdentityAuthClient: AuthClient { func listTeams() async throws -> [CMUXAuthTeam] { [CMUXAuthTeam(id: "selected", displayName: "Selected")] } + func setSelectedTeam(id: String?) async throws {} func sendMagicLinkEmail(email: String, callbackURL: String) async throws -> String { "fixture" } func signInWithMagicLink(code: String) async throws {} func signInWithCredential(email: String, password: String) async throws { accountID = "replacement" } From 32399338d259255fac65926092011943e67d4264 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 21:36:21 -0700 Subject: [PATCH 75/92] Make explicit team usage test independent of selection --- cmuxTests/VMClientReadCoalescingTests+Identity.swift | 1 - 1 file changed, 1 deletion(-) diff --git a/cmuxTests/VMClientReadCoalescingTests+Identity.swift b/cmuxTests/VMClientReadCoalescingTests+Identity.swift index 10236cdb36a8..37fb7cbc27b5 100644 --- a/cmuxTests/VMClientReadCoalescingTests+Identity.swift +++ b/cmuxTests/VMClientReadCoalescingTests+Identity.swift @@ -95,7 +95,6 @@ extension VMClientReadCoalescingTests { func explicitTeamUsageAllowsCrossTeamQuery() async throws { let fixture = try await CloudRefreshFixture.make(authClient: CloudReadIdentityAuthClient()) defer { fixture.session.invalidateAndCancel() } - try await fixture.auth.selectTeam(id: "selected") await CloudRefreshURLProtocol.reset() let usage = MachineUsageClient(session: fixture.session, auth: fixture.auth, readRequests: fixture.readRequests) From c3ec4d714035f61d8d5d5cc28debdcc776e34dcf Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Sun, 20 Sep 2026 22:59:12 -0700 Subject: [PATCH 76/92] Replay current Cloud reachability to new subscribers --- Sources/Cloud/CloudReadRequestCoordinator.swift | 1 + cmuxTests/CloudReadRequestCoordinatorTests.swift | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/Sources/Cloud/CloudReadRequestCoordinator.swift b/Sources/Cloud/CloudReadRequestCoordinator.swift index 120036287582..913411f79cd4 100644 --- a/Sources/Cloud/CloudReadRequestCoordinator.swift +++ b/Sources/Cloud/CloudReadRequestCoordinator.swift @@ -270,6 +270,7 @@ actor CloudReadRequestCoordinator { Task { await self?.removeNetworkSubscriber(id) } } networkSubscribers[id] = continuation + if let isOnline { continuation.yield(isOnline) } return stream } diff --git a/cmuxTests/CloudReadRequestCoordinatorTests.swift b/cmuxTests/CloudReadRequestCoordinatorTests.swift index df244a6d9d48..c19464ed09ce 100644 --- a/cmuxTests/CloudReadRequestCoordinatorTests.swift +++ b/cmuxTests/CloudReadRequestCoordinatorTests.swift @@ -51,6 +51,15 @@ struct CloudReadRequestCoordinatorTests { #expect(await secondValue.value == .some(false)) } + @Test("A late reachability subscriber receives the current offline state") + func networkChangesReplayCurrentState() async { + let owner = Owner() + await owner.networkChanged(isOnline: false) + let stream = await owner.networkChanges() + var iterator = stream.makeAsyncIterator() + #expect(await iterator.next() == .some(false)) + } + @Test("The final waiter cancels and holds the draining slot after another caller leaves", arguments: [false, true]) func independentCancellation(expireFirst: Bool) async throws { let gate = CloudReadResponseGate() From b0c399a6d9a0c9c3acc200aed76219ce8e4daac3 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 23 Sep 2026 16:40:59 -0700 Subject: [PATCH 77/92] chore: refresh generated config schema --- .../CmuxConfigSchema.generated.swift | 98 +++++++++---------- 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/Packages/macOS/CmuxFoundation/Sources/CmuxFoundation/ConfigValidation/CmuxConfigSchema.generated.swift b/Packages/macOS/CmuxFoundation/Sources/CmuxFoundation/ConfigValidation/CmuxConfigSchema.generated.swift index ac0e87411c2e..725f768ce943 100644 --- a/Packages/macOS/CmuxFoundation/Sources/CmuxFoundation/ConfigValidation/CmuxConfigSchema.generated.swift +++ b/Packages/macOS/CmuxFoundation/Sources/CmuxFoundation/ConfigValidation/CmuxConfigSchema.generated.swift @@ -273,55 +273,55 @@ enum CmuxEmbeddedConfigSchema { "a3NwYWNlVGVybWluYWxGb250U2l6ZSIsCiAgICAgICAgICAgICAgImRlY3JlYXNlV29ya3NwYWNlVGVybWluYWxGb250U2l6ZSIsCiAgICAgICAgICAgICAgInJlc2V0V29ya3NwYWNlVGVybWluYWxGb250U2l6ZSIsCiAgICAgICAgICAgICAgImVxdWFsaXplU3BsaXRzIiwKICAgICAgICAgICAgICAicmVzaXplLXBhbmUtbGVmdCIsCiAgICAgICAgICAgICAgInJlc2l6ZS1wYW5lLXJpZ2h0IiwKICAgICAgICAgICAgICAicmVzaXplLXBhbmUtdXAiLAogICAgICAgICAgICAgICJyZXNpemUtcGFuZS1kb3duIiwKICAgICAgICAg", "ICAgICAic3BsaXRCcm93c2VyUmlnaHQiLAogICAgICAgICAgICAgICJzcGxpdEJyb3dzZXJEb3duIiwKICAgICAgICAgICAgICAidG9nZ2xlQ2FudmFzTGF5b3V0IiwKICAgICAgICAgICAgICAiY2FudmFzUmV2ZWFsRm9jdXNlZFBhbmUiLAogICAgICAgICAgICAgICJjYW52YXNPdmVydmlldyIsCiAgICAgICAgICAgICAgImNhbnZhc1pvb21JbiIsCiAgICAgICAgICAgICAgImNhbnZhc1pvb21PdXQiLAogICAgICAgICAgICAgICJjYW52YXNab29tUmVzZXQiLAogICAgICAgICAgICAgICJjYW52YXNUaWR5IiwKICAgICAgICAg", "ICAgICAiY2FudmFzQWxpZ25MZWZ0IiwKICAgICAgICAgICAgICAiY2FudmFzQWxpZ25SaWdodCIsCiAgICAgICAgICAgICAgImNhbnZhc0FsaWduVG9wIiwKICAgICAgICAgICAgICAiY2FudmFzQWxpZ25Cb3R0b20iLAogICAgICAgICAgICAgICJjYW52YXNFcXVhbGl6ZVdpZHRocyIsCiAgICAgICAgICAgICAgImNhbnZhc0VxdWFsaXplSGVpZ2h0cyIsCiAgICAgICAgICAgICAgImNhbnZhc0Rpc3RyaWJ1dGVIb3Jpem9udGFsbHkiLAogICAgICAgICAgICAgICJjYW52YXNEaXN0cmlidXRlVmVydGljYWxseSIsCiAgICAgICAg", - "ICAgICAgInRvZ2dsZUZpbGVFeHBsb3JlciIsCiAgICAgICAgICAgICAgImZpbGVFeHBsb3Jlck9wZW5TZWxlY3Rpb24iLAogICAgICAgICAgICAgICJmaWxlRXhwbG9yZXJPcGVuU2VsZWN0aW9uRmluZGVyQWxpYXMiLAogICAgICAgICAgICAgICJzYXZlRmlsZVByZXZpZXciLAogICAgICAgICAgICAgICJvcGVuQnJvd3NlciIsCiAgICAgICAgICAgICAgImZvY3VzQnJvd3NlckFkZHJlc3NCYXIiLAogICAgICAgICAgICAgICJicm93c2VyQmFjayIsCiAgICAgICAgICAgICAgImJyb3dzZXJGb3J3YXJkIiwKICAgICAgICAgICAg", - "ICAiYnJvd3NlclJlbG9hZCIsCiAgICAgICAgICAgICAgImJyb3dzZXJIYXJkUmVsb2FkIiwKICAgICAgICAgICAgICAiYnJvd3Nlclpvb21JbiIsCiAgICAgICAgICAgICAgImJyb3dzZXJab29tT3V0IiwKICAgICAgICAgICAgICAiYnJvd3Nlclpvb21SZXNldCIsCiAgICAgICAgICAgICAgIm1hcmtkb3duWm9vbUluIiwKICAgICAgICAgICAgICAibWFya2Rvd25ab29tT3V0IiwKICAgICAgICAgICAgICAibWFya2Rvd25ab29tUmVzZXQiLAogICAgICAgICAgICAgICJmaW5kIiwKICAgICAgICAgICAgICAiZmluZEluRGlyZWN0", - "b3J5IiwKICAgICAgICAgICAgICAiZmluZE5leHQiLAogICAgICAgICAgICAgICJmaW5kUHJldmlvdXMiLAogICAgICAgICAgICAgICJoaWRlRmluZCIsCiAgICAgICAgICAgICAgInVzZVNlbGVjdGlvbkZvckZpbmQiLAogICAgICAgICAgICAgICJ0b2dnbGVCcm93c2VyRGV2ZWxvcGVyVG9vbHMiLAogICAgICAgICAgICAgICJzaG93QnJvd3NlckphdmFTY3JpcHRDb25zb2xlIiwKICAgICAgICAgICAgICAidG9nZ2xlQnJvd3NlckZvY3VzTW9kZSIsCiAgICAgICAgICAgICAgInRvZ2dsZUJyb3dzZXJEZXNpZ25Nb2RlIiwKICAg", - "ICAgICAgICAgICAidG9nZ2xlUmVhY3RHcmFiIiwKICAgICAgICAgICAgICAib3BlbkRpZmZWaWV3ZXIiLAogICAgICAgICAgICAgICJkaWZmVmlld2VyU2Nyb2xsRG93biIsCiAgICAgICAgICAgICAgImRpZmZWaWV3ZXJTY3JvbGxVcCIsCiAgICAgICAgICAgICAgImRpZmZWaWV3ZXJTY3JvbGxIYWxmUGFnZURvd24iLAogICAgICAgICAgICAgICJkaWZmVmlld2VyU2Nyb2xsSGFsZlBhZ2VVcCIsCiAgICAgICAgICAgICAgImRpZmZWaWV3ZXJTY3JvbGxEb3duRW1hY3MiLAogICAgICAgICAgICAgICJkaWZmVmlld2VyU2Nyb2xs", - "VXBFbWFjcyIsCiAgICAgICAgICAgICAgImRpZmZWaWV3ZXJTY3JvbGxUb0JvdHRvbSIsCiAgICAgICAgICAgICAgImRpZmZWaWV3ZXJTY3JvbGxUb1RvcCIsCiAgICAgICAgICAgICAgImRpZmZWaWV3ZXJPcGVuRmlsZVNlYXJjaCIsCiAgICAgICAgICAgICAgImRpZmZWaWV3ZXJOZXh0RmlsZSIsCiAgICAgICAgICAgICAgImRpZmZWaWV3ZXJQcmV2aW91c0ZpbGUiCiAgICAgICAgICAgIF0KICAgICAgICAgIH0sCiAgICAgICAgICAicHJvcGVydGllcyI6IHsKICAgICAgICAgICAgImZpbGVFeHBsb3Jlck9wZW5TZWxlY3Rpb24i", - "OiB7CiAgICAgICAgICAgICAgIiRyZWYiOiAiIy8kZGVmcy9zaW5nbGVCYXJlRmlyc3RTdHJva2VTaG9ydGN1dEJpbmRpbmdOdWxsYWJsZSIKICAgICAgICAgICAgfSwKICAgICAgICAgICAgImZpbGVFeHBsb3Jlck9wZW5TZWxlY3Rpb25GaW5kZXJBbGlhcyI6IHsKICAgICAgICAgICAgICAiJHJlZiI6ICIjLyRkZWZzL3NpbmdsZUJhcmVGaXJzdFN0cm9rZVNob3J0Y3V0QmluZGluZ051bGxhYmxlIgogICAgICAgICAgICB9LAogICAgICAgICAgICAiZGlmZlZpZXdlclNjcm9sbERvd24iOiB7CiAgICAgICAgICAgICAgIiRyZWYi", - "OiAiIy8kZGVmcy9iYXJlRmlyc3RTdHJva2VTaG9ydGN1dEJpbmRpbmdOdWxsYWJsZSIKICAgICAgICAgICAgfSwKICAgICAgICAgICAgImRpZmZWaWV3ZXJTY3JvbGxVcCI6IHsKICAgICAgICAgICAgICAiJHJlZiI6ICIjLyRkZWZzL2JhcmVGaXJzdFN0cm9rZVNob3J0Y3V0QmluZGluZ051bGxhYmxlIgogICAgICAgICAgICB9LAogICAgICAgICAgICAiZGlmZlZpZXdlclNjcm9sbEhhbGZQYWdlRG93biI6IHsKICAgICAgICAgICAgICAiJHJlZiI6ICIjLyRkZWZzL2JhcmVGaXJzdFN0cm9rZVNob3J0Y3V0QmluZGluZ051bGxh", - "YmxlIgogICAgICAgICAgICB9LAogICAgICAgICAgICAiZGlmZlZpZXdlclNjcm9sbEhhbGZQYWdlVXAiOiB7CiAgICAgICAgICAgICAgIiRyZWYiOiAiIy8kZGVmcy9iYXJlRmlyc3RTdHJva2VTaG9ydGN1dEJpbmRpbmdOdWxsYWJsZSIKICAgICAgICAgICAgfSwKICAgICAgICAgICAgImRpZmZWaWV3ZXJTY3JvbGxEb3duRW1hY3MiOiB7CiAgICAgICAgICAgICAgIiRyZWYiOiAiIy8kZGVmcy9iYXJlRmlyc3RTdHJva2VTaG9ydGN1dEJpbmRpbmdOdWxsYWJsZSIKICAgICAgICAgICAgfSwKICAgICAgICAgICAgImRpZmZWaWV3", - "ZXJTY3JvbGxVcEVtYWNzIjogewogICAgICAgICAgICAgICIkcmVmIjogIiMvJGRlZnMvYmFyZUZpcnN0U3Ryb2tlU2hvcnRjdXRCaW5kaW5nTnVsbGFibGUiCiAgICAgICAgICAgIH0sCiAgICAgICAgICAgICJkaWZmVmlld2VyU2Nyb2xsVG9Cb3R0b20iOiB7CiAgICAgICAgICAgICAgIiRyZWYiOiAiIy8kZGVmcy9iYXJlRmlyc3RTdHJva2VTaG9ydGN1dEJpbmRpbmdOdWxsYWJsZSIKICAgICAgICAgICAgfSwKICAgICAgICAgICAgImRpZmZWaWV3ZXJTY3JvbGxUb1RvcCI6IHsKICAgICAgICAgICAgICAiJHJlZiI6ICIjLyRk", - "ZWZzL2JhcmVGaXJzdFN0cm9rZVNob3J0Y3V0QmluZGluZ051bGxhYmxlIgogICAgICAgICAgICB9LAogICAgICAgICAgICAiZGlmZlZpZXdlck9wZW5GaWxlU2VhcmNoIjogewogICAgICAgICAgICAgICIkcmVmIjogIiMvJGRlZnMvYmFyZUZpcnN0U3Ryb2tlU2hvcnRjdXRCaW5kaW5nTnVsbGFibGUiCiAgICAgICAgICAgIH0sCiAgICAgICAgICAgICJkaWZmVmlld2VyTmV4dEZpbGUiOiB7CiAgICAgICAgICAgICAgIiRyZWYiOiAiIy8kZGVmcy9iYXJlRmlyc3RTdHJva2VTaG9ydGN1dEJpbmRpbmdOdWxsYWJsZSIKICAgICAg", - "ICAgICAgfSwKICAgICAgICAgICAgImRpZmZWaWV3ZXJQcmV2aW91c0ZpbGUiOiB7CiAgICAgICAgICAgICAgIiRyZWYiOiAiIy8kZGVmcy9iYXJlRmlyc3RTdHJva2VTaG9ydGN1dEJpbmRpbmdOdWxsYWJsZSIKICAgICAgICAgICAgfQogICAgICAgICAgfSwKICAgICAgICAgICJhZGRpdGlvbmFsUHJvcGVydGllcyI6IHsKICAgICAgICAgICAgIiRyZWYiOiAiIy8kZGVmcy9zaG9ydGN1dEJpbmRpbmdOdWxsYWJsZSIKICAgICAgICAgIH0KICAgICAgICB9LAogICAgICAgICJ3aGVuIjogewogICAgICAgICAgInR5cGUiOiAib2Jq", - "ZWN0IiwKICAgICAgICAgICJkZWZhdWx0Ijoge30sCiAgICAgICAgICAiZGVzY3JpcHRpb24iOiAiT3B0aW9uYWwgcGVyLWFjdGlvbiBjb250ZXh0IHByZWRpY2F0ZXMgKFZTIENvZGUtc3R5bGUgYHdoZW5gIGNsYXVzZXMpLCBrZXllZCBieSBjbXV4IGFjdGlvbiBpZC4gRWFjaCB2YWx1ZSBpcyBhIGJvb2xlYW4gZXhwcmVzc2lvbiBvdmVyIGNvbnRleHQga2V5cyBjb21iaW5lZCB3aXRoICEsICYmLCB8fCwgYW5kIHBhcmVudGhlc2VzLiBCb29sZWFuIGtleXM6IHNpZGViYXJGb2N1cywgYnJvd3NlckZvY3VzLCBtYXJrZG93bkZv", - "Y3VzLCBmaWxlUHJldmlld1RleHRFZGl0b3JGb2N1cywgc2ltdWxhdG9yRm9jdXMsIHRlcm1pbmFsRm9jdXMsIGNvbW1hbmRQYWxldHRlVmlzaWJsZSwgdGVybWluYWxGaW5kVmlzaWJsZSwgd29ya3NwYWNlQ2FudmFzTGF5b3V0LiBUeXBlZCBrZXlzIHN1cHBvcnQgY29tcGFyaXNvbnM6IHRoZSBzdHJpbmcgc2lkZWJhck1vZGUgKGZpbGVzLCBmaW5kLCBzZXNzaW9ucywgZmVlZCwgb3IgZG9jaykgYW5kIHRoZSBpbnRlZ2VycyBwYW5lQ291bnQgYW5kIHdvcmtzcGFjZUNvdW50LiBDb21wYXJpc29uIG9wZXJhdG9ycyBhcmUgPT0s", - "ICE9LCA9fiAocmVnZXgpLCA8LCA8PSwgPiwgPj0sIGFuZCBgaW4gW2EsIGJdYDsgYW4gdW5rbm93biBvciBhYnNlbnQga2V5IHJlYWRzIGFzIGZhbHNlLiBUaGUgYm9vbGVhbiBsaXRlcmFscyB0cnVlIGFuZCBmYWxzZSBhcmUgYWxzbyBhY2NlcHRlZDsgYGtleSA9PSBmYWxzZWAgaXMgdGhlIHNhbWUgYXMgYCFrZXlgLiBUaGUgYWN0aW9uJ3Mgc2hvcnRjdXQgb25seSBmaXJlcyAoYW5kIG9ubHkgY29uZmxpY3RzIHdpdGggb3RoZXIgc2hvcnRjdXRzKSB3aGVuIHRoZSBjbGF1c2UgaG9sZHMuIEV4YW1wbGVzOiB7IFwic2VsZWN0", - "V29ya3NwYWNlQnlOdW1iZXJcIjogXCIhc2lkZWJhckZvY3VzXCIgfSBzZWxlY3RzIHdvcmtzcGFjZXMgd2l0aCBDdHJsKzHigJM5IGV2ZXJ5d2hlcmUgZXhjZXB0IHdoZW4gdGhlIHJpZ2h0IHNpZGViYXIgaXMgZm9jdXNlZDsgeyBcInNlbGVjdFN1cmZhY2VCeU51bWJlclwiOiBcInNpZGViYXJNb2RlID09ICdmaW5kJyAmJiBwYW5lQ291bnQgPiAxXCIgfSBzY29wZXMgYSBiaW5kaW5nIHRvIHRoZSBGaW5kIHNpZGViYXIgd2hlbiB0aGUgd29ya3NwYWNlIGhhcyBtdWx0aXBsZSBwYW5lcy4iLAogICAgICAgICAgImRlc2NyaXB0", - "aW9uS2V5IjogInNjaGVtYURlc2NyaXB0aW9ucy5zaG9ydGN1dHMud2hlbiIsCiAgICAgICAgICAiYWRkaXRpb25hbFByb3BlcnRpZXMiOiB7CiAgICAgICAgICAgICJ0eXBlIjogInN0cmluZyIKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIH0KICAgIH0KICB9LAogICIkZGVmcyI6IHsKICAgICJjb2xvckhleCI6IHsKICAgICAgInR5cGUiOiAic3RyaW5nIiwKICAgICAgInBhdHRlcm4iOiAiXiNbMC05QS1GYS1mXXs2fSQiLAogICAgICAiZGVzY3JpcHRpb24iOiAiVXBwZXJjYXNlIG9yIGxvd2VyY2FzZSA2LWRpZ2l0IGhl", - "eCBjb2xvci4iCiAgICB9LAogICAgImNvbG9ySGV4T3JOdWxsIjogewogICAgICAib25lT2YiOiBbCiAgICAgICAgewogICAgICAgICAgIiRyZWYiOiAiIy8kZGVmcy9jb2xvckhleCIKICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJ0eXBlIjogIm51bGwiCiAgICAgICAgfQogICAgICBdCiAgICB9LAogICAgIm5vdGlmaWNhdGlvblNvdW5kT3ZlcnJpZGUiOiB7CiAgICAgICJvbmVPZiI6IFsKICAgICAgICB7CiAgICAgICAgICAidHlwZSI6ICJvYmplY3QiLAogICAgICAgICAgImFkZGl0aW9uYWxQcm9wZXJ0aWVzIjog", - "ZmFsc2UsCiAgICAgICAgICAicmVxdWlyZWQiOiBbInNvdW5kIl0sCiAgICAgICAgICAicHJvcGVydGllcyI6IHsKICAgICAgICAgICAgInNvdW5kIjogewogICAgICAgICAgICAgICJ0eXBlIjogInN0cmluZyIsCiAgICAgICAgICAgICAgImVudW0iOiBbImRlZmF1bHQiLCAiQmFzc28iLCAiQmxvdyIsICJCb3R0bGUiLCAiRnJvZyIsICJGdW5rIiwgIkdsYXNzIiwgIkhlcm8iLCAiTW9yc2UiLCAiUGluZyIsICJQb3AiLCAiUHVyciIsICJTb3N1bWkiLCAiU3VibWFyaW5lIiwgIlRpbmsiLCAibm9uZSJdCiAgICAgICAgICAgIH0K", - "ICAgICAgICAgIH0KICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJ0eXBlIjogIm9iamVjdCIsCiAgICAgICAgICAiYWRkaXRpb25hbFByb3BlcnRpZXMiOiBmYWxzZSwKICAgICAgICAgICJyZXF1aXJlZCI6IFsic291bmQiLCAiY3VzdG9tU291bmRGaWxlUGF0aCJdLAogICAgICAgICAgInByb3BlcnRpZXMiOiB7CiAgICAgICAgICAgICJzb3VuZCI6IHsgImNvbnN0IjogImN1c3RvbV9maWxlIiB9LAogICAgICAgICAgICAiY3VzdG9tU291bmRGaWxlUGF0aCI6IHsKICAgICAgICAgICAgICAidHlwZSI6ICJzdHJpbmci", - "LAogICAgICAgICAgICAgICJtaW5MZW5ndGgiOiAxLAogICAgICAgICAgICAgICJwYXR0ZXJuIjogIi4qXFxTLioiCiAgICAgICAgICAgIH0KICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJkZXNjcmlwdGlvbktleSI6ICJzY2hlbWFEZXNjcmlwdGlvbnMubm90aWZpY2F0aW9ucy5zb3VuZE92ZXJyaWRlIiwKICAgICAgImRlc2NyaXB0aW9uIjogIkEgbm90aWZpY2F0aW9uIHNvdW5kIG92ZXJyaWRlLiBgZGVmYXVsdGAgdXNlcyBub3RpZmljYXRpb25zLnNvdW5kLCBgbm9uZWAgZGlzYWJsZXMgcGxheWJhY2ss", - "IGFuZCB0aGUgcmVtYWluaW5nIHZhbHVlcyBzZWxlY3QgYSBidWlsdC1pbiBvciBjdXN0b20gc291bmQuIgogICAgfSwKICAgICJzaG9ydGN1dEJpbmRpbmciOiB7CiAgICAgICJvbmVPZiI6IFsKICAgICAgICB7CiAgICAgICAgICAiJHJlZiI6ICIjLyRkZWZzL3VuYm91bmRTaG9ydGN1dEJpbmRpbmciLAogICAgICAgICAgImRlc2NyaXB0aW9uIjogIlVuYmluZCB0aGlzIHNob3J0Y3V0LiBBY2NlcHRlZCB2YWx1ZXMgYXJlIGFuIGVtcHR5IHN0cmluZywgbm9uZSwgY2xlYXIsIHVuYm91bmQsIG9yIGRpc2FibGVkLiIKICAgICAg", - "ICB9LAogICAgICAgIHsKICAgICAgICAgICIkcmVmIjogIiMvJGRlZnMvc2hvcnRjdXRGaXJzdFN0cm9rZSIsCiAgICAgICAgICAiZGVzY3JpcHRpb24iOiAiU2luZ2xlLXN0cm9rZSBzaG9ydGN1dCwgZm9yIGV4YW1wbGUgY21kK24gb3IgY21kK3NwYWNlLiIKICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJ0eXBlIjogImFycmF5IiwKICAgICAgICAgICJtaW5JdGVtcyI6IDEsCiAgICAgICAgICAibWF4SXRlbXMiOiAyLAogICAgICAgICAgInByZWZpeEl0ZW1zIjogWwogICAgICAgICAgICB7CiAgICAgICAgICAgICAg", - "IiRyZWYiOiAiIy8kZGVmcy9zaG9ydGN1dEZpcnN0U3Ryb2tlIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgIiRyZWYiOiAiIy8kZGVmcy9zaG9ydGN1dFN0cm9rZSIKICAgICAgICAgICAgfQogICAgICAgICAgXSwKICAgICAgICAgICJkZXNjcmlwdGlvbiI6ICJDaG9yZGVkIHNob3J0Y3V0LiBFeGFtcGxlOiBbXCJjdHJsK2JcIiwgXCJjXCJdLiIKICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICAic2hvcnRjdXRCaW5kaW5nTnVsbGFibGUiOiB7CiAgICAgICJvbmVPZiI6IFsKICAgICAgICB7", - "CiAgICAgICAgICAiJHJlZiI6ICIjLyRkZWZzL3Nob3J0Y3V0QmluZGluZyIKICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJ0eXBlIjogIm51bGwiLAogICAgICAgICAgImRlc2NyaXB0aW9uIjogIlVuYmluZCB0aGlzIHNob3J0Y3V0LiIKICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICAic2luZ2xlQmFyZUZpcnN0U3Ryb2tlU2hvcnRjdXRCaW5kaW5nIjogewogICAgICAib25lT2YiOiBbCiAgICAgICAgewogICAgICAgICAgIiRyZWYiOiAiIy8kZGVmcy91bmJvdW5kU2hvcnRjdXRCaW5kaW5nIiwKICAgICAgICAg", - "ICJkZXNjcmlwdGlvbiI6ICJVbmJpbmQgdGhpcyBzaG9ydGN1dC4gQWNjZXB0ZWQgdmFsdWVzIGFyZSBhbiBlbXB0eSBzdHJpbmcsIG5vbmUsIGNsZWFyLCB1bmJvdW5kLCBvciBkaXNhYmxlZC4iCiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAiJHJlZiI6ICIjLyRkZWZzL2JhcmVGaXJzdFN0cm9rZVNob3J0Y3V0Rmlyc3RTdHJva2UiLAogICAgICAgICAgImRlc2NyaXB0aW9uIjogIlNpbmdsZS1zdHJva2Ugc2hvcnRjdXQuIEJhcmUgZmlyc3Qgc3Ryb2tlcyBhcmUgYWNjZXB0ZWQgZm9yIHRoaXMgYWN0aW9uLiIKICAg", - "ICAgICB9CiAgICAgIF0KICAgIH0sCiAgICAic2luZ2xlQmFyZUZpcnN0U3Ryb2tlU2hvcnRjdXRCaW5kaW5nTnVsbGFibGUiOiB7CiAgICAgICJvbmVPZiI6IFsKICAgICAgICB7CiAgICAgICAgICAiJHJlZiI6ICIjLyRkZWZzL3NpbmdsZUJhcmVGaXJzdFN0cm9rZVNob3J0Y3V0QmluZGluZyIKICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJ0eXBlIjogIm51bGwiLAogICAgICAgICAgImRlc2NyaXB0aW9uIjogIlVuYmluZCB0aGlzIHNob3J0Y3V0LiIKICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICAiYmFyZUZp", - "cnN0U3Ryb2tlU2hvcnRjdXRCaW5kaW5nIjogewogICAgICAib25lT2YiOiBbCiAgICAgICAgewogICAgICAgICAgIiRyZWYiOiAiIy8kZGVmcy91bmJvdW5kU2hvcnRjdXRCaW5kaW5nIiwKICAgICAgICAgICJkZXNjcmlwdGlvbiI6ICJVbmJpbmQgdGhpcyBzaG9ydGN1dC4gQWNjZXB0ZWQgdmFsdWVzIGFyZSBhbiBlbXB0eSBzdHJpbmcsIG5vbmUsIGNsZWFyLCB1bmJvdW5kLCBvciBkaXNhYmxlZC4iCiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAiJHJlZiI6ICIjLyRkZWZzL2JhcmVGaXJzdFN0cm9rZVNob3J0Y3V0", - "Rmlyc3RTdHJva2UiLAogICAgICAgICAgImRlc2NyaXB0aW9uIjogIlNpbmdsZS1zdHJva2Ugc2hvcnRjdXQuIEJhcmUgZmlyc3Qgc3Ryb2tlcyBhcmUgYWNjZXB0ZWQgZm9yIHRoaXMgYWN0aW9uLiIKICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJ0eXBlIjogImFycmF5IiwKICAgICAgICAgICJtaW5JdGVtcyI6IDEsCiAgICAgICAgICAibWF4SXRlbXMiOiAyLAogICAgICAgICAgInByZWZpeEl0ZW1zIjogWwogICAgICAgICAgICB7CiAgICAgICAgICAgICAgIiRyZWYiOiAiIy8kZGVmcy9iYXJlRmlyc3RTdHJva2VT", - "aG9ydGN1dEZpcnN0U3Ryb2tlIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgIiRyZWYiOiAiIy8kZGVmcy9zaG9ydGN1dFN0cm9rZSIKICAgICAgICAgICAgfQogICAgICAgICAgXSwKICAgICAgICAgICJkZXNjcmlwdGlvbiI6ICJDaG9yZGVkIHNob3J0Y3V0LiBCYXJlIGZpcnN0IHN0cm9rZXMgYXJlIGFjY2VwdGVkIGZvciB0aGlzIGFjdGlvbi4gRXhhbXBsZTogW1wiZ1wiLCBcImdcIl0uIgogICAgICAgIH0KICAgICAgXQogICAgfSwKICAgICJiYXJlRmlyc3RTdHJva2VTaG9ydGN1dEJpbmRp", - "bmdOdWxsYWJsZSI6IHsKICAgICAgIm9uZU9mIjogWwogICAgICAgIHsKICAgICAgICAgICIkcmVmIjogIiMvJGRlZnMvYmFyZUZpcnN0U3Ryb2tlU2hvcnRjdXRCaW5kaW5nIgogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgInR5cGUiOiAibnVsbCIsCiAgICAgICAgICAiZGVzY3JpcHRpb24iOiAiVW5iaW5kIHRoaXMgc2hvcnRjdXQuIgogICAgICAgIH0KICAgICAgXQogICAgfSwKICAgICJ1bmJvdW5kU2hvcnRjdXRCaW5kaW5nIjogewogICAgICAidHlwZSI6ICJzdHJpbmciLAogICAgICAiZW51bSI6IFsiIiwgIm5v", - "bmUiLCAiY2xlYXIiLCAidW5ib3VuZCIsICJkaXNhYmxlZCJdCiAgICB9LAogICAgInNob3J0Y3V0Rmlyc3RTdHJva2UiOiB7CiAgICAgICJhbGxPZiI6IFsKICAgICAgICB7CiAgICAgICAgICAiJHJlZiI6ICIjLyRkZWZzL3Nob3J0Y3V0U3Ryb2tlIgogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImFueU9mIjogWwogICAgICAgICAgICB7CiAgICAgICAgICAgICAgInBhdHRlcm4iOiAiXig/Oig/OltjQ11bbU1dW2REXXxbY0NdW29PXVttTV1bbU1dW2FBXVtuTl1bZERdfFtzU11baEhdW2lJXVtmRl1bdFRdfFtvT11b", - "cFBdW3RUXXxbb09dW3BQXVt0VF1baUldW29PXVtuTl18W2FBXVtsTF1bdFRdfFtjQ11bdFRdW3JSXVtsTF18W2NDXVtvT11bbk5dW3RUXVtyUl1bb09dW2xMXXxbY0NdW3RUXVtsTF184oyYfOKHp3zijKV84oyDKVxcKykrIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgInBhdHRlcm4iOiAiXig/OiB8W3NTXVtwUF1bYUFdW2NDXVtlRV18W3NTXVtwUF1bYUFdW2NDXVtlRV1bYkJdW2FBXVtyUl18PFtzU11bcFBdW2FBXVtjQ11bZUVdPikkIgogICAgICAgICAgICB9CiAgICAgICAgICBdCiAgICAg", - "ICAgfQogICAgICBdLAogICAgICAiZGVzY3JpcHRpb24iOiAiRmlyc3Qgb3Igb25seSBzaG9ydGN1dCBzdHJva2UuIE11c3QgaW5jbHVkZSBhIG1vZGlmaWVyIHVubGVzcyB0aGUga2V5IGlzIFNwYWNlLiIKICAgIH0sCiAgICAiYmFyZUZpcnN0U3Ryb2tlU2hvcnRjdXRGaXJzdFN0cm9rZSI6IHsKICAgICAgImFsbE9mIjogWwogICAgICAgIHsKICAgICAgICAgICIkcmVmIjogIiMvJGRlZnMvc2hvcnRjdXRTdHJva2UiCiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAiYW55T2YiOiBbCiAgICAgICAgICAgIHsKICAgICAg", - "ICAgICAgICAicGF0dGVybiI6ICJeKD86KD86W2NDXVttTV1bZERdfFtjQ11bb09dW21NXVttTV1bYUFdW25OXVtkRF18W3NTXVtoSF1baUldW2ZGXVt0VF18W29PXVtwUF1bdFRdfFtvT11bcFBdW3RUXVtpSV1bb09dW25OXXxbYUFdW2xMXVt0VF18W2NDXVt0VF1bclJdW2xMXXxbY0NdW29PXVtuTl1bdFRdW3JSXVtvT11bbExdfFtjQ11bdFRdW2xMXXzijJh84oenfOKMpXzijIMpXFwrKSsiCiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAicGF0dGVybiI6ICJeKD86IHxbc1NdW3BQXVthQV1bY0Nd", - "W2VFXXxbc1NdW3BQXVthQV1bY0NdW2VFXVtiQl1bYUFdW3JSXXw8W3NTXVtwUF1bYUFdW2NDXVtlRV0+KSQiCiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAicGF0dGVybiI6ICJeKD86W3JSXVtlRV1bdFRdW3VVXVtyUl1bbk5dfFtlRV1bbk5dW3RUXVtlRV1bclJdfOKGqSkkIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgInBhdHRlcm4iOiAiXig/OltsTF1bZUVdW2ZGXVt0VF18W2FBXVtyUl1bclJdW29PXVt3V11bbExdW2VFXVtmRl1bdFRdfFtsTF1bZUVdW2ZG", - "XVt0VF1bYUFdW3JSXVtyUl1bb09dW3dXXXzihpB8W3JSXVtpSV1bZ0ddW2hIXVt0VF18W2FBXVtyUl1bclJdW29PXVt3V11bclJdW2lJXVtnR11baEhdW3RUXXxbclJdW2lJXVtnR11baEhdW3RUXVthQV1bclJdW3JSXVtvT11bd1ddfOKGknxbdVVdW3BQXXxbYUFdW3JSXVtyUl1bb09dW3dXXVt1VV1bcFBdfFt1VV1bcFBdW2FBXVtyUl1bclJdW29PXVt3V1184oaRfFtkRF1bb09dW3dXXVtuTl18W2FBXVtyUl1bclJdW29PXVt3V11bZERdW29PXVt3V11bbk5dfFtkRF1bb09dW3dXXVtuTl1bYUFdW3JSXVtyUl1bb09dW3dXXXzi", - "hpMpJCIKICAgICAgICAgICAgfSwKICAgICAgICAgICAgewogICAgICAgICAgICAgICJwYXR0ZXJuIjogIl4oPzpbQS1aYS16MC05XXxbLC4vXFxcXDsnYD1cXFtcXF0tXXxbY0NdW29PXVttTV1bbU1dW2FBXXxbcFBdW2VFXVtyUl1baUldW29PXVtkRF18W2REXVtvT11bdFRdfFtzU11bbExdW2FBXVtzU11baEhdfFtiQl1bYUFdW2NDXVtrS11bc1NdW2xMXVthQV1bc1NdW2hIXXxbc1NdW2VFXVttTV1baUldW2NDXVtvT11bbExdW29PXVtuTl18W3FRXVt1VV1bb09dW3RUXVtlRV18W2FBXVtwUF1bb09dW3NTXVt0VF1bclJdW29P", - "XVtwUF1baEhdW2VFXXxbYkJdW2FBXVtjQ11ba0tdW3RUXVtpSV1bY0NdW2tLXXxbZ0ddW3JSXVthQV1bdlZdW2VFXXxbbU1dW2lJXVtuTl1bdVVdW3NTXXxbaEhdW3lZXVtwUF1baEhdW2VFXVtuTl18W3BQXVtsTF1bdVVdW3NTXXxbZUVdW3FRXVt1VV1bYUFdW2xMXVtzU118W2xMXVtlRV1bZkZdW3RUXVtiQl1bclJdW2FBXVtjQ11ba0tdW2VFXVt0VF18W29PXVtwUF1bZUVdW25OXVtiQl1bclJdW2FBXVtjQ11ba0tdW2VFXVt0VF18W3JSXVtpSV1bZ0ddW2hIXVt0VF1bYkJdW3JSXVthQV1bY0NdW2tLXVtlRV1bdFRdfFtjQ11b", - "bExdW29PXVtzU11bZUVdW2JCXVtyUl1bYUFdW2NDXVtrS11bZUVdW3RUXSkkIgogICAgICAgICAgICB9CiAgICAgICAgICBdCiAgICAgICAgfQogICAgICBdLAogICAgICAiZGVzY3JpcHRpb24iOiAiRmlyc3Qgb3Igb25seSBzaG9ydGN1dCBzdHJva2UuIEJhcmUgcHJpbnRhYmxlIGtleXMgYXJlIGFjY2VwdGVkIGZvciBhY3Rpb25zIHRoYXQgb3B0IGludG8gYmFyZSBmaXJzdCBzdHJva2VzLiIKICAgIH0sCiAgICAic2hvcnRjdXRTdHJva2UiOiB7CiAgICAgICJ0eXBlIjogInN0cmluZyIsCiAgICAgICJwYXR0ZXJuIjogIl4o", - "PzooPzpbY0NdW21NXVtkRF18W2NDXVtvT11bbU1dW21NXVthQV1bbk5dW2REXXxbc1NdW2hIXVtpSV1bZkZdW3RUXXxbb09dW3BQXVt0VF18W29PXVtwUF1bdFRdW2lJXVtvT11bbk5dfFthQV1bbExdW3RUXXxbY0NdW3RUXVtyUl1bbExdfFtjQ11bb09dW25OXVt0VF1bclJdW29PXVtsTF18W2NDXVt0VF1bbExdfOKMmHzih6d84oylfOKMgylcXCspKig/OiB8W0EtWmEtejAtOV18W2xMXVtlRV1bZkZdW3RUXXxbYUFdW3JSXVtyUl1bb09dW3dXXVtsTF1bZUVdW2ZGXVt0VF18W2xMXVtlRV1bZkZdW3RUXVthQV1bclJdW3JSXVtv", - "T11bd1ddfOKGkHxbclJdW2lJXVtnR11baEhdW3RUXXxbYUFdW3JSXVtyUl1bb09dW3dXXVtyUl1baUldW2dHXVtoSF1bdFRdfFtyUl1baUldW2dHXVtoSF1bdFRdW2FBXVtyUl1bclJdW29PXVt3V1184oaSfFt1VV1bcFBdfFthQV1bclJdW3JSXVtvT11bd1ddW3VVXVtwUF18W3VVXVtwUF1bYUFdW3JSXVtyUl1bb09dW3dXXXzihpF8W2REXVtvT11bd1ddW25OXXxbYUFdW3JSXVtyUl1bb09dW3dXXVtkRF1bb09dW3dXXVtuTl18W2REXVtvT11bd1ddW25OXVthQV1bclJdW3JSXVtvT11bd1ddfOKGk3xbdFRdW2FBXVtiQl18W3JS", - "XVtlRV1bdFRdW3VVXVtyUl1bbk5dfFtlRV1bbk5dW3RUXVtlRV1bclJdfOKGqXxbc1NdW3BQXVthQV1bY0NdW2VFXXxbc1NdW3BQXVthQV1bY0NdW2VFXVtiQl1bYUFdW3JSXXw8W3NTXVtwUF1bYUFdW2NDXVtlRV0+fFssLi9cXFxcOydgPVxcW1xcXS1dfFtjQ11bb09dW21NXVttTV1bYUFdfFtwUF1bZUVdW3JSXVtpSV1bb09dW2REXXxbZERdW29PXVt0VF18W3NTXVtsTF1bYUFdW3NTXVtoSF18W2JCXVthQV1bY0NdW2tLXVtzU11bbExdW2FBXVtzU11baEhdfFtzU11bZUVdW21NXVtpSV1bY0NdW29PXVtsTF1bb09dW25OXXxb", - "cVFdW3VVXVtvT11bdFRdW2VFXXxbYUFdW3BQXVtvT11bc1NdW3RUXVtyUl1bb09dW3BQXVtoSF1bZUVdfFtiQl1bYUFdW2NDXVtrS11bdFRdW2lJXVtjQ11ba0tdfFtnR11bclJdW2FBXVt2Vl1bZUVdfFttTV1baUldW25OXVt1VV1bc1NdfFtoSF1beVldW3BQXVtoSF1bZUVdW25OXXxbcFBdW2xMXVt1VV1bc1NdfFtlRV1bcVFdW3VVXVthQV1bbExdW3NTXXxbbExdW2VFXVtmRl1bdFRdW2JCXVtyUl1bYUFdW2NDXVtrS11bZUVdW3RUXXxbb09dW3BQXVtlRV1bbk5dW2JCXVtyUl1bYUFdW2NDXVtrS11bZUVdW3RUXXxbclJdW2lJ", - "XVtnR11baEhdW3RUXVtiQl1bclJdW2FBXVtjQ11ba0tdW2VFXVt0VF18W2NDXVtsTF1bb09dW3NTXVtlRV1bYkJdW3JSXVthQV1bY0NdW2tLXVtlRV1bdFRdfFtmRl0oPzpbMS05XXwxWzAtOV18MjApfFt2Vl1bb09dW2xMXVt1VV1bbU1dW2VFXVt1VV1bcFBdfFttTV1bZUVdW2REXVtpSV1bYUFdW3ZWXVtvT11bbExdW3VVXVttTV1bZUVdW3VVXVtwUF18W21NXVtlRV1bZERdW2lJXVthQV1cXC5bdlZdW29PXVtsTF1bdVVdW21NXVtlRV1bdVVdW3BQXXxbdlZdW29PXVtsTF1bdVVdW21NXVtlRV1bZERdW29PXVt3V11bbk5dfFtt", - "TV1bZUVdW2REXVtpSV1bYUFdW3ZWXVtvT11bbExdW3VVXVttTV1bZUVdW2REXVtvT11bd1ddW25OXXxbbU1dW2VFXVtkRF1baUldW2FBXVxcLlt2Vl1bb09dW2xMXVt1VV1bbU1dW2VFXVtkRF1bb09dW3dXXVtuTl18W2JCXVtyUl1baUldW2dHXVtoSF1bdFRdW25OXVtlRV1bc1NdW3NTXVt1VV1bcFBdfFttTV1bZUVdW2REXVtpSV1bYUFdW2JCXVtyUl1baUldW2dHXVtoSF1bdFRdW25OXVtlRV1bc1NdW3NTXVt1VV1bcFBdfFttTV1bZUVdW2REXVtpSV1bYUFdXFwuW2JCXVtyUl1baUldW2dHXVtoSF1bdFRdW25OXVtlRV1bc1Nd", - "W3NTXVt1VV1bcFBdfFtiQl1bclJdW2lJXVtnR11baEhdW3RUXVtuTl1bZUVdW3NTXVtzU11bZERdW29PXVt3V11bbk5dfFttTV1bZUVdW2REXVtpSV1bYUFdW2JCXVtyUl1baUldW2dHXVtoSF1bdFRdW25OXVtlRV1bc1NdW3NTXVtkRF1bb09dW3dXXVtuTl18W21NXVtlRV1bZERdW2lJXVthQV1cXC5bYkJdW3JSXVtpSV1bZ0ddW2hIXVt0VF1bbk5dW2VFXVtzU11bc1NdW2REXVtvT11bd1ddW25OXXxbbU1dW3VVXVt0VF1bZUVdfFttTV1bZUVdW2REXVtpSV1bYUFdW21NXVt1VV1bdFRdW2VFXXxbbU1dW2VFXVtkRF1baUldW2FB", - "XVxcLlttTV1bdVVdW3RUXVtlRV18W3BQXVtsTF1bYUFdW3lZXVtwUF1bYUFdW3VVXVtzU11bZUVdfFttTV1bZUVdW2REXVtpSV1bYUFdW3BQXVtsTF1bYUFdW3lZXVtwUF1bYUFdW3VVXVtzU11bZUVdfFttTV1bZUVdW2REXVtpSV1bYUFdXFwuW3BQXVtsTF1bYUFdW3lZXVtwUF1bYUFdW3VVXVtzU11bZUVdfFtuTl1bZUVdW3hYXVt0VF1bdFRdW3JSXVthQV1bY0NdW2tLXXxbbU1dW2VFXVtkRF1baUldW2FBXVtuTl1bZUVdW3hYXVt0VF18W21NXVtlRV1bZERdW2lJXVthQV1cXC5bbk5dW2VFXVt4WF1bdFRdfFttTV1bZUVdW2RE", - "XVtpSV1bYUFdXFwuW25OXVtlRV1beFhdW3RUXVt0VF1bclJdW2FBXVtjQ11ba0tdfFtwUF1bclJdW2VFXVt2Vl1baUldW29PXVt1VV1bc1NdW3RUXVtyUl1bYUFdW2NDXVtrS118W21NXVtlRV1bZERdW2lJXVthQV1bcFBdW3JSXVtlRV1bdlZdW2lJXVtvT11bdVVdW3NTXXxbbU1dW2VFXVtkRF1baUldW2FBXVxcLltwUF1bclJdW2VFXVt2Vl1baUldW29PXVt1VV1bc1NdfFttTV1bZUVdW2REXVtpSV1bYUFdXFwuW3BQXVtyUl1bZUVdW3ZWXVtpSV1bb09dW3VVXVtzU11bdFRdW3JSXVthQV1bY0NdW2tLXSkkIiwKICAgICAgImRl", - "c2NyaXB0aW9uIjogIk9uZSBrZXlib2FyZCBzaG9ydGN1dCBzdHJva2UgdXNpbmcgbW9kaWZpZXIra2V5IHN5bnRheC4gU3VwcG9ydGVkIGtleSBuYW1lcyBpbmNsdWRlIHNwYWNlLCBTcGFjZSwgPHNwYWNlPiwgYW5kIDxTcGFjZT4uIgogICAgfQogIH0KfQo=", + "ICAgICAgInRvZ2dsZUZpbGVFeHBsb3JlciIsCiAgICAgICAgICAgICAgImZpbGVFeHBsb3Jlck9wZW5TZWxlY3Rpb24iLAogICAgICAgICAgICAgICJmaWxlRXhwbG9yZXJPcGVuU2VsZWN0aW9uRmluZGVyQWxpYXMiLAogICAgICAgICAgICAgICJzYXZlRmlsZVByZXZpZXciLAogICAgICAgICAgICAgICJ0b2dnbGVGaWxlRWRpdG9yV29yZFdyYXAiLAogICAgICAgICAgICAgICJvcGVuQnJvd3NlciIsCiAgICAgICAgICAgICAgImZvY3VzQnJvd3NlckFkZHJlc3NCYXIiLAogICAgICAgICAgICAgICJicm93c2VyQmFjayIsCiAg", + "ICAgICAgICAgICAgImJyb3dzZXJGb3J3YXJkIiwKICAgICAgICAgICAgICAiYnJvd3NlclJlbG9hZCIsCiAgICAgICAgICAgICAgImJyb3dzZXJIYXJkUmVsb2FkIiwKICAgICAgICAgICAgICAiYnJvd3Nlclpvb21JbiIsCiAgICAgICAgICAgICAgImJyb3dzZXJab29tT3V0IiwKICAgICAgICAgICAgICAiYnJvd3Nlclpvb21SZXNldCIsCiAgICAgICAgICAgICAgIm1hcmtkb3duWm9vbUluIiwKICAgICAgICAgICAgICAibWFya2Rvd25ab29tT3V0IiwKICAgICAgICAgICAgICAibWFya2Rvd25ab29tUmVzZXQiLAogICAgICAg", + "ICAgICAgICJmaW5kIiwKICAgICAgICAgICAgICAiZmluZEluRGlyZWN0b3J5IiwKICAgICAgICAgICAgICAiZmluZE5leHQiLAogICAgICAgICAgICAgICJmaW5kUHJldmlvdXMiLAogICAgICAgICAgICAgICJoaWRlRmluZCIsCiAgICAgICAgICAgICAgInVzZVNlbGVjdGlvbkZvckZpbmQiLAogICAgICAgICAgICAgICJ0b2dnbGVCcm93c2VyRGV2ZWxvcGVyVG9vbHMiLAogICAgICAgICAgICAgICJzaG93QnJvd3NlckphdmFTY3JpcHRDb25zb2xlIiwKICAgICAgICAgICAgICAidG9nZ2xlQnJvd3NlckZvY3VzTW9kZSIsCiAg", + "ICAgICAgICAgICAgInRvZ2dsZUJyb3dzZXJEZXNpZ25Nb2RlIiwKICAgICAgICAgICAgICAidG9nZ2xlUmVhY3RHcmFiIiwKICAgICAgICAgICAgICAib3BlbkRpZmZWaWV3ZXIiLAogICAgICAgICAgICAgICJkaWZmVmlld2VyU2Nyb2xsRG93biIsCiAgICAgICAgICAgICAgImRpZmZWaWV3ZXJTY3JvbGxVcCIsCiAgICAgICAgICAgICAgImRpZmZWaWV3ZXJTY3JvbGxIYWxmUGFnZURvd24iLAogICAgICAgICAgICAgICJkaWZmVmlld2VyU2Nyb2xsSGFsZlBhZ2VVcCIsCiAgICAgICAgICAgICAgImRpZmZWaWV3ZXJTY3JvbGxE", + "b3duRW1hY3MiLAogICAgICAgICAgICAgICJkaWZmVmlld2VyU2Nyb2xsVXBFbWFjcyIsCiAgICAgICAgICAgICAgImRpZmZWaWV3ZXJTY3JvbGxUb0JvdHRvbSIsCiAgICAgICAgICAgICAgImRpZmZWaWV3ZXJTY3JvbGxUb1RvcCIsCiAgICAgICAgICAgICAgImRpZmZWaWV3ZXJPcGVuRmlsZVNlYXJjaCIsCiAgICAgICAgICAgICAgImRpZmZWaWV3ZXJOZXh0RmlsZSIsCiAgICAgICAgICAgICAgImRpZmZWaWV3ZXJQcmV2aW91c0ZpbGUiCiAgICAgICAgICAgIF0KICAgICAgICAgIH0sCiAgICAgICAgICAicHJvcGVydGllcyI6", + "IHsKICAgICAgICAgICAgImZpbGVFeHBsb3Jlck9wZW5TZWxlY3Rpb24iOiB7CiAgICAgICAgICAgICAgIiRyZWYiOiAiIy8kZGVmcy9zaW5nbGVCYXJlRmlyc3RTdHJva2VTaG9ydGN1dEJpbmRpbmdOdWxsYWJsZSIKICAgICAgICAgICAgfSwKICAgICAgICAgICAgImZpbGVFeHBsb3Jlck9wZW5TZWxlY3Rpb25GaW5kZXJBbGlhcyI6IHsKICAgICAgICAgICAgICAiJHJlZiI6ICIjLyRkZWZzL3NpbmdsZUJhcmVGaXJzdFN0cm9rZVNob3J0Y3V0QmluZGluZ051bGxhYmxlIgogICAgICAgICAgICB9LAogICAgICAgICAgICAiZGlm", + "ZlZpZXdlclNjcm9sbERvd24iOiB7CiAgICAgICAgICAgICAgIiRyZWYiOiAiIy8kZGVmcy9iYXJlRmlyc3RTdHJva2VTaG9ydGN1dEJpbmRpbmdOdWxsYWJsZSIKICAgICAgICAgICAgfSwKICAgICAgICAgICAgImRpZmZWaWV3ZXJTY3JvbGxVcCI6IHsKICAgICAgICAgICAgICAiJHJlZiI6ICIjLyRkZWZzL2JhcmVGaXJzdFN0cm9rZVNob3J0Y3V0QmluZGluZ051bGxhYmxlIgogICAgICAgICAgICB9LAogICAgICAgICAgICAiZGlmZlZpZXdlclNjcm9sbEhhbGZQYWdlRG93biI6IHsKICAgICAgICAgICAgICAiJHJlZiI6ICIj", + "LyRkZWZzL2JhcmVGaXJzdFN0cm9rZVNob3J0Y3V0QmluZGluZ051bGxhYmxlIgogICAgICAgICAgICB9LAogICAgICAgICAgICAiZGlmZlZpZXdlclNjcm9sbEhhbGZQYWdlVXAiOiB7CiAgICAgICAgICAgICAgIiRyZWYiOiAiIy8kZGVmcy9iYXJlRmlyc3RTdHJva2VTaG9ydGN1dEJpbmRpbmdOdWxsYWJsZSIKICAgICAgICAgICAgfSwKICAgICAgICAgICAgImRpZmZWaWV3ZXJTY3JvbGxEb3duRW1hY3MiOiB7CiAgICAgICAgICAgICAgIiRyZWYiOiAiIy8kZGVmcy9iYXJlRmlyc3RTdHJva2VTaG9ydGN1dEJpbmRpbmdOdWxs", + "YWJsZSIKICAgICAgICAgICAgfSwKICAgICAgICAgICAgImRpZmZWaWV3ZXJTY3JvbGxVcEVtYWNzIjogewogICAgICAgICAgICAgICIkcmVmIjogIiMvJGRlZnMvYmFyZUZpcnN0U3Ryb2tlU2hvcnRjdXRCaW5kaW5nTnVsbGFibGUiCiAgICAgICAgICAgIH0sCiAgICAgICAgICAgICJkaWZmVmlld2VyU2Nyb2xsVG9Cb3R0b20iOiB7CiAgICAgICAgICAgICAgIiRyZWYiOiAiIy8kZGVmcy9iYXJlRmlyc3RTdHJva2VTaG9ydGN1dEJpbmRpbmdOdWxsYWJsZSIKICAgICAgICAgICAgfSwKICAgICAgICAgICAgImRpZmZWaWV3ZXJT", + "Y3JvbGxUb1RvcCI6IHsKICAgICAgICAgICAgICAiJHJlZiI6ICIjLyRkZWZzL2JhcmVGaXJzdFN0cm9rZVNob3J0Y3V0QmluZGluZ051bGxhYmxlIgogICAgICAgICAgICB9LAogICAgICAgICAgICAiZGlmZlZpZXdlck9wZW5GaWxlU2VhcmNoIjogewogICAgICAgICAgICAgICIkcmVmIjogIiMvJGRlZnMvYmFyZUZpcnN0U3Ryb2tlU2hvcnRjdXRCaW5kaW5nTnVsbGFibGUiCiAgICAgICAgICAgIH0sCiAgICAgICAgICAgICJkaWZmVmlld2VyTmV4dEZpbGUiOiB7CiAgICAgICAgICAgICAgIiRyZWYiOiAiIy8kZGVmcy9iYXJl", + "Rmlyc3RTdHJva2VTaG9ydGN1dEJpbmRpbmdOdWxsYWJsZSIKICAgICAgICAgICAgfSwKICAgICAgICAgICAgImRpZmZWaWV3ZXJQcmV2aW91c0ZpbGUiOiB7CiAgICAgICAgICAgICAgIiRyZWYiOiAiIy8kZGVmcy9iYXJlRmlyc3RTdHJva2VTaG9ydGN1dEJpbmRpbmdOdWxsYWJsZSIKICAgICAgICAgICAgfQogICAgICAgICAgfSwKICAgICAgICAgICJhZGRpdGlvbmFsUHJvcGVydGllcyI6IHsKICAgICAgICAgICAgIiRyZWYiOiAiIy8kZGVmcy9zaG9ydGN1dEJpbmRpbmdOdWxsYWJsZSIKICAgICAgICAgIH0KICAgICAgICB9", + "LAogICAgICAgICJ3aGVuIjogewogICAgICAgICAgInR5cGUiOiAib2JqZWN0IiwKICAgICAgICAgICJkZWZhdWx0Ijoge30sCiAgICAgICAgICAiZGVzY3JpcHRpb24iOiAiT3B0aW9uYWwgcGVyLWFjdGlvbiBjb250ZXh0IHByZWRpY2F0ZXMgKFZTIENvZGUtc3R5bGUgYHdoZW5gIGNsYXVzZXMpLCBrZXllZCBieSBjbXV4IGFjdGlvbiBpZC4gRWFjaCB2YWx1ZSBpcyBhIGJvb2xlYW4gZXhwcmVzc2lvbiBvdmVyIGNvbnRleHQga2V5cyBjb21iaW5lZCB3aXRoICEsICYmLCB8fCwgYW5kIHBhcmVudGhlc2VzLiBCb29sZWFuIGtl", + "eXM6IHNpZGViYXJGb2N1cywgYnJvd3NlckZvY3VzLCBtYXJrZG93bkZvY3VzLCBmaWxlUHJldmlld1RleHRFZGl0b3JGb2N1cywgc2ltdWxhdG9yRm9jdXMsIHRlcm1pbmFsRm9jdXMsIGNvbW1hbmRQYWxldHRlVmlzaWJsZSwgdGVybWluYWxGaW5kVmlzaWJsZSwgd29ya3NwYWNlQ2FudmFzTGF5b3V0LiBUeXBlZCBrZXlzIHN1cHBvcnQgY29tcGFyaXNvbnM6IHRoZSBzdHJpbmcgc2lkZWJhck1vZGUgKGZpbGVzLCBmaW5kLCBzZXNzaW9ucywgZmVlZCwgb3IgZG9jaykgYW5kIHRoZSBpbnRlZ2VycyBwYW5lQ291bnQgYW5kIHdv", + "cmtzcGFjZUNvdW50LiBDb21wYXJpc29uIG9wZXJhdG9ycyBhcmUgPT0sICE9LCA9fiAocmVnZXgpLCA8LCA8PSwgPiwgPj0sIGFuZCBgaW4gW2EsIGJdYDsgYW4gdW5rbm93biBvciBhYnNlbnQga2V5IHJlYWRzIGFzIGZhbHNlLiBUaGUgYm9vbGVhbiBsaXRlcmFscyB0cnVlIGFuZCBmYWxzZSBhcmUgYWxzbyBhY2NlcHRlZDsgYGtleSA9PSBmYWxzZWAgaXMgdGhlIHNhbWUgYXMgYCFrZXlgLiBUaGUgYWN0aW9uJ3Mgc2hvcnRjdXQgb25seSBmaXJlcyAoYW5kIG9ubHkgY29uZmxpY3RzIHdpdGggb3RoZXIgc2hvcnRjdXRzKSB3", + "aGVuIHRoZSBjbGF1c2UgaG9sZHMuIEV4YW1wbGVzOiB7IFwic2VsZWN0V29ya3NwYWNlQnlOdW1iZXJcIjogXCIhc2lkZWJhckZvY3VzXCIgfSBzZWxlY3RzIHdvcmtzcGFjZXMgd2l0aCBDdHJsKzHigJM5IGV2ZXJ5d2hlcmUgZXhjZXB0IHdoZW4gdGhlIHJpZ2h0IHNpZGViYXIgaXMgZm9jdXNlZDsgeyBcInNlbGVjdFN1cmZhY2VCeU51bWJlclwiOiBcInNpZGViYXJNb2RlID09ICdmaW5kJyAmJiBwYW5lQ291bnQgPiAxXCIgfSBzY29wZXMgYSBiaW5kaW5nIHRvIHRoZSBGaW5kIHNpZGViYXIgd2hlbiB0aGUgd29ya3NwYWNl", + "IGhhcyBtdWx0aXBsZSBwYW5lcy4iLAogICAgICAgICAgImRlc2NyaXB0aW9uS2V5IjogInNjaGVtYURlc2NyaXB0aW9ucy5zaG9ydGN1dHMud2hlbiIsCiAgICAgICAgICAiYWRkaXRpb25hbFByb3BlcnRpZXMiOiB7CiAgICAgICAgICAgICJ0eXBlIjogInN0cmluZyIKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIH0KICAgIH0KICB9LAogICIkZGVmcyI6IHsKICAgICJjb2xvckhleCI6IHsKICAgICAgInR5cGUiOiAic3RyaW5nIiwKICAgICAgInBhdHRlcm4iOiAiXiNbMC05QS1GYS1mXXs2fSQiLAogICAgICAiZGVzY3Jp", + "cHRpb24iOiAiVXBwZXJjYXNlIG9yIGxvd2VyY2FzZSA2LWRpZ2l0IGhleCBjb2xvci4iCiAgICB9LAogICAgImNvbG9ySGV4T3JOdWxsIjogewogICAgICAib25lT2YiOiBbCiAgICAgICAgewogICAgICAgICAgIiRyZWYiOiAiIy8kZGVmcy9jb2xvckhleCIKICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJ0eXBlIjogIm51bGwiCiAgICAgICAgfQogICAgICBdCiAgICB9LAogICAgIm5vdGlmaWNhdGlvblNvdW5kT3ZlcnJpZGUiOiB7CiAgICAgICJvbmVPZiI6IFsKICAgICAgICB7CiAgICAgICAgICAidHlwZSI6ICJv", + "YmplY3QiLAogICAgICAgICAgImFkZGl0aW9uYWxQcm9wZXJ0aWVzIjogZmFsc2UsCiAgICAgICAgICAicmVxdWlyZWQiOiBbInNvdW5kIl0sCiAgICAgICAgICAicHJvcGVydGllcyI6IHsKICAgICAgICAgICAgInNvdW5kIjogewogICAgICAgICAgICAgICJ0eXBlIjogInN0cmluZyIsCiAgICAgICAgICAgICAgImVudW0iOiBbImRlZmF1bHQiLCAiQmFzc28iLCAiQmxvdyIsICJCb3R0bGUiLCAiRnJvZyIsICJGdW5rIiwgIkdsYXNzIiwgIkhlcm8iLCAiTW9yc2UiLCAiUGluZyIsICJQb3AiLCAiUHVyciIsICJTb3N1bWkiLCAi", + "U3VibWFyaW5lIiwgIlRpbmsiLCAibm9uZSJdCiAgICAgICAgICAgIH0KICAgICAgICAgIH0KICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJ0eXBlIjogIm9iamVjdCIsCiAgICAgICAgICAiYWRkaXRpb25hbFByb3BlcnRpZXMiOiBmYWxzZSwKICAgICAgICAgICJyZXF1aXJlZCI6IFsic291bmQiLCAiY3VzdG9tU291bmRGaWxlUGF0aCJdLAogICAgICAgICAgInByb3BlcnRpZXMiOiB7CiAgICAgICAgICAgICJzb3VuZCI6IHsgImNvbnN0IjogImN1c3RvbV9maWxlIiB9LAogICAgICAgICAgICAiY3VzdG9tU291bmRG", + "aWxlUGF0aCI6IHsKICAgICAgICAgICAgICAidHlwZSI6ICJzdHJpbmciLAogICAgICAgICAgICAgICJtaW5MZW5ndGgiOiAxLAogICAgICAgICAgICAgICJwYXR0ZXJuIjogIi4qXFxTLioiCiAgICAgICAgICAgIH0KICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJkZXNjcmlwdGlvbktleSI6ICJzY2hlbWFEZXNjcmlwdGlvbnMubm90aWZpY2F0aW9ucy5zb3VuZE92ZXJyaWRlIiwKICAgICAgImRlc2NyaXB0aW9uIjogIkEgbm90aWZpY2F0aW9uIHNvdW5kIG92ZXJyaWRlLiBgZGVmYXVsdGAgdXNlcyBub3Rp", + "ZmljYXRpb25zLnNvdW5kLCBgbm9uZWAgZGlzYWJsZXMgcGxheWJhY2ssIGFuZCB0aGUgcmVtYWluaW5nIHZhbHVlcyBzZWxlY3QgYSBidWlsdC1pbiBvciBjdXN0b20gc291bmQuIgogICAgfSwKICAgICJzaG9ydGN1dEJpbmRpbmciOiB7CiAgICAgICJvbmVPZiI6IFsKICAgICAgICB7CiAgICAgICAgICAiJHJlZiI6ICIjLyRkZWZzL3VuYm91bmRTaG9ydGN1dEJpbmRpbmciLAogICAgICAgICAgImRlc2NyaXB0aW9uIjogIlVuYmluZCB0aGlzIHNob3J0Y3V0LiBBY2NlcHRlZCB2YWx1ZXMgYXJlIGFuIGVtcHR5IHN0cmluZywg", + "bm9uZSwgY2xlYXIsIHVuYm91bmQsIG9yIGRpc2FibGVkLiIKICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICIkcmVmIjogIiMvJGRlZnMvc2hvcnRjdXRGaXJzdFN0cm9rZSIsCiAgICAgICAgICAiZGVzY3JpcHRpb24iOiAiU2luZ2xlLXN0cm9rZSBzaG9ydGN1dCwgZm9yIGV4YW1wbGUgY21kK24gb3IgY21kK3NwYWNlLiIKICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJ0eXBlIjogImFycmF5IiwKICAgICAgICAgICJtaW5JdGVtcyI6IDEsCiAgICAgICAgICAibWF4SXRlbXMiOiAyLAogICAgICAgICAgInBy", + "ZWZpeEl0ZW1zIjogWwogICAgICAgICAgICB7CiAgICAgICAgICAgICAgIiRyZWYiOiAiIy8kZGVmcy9zaG9ydGN1dEZpcnN0U3Ryb2tlIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgIiRyZWYiOiAiIy8kZGVmcy9zaG9ydGN1dFN0cm9rZSIKICAgICAgICAgICAgfQogICAgICAgICAgXSwKICAgICAgICAgICJkZXNjcmlwdGlvbiI6ICJDaG9yZGVkIHNob3J0Y3V0LiBFeGFtcGxlOiBbXCJjdHJsK2JcIiwgXCJjXCJdLiIKICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICAic2hvcnRjdXRCaW5k", + "aW5nTnVsbGFibGUiOiB7CiAgICAgICJvbmVPZiI6IFsKICAgICAgICB7CiAgICAgICAgICAiJHJlZiI6ICIjLyRkZWZzL3Nob3J0Y3V0QmluZGluZyIKICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJ0eXBlIjogIm51bGwiLAogICAgICAgICAgImRlc2NyaXB0aW9uIjogIlVuYmluZCB0aGlzIHNob3J0Y3V0LiIKICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICAic2luZ2xlQmFyZUZpcnN0U3Ryb2tlU2hvcnRjdXRCaW5kaW5nIjogewogICAgICAib25lT2YiOiBbCiAgICAgICAgewogICAgICAgICAgIiRyZWYiOiAi", + "Iy8kZGVmcy91bmJvdW5kU2hvcnRjdXRCaW5kaW5nIiwKICAgICAgICAgICJkZXNjcmlwdGlvbiI6ICJVbmJpbmQgdGhpcyBzaG9ydGN1dC4gQWNjZXB0ZWQgdmFsdWVzIGFyZSBhbiBlbXB0eSBzdHJpbmcsIG5vbmUsIGNsZWFyLCB1bmJvdW5kLCBvciBkaXNhYmxlZC4iCiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAiJHJlZiI6ICIjLyRkZWZzL2JhcmVGaXJzdFN0cm9rZVNob3J0Y3V0Rmlyc3RTdHJva2UiLAogICAgICAgICAgImRlc2NyaXB0aW9uIjogIlNpbmdsZS1zdHJva2Ugc2hvcnRjdXQuIEJhcmUgZmlyc3Qg", + "c3Ryb2tlcyBhcmUgYWNjZXB0ZWQgZm9yIHRoaXMgYWN0aW9uLiIKICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICAic2luZ2xlQmFyZUZpcnN0U3Ryb2tlU2hvcnRjdXRCaW5kaW5nTnVsbGFibGUiOiB7CiAgICAgICJvbmVPZiI6IFsKICAgICAgICB7CiAgICAgICAgICAiJHJlZiI6ICIjLyRkZWZzL3NpbmdsZUJhcmVGaXJzdFN0cm9rZVNob3J0Y3V0QmluZGluZyIKICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJ0eXBlIjogIm51bGwiLAogICAgICAgICAgImRlc2NyaXB0aW9uIjogIlVuYmluZCB0aGlzIHNob3J0", + "Y3V0LiIKICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICAiYmFyZUZpcnN0U3Ryb2tlU2hvcnRjdXRCaW5kaW5nIjogewogICAgICAib25lT2YiOiBbCiAgICAgICAgewogICAgICAgICAgIiRyZWYiOiAiIy8kZGVmcy91bmJvdW5kU2hvcnRjdXRCaW5kaW5nIiwKICAgICAgICAgICJkZXNjcmlwdGlvbiI6ICJVbmJpbmQgdGhpcyBzaG9ydGN1dC4gQWNjZXB0ZWQgdmFsdWVzIGFyZSBhbiBlbXB0eSBzdHJpbmcsIG5vbmUsIGNsZWFyLCB1bmJvdW5kLCBvciBkaXNhYmxlZC4iCiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAg", + "ICAiJHJlZiI6ICIjLyRkZWZzL2JhcmVGaXJzdFN0cm9rZVNob3J0Y3V0Rmlyc3RTdHJva2UiLAogICAgICAgICAgImRlc2NyaXB0aW9uIjogIlNpbmdsZS1zdHJva2Ugc2hvcnRjdXQuIEJhcmUgZmlyc3Qgc3Ryb2tlcyBhcmUgYWNjZXB0ZWQgZm9yIHRoaXMgYWN0aW9uLiIKICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJ0eXBlIjogImFycmF5IiwKICAgICAgICAgICJtaW5JdGVtcyI6IDEsCiAgICAgICAgICAibWF4SXRlbXMiOiAyLAogICAgICAgICAgInByZWZpeEl0ZW1zIjogWwogICAgICAgICAgICB7CiAgICAg", + "ICAgICAgICAgIiRyZWYiOiAiIy8kZGVmcy9iYXJlRmlyc3RTdHJva2VTaG9ydGN1dEZpcnN0U3Ryb2tlIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgIiRyZWYiOiAiIy8kZGVmcy9zaG9ydGN1dFN0cm9rZSIKICAgICAgICAgICAgfQogICAgICAgICAgXSwKICAgICAgICAgICJkZXNjcmlwdGlvbiI6ICJDaG9yZGVkIHNob3J0Y3V0LiBCYXJlIGZpcnN0IHN0cm9rZXMgYXJlIGFjY2VwdGVkIGZvciB0aGlzIGFjdGlvbi4gRXhhbXBsZTogW1wiZ1wiLCBcImdcIl0uIgogICAgICAgIH0KICAgICAg", + "XQogICAgfSwKICAgICJiYXJlRmlyc3RTdHJva2VTaG9ydGN1dEJpbmRpbmdOdWxsYWJsZSI6IHsKICAgICAgIm9uZU9mIjogWwogICAgICAgIHsKICAgICAgICAgICIkcmVmIjogIiMvJGRlZnMvYmFyZUZpcnN0U3Ryb2tlU2hvcnRjdXRCaW5kaW5nIgogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgInR5cGUiOiAibnVsbCIsCiAgICAgICAgICAiZGVzY3JpcHRpb24iOiAiVW5iaW5kIHRoaXMgc2hvcnRjdXQuIgogICAgICAgIH0KICAgICAgXQogICAgfSwKICAgICJ1bmJvdW5kU2hvcnRjdXRCaW5kaW5nIjogewogICAg", + "ICAidHlwZSI6ICJzdHJpbmciLAogICAgICAiZW51bSI6IFsiIiwgIm5vbmUiLCAiY2xlYXIiLCAidW5ib3VuZCIsICJkaXNhYmxlZCJdCiAgICB9LAogICAgInNob3J0Y3V0Rmlyc3RTdHJva2UiOiB7CiAgICAgICJhbGxPZiI6IFsKICAgICAgICB7CiAgICAgICAgICAiJHJlZiI6ICIjLyRkZWZzL3Nob3J0Y3V0U3Ryb2tlIgogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImFueU9mIjogWwogICAgICAgICAgICB7CiAgICAgICAgICAgICAgInBhdHRlcm4iOiAiXig/Oig/OltjQ11bbU1dW2REXXxbY0NdW29PXVttTV1b", + "bU1dW2FBXVtuTl1bZERdfFtzU11baEhdW2lJXVtmRl1bdFRdfFtvT11bcFBdW3RUXXxbb09dW3BQXVt0VF1baUldW29PXVtuTl18W2FBXVtsTF1bdFRdfFtjQ11bdFRdW3JSXVtsTF18W2NDXVtvT11bbk5dW3RUXVtyUl1bb09dW2xMXXxbY0NdW3RUXVtsTF184oyYfOKHp3zijKV84oyDKVxcKykrIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgInBhdHRlcm4iOiAiXig/OiB8W3NTXVtwUF1bYUFdW2NDXVtlRV18W3NTXVtwUF1bYUFdW2NDXVtlRV1bYkJdW2FBXVtyUl18PFtzU11bcFBdW2FBXVtj", + "Q11bZUVdPikkIgogICAgICAgICAgICB9CiAgICAgICAgICBdCiAgICAgICAgfQogICAgICBdLAogICAgICAiZGVzY3JpcHRpb24iOiAiRmlyc3Qgb3Igb25seSBzaG9ydGN1dCBzdHJva2UuIE11c3QgaW5jbHVkZSBhIG1vZGlmaWVyIHVubGVzcyB0aGUga2V5IGlzIFNwYWNlLiIKICAgIH0sCiAgICAiYmFyZUZpcnN0U3Ryb2tlU2hvcnRjdXRGaXJzdFN0cm9rZSI6IHsKICAgICAgImFsbE9mIjogWwogICAgICAgIHsKICAgICAgICAgICIkcmVmIjogIiMvJGRlZnMvc2hvcnRjdXRTdHJva2UiCiAgICAgICAgfSwKICAgICAgICB7", + "CiAgICAgICAgICAiYW55T2YiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAicGF0dGVybiI6ICJeKD86KD86W2NDXVttTV1bZERdfFtjQ11bb09dW21NXVttTV1bYUFdW25OXVtkRF18W3NTXVtoSF1baUldW2ZGXVt0VF18W29PXVtwUF1bdFRdfFtvT11bcFBdW3RUXVtpSV1bb09dW25OXXxbYUFdW2xMXVt0VF18W2NDXVt0VF1bclJdW2xMXXxbY0NdW29PXVtuTl1bdFRdW3JSXVtvT11bbExdfFtjQ11bdFRdW2xMXXzijJh84oenfOKMpXzijIMpXFwrKSsiCiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAg", + "ICAgICAgICAicGF0dGVybiI6ICJeKD86IHxbc1NdW3BQXVthQV1bY0NdW2VFXXxbc1NdW3BQXVthQV1bY0NdW2VFXVtiQl1bYUFdW3JSXXw8W3NTXVtwUF1bYUFdW2NDXVtlRV0+KSQiCiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAicGF0dGVybiI6ICJeKD86W3JSXVtlRV1bdFRdW3VVXVtyUl1bbk5dfFtlRV1bbk5dW3RUXVtlRV1bclJdfOKGqSkkIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgInBhdHRlcm4iOiAiXig/OltsTF1bZUVdW2ZGXVt0VF18W2FBXVty", + "Ul1bclJdW29PXVt3V11bbExdW2VFXVtmRl1bdFRdfFtsTF1bZUVdW2ZGXVt0VF1bYUFdW3JSXVtyUl1bb09dW3dXXXzihpB8W3JSXVtpSV1bZ0ddW2hIXVt0VF18W2FBXVtyUl1bclJdW29PXVt3V11bclJdW2lJXVtnR11baEhdW3RUXXxbclJdW2lJXVtnR11baEhdW3RUXVthQV1bclJdW3JSXVtvT11bd1ddfOKGknxbdVVdW3BQXXxbYUFdW3JSXVtyUl1bb09dW3dXXVt1VV1bcFBdfFt1VV1bcFBdW2FBXVtyUl1bclJdW29PXVt3V1184oaRfFtkRF1bb09dW3dXXVtuTl18W2FBXVtyUl1bclJdW29PXVt3V11bZERdW29PXVt3V11b", + "bk5dfFtkRF1bb09dW3dXXVtuTl1bYUFdW3JSXVtyUl1bb09dW3dXXXzihpMpJCIKICAgICAgICAgICAgfSwKICAgICAgICAgICAgewogICAgICAgICAgICAgICJwYXR0ZXJuIjogIl4oPzpbQS1aYS16MC05XXxbLC4vXFxcXDsnYD1cXFtcXF0tXXxbY0NdW29PXVttTV1bbU1dW2FBXXxbcFBdW2VFXVtyUl1baUldW29PXVtkRF18W2REXVtvT11bdFRdfFtzU11bbExdW2FBXVtzU11baEhdfFtiQl1bYUFdW2NDXVtrS11bc1NdW2xMXVthQV1bc1NdW2hIXXxbc1NdW2VFXVttTV1baUldW2NDXVtvT11bbExdW29PXVtuTl18W3FRXVt1", + "VV1bb09dW3RUXVtlRV18W2FBXVtwUF1bb09dW3NTXVt0VF1bclJdW29PXVtwUF1baEhdW2VFXXxbYkJdW2FBXVtjQ11ba0tdW3RUXVtpSV1bY0NdW2tLXXxbZ0ddW3JSXVthQV1bdlZdW2VFXXxbbU1dW2lJXVtuTl1bdVVdW3NTXXxbaEhdW3lZXVtwUF1baEhdW2VFXVtuTl18W3BQXVtsTF1bdVVdW3NTXXxbZUVdW3FRXVt1VV1bYUFdW2xMXVtzU118W2xMXVtlRV1bZkZdW3RUXVtiQl1bclJdW2FBXVtjQ11ba0tdW2VFXVt0VF18W29PXVtwUF1bZUVdW25OXVtiQl1bclJdW2FBXVtjQ11ba0tdW2VFXVt0VF18W3JSXVtpSV1bZ0dd", + "W2hIXVt0VF1bYkJdW3JSXVthQV1bY0NdW2tLXVtlRV1bdFRdfFtjQ11bbExdW29PXVtzU11bZUVdW2JCXVtyUl1bYUFdW2NDXVtrS11bZUVdW3RUXSkkIgogICAgICAgICAgICB9CiAgICAgICAgICBdCiAgICAgICAgfQogICAgICBdLAogICAgICAiZGVzY3JpcHRpb24iOiAiRmlyc3Qgb3Igb25seSBzaG9ydGN1dCBzdHJva2UuIEJhcmUgcHJpbnRhYmxlIGtleXMgYXJlIGFjY2VwdGVkIGZvciBhY3Rpb25zIHRoYXQgb3B0IGludG8gYmFyZSBmaXJzdCBzdHJva2VzLiIKICAgIH0sCiAgICAic2hvcnRjdXRTdHJva2UiOiB7CiAg", + "ICAgICJ0eXBlIjogInN0cmluZyIsCiAgICAgICJwYXR0ZXJuIjogIl4oPzooPzpbY0NdW21NXVtkRF18W2NDXVtvT11bbU1dW21NXVthQV1bbk5dW2REXXxbc1NdW2hIXVtpSV1bZkZdW3RUXXxbb09dW3BQXVt0VF18W29PXVtwUF1bdFRdW2lJXVtvT11bbk5dfFthQV1bbExdW3RUXXxbY0NdW3RUXVtyUl1bbExdfFtjQ11bb09dW25OXVt0VF1bclJdW29PXVtsTF18W2NDXVt0VF1bbExdfOKMmHzih6d84oylfOKMgylcXCspKig/OiB8W0EtWmEtejAtOV18W2xMXVtlRV1bZkZdW3RUXXxbYUFdW3JSXVtyUl1bb09dW3dXXVtsTF1b", + "ZUVdW2ZGXVt0VF18W2xMXVtlRV1bZkZdW3RUXVthQV1bclJdW3JSXVtvT11bd1ddfOKGkHxbclJdW2lJXVtnR11baEhdW3RUXXxbYUFdW3JSXVtyUl1bb09dW3dXXVtyUl1baUldW2dHXVtoSF1bdFRdfFtyUl1baUldW2dHXVtoSF1bdFRdW2FBXVtyUl1bclJdW29PXVt3V1184oaSfFt1VV1bcFBdfFthQV1bclJdW3JSXVtvT11bd1ddW3VVXVtwUF18W3VVXVtwUF1bYUFdW3JSXVtyUl1bb09dW3dXXXzihpF8W2REXVtvT11bd1ddW25OXXxbYUFdW3JSXVtyUl1bb09dW3dXXVtkRF1bb09dW3dXXVtuTl18W2REXVtvT11bd1ddW25O", + "XVthQV1bclJdW3JSXVtvT11bd1ddfOKGk3xbdFRdW2FBXVtiQl18W3JSXVtlRV1bdFRdW3VVXVtyUl1bbk5dfFtlRV1bbk5dW3RUXVtlRV1bclJdfOKGqXxbc1NdW3BQXVthQV1bY0NdW2VFXXxbc1NdW3BQXVthQV1bY0NdW2VFXVtiQl1bYUFdW3JSXXw8W3NTXVtwUF1bYUFdW2NDXVtlRV0+fFssLi9cXFxcOydgPVxcW1xcXS1dfFtjQ11bb09dW21NXVttTV1bYUFdfFtwUF1bZUVdW3JSXVtpSV1bb09dW2REXXxbZERdW29PXVt0VF18W3NTXVtsTF1bYUFdW3NTXVtoSF18W2JCXVthQV1bY0NdW2tLXVtzU11bbExdW2FBXVtzU11b", + "aEhdfFtzU11bZUVdW21NXVtpSV1bY0NdW29PXVtsTF1bb09dW25OXXxbcVFdW3VVXVtvT11bdFRdW2VFXXxbYUFdW3BQXVtvT11bc1NdW3RUXVtyUl1bb09dW3BQXVtoSF1bZUVdfFtiQl1bYUFdW2NDXVtrS11bdFRdW2lJXVtjQ11ba0tdfFtnR11bclJdW2FBXVt2Vl1bZUVdfFttTV1baUldW25OXVt1VV1bc1NdfFtoSF1beVldW3BQXVtoSF1bZUVdW25OXXxbcFBdW2xMXVt1VV1bc1NdfFtlRV1bcVFdW3VVXVthQV1bbExdW3NTXXxbbExdW2VFXVtmRl1bdFRdW2JCXVtyUl1bYUFdW2NDXVtrS11bZUVdW3RUXXxbb09dW3BQXVtl", + "RV1bbk5dW2JCXVtyUl1bYUFdW2NDXVtrS11bZUVdW3RUXXxbclJdW2lJXVtnR11baEhdW3RUXVtiQl1bclJdW2FBXVtjQ11ba0tdW2VFXVt0VF18W2NDXVtsTF1bb09dW3NTXVtlRV1bYkJdW3JSXVthQV1bY0NdW2tLXVtlRV1bdFRdfFtmRl0oPzpbMS05XXwxWzAtOV18MjApfFt2Vl1bb09dW2xMXVt1VV1bbU1dW2VFXVt1VV1bcFBdfFttTV1bZUVdW2REXVtpSV1bYUFdW3ZWXVtvT11bbExdW3VVXVttTV1bZUVdW3VVXVtwUF18W21NXVtlRV1bZERdW2lJXVthQV1cXC5bdlZdW29PXVtsTF1bdVVdW21NXVtlRV1bdVVdW3BQXXxb", + "dlZdW29PXVtsTF1bdVVdW21NXVtlRV1bZERdW29PXVt3V11bbk5dfFttTV1bZUVdW2REXVtpSV1bYUFdW3ZWXVtvT11bbExdW3VVXVttTV1bZUVdW2REXVtvT11bd1ddW25OXXxbbU1dW2VFXVtkRF1baUldW2FBXVxcLlt2Vl1bb09dW2xMXVt1VV1bbU1dW2VFXVtkRF1bb09dW3dXXVtuTl18W2JCXVtyUl1baUldW2dHXVtoSF1bdFRdW25OXVtlRV1bc1NdW3NTXVt1VV1bcFBdfFttTV1bZUVdW2REXVtpSV1bYUFdW2JCXVtyUl1baUldW2dHXVtoSF1bdFRdW25OXVtlRV1bc1NdW3NTXVt1VV1bcFBdfFttTV1bZUVdW2REXVtpSV1b", + "YUFdXFwuW2JCXVtyUl1baUldW2dHXVtoSF1bdFRdW25OXVtlRV1bc1NdW3NTXVt1VV1bcFBdfFtiQl1bclJdW2lJXVtnR11baEhdW3RUXVtuTl1bZUVdW3NTXVtzU11bZERdW29PXVt3V11bbk5dfFttTV1bZUVdW2REXVtpSV1bYUFdW2JCXVtyUl1baUldW2dHXVtoSF1bdFRdW25OXVtlRV1bc1NdW3NTXVtkRF1bb09dW3dXXVtuTl18W21NXVtlRV1bZERdW2lJXVthQV1cXC5bYkJdW3JSXVtpSV1bZ0ddW2hIXVt0VF1bbk5dW2VFXVtzU11bc1NdW2REXVtvT11bd1ddW25OXXxbbU1dW3VVXVt0VF1bZUVdfFttTV1bZUVdW2REXVtp", + "SV1bYUFdW21NXVt1VV1bdFRdW2VFXXxbbU1dW2VFXVtkRF1baUldW2FBXVxcLlttTV1bdVVdW3RUXVtlRV18W3BQXVtsTF1bYUFdW3lZXVtwUF1bYUFdW3VVXVtzU11bZUVdfFttTV1bZUVdW2REXVtpSV1bYUFdW3BQXVtsTF1bYUFdW3lZXVtwUF1bYUFdW3VVXVtzU11bZUVdfFttTV1bZUVdW2REXVtpSV1bYUFdXFwuW3BQXVtsTF1bYUFdW3lZXVtwUF1bYUFdW3VVXVtzU11bZUVdfFtuTl1bZUVdW3hYXVt0VF1bdFRdW3JSXVthQV1bY0NdW2tLXXxbbU1dW2VFXVtkRF1baUldW2FBXVtuTl1bZUVdW3hYXVt0VF18W21NXVtlRV1b", + "ZERdW2lJXVthQV1cXC5bbk5dW2VFXVt4WF1bdFRdfFttTV1bZUVdW2REXVtpSV1bYUFdXFwuW25OXVtlRV1beFhdW3RUXVt0VF1bclJdW2FBXVtjQ11ba0tdfFtwUF1bclJdW2VFXVt2Vl1baUldW29PXVt1VV1bc1NdW3RUXVtyUl1bYUFdW2NDXVtrS118W21NXVtlRV1bZERdW2lJXVthQV1bcFBdW3JSXVtlRV1bdlZdW2lJXVtvT11bdVVdW3NTXXxbbU1dW2VFXVtkRF1baUldW2FBXVxcLltwUF1bclJdW2VFXVt2Vl1baUldW29PXVt1VV1bc1NdfFttTV1bZUVdW2REXVtpSV1bYUFdXFwuW3BQXVtyUl1bZUVdW3ZWXVtpSV1bb09d", + "W3VVXVtzU11bdFRdW3JSXVthQV1bY0NdW2tLXSkkIiwKICAgICAgImRlc2NyaXB0aW9uIjogIk9uZSBrZXlib2FyZCBzaG9ydGN1dCBzdHJva2UgdXNpbmcgbW9kaWZpZXIra2V5IHN5bnRheC4gU3VwcG9ydGVkIGtleSBuYW1lcyBpbmNsdWRlIHNwYWNlLCBTcGFjZSwgPHNwYWNlPiwgYW5kIDxTcGFjZT4uIgogICAgfQogIH0KfQo=", ].joined() guard let data = Data(base64Encoded: base64) else { preconditionFailure("embedded cmux.json schema is corrupt") From a118484f26f86725ee05f478e12d74b3852848ff Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 23 Sep 2026 17:02:17 -0700 Subject: [PATCH 78/92] fix: align Cloud CI gates with current workflows --- .github/workflows/ci-macos.yml | 4 +- .github/workflows/ci.yml | 61 ---------------------- scripts/ci/cmux_unit_test_shard.py | 5 -- web/tests/fixtures/freestyleGuest.ts | 38 ++++++++++++-- web/tests/vm-guest-install-runtime.test.ts | 16 +++--- web/tests/vm-guest-install.test.ts | 10 ++-- web/tests/vm-workflows.test.ts | 16 +++--- 7 files changed, 59 insertions(+), 91 deletions(-) diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml index 32e3279d75ab..71be125d6ead 100644 --- a/.github/workflows/ci-macos.yml +++ b/.github/workflows/ci-macos.yml @@ -1679,7 +1679,9 @@ jobs: # Separate invocations prove each selector executed tests; a healthy # suite cannot mask a miswired or accidentally empty selector. set -euo pipefail - for suite in CloudMachineDragSourceTests CloudMachineOrderingTests; do + for suite in CloudMachineDragSourceTests CloudMachineOrderingTests \ + CloudOperationRecorderTests CloudReadRequestCoordinatorTests \ + VMClientReadCoalescingTests; do test_log="$RUNNER_TEMP/cmux-cloud-ordering-${suite}.txt" bash scripts/ci/run-and-capture.sh "$test_log" \ scripts/ci/run-in-console-session.sh \ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4c42d8a8d983..04018a1074d8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -852,67 +852,6 @@ jobs: with: node-version: "20" - - name: Run Ghostty numeric locale tests - if: ${{ matrix.shard == fromJSON(env.CMUX_APP_HOST_FOCUSED_REGRESSION_B_SHARD) }} - run: | - # LC_NUMERIC is process-global. Keep these tests in their own app-host - # invocation and disable Xcode's parallel test execution for it. - set -euo pipefail - scripts/ci/run-in-console-session.sh \ - scripts/ci/run-app-host-xcodebuild.sh \ - -xctestrun "$CMUX_NUMERIC_LOCALE_XCTESTRUN" \ - -destination "platform=macOS" \ - -only-testing:cmuxTests/GhosttyNumericLocaleTests \ - -parallel-testing-enabled NO \ - CMUX_SKIP_ZIG_BUILD=1 \ - test-without-building - - - name: Run global search shortcut regressions - if: ${{ matrix.shard == fromJSON(env.CMUX_APP_HOST_GLOBAL_SEARCH_SHARD) }} - timeout-minutes: 40 - run: | - # This serialized container owns global shortcut settings. Keep its - # mutations out of unrelated UI suites' retained window graphs; it - # exceeded thirteen minutes in a shared app host. Run the whole - # container in its own non-tolerant app host, including every nested - # test, as a plain `xcodebuild test-without-building` gate like every - # other focused gate. The compile admission job has already produced - # the test bundle, so this shard only spends time running tests. - set -euo pipefail - test_log="$RUNNER_TEMP/cmux-global-search-shortcuts.txt" - scripts/ci/run-in-console-session.sh \ - scripts/ci/run-app-host-xcodebuild.sh \ - -xctestrun "$CMUX_APP_HOST_XCTESTRUN" \ - -destination "platform=macOS" \ - CMUX_SKIP_ZIG_BUILD=1 \ - -only-testing:cmuxTests/GlobalSearchShortcutBehaviorTests \ - test-without-building 2>&1 | tee "$test_log" - scripts/ci/require_selected_test_execution.sh \ - "$test_log" cmuxTests/GlobalSearchShortcutBehaviorTests - - - name: Run Cloud machine ordering acceptance - if: ${{ !cancelled() && steps.prepare-app-host-home.outcome == 'success' && matrix.shard == fromJSON(env.CMUX_APP_HOST_FOCUSED_REGRESSION_SHARD) }} - timeout-minutes: 10 - run: | - # Separate invocations prove each selector executed tests; a healthy - # suite cannot mask a miswired or accidentally empty selector. - set -euo pipefail - for suite in CloudMachineDragSourceTests CloudMachineOrderingTests \ - CloudOperationRecorderTests CloudReadRequestCoordinatorTests VMClientReadCoalescingTests; do - test_log="$RUNNER_TEMP/cmux-cloud-acceptance-${suite}.txt" - scripts/ci/run-in-console-session.sh \ - scripts/ci/run-app-host-xcodebuild.sh \ - -xctestrun "$CMUX_APP_HOST_XCTESTRUN" \ - -destination "platform=macOS" \ - CMUX_SKIP_ZIG_BUILD=1 \ - -test-timeouts-enabled YES \ - -default-test-execution-time-allowance 60 \ - -maximum-test-execution-time-allowance 60 \ - -only-testing:"cmuxTests/$suite" \ - test-without-building 2>&1 | tee "$test_log" - scripts/ci/require_selected_test_execution.sh "$test_log" "cmuxTests/$suite" - done - - name: Run standalone Claude wrapper regressions run: | set -euo pipefail diff --git a/scripts/ci/cmux_unit_test_shard.py b/scripts/ci/cmux_unit_test_shard.py index 70a996958f29..45776c3f3979 100755 --- a/scripts/ci/cmux_unit_test_shard.py +++ b/scripts/ci/cmux_unit_test_shard.py @@ -45,12 +45,8 @@ "cmuxTests/ClaudeBackgroundWorkNotifyTests", "cmuxTests/CloudMachineDragSourceTests", "cmuxTests/CloudMachineOrderingTests", - "cmuxTests/CloudNotificationDismissParityTests", "cmuxTests/CloudOperationRecorderTests", "cmuxTests/CloudReadRequestCoordinatorTests", - "cmuxTests/CloudWorkspaceRenameSurfaceParityTests", - "cmuxTests/CmuxBundledBinPathIntegrationTests", - "cmuxTests/DockNotificationAttentionTests", "cmuxTests/DeviceDirectoryLifecycleTests", "cmuxTests/DeviceDirectoryMergeTests", "cmuxTests/DeviceLinkReconnectPolicyTests", @@ -78,7 +74,6 @@ "cmuxTests/SidebarIssue8373StressTests", "cmuxTests/SidebarWorkspaceSwitchLayoutFaultTests", "cmuxTests/SocketACLReloadRegressionTests", - "cmuxTests/SurfaceResumeAgentHookDowngradeTests", "cmuxTests/VMClientReadCoalescingTests", "cmuxTests/SurfaceMachineIDDeviceEncodingTests", diff --git a/web/tests/fixtures/freestyleGuest.ts b/web/tests/fixtures/freestyleGuest.ts index d438d927e2d1..fd66c9d0fb6b 100644 --- a/web/tests/fixtures/freestyleGuest.ts +++ b/web/tests/fixtures/freestyleGuest.ts @@ -4,8 +4,12 @@ import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { pathToFileURL } from "node:url"; +import { Effect } from "effect"; import { Freestyle } from "freestyle"; import { FreestyleProvider } from "../../services/vms/drivers/freestyle"; +import { installFreestyleGuestCli } from "../../services/vms/drivers/freestyleGuestCli"; +import { rollbackFreestyleCreate } from "../../services/vms/drivers/providerCreateCleanup"; +import { ProviderError, type CreateOptions } from "../../services/vms/drivers/types"; import type { GuestCliDistribution } from "../../services/vms/guestCliDistribution"; export type GuestExecRequest = { @@ -96,10 +100,38 @@ export function freestyleGuestFixture(options: { }); const provider = new FreestyleProvider({ client, - resolveDaemonSource: async () => { throw new Error("fixture must never resolve a live daemon"); }, - guestCliDistribution: options.guestCliDistribution ?? testDistribution, }); - return { provider, client, requests, writes, removals, liveVms, allocations: () => allocations }; + /** + * Exercise guest bootstrap as a separate workflow. The production provider's + * create path follows snapshot-v2 and intentionally performs no guest setup. + */ + const createWithGuestInstall = async (createOptions: CreateOptions) => { + const handle = await provider.create(createOptions); + const install = await Effect.runPromise(Effect.either(installFreestyleGuestCli( + client, + handle.providerVmId, + createOptions.promptIdentity, + options.guestCliDistribution ?? testDistribution, + ))); + if (install._tag === "Right") { + return handle; + } + const rollback = await Effect.runPromise(Effect.either( + rollbackFreestyleCreate(client, handle.providerVmId, install.left), + )); + if (rollback._tag === "Left") throw rollback.left; + throw new ProviderError("freestyle", "guest bootstrap failed", install.left); + }; + return { + provider, + client, + createWithGuestInstall, + requests, + writes, + removals, + liveVms, + allocations: () => allocations, + }; } export const guestCreateOptions = { diff --git a/web/tests/vm-guest-install-runtime.test.ts b/web/tests/vm-guest-install-runtime.test.ts index c91886461ed0..f3b59de6e72f 100644 --- a/web/tests/vm-guest-install-runtime.test.ts +++ b/web/tests/vm-guest-install-runtime.test.ts @@ -57,7 +57,7 @@ function guest(options: { corruptUpload?: boolean; promptFails?: boolean; publis describe("guest CLI publication in an isolated filesystem", () => { test("successful create installs the complete executable CLI and answers help", async () => { const { fixture, target, root } = guest(); - const handle = await fixture.provider.create(guestCreateOptions); + const handle = await fixture.createWithGuestInstall(guestCreateOptions); expect(handle.status).toBe("running"); expect(readFileSync(target, "utf8")).toBe(GUEST_CMUX_SHIM); for (const file of GUEST_BROWSER_FILES.filter((file) => file.path.startsWith("/usr/local/bin/"))) { @@ -84,11 +84,11 @@ describe("guest CLI publication in an isolated filesystem", () => { test("a fresh prompt install keeps its identity when an older revision attaches", async () => { const { fixture, root } = guest(); - await fixture.provider.create({ + await fixture.createWithGuestInstall({ ...guestCreateOptions, promptIdentity: { machineId: "synthetic", name: "fresh-machine", revision: 10 }, }); - await fixture.provider.create({ + await fixture.createWithGuestInstall({ ...guestCreateOptions, promptIdentity: { machineId: "synthetic", name: "stale-machine", revision: 9 }, }); @@ -103,7 +103,7 @@ describe("guest CLI publication in an isolated filesystem", () => { const unrelated = join(root, "unrelated"); writeFileSync(unrelated, "preserve me"); symlinkSync(unrelated, target); - await fixture.provider.create(guestCreateOptions); + await fixture.createWithGuestInstall(guestCreateOptions); expect(readFileSync(unrelated, "utf8")).toBe("preserve me"); expect(readFileSync(target, "utf8")).toBe(GUEST_CMUX_SHIM); }); @@ -111,7 +111,7 @@ describe("guest CLI publication in an isolated filesystem", () => { test("a directory destination fails instead of moving the shim inside and reporting ready", async () => { const { fixture, target } = guest(); mkdirSync(target); - const result = await fixture.provider.create(guestCreateOptions).then(() => "ready", () => "failed"); + const result = await fixture.createWithGuestInstall(guestCreateOptions).then(() => "ready", () => "failed"); expect(result).toBe("failed"); expect(fixture.liveVms.size).toBe(0); }); @@ -119,7 +119,7 @@ describe("guest CLI publication in an isolated filesystem", () => { test("rejects a corrupted upload before replacing the previous generation", async () => { const { fixture, target } = guest({ corruptUpload: true }); writeFileSync(target, "previous generation"); - const result = await fixture.provider.create(guestCreateOptions).then(() => "ready", () => "failed"); + const result = await fixture.createWithGuestInstall(guestCreateOptions).then(() => "ready", () => "failed"); expect(result).toBe("failed"); expect(readFileSync(target, "utf8")).toBe("previous generation"); expect(fixture.liveVms.size).toBe(0); @@ -135,7 +135,7 @@ describe("guest CLI publication in an isolated filesystem", () => { writeFileSync(join(root, "etc/bashrc"), "previous bashrc generation"); writeFileSync(join(root, "etc/.prompt-identity"), "previous identity"); writeFileSync(join(root, "etc/vm-name"), "previous name\n"); - const failure = await fixture.provider.create({ + const failure = await fixture.createWithGuestInstall({ ...guestCreateOptions, promptIdentity: { machineId: "synthetic", name: "synthetic", revision: 1 }, }).then(() => undefined, (error) => error); @@ -164,7 +164,7 @@ describe("guest CLI publication in an isolated filesystem", () => { writeFileSync(join(root, "etc/bashrc"), "previous bashrc generation"); writeFileSync(join(root, "etc/.prompt-identity"), "previous identity"); writeFileSync(join(root, "etc/vm-name"), "previous name\n"); - const failure = await fixture.provider.create({ + const failure = await fixture.createWithGuestInstall({ ...guestCreateOptions, promptIdentity: { machineId: "synthetic", name: "synthetic", revision: 1 }, }).then(() => undefined, (error) => error); diff --git a/web/tests/vm-guest-install.test.ts b/web/tests/vm-guest-install.test.ts index 3cbebedf86de..549a7a5b2eeb 100644 --- a/web/tests/vm-guest-install.test.ts +++ b/web/tests/vm-guest-install.test.ts @@ -4,7 +4,7 @@ import { freestyleGuestFixture, guestCreateOptions } from "./fixtures/freestyleG describe("Freestyle guest install result contract (SDK 0.2.10)", () => { test("a reported zero exit publishes a ready machine", async () => { const fixture = freestyleGuestFixture(); - const handle = await fixture.provider.create(guestCreateOptions); + const handle = await fixture.createWithGuestInstall(guestCreateOptions); expect(handle.status).toBe("running"); expect(fixture.liveVms.size).toBe(1); expect(fixture.allocations()).toBe(1); @@ -22,7 +22,7 @@ describe("Freestyle guest install result contract (SDK 0.2.10)", () => { ] as Array<[Record, string, number | null | undefined]>) ("preserves outcome %s without a false ready result", async (body, outcome, statusCode) => { const fixture = freestyleGuestFixture({ exec: async () => Response.json(body) }); - const error = await fixture.provider.create(guestCreateOptions).catch((error: unknown) => error); + const error = await fixture.createWithGuestInstall(guestCreateOptions).catch((error: unknown) => error); expect(error).toBeInstanceOf(Error); const installError = (error as { cause: { name: string; outcome: string; exitCode?: number | null } }).cause; expect(installError.name).toBe("GuestCliInstallError"); @@ -39,7 +39,7 @@ describe("Freestyle guest install result contract (SDK 0.2.10)", () => { ])("preserves transport %s separately from a guest exit", async (name, outcome) => { const cause = new DOMException("synthetic transport interruption", name); const fixture = freestyleGuestFixture({ exec: async () => { throw cause; } }); - const error = await fixture.provider.create(guestCreateOptions).catch((error: unknown) => error); + const error = await fixture.createWithGuestInstall(guestCreateOptions).catch((error: unknown) => error); expect((error as { cause?: unknown }).cause).toMatchObject({ name: "GuestCliInstallError", outcome, cause }); expect(fixture.liveVms.size).toBe(0); expect(fixture.removals).toEqual(fixture.writes); @@ -50,7 +50,7 @@ describe("Freestyle guest install result contract (SDK 0.2.10)", () => { exec: async () => Response.json({ statusCode: 1, stderr: "synthetic install failure" }), deleteFailure: true, }); - const error = await fixture.provider.create(guestCreateOptions).catch((error: unknown) => error); + const error = await fixture.createWithGuestInstall(guestCreateOptions).catch((error: unknown) => error); expect(error).toMatchObject({ name: "ProviderCreateCleanupError", providerVmId: "vm-fixture-1", @@ -69,7 +69,7 @@ describe("Freestyle guest install result contract (SDK 0.2.10)", () => { })}`, }), }); - const error = await fixture.provider.create(guestCreateOptions).catch((error: unknown) => error); + const error = await fixture.createWithGuestInstall(guestCreateOptions).catch((error: unknown) => error); expect((error as { cause?: unknown }).cause).toMatchObject({ name: "GuestCliInstallError", stage: "publish", diff --git a/web/tests/vm-workflows.test.ts b/web/tests/vm-workflows.test.ts index a9c2b999a918..1ab80fc6c902 100644 --- a/web/tests/vm-workflows.test.ts +++ b/web/tests/vm-workflows.test.ts @@ -204,7 +204,7 @@ describe("VM Effect workflows", () => { const layer = providerLayer({ ...unusedProviderGateway(), create: (_, options) => Effect.tryPromise({ - try: () => fixture.provider.create(options), + try: () => fixture.createWithGuestInstall(options), catch: (cause) => new VmProviderOperationError({ provider: "freestyle", operation: "create", cause }), }), }); @@ -238,7 +238,7 @@ describe("VM Effect workflows", () => { const layer = providerLayer({ ...unusedProviderGateway(), create: (_, options) => Effect.tryPromise({ - try: () => fixture.provider.create(options), + try: () => fixture.createWithGuestInstall(options), catch: (cause) => new VmProviderOperationError({ provider: "freestyle", operation: "create", cause }), }), }); @@ -282,7 +282,7 @@ describe("VM Effect workflows", () => { const layer = providerLayer({ ...unusedProviderGateway(), create: (_, options) => Effect.tryPromise({ - try: () => fixture.provider.create(options), + try: () => fixture.createWithGuestInstall(options), catch: (cause) => new VmProviderOperationError({ provider: "freestyle", operation: "create", cause }), }), destroy: (_, providerVmId) => Effect.sync(() => { @@ -327,7 +327,7 @@ describe("VM Effect workflows", () => { const layer = providerLayer({ ...unusedProviderGateway(), create: (_, options) => Effect.tryPromise({ - try: () => fixture.provider.create(options), + try: () => fixture.createWithGuestInstall(options), catch: (cause) => new VmProviderOperationError({ provider: "freestyle", operation: "create", cause }), }), destroy: () => Effect.sync(() => { @@ -371,7 +371,7 @@ describe("VM Effect workflows", () => { const layer = providerLayer({ ...unusedProviderGateway(), create: (_, options) => Effect.tryPromise({ - try: () => fixture.provider.create(options), + try: () => fixture.createWithGuestInstall(options), catch: (cause) => new VmProviderOperationError({ provider: "freestyle", operation: "create", cause }), }), destroy: (_, providerVmId) => Effect.sync(() => { @@ -413,7 +413,7 @@ describe("VM Effect workflows", () => { const layer = providerLayer({ ...unusedProviderGateway(), create: (_, options) => Effect.tryPromise({ - try: () => fixture.provider.create(options), + try: () => fixture.createWithGuestInstall(options), catch: (cause) => new VmProviderOperationError({ provider: "freestyle", operation: "create", cause }), }), }); @@ -470,7 +470,7 @@ describe("VM Effect workflows", () => { const layer = providerLayer({ ...unusedProviderGateway(), create: (_, options) => Effect.tryPromise({ - try: () => fixture.provider.create({ ...options, imageSize: guestCreateOptions.imageSize }), + try: () => fixture.createWithGuestInstall({ ...options, imageSize: guestCreateOptions.imageSize }), catch: (cause) => new VmProviderOperationError({ provider: "freestyle", operation: "create", cause }), }), }); @@ -506,7 +506,7 @@ describe("VM Effect workflows", () => { const layer = providerLayer({ ...unusedProviderGateway(), create: (_, options) => Effect.tryPromise({ - try: () => fixture.provider.create(options), + try: () => fixture.createWithGuestInstall(options), catch: (cause) => new VmProviderOperationError({ provider: "freestyle", operation: "create", cause }), }), destroy: (_, providerVmId) => Effect.sync(() => { From 61472b2a5948414df32eb1d7f8f1d1b87c52a261 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 23 Sep 2026 17:07:43 -0700 Subject: [PATCH 79/92] fix: close current Cloud review findings --- .../MachinesPanelViewModel+Refresh.swift | 24 +++++++++++++++++++ Sources/Cloud/MachinesPanelViewModel.swift | 23 ------------------ cmux.xcodeproj/project.pbxproj | 4 ++++ cmuxTests/CloudOperationRecorderTests.swift | 5 +++- .../CloudReadRequestCoordinatorTests.swift | 17 ------------- cmuxTests/CloudReadResponseGate.swift | 20 ++++++++++++++++ .../vms/drivers/guestCliInstallCommand.ts | 4 +++- 7 files changed, 55 insertions(+), 42 deletions(-) create mode 100644 cmuxTests/CloudReadResponseGate.swift diff --git a/Sources/Cloud/MachinesPanelViewModel+Refresh.swift b/Sources/Cloud/MachinesPanelViewModel+Refresh.swift index 5123c74713d9..3d660293fd92 100644 --- a/Sources/Cloud/MachinesPanelViewModel+Refresh.swift +++ b/Sources/Cloud/MachinesPanelViewModel+Refresh.swift @@ -22,6 +22,30 @@ extension MachinesPanelViewModel { } } + /// Samples machines advertising stats support. Sleeping machines report + /// `asleep` without being woken, so polling never costs the user anything. + /// Older servers omitting the flag retain the desktop-only polling policy + /// through capability decoding; explicit support overrides that fallback. + func refreshUsage() { + guard isCloudEnabled(), usageTask == nil else { return } + if let retryNotBefore = usageRetryNotBefore, retryNotBefore > Date() { return } + guard let client = MachineUsageClient.shared else { return } + let generation = refreshGeneration + usageTask = Task { [weak self] in + defer { if generation == self?.refreshGeneration { self?.usageTask = nil } } + do { + let usage = (try await client.teamUsage()).byMachineID + guard !Task.isCancelled, let self, self.isCloudEnabled() else { return } + self.usageFailureCount = 0; self.usageRetryNotBefore = nil + self.applyUsage(usage) + } catch is CancellationError { return } catch { + guard !Task.isCancelled, let self else { return } + self.usageFailureCount = min(self.usageFailureCount + 1, 4) + self.usageRetryNotBefore = Date().addingTimeInterval(Self.usageBackoffDelay(failureCount: self.usageFailureCount)) + } + } + } + func startPolling() { wantsPolling = true guard isCloudEnabled() else { pausePolling(); return } diff --git a/Sources/Cloud/MachinesPanelViewModel.swift b/Sources/Cloud/MachinesPanelViewModel.swift index def26d7c4e76..706d1595fa78 100644 --- a/Sources/Cloud/MachinesPanelViewModel.swift +++ b/Sources/Cloud/MachinesPanelViewModel.swift @@ -345,29 +345,6 @@ final class MachinesPanelViewModel: ObservableObject { refreshTree(force: forceTree) } func refreshMachine(_ machine: SurfaceMachineID) { machineRefreshes.refresh(machine) } - /// Samples machines advertising stats support. Sleeping machines report - /// `asleep` without being woken, so polling never costs the user anything. - /// Older servers omitting the flag retain the desktop-only polling policy - /// through capability decoding; explicit support overrides that fallback. - func refreshUsage() { - guard CloudMachinesFeature.isEnabled, usageTask == nil else { return } - if let retryNotBefore = usageRetryNotBefore, retryNotBefore > Date() { return } - guard let client = MachineUsageClient.shared else { return } - let generation = refreshGeneration - usageTask = Task { [weak self] in - defer { if generation == self?.refreshGeneration { self?.usageTask = nil } } - do { - let usage = (try await client.teamUsage()).byMachineID - guard !Task.isCancelled, CloudMachinesFeature.isEnabled, let self else { return } - self.usageFailureCount = 0; self.usageRetryNotBefore = nil - self.applyUsage(usage) - } catch is CancellationError { return } catch { - guard !Task.isCancelled, let self else { return } - self.usageFailureCount = min(self.usageFailureCount + 1, 4) - self.usageRetryNotBefore = Date().addingTimeInterval(Self.usageBackoffDelay(failureCount: self.usageFailureCount)) - } - } - } nonisolated static func usageBackoffDelay(failureCount: Int) -> TimeInterval { [30, 30, 60, 120, 300][min(max(failureCount, 0), 4)] } diff --git a/cmux.xcodeproj/project.pbxproj b/cmux.xcodeproj/project.pbxproj index d58e06c7c596..273d70770bff 100644 --- a/cmux.xcodeproj/project.pbxproj +++ b/cmux.xcodeproj/project.pbxproj @@ -870,6 +870,7 @@ C13151C0000000000000000C /* CloudReadRequestCoordinator+Waiter.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C0000000000000000B /* CloudReadRequestCoordinator+Waiter.swift */; }; C12625A00000000000000004 /* CloudReadRequestCoordinator.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12625A00000000000000003 /* CloudReadRequestCoordinator.swift */; }; C12625B00000000000000004 /* CloudReadRequestCoordinatorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12625B00000000000000003 /* CloudReadRequestCoordinatorTests.swift */; }; + 9F046FC7A14BBDB098F471B3 /* CloudReadResponseGate.swift in Sources */ = {isa = PBXBuildFile; fileRef = E9A79D080D5B1E7996CC8586 /* CloudReadResponseGate.swift */; }; C13151C00000000000000023 /* CloudRefreshAuthClient.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C00000000000000022 /* CloudRefreshAuthClient.swift */; }; C13151C00000000000000021 /* CloudRefreshFixture.swift in Sources */ = {isa = PBXBuildFile; fileRef = C13151C00000000000000020 /* CloudRefreshFixture.swift */; }; C12625000000000000000004 /* CloudRefreshURLProtocol.swift in Sources */ = {isa = PBXBuildFile; fileRef = C12625000000000000000003 /* CloudRefreshURLProtocol.swift */; }; @@ -5145,6 +5146,7 @@ C13151C0000000000000000B /* CloudReadRequestCoordinator+Waiter.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudReadRequestCoordinator+Waiter.swift"; sourceTree = ""; }; C12625A00000000000000003 /* CloudReadRequestCoordinator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudReadRequestCoordinator.swift; sourceTree = ""; }; C12625B00000000000000003 /* CloudReadRequestCoordinatorTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudReadRequestCoordinatorTests.swift; sourceTree = ""; }; + E9A79D080D5B1E7996CC8586 /* CloudReadResponseGate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CloudReadResponseGate.swift"; sourceTree = ""; }; C13151C00000000000000022 /* CloudRefreshAuthClient.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudRefreshAuthClient.swift; sourceTree = ""; }; C13151C00000000000000020 /* CloudRefreshFixture.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudRefreshFixture.swift; sourceTree = ""; }; C12625000000000000000003 /* CloudRefreshURLProtocol.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudRefreshURLProtocol.swift; sourceTree = ""; }; @@ -12577,6 +12579,7 @@ 52476C8E96AAA6928BA35B95 /* CloudDesktopOpenActionTests.swift */, AC9CBF6AD1163240A7A1E161 /* CloudDesktopOpenFixture.swift */, C09C35915CA32C6A8B13F0A0 /* CloudDesktopOpenTestProvider.swift */, + E9A79D080D5B1E7996CC8586 /* CloudReadResponseGate.swift */, ); path = cmuxTests; sourceTree = ""; @@ -16584,6 +16587,7 @@ C13151C00000000000000031 /* CloudReadIdentityAuthClient.swift in Sources */, C12625B00000000000000006 /* CloudReadManualClock.swift in Sources */, C12625B00000000000000004 /* CloudReadRequestCoordinatorTests.swift in Sources */, + 9F046FC7A14BBDB098F471B3 /* CloudReadResponseGate.swift in Sources */, C13151C00000000000000023 /* CloudRefreshAuthClient.swift in Sources */, C13151C00000000000000021 /* CloudRefreshFixture.swift in Sources */, C12625000000000000000004 /* CloudRefreshURLProtocol.swift in Sources */, diff --git a/cmuxTests/CloudOperationRecorderTests.swift b/cmuxTests/CloudOperationRecorderTests.swift index 383226bff192..5fdbb54f1596 100644 --- a/cmuxTests/CloudOperationRecorderTests.swift +++ b/cmuxTests/CloudOperationRecorderTests.swift @@ -36,7 +36,10 @@ struct CloudOperationRecorderTests { while ContinuousClock.now < deadline { let data = try Data(contentsOf: queueURL) let entries = try #require(JSONSerialization.jsonObject(with: data) as? [[String: Any]]) - if entries.isEmpty { acknowledged = true; break } + if entries.isEmpty, await PlacementReceiptURLProtocol.captured(status: status) != nil { + acknowledged = true + break + } try await Task.sleep(for: .milliseconds(10)) } #expect(acknowledged) diff --git a/cmuxTests/CloudReadRequestCoordinatorTests.swift b/cmuxTests/CloudReadRequestCoordinatorTests.swift index c19464ed09ce..45dc89a7d5ad 100644 --- a/cmuxTests/CloudReadRequestCoordinatorTests.swift +++ b/cmuxTests/CloudReadRequestCoordinatorTests.swift @@ -487,20 +487,3 @@ struct CloudMachinesOfflineStateTests { model.stopPolling() } } - -actor CloudReadResponseGate { - private(set) var requests = 0 - private var released = false - private var waiting: [(CloudReadRequestCoordinator.Response, CheckedContinuation)] = [] - func read(_ response: CloudReadRequestCoordinator.Response) async -> CloudReadRequestCoordinator.Response { - requests += 1 - if released { return response } - return await withCheckedContinuation { waiting.append((response, $0)) } - } - func release() { - released = true - let pending = waiting - waiting.removeAll() - for (response, continuation) in pending { continuation.resume(returning: response) } - } -} diff --git a/cmuxTests/CloudReadResponseGate.swift b/cmuxTests/CloudReadResponseGate.swift new file mode 100644 index 000000000000..3d636c0a9b83 --- /dev/null +++ b/cmuxTests/CloudReadResponseGate.swift @@ -0,0 +1,20 @@ +import Foundation + +actor CloudReadResponseGate { + private(set) var requests = 0 + private var released = false + private var waiting: [(CloudReadRequestCoordinator.Response, CheckedContinuation)] = [] + + func read(_ response: CloudReadRequestCoordinator.Response) async -> CloudReadRequestCoordinator.Response { + requests += 1 + if released { return response } + return await withCheckedContinuation { waiting.append((response, $0)) } + } + + func release() { + released = true + let pending = waiting + waiting.removeAll() + for (response, continuation) in pending { continuation.resume(returning: response) } + } +} diff --git a/web/services/vms/drivers/guestCliInstallCommand.ts b/web/services/vms/drivers/guestCliInstallCommand.ts index 4c638f540f30..532f76548517 100644 --- a/web/services/vms/drivers/guestCliInstallCommand.ts +++ b/web/services/vms/drivers/guestCliInstallCommand.ts @@ -250,6 +250,7 @@ try: stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) except Exception as error: rollback_error = None + restore_failed = False try: cleanup_generated() except Exception as cleanup_error: @@ -259,11 +260,12 @@ except Exception as error: restore_paths() except Exception as restore_error: rollback_error = restore_error + restore_failed = True try: cleanup_created_directories(before_directories if "before_directories" in globals() else {}) except Exception as directory_error: rollback_error = directory_error - if backup_root: + if backup_root and not restore_failed: try: shutil.rmtree(backup_root) except Exception as backup_error: From 9c2ca272cf685894985240281075b582f3489c74 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 23 Sep 2026 17:10:37 -0700 Subject: [PATCH 80/92] ci: gate dual Xcode runner fallback --- .github/workflows/auth-refresh-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/auth-refresh-tests.yml b/.github/workflows/auth-refresh-tests.yml index d683e498b7e3..3e2890f9236c 100644 --- a/.github/workflows/auth-refresh-tests.yml +++ b/.github/workflows/auth-refresh-tests.yml @@ -18,7 +18,7 @@ concurrency: jobs: auth-refresh-tests: - runs-on: ${{ vars.MACOS_RUNNER_DUAL_XCODE || 'blacksmith-6vcpu-macos-15' }} + runs-on: ${{ vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_DUAL_XCODE || 'blacksmith-6vcpu-macos-15' }} timeout-minutes: 15 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd From 5fbf36afe83ab8219e3444c2acc642d473408868 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 23 Sep 2026 18:38:30 -0700 Subject: [PATCH 81/92] fix: import Cloud read test fixture module --- cmuxTests/CloudReadResponseGate.swift | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cmuxTests/CloudReadResponseGate.swift b/cmuxTests/CloudReadResponseGate.swift index 3d636c0a9b83..a91840783703 100644 --- a/cmuxTests/CloudReadResponseGate.swift +++ b/cmuxTests/CloudReadResponseGate.swift @@ -1,5 +1,13 @@ import Foundation +#if canImport(cmux_DEV) +@testable import cmux_DEV +#elseif canImport(cmux) +@testable import cmux +#else +@testable import CloudReadFixture +#endif + actor CloudReadResponseGate { private(set) var requests = 0 private var released = false From e8bfb107f23ad2cc8c54df6a7d5d37952f50e6d0 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 23 Sep 2026 19:19:04 -0700 Subject: [PATCH 82/92] test: cover Canvas zoom while a Markdown source editor is focused Cmd-Opt-= and Cmd-Opt-- must zoom the canvas when the focused panel is a Markdown file in text mode. That editor is a SavingTextView but not a text file preview, so the canvas layout clause should still allow canvas zoom. Records the canvas zoom factors and checks the editor font is unchanged. This test fails on the current branch: #12814 widened filePreviewTextEditorFocused to every SavingTextView, which blocks canvasLayoutOutsideFocusedContent (and Cmd-0, covered by cmdZeroInCanvasResetsCanvasZoomWhenMarkdownSourceEditorIsFocused). Co-Authored-By: Claude Opus 5.5 --- cmuxTests/CanvasShortcutContextTests.swift | 68 +++++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/cmuxTests/CanvasShortcutContextTests.swift b/cmuxTests/CanvasShortcutContextTests.swift index 8a22e37bacd7..372b40dfc33b 100644 --- a/cmuxTests/CanvasShortcutContextTests.swift +++ b/cmuxTests/CanvasShortcutContextTests.swift @@ -14,13 +14,14 @@ import Testing private final class CanvasRoutingViewportSpy: CanvasViewportControlling { var revealedPanelIds: [UUID] = [] var resetZoomCount = 0 + var zoomFactors: [CGFloat] = [] var currentMagnification: CGFloat = 1 var currentCenterInCanvas: CGPoint = .zero func revealPane(_ panelId: UUID, animated: Bool) { revealedPanelIds.append(panelId) } func resetZoom() { resetZoomCount += 1 } func toggleOverview() {} - func zoom(by factor: CGFloat) {} + func zoom(by factor: CGFloat) { zoomFactors.append(factor) } func setViewport(center: CGPoint, magnification: CGFloat?) {} func modelDidChangeExternally(animated: Bool) {} } @@ -331,6 +332,71 @@ struct CanvasShortcutRoutingFeedbackTests { } } + /// Canvas zoom in/out must reach the canvas, not the Markdown source editor: + /// the editor is a file editor for word wrap, but not a text file preview. + @Test func canvasZoomShortcutsZoomCanvasWhenMarkdownSourceEditorIsFocused() throws { + try withIsolatedShortcutSettings { + let appDelegate = try #require(AppDelegate.shared) + let windowId = appDelegate.createMainWindow() + defer { closeWindow(withId: windowId) } + + let window = try #require(mainWindow(for: windowId)) + let manager = try #require(appDelegate.tabManagerFor(windowId: windowId)) + let workspace = try #require(manager.selectedWorkspace) + let zoomIn = try #require(makeKeyDownEvent( + key: "=", + modifiers: [.command, .option], + keyCode: UInt16(kVK_ANSI_Equal), + windowNumber: window.windowNumber + )) + let zoomOut = try #require(makeKeyDownEvent( + key: "-", + modifiers: [.command, .option], + keyCode: UInt16(kVK_ANSI_Minus), + windowNumber: window.windowNumber + )) + + window.makeKeyAndOrderFront(nil) + workspace.setLayoutMode(.canvas) + let viewport = CanvasRoutingViewportSpy() + workspace.canvasModel.viewport = viewport + + let firstPane = try #require(workspace.bonsplitController.allPaneIds.first) + let fileURL = try temporaryMarkdownFile(contents: "# Preview\n") + defer { try? FileManager.default.removeItem(at: fileURL) } + let panel = try #require(workspace.newMarkdownSurface( + inPane: firstPane, + filePath: fileURL.path, + focus: true + )) + panel.setDisplayMode(.text) + + let textView = SavingTextView.makeFilePreviewTextView() + textView.frame = NSRect(x: 0, y: 0, width: 200, height: 120) + textView.string = panel.textContent + textView.panel = panel + panel.attachTextView(textView) + window.contentView?.addSubview(textView) + defer { textView.removeFromSuperview() } + #expect(window.makeFirstResponder(textView)) + #expect(workspace.focusedPanelId == panel.id) + #expect(manager.focusedTextFilePreviewPanel == nil) + let initialPointSize = try #require(textView.font?.pointSize) + +#if DEBUG + #expect(appDelegate.debugHandleCustomShortcut(event: zoomIn)) + #expect(appDelegate.debugHandleCustomShortcut(event: zoomOut)) +#else + Issue.record("debugHandleCustomShortcut is only available in DEBUG") +#endif + + #expect(viewport.zoomFactors.count == 2) + #expect((viewport.zoomFactors.first ?? 1) > 1) + #expect((viewport.zoomFactors.last ?? 1) < 1) + #expect(abs((textView.font?.pointSize ?? 0) - initialPointSize) < 0.01) + } + } + @Test func chordedViewZoomShortcutZoomsFocusedTextPreviewThroughAppRouter() throws { try withIsolatedShortcutSettings { KeyboardShortcutSettings.setShortcut( From 4d38f452bc7edc628e00a03a040a0aa7cc82db65 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 23 Sep 2026 19:19:31 -0700 Subject: [PATCH 83/92] fix: scope file-editor focus to file-editor shortcuts only #7157 limited filePreviewTextEditorFocused to the focused text file preview's editor, matching the command palette's panelIsFilePreviewTextEditor, so Markdown source and Dock editors keep Canvas zoom and do not take preview zoom. #12814 widened the flag to every SavingTextView so word wrap works in those editors, which also turned off canvasLayoutOutsideFocusedContent there: Cmd-0, Cmd-Opt-= and Cmd-Opt-- stopped reaching the canvas. Keep both scopes. filePreviewTextEditorFocused is narrow again and drives the shared shortcut context, preview zoom and Canvas routing. The new fileEditorFocused covers every file editor and applies only to actions in the .filePreviewTextEditor context (word wrap): whenClauseContext(for:) projects it onto the file-editor atom for their `when` clause, both at the keyDown gate and when arming chords, and isAvailable uses it for their menu and palette state. FileEditorWordWrapShortcutTests.appShortcutRouting still covers Opt-Z in a bare file editor. Co-Authored-By: Claude Opus 5.5 --- Sources/AppDelegate.swift | 7 ++-- Sources/KeyboardShortcutActionContext.swift | 4 +- Sources/KeyboardShortcutContext.swift | 44 +++++++++++++++++++-- cmuxTests/CanvasShortcutContextTests.swift | 39 ++++++++++++++++++ 4 files changed, 86 insertions(+), 8 deletions(-) diff --git a/Sources/AppDelegate.swift b/Sources/AppDelegate.swift index c1e74c76ddfe..83921a32720c 100644 --- a/Sources/AppDelegate.swift +++ b/Sources/AppDelegate.swift @@ -14889,13 +14889,14 @@ final class AppDelegate: NSObject, NSApplicationDelegate, UNUserNotificationCent } if activeConfiguredShortcutChordPrefixForCurrentEvent == nil { - let shortcutContext = shortcutEventFocusContext(event).shortcutContext + let focusContext = shortcutEventFocusContext(event) let availableChordActions = currentConfiguredShortcutChordActions().filter { action in // Arm by the effective `when` clause (its shortcuts.when override or // the built-in context default), matching the keyDown gate, so a // `when`-broadened chord arms in its allowed context and a narrowed // one does not swallow its first stroke elsewhere (issue #5189). - KeyboardShortcutSettings.effectiveWhenClause(for: action).evaluate(shortcutContext) + KeyboardShortcutSettings.effectiveWhenClause(for: action) + .evaluate(focusContext.whenClauseContext(for: action)) } if armConfiguredShortcutChordIfNeeded(event: event, actions: availableChordActions) { return true @@ -17168,7 +17169,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate, UNUserNotificationCent /// handlers that previously ignored context (issue #5189). func shortcutWhenClauseAllows(action: KeyboardShortcutSettings.Action, event: NSEvent) -> Bool { KeyboardShortcutSettings.effectiveWhenClause(for: action) - .evaluate(shortcutEventFocusContext(event).shortcutContext) + .evaluate(shortcutEventFocusContext(event).whenClauseContext(for: action)) } /// Resolves a right-sidebar mode shortcut after applying the action's diff --git a/Sources/KeyboardShortcutActionContext.swift b/Sources/KeyboardShortcutActionContext.swift index e3bebbf99232..29df6de227f4 100644 --- a/Sources/KeyboardShortcutActionContext.swift +++ b/Sources/KeyboardShortcutActionContext.swift @@ -105,7 +105,9 @@ extension KeyboardShortcutSettings.Action { focusedBrowserPanel: context.browserPanel != nil, focusedMarkdownPanel: context.markdownPanel != nil, focusedSimulatorPanel: context.shortcutContext.bool(ShortcutContextKnownKey.simulatorFocus.rawValue), - focusedFilePreviewTextEditor: context.filePreviewTextEditorFocused, + focusedFilePreviewTextEditor: self == .filePreviewTextEditor + ? context.fileEditorFocused + : context.filePreviewTextEditorFocused, rightSidebarFocused: context.rightSidebarFocused, workspaceCanvasLayout: context.shortcutContext.bool(ShortcutContextKnownKey.workspaceCanvasLayout.rawValue) ) diff --git a/Sources/KeyboardShortcutContext.swift b/Sources/KeyboardShortcutContext.swift index 62d7297942cf..58fccab7e90e 100644 --- a/Sources/KeyboardShortcutContext.swift +++ b/Sources/KeyboardShortcutContext.swift @@ -6,7 +6,12 @@ import WebKit struct ShortcutEventFocusContext { let browserPanel: BrowserPanel? let markdownPanel: MarkdownPanel? + /// The focused text file preview's editor owns the responder. Preview zoom + /// and Canvas zoom routing use this narrow scope. let filePreviewTextEditorFocused: Bool + /// Any file editor owns the responder, including editors in the Dock and + /// Markdown text mode. File-editor actions such as word wrap use it. + let fileEditorFocused: Bool let simulatorFocused: Bool let simulatorPanel: SimulatorPanel? let simulatorTextEditorFocused: Bool @@ -18,6 +23,7 @@ struct ShortcutEventFocusContext { browserPanel: BrowserPanel?, markdownPanel: MarkdownPanel?, filePreviewTextEditorFocused: Bool, + fileEditorFocused: Bool = false, simulatorFocused: Bool, simulatorPanel: SimulatorPanel? = nil, simulatorTextEditorFocused: Bool = false, @@ -27,6 +33,7 @@ struct ShortcutEventFocusContext { self.browserPanel = browserPanel self.markdownPanel = markdownPanel self.filePreviewTextEditorFocused = filePreviewTextEditorFocused + self.fileEditorFocused = fileEditorFocused || filePreviewTextEditorFocused self.simulatorFocused = simulatorFocused self.simulatorPanel = simulatorPanel self.simulatorTextEditorFocused = simulatorTextEditorFocused @@ -49,6 +56,23 @@ struct ShortcutEventFocusContext { simulator: simulatorFocused ) } + + /// ``shortcutContext`` with any focused file editor projected onto the + /// file-editor atom, so a file-editor action's `when` clause holds in + /// editors outside a text file preview. + var fileEditorShortcutContext: ShortcutContext { + guard fileEditorFocused, !filePreviewTextEditorFocused else { return shortcutContext } + var context = shortcutContext + context.setBool(ShortcutFocusAtom.filePreviewTextEditorFocus.rawValue, true) + context.setBool(ShortcutFocusAtom.terminalFocus.rawValue, false) + return context + } + + /// The context `action`'s `when` clause evaluates against: file-editor + /// actions see every file editor, everything else the narrow preview scope. + func whenClauseContext(for action: KeyboardShortcutSettings.Action) -> ShortcutContext { + action.shortcutContext == .filePreviewTextEditor ? fileEditorShortcutContext : shortcutContext + } } func shortcutResponderAcceptsTextEditing(_ responder: NSResponder) -> Bool { @@ -110,9 +134,11 @@ extension AppDelegate { // Only treat a markdown panel as focused when no browser panel owns the // event, so a focused browser never routes markdown shortcuts. let markdownPanel = browserPanel == nil ? shortcutFocusedMarkdownPanel(in: shortcutWindow) : nil - let filePreviewTextEditorFocused = browserPanel == nil && markdownPanel == nil - ? shortcutFocusedFilePreviewTextEditor(in: shortcutWindow) + let fileEditorFocused = browserPanel == nil && markdownPanel == nil + ? shortcutFocusedSavingTextView(in: shortcutWindow) != nil : false + let filePreviewTextEditorFocused = fileEditorFocused + && shortcutFocusedFilePreviewTextEditor(in: shortcutWindow) let rightSidebarFocused = !simulatorFocused && (shortcutWindow.map { shouldRouteRightSidebarModeShortcut(in: $0) } ?? false) let focusState = ShortcutFocusState( @@ -126,6 +152,7 @@ extension AppDelegate { browserPanel: browserPanel, markdownPanel: markdownPanel, filePreviewTextEditorFocused: filePreviewTextEditorFocused, + fileEditorFocused: fileEditorFocused, simulatorFocused: simulatorFocused, simulatorPanel: simulatorPanel, simulatorTextEditorFocused: simulatorTextEditorFocused, @@ -191,9 +218,18 @@ extension AppDelegate { return tabManager?.focusedMarkdownPanel } - /// Includes text editors in the Dock and Markdown text mode by responder ownership. + /// Matches only the focused text file preview's editor, the same scope as the + /// command palette's `panelIsFilePreviewTextEditor`. Editors in the Dock and + /// Markdown text mode are file editors but not previews, so preview zoom and + /// Canvas zoom routing leave them alone. private func shortcutFocusedFilePreviewTextEditor(in window: NSWindow?) -> Bool { - shortcutFocusedSavingTextView(in: window) != nil + guard let focusedFilePreviewPanel = shortcutContextTabManager(in: window)?.focusedTextFilePreviewPanel, + let textView = shortcutFocusedSavingTextView(in: window), + let owningFilePreviewPanel = textView.panel as? FilePreviewPanel, + owningFilePreviewPanel === focusedFilePreviewPanel else { + return false + } + return true } /// Resolves the editor that owns the requested window’s keyboard responder. diff --git a/cmuxTests/CanvasShortcutContextTests.swift b/cmuxTests/CanvasShortcutContextTests.swift index 372b40dfc33b..471ffb849e3e 100644 --- a/cmuxTests/CanvasShortcutContextTests.swift +++ b/cmuxTests/CanvasShortcutContextTests.swift @@ -54,6 +54,45 @@ struct CanvasShortcutContextTests { } } + @MainActor + @Test + func fileEditorFocusWidensOnlyFileEditorActions() { + var shortcutContext = ShortcutFocusState(browser: false, markdown: false, sidebar: false).context + shortcutContext.setBool(ShortcutContextKnownKey.workspaceCanvasLayout.rawValue, true) + // A Markdown source or Dock editor: a file editor, but not a text file preview. + let focus = ShortcutEventFocusContext( + browserPanel: nil, + markdownPanel: nil, + filePreviewTextEditorFocused: false, + fileEditorFocused: true, + simulatorFocused: false, + rightSidebarFocused: false, + shortcutContext: shortcutContext + ) + + let wordWrap = KeyboardShortcutSettings.Action.toggleFileEditorWordWrap + #expect(KeyboardShortcutSettings.effectiveWhenClause(for: wordWrap).evaluate(focus.whenClauseContext(for: wordWrap))) + #expect(wordWrap.shortcutContext.isAvailable(focus)) + + let canvasZoomActions: [KeyboardShortcutSettings.Action] = [.canvasZoomIn, .canvasZoomOut, .canvasZoomReset] + for action in canvasZoomActions { + #expect( + KeyboardShortcutSettings.effectiveWhenClause(for: action).evaluate(focus.whenClauseContext(for: action)), + "\(action.rawValue) must reach the canvas while a non-preview file editor is focused" + ) + #expect(action.shortcutContext.isAvailable(focus)) + } + + let previewZoomActions: [KeyboardShortcutSettings.Action] = [.browserZoomIn, .browserZoomOut, .browserZoomReset] + for action in previewZoomActions { + #expect( + !KeyboardShortcutSettings.effectiveWhenClause(for: action).evaluate(focus.whenClauseContext(for: action)), + "\(action.rawValue) is scoped to text file previews, not every file editor" + ) + #expect(!action.shortcutContext.isAvailable(focus)) + } + } + @Test func canvasLayoutContextOverlapsNormalTerminalFocusShortcuts() { let canvas = KeyboardShortcutSettings.Action.canvasOverview.shortcutContext From 098702553db09af1fe1442a4dfb82b00d796bbb1 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Wed, 23 Sep 2026 19:07:27 -0700 Subject: [PATCH 84/92] test: stop racing the hook queue's drain task for tool admission "Best-effort telemetry cannot monopolize resident delivery capacity" enqueued three tool events back to back. Tool telemetry has one ingress slot, so the second and third are admitted only if the drain task has already moved the previous one into a lane. On a loaded app host it had not: one tool event was dropped, tool-4 was then admitted in its place, and the lifecycle event queued behind tool-4 in the surface-4 lane never started, so waitUntilStarted(count: 3) timed out. Wait for each of the first two tool deliveries to start before the next enqueue. Tool-4 is then rejected by the three-event best-effort reservation whether or not tool-3 has left ingress. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit f385fa6d9c634a20f3707d6a2d6456a6fe1527cc) --- cmuxTests/AgentHookDeliveryQueueTests.swift | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/cmuxTests/AgentHookDeliveryQueueTests.swift b/cmuxTests/AgentHookDeliveryQueueTests.swift index d56930f05de7..5f9d44e2289d 100644 --- a/cmuxTests/AgentHookDeliveryQueueTests.swift +++ b/cmuxTests/AgentHookDeliveryQueueTests.swift @@ -321,16 +321,29 @@ struct AgentHookDeliveryQueueTests { await probe.deliver(event) } - for index in 1...3 { + // Tool telemetry has a single ingress slot, so a second tool event is + // admitted only after the drain task has moved the first one into a + // lane. Waiting for each delivery to start makes that hand-off + // observable instead of racing the drain task. Only two tool + // deliveries may run, so the third stays resident behind them. + for index in 1...2 { #expect(queue.enqueue(try makeEvent( agent: "cursor", subcommand: "shell-exec", payload: "tool-\(index)", surfaceID: "surface-\(index)" ))) + try await probe.waitUntilStarted(count: index) } - try await probe.waitUntilStarted(count: 2) + #expect(queue.enqueue(try makeEvent( + agent: "cursor", + subcommand: "shell-exec", + payload: "tool-3", + surfaceID: "surface-3" + ))) + // Three outstanding tool events exhaust the best-effort reservation + // whether or not tool-3 has left ingress yet. #expect(!queue.enqueue(try makeEvent( agent: "cursor", subcommand: "shell-exec", From 2346292452b93b376027280892755099e6696bf2 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Wed, 23 Sep 2026 16:50:47 -0700 Subject: [PATCH 85/92] test: wait for child processes without pinning a global-pool thread 24 test helpers waited for a child process by queueing a blocking waitUntilExit() on DispatchQueue.global() and waiting on a semaphore with a timeout. Every queued waiter holds a pool thread until its child exits. Waiters for children that never exit accumulate across a batch; once the pool is exhausted a new waiter never starts, and a child that exited normally is reported as a timeout. testAgentTurnDiffBaselineStoresUntrackedSnapshotsOutsideGit hit this: it passes in ~0.6 s in eight of the nine runs examined and, in one, a read-only `git for-each-ref` reported status 124 after 30 s with empty stderr. cd7d25c13c fixed the same failure in one Claude hook helper. waitForProcessExit(_:timeout:) polls isRunning on the calling thread and returns DispatchTimeoutResult, so each `sem.wait(timeout: .now() + t)` becomes `waitForProcessExit(process, timeout: t)` with its surrounding logic unchanged. No test in cmuxTests queues a waitUntilExit() anymore. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit dba68ec4166889ba6b57abd2c1556f486b6ce9da) --- cmux.xcodeproj/project.pbxproj | 4 +++ ...ifyProcessIntegrationRegressionTests.swift | 21 ++---------- ...LIRemoteShellStartupPerformanceTests.swift | 9 ++---- cmuxTests/CLISSHPTYResizeInputTests.swift | 7 +--- .../CLISSHSessionAttachAnchorTests.swift | 11 ++----- cmuxTests/CLISendQueuedOutputTests.swift | 11 ++----- .../CLIStdioSIGPIPERegressionTests.swift | 16 ++-------- .../CLITmuxCompatStoreConcurrencyTests.swift | 11 ++----- cmuxTests/CLIVMLayoutEnvTests.swift | 11 ++----- cmuxTests/CMUXOpenCommandTests.swift | 20 +++--------- cmuxTests/CampfireHookNotificationTests.swift | 11 ++----- .../CodexTerminalErrorNotificationTests.swift | 11 ++----- cmuxTests/FishShellIntegrationTests.swift | 9 ++---- cmuxTests/GhosttyConfigTests.swift | 9 ++---- cmuxTests/OpenCodeHookRegressionTests.swift | 9 ++---- cmuxTests/ProcessExitWait.swift | 21 ++++++++++++ cmuxTests/SSHDeepSleepReattachTests.swift | 9 ++---- cmuxTests/SSHRemoteCWDRegressionTests.swift | 9 ++---- .../WorkspaceRemoteConnectionTests.swift | 32 ++++--------------- 19 files changed, 73 insertions(+), 168 deletions(-) create mode 100644 cmuxTests/ProcessExitWait.swift diff --git a/cmux.xcodeproj/project.pbxproj b/cmux.xcodeproj/project.pbxproj index 273d70770bff..8bb12ee13fd0 100644 --- a/cmux.xcodeproj/project.pbxproj +++ b/cmux.xcodeproj/project.pbxproj @@ -3299,6 +3299,7 @@ FCB4880428BE4BB3990BCD51 /* SSHRemoteCommandCLIIntegrationTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = BB6498DB903042C8B9EB34C5 /* SSHRemoteCommandCLIIntegrationTests.swift */; }; D7AB1204800000000000000A /* SSHRemoteCWDRegressionTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7AB1204800000000000000B /* SSHRemoteCWDRegressionTests.swift */; }; FF16E760F7BC847FDBB97489 /* SSHStartupCommandTestSupport.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4575F4BB7F52CF88AB6D0022 /* SSHStartupCommandTestSupport.swift */; }; + AC0FCEFB6665CF134634D60B /* ProcessExitWait.swift in Sources */ = {isa = PBXBuildFile; fileRef = E06E7588C05AC6A8659CFA51 /* ProcessExitWait.swift */; }; D6B29560DEE34B08842A143A /* SSHStartupManualReconnectTests+Fixtures.swift in Sources */ = {isa = PBXBuildFile; fileRef = 37B04BC7D56945E59781D11F /* SSHStartupManualReconnectTests+Fixtures.swift */; }; EF70B7123200D1FB6F03DB26 /* SSHStartupManualReconnectTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4F73A5C0BB93507E18261385 /* SSHStartupManualReconnectTests.swift */; }; F6355600A1B2C3D4E5F60718 /* SSHStartupSignalLifecycleTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = F6355601A1B2C3D4E5F60718 /* SSHStartupSignalLifecycleTests.swift */; }; @@ -7444,6 +7445,7 @@ BB6498DB903042C8B9EB34C5 /* SSHRemoteCommandCLIIntegrationTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SSHRemoteCommandCLIIntegrationTests.swift; sourceTree = ""; }; D7AB1204800000000000000B /* SSHRemoteCWDRegressionTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SSHRemoteCWDRegressionTests.swift; sourceTree = ""; }; 4575F4BB7F52CF88AB6D0022 /* SSHStartupCommandTestSupport.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SSHStartupCommandTestSupport.swift; sourceTree = ""; }; + E06E7588C05AC6A8659CFA51 /* ProcessExitWait.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProcessExitWait.swift; sourceTree = ""; }; 37B04BC7D56945E59781D11F /* SSHStartupManualReconnectTests+Fixtures.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "SSHStartupManualReconnectTests+Fixtures.swift"; sourceTree = ""; }; 4F73A5C0BB93507E18261385 /* SSHStartupManualReconnectTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SSHStartupManualReconnectTests.swift; sourceTree = ""; }; F6355601A1B2C3D4E5F60718 /* SSHStartupSignalLifecycleTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SSHStartupSignalLifecycleTests.swift; sourceTree = ""; }; @@ -11787,6 +11789,7 @@ F6724603A1B2C3D4E5F67246 /* RemoteTmuxHostRemoteCommandOverrideTests.swift */, F6724601A1B2C3D4E5F67246 /* SSHConfiguredRemoteCommandHostTests.swift */, 4575F4BB7F52CF88AB6D0022 /* SSHStartupCommandTestSupport.swift */, + E06E7588C05AC6A8659CFA51 /* ProcessExitWait.swift */, 37B04BC7D56945E59781D11F /* SSHStartupManualReconnectTests+Fixtures.swift */, EDF9EEAA233347858BCF3320 /* SSHConfiguredRemoteCommandHostTests+Harness.swift */, A82715B8BDE347A18031067C /* CLINotifyProcessIntegrationRegressionTests+JSON.swift */, @@ -17183,6 +17186,7 @@ FCB4880428BE4BB3990BCD51 /* SSHRemoteCommandCLIIntegrationTests.swift in Sources */, D7AB1204800000000000000A /* SSHRemoteCWDRegressionTests.swift in Sources */, FF16E760F7BC847FDBB97489 /* SSHStartupCommandTestSupport.swift in Sources */, + AC0FCEFB6665CF134634D60B /* ProcessExitWait.swift in Sources */, D6B29560DEE34B08842A143A /* SSHStartupManualReconnectTests+Fixtures.swift in Sources */, EF70B7123200D1FB6F03DB26 /* SSHStartupManualReconnectTests.swift in Sources */, F6355600A1B2C3D4E5F60718 /* SSHStartupSignalLifecycleTests.swift in Sources */, diff --git a/cmuxTests/CLINotifyProcessIntegrationRegressionTests.swift b/cmuxTests/CLINotifyProcessIntegrationRegressionTests.swift index e455fdbb68ee..aa9b66894fad 100644 --- a/cmuxTests/CLINotifyProcessIntegrationRegressionTests.swift +++ b/cmuxTests/CLINotifyProcessIntegrationRegressionTests.swift @@ -5032,12 +5032,7 @@ final class CLINotifyProcessIntegrationRegressionTests: XCTestCase { allowBridgeResponse.signal() XCTAssertEqual(handshakeReceived.wait(timeout: .now() + 5), .success) - let exited = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .userInitiated).async { - process.waitUntilExit() - exited.signal() - } - XCTAssertEqual(exited.wait(timeout: .now() + 5), .success) + XCTAssertEqual(waitForProcessExit(process, timeout: 5), .success) wait(for: [socketHandled, bridgeHandled], timeout: 5) let stderr = String(data: stderrPipe.fileHandleForReading.readDataToEndOfFile(), encoding: .utf8) ?? "" @@ -5244,12 +5239,7 @@ final class CLINotifyProcessIntegrationRegressionTests: XCTestCase { wait(for: [bridgeHandled], timeout: 5) allowResizeResponse.signal() - let exited = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .userInitiated).async { - process.waitUntilExit() - exited.signal() - } - XCTAssertEqual(exited.wait(timeout: .now() + 5), .success) + XCTAssertEqual(waitForProcessExit(process, timeout: 5), .success) wait(for: [socketHandled, unexpectedReadinessAfterAcknowledgement], timeout: 0.5) let stdout = String(data: stdoutPipe.fileHandleForReading.readDataToEndOfFile(), encoding: .utf8) ?? "" @@ -5436,12 +5426,7 @@ final class CLINotifyProcessIntegrationRegressionTests: XCTestCase { closeBridge.signal() wait(for: [bridgeHandled], timeout: 5) - let exited = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .userInitiated).async { - process.waitUntilExit() - exited.signal() - } - XCTAssertEqual(exited.wait(timeout: .now() + 5), .success) + XCTAssertEqual(waitForProcessExit(process, timeout: 5), .success) wait(for: [socketHandled], timeout: 5) let stdout = String( diff --git a/cmuxTests/CLIRemoteShellStartupPerformanceTests.swift b/cmuxTests/CLIRemoteShellStartupPerformanceTests.swift index 513211cda6f3..4efc72afeb71 100644 --- a/cmuxTests/CLIRemoteShellStartupPerformanceTests.swift +++ b/cmuxTests/CLIRemoteShellStartupPerformanceTests.swift @@ -395,15 +395,10 @@ struct CLIRemoteShellStartupPerformanceTests { } private func waitForProcess(_ running: RunningProcess, timeout: TimeInterval) -> ProcessRunResult { - let done = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .userInitiated).async { - running.process.waitUntilExit() - done.signal() - } - let timedOut = done.wait(timeout: .now() + timeout) == .timedOut + let timedOut = waitForProcessExit(running.process, timeout: timeout) == .timedOut if timedOut { running.process.terminate() - _ = done.wait(timeout: .now() + 1) + _ = waitForProcessExit(running.process, timeout: 1) } let stderr = String(data: running.stderrPipe.fileHandleForReading.readDataToEndOfFile(), encoding: .utf8) ?? "" return ProcessRunResult( diff --git a/cmuxTests/CLISSHPTYResizeInputTests.swift b/cmuxTests/CLISSHPTYResizeInputTests.swift index 3cd6701099cb..17e4ca1b39c6 100644 --- a/cmuxTests/CLISSHPTYResizeInputTests.swift +++ b/cmuxTests/CLISSHPTYResizeInputTests.swift @@ -149,12 +149,7 @@ struct CLISSHPTYResizeInputTests { closeBridge.signal() #expect(bridgeCloseObserved.wait(timeout: .now() + 5) == .success) - let exited = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .userInitiated).async { - process.waitUntilExit() - exited.signal() - } - let didExit = exited.wait(timeout: .now() + 5) == .success + let didExit = waitForProcessExit(process, timeout: 5) == .success #expect(didExit, "Expected ssh-pty-attach to exit") guard didExit else { if process.isRunning { diff --git a/cmuxTests/CLISSHSessionAttachAnchorTests.swift b/cmuxTests/CLISSHSessionAttachAnchorTests.swift index ace925b7876b..890a4024a953 100644 --- a/cmuxTests/CLISSHSessionAttachAnchorTests.swift +++ b/cmuxTests/CLISSHSessionAttachAnchorTests.swift @@ -516,18 +516,13 @@ struct CLISSHSessionAttachAnchorTests { return ProcessRunResult(status: -1, stdout: "", stderr: String(describing: error), timedOut: false) } - let exitSignal = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .userInitiated).async { - process.waitUntilExit() - exitSignal.signal() - } - let timedOut = exitSignal.wait(timeout: .now() + timeout) == .timedOut + let timedOut = waitForProcessExit(process, timeout: timeout) == .timedOut if timedOut { process.terminate() - if exitSignal.wait(timeout: .now() + 1) == .timedOut { + if waitForProcessExit(process, timeout: 1) == .timedOut { kill(process.processIdentifier, SIGKILL) - _ = exitSignal.wait(timeout: .now() + 1) + _ = waitForProcessExit(process, timeout: 1) } } diff --git a/cmuxTests/CLISendQueuedOutputTests.swift b/cmuxTests/CLISendQueuedOutputTests.swift index 1b9b34443e3f..2dc93e4a38e1 100644 --- a/cmuxTests/CLISendQueuedOutputTests.swift +++ b/cmuxTests/CLISendQueuedOutputTests.swift @@ -201,18 +201,13 @@ struct CLISendQueuedOutputTests { return ProcessRunResult(status: -1, stdout: "", stderr: String(describing: error), timedOut: false) } - let exitSignal = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .userInitiated).async { - process.waitUntilExit() - exitSignal.signal() - } - let timedOut = exitSignal.wait(timeout: .now() + timeout) == .timedOut + let timedOut = waitForProcessExit(process, timeout: timeout) == .timedOut if timedOut { process.terminate() - if exitSignal.wait(timeout: .now() + 1) == .timedOut { + if waitForProcessExit(process, timeout: 1) == .timedOut { kill(process.processIdentifier, SIGKILL) - _ = exitSignal.wait(timeout: .now() + 1) + _ = waitForProcessExit(process, timeout: 1) } } diff --git a/cmuxTests/CLIStdioSIGPIPERegressionTests.swift b/cmuxTests/CLIStdioSIGPIPERegressionTests.swift index 166becfce49a..fd86c5e261c9 100644 --- a/cmuxTests/CLIStdioSIGPIPERegressionTests.swift +++ b/cmuxTests/CLIStdioSIGPIPERegressionTests.swift @@ -56,12 +56,7 @@ final class CLIStdioSIGPIPERegressionTests: XCTestCase { description: String, timeout: TimeInterval = 5 ) -> XCTWaiter.Result { - let exited = expectation(description: description) - DispatchQueue.global(qos: .userInitiated).async { - process.waitUntilExit() - exited.fulfill() - } - return XCTWaiter().wait(for: [exited], timeout: timeout) + waitForProcessExit(process, timeout: timeout) == .success ? .completed : .timedOut } private func runProcess( @@ -91,16 +86,11 @@ final class CLIStdioSIGPIPERegressionTests: XCTestCase { ) } - let exitSignal = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .userInitiated).async { - process.waitUntilExit() - exitSignal.signal() - } - let timedOut = exitSignal.wait(timeout: .now() + timeout) == .timedOut + let timedOut = waitForProcessExit(process, timeout: timeout) == .timedOut if timedOut { process.terminate() - _ = exitSignal.wait(timeout: .now() + 1) + _ = waitForProcessExit(process, timeout: 1) } let stdout = String(data: stdoutPipe.fileHandleForReading.readDataToEndOfFile(), encoding: .utf8) ?? "" diff --git a/cmuxTests/CLITmuxCompatStoreConcurrencyTests.swift b/cmuxTests/CLITmuxCompatStoreConcurrencyTests.swift index 65f2f4c97724..219295f4ff6a 100644 --- a/cmuxTests/CLITmuxCompatStoreConcurrencyTests.swift +++ b/cmuxTests/CLITmuxCompatStoreConcurrencyTests.swift @@ -206,17 +206,12 @@ struct CLITmuxCompatStoreConcurrencyTests { return ProcessRunResult(status: -1, stderr: String(describing: error), timedOut: false) } - let exited = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .userInitiated).async { - process.waitUntilExit() - exited.signal() - } - let timedOut = exited.wait(timeout: .now() + timeout) == .timedOut + let timedOut = waitForProcessExit(process, timeout: timeout) == .timedOut if timedOut { process.terminate() - if exited.wait(timeout: .now() + 1) == .timedOut { + if waitForProcessExit(process, timeout: 1) == .timedOut { kill(process.processIdentifier, SIGKILL) - _ = exited.wait(timeout: .now() + 1) + _ = waitForProcessExit(process, timeout: 1) } } let stderr = String( diff --git a/cmuxTests/CLIVMLayoutEnvTests.swift b/cmuxTests/CLIVMLayoutEnvTests.swift index de291847a0cc..5a7a3889e011 100644 --- a/cmuxTests/CLIVMLayoutEnvTests.swift +++ b/cmuxTests/CLIVMLayoutEnvTests.swift @@ -168,17 +168,12 @@ extension CLINotifyProcessIntegrationRegressionTests { } stdinPipe.fileHandleForWriting.write(Data(standardInput.utf8)) try? stdinPipe.fileHandleForWriting.close() - let exitSignal = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .userInitiated).async { - process.waitUntilExit() - exitSignal.signal() - } - let timedOut = exitSignal.wait(timeout: .now() + timeout) == .timedOut + let timedOut = waitForProcessExit(process, timeout: timeout) == .timedOut if timedOut { process.terminate() - if exitSignal.wait(timeout: .now() + 1) == .timedOut { + if waitForProcessExit(process, timeout: 1) == .timedOut { kill(process.processIdentifier, SIGKILL) - _ = exitSignal.wait(timeout: .now() + 1) + _ = waitForProcessExit(process, timeout: 1) } } let stdout = String(data: stdoutPipe.fileHandleForReading.readDataToEndOfFile(), encoding: .utf8) ?? "" diff --git a/cmuxTests/CMUXOpenCommandTests.swift b/cmuxTests/CMUXOpenCommandTests.swift index ab0e3cb91bf0..3c340490099c 100644 --- a/cmuxTests/CMUXOpenCommandTests.swift +++ b/cmuxTests/CMUXOpenCommandTests.swift @@ -2930,22 +2930,17 @@ final class CMUXOpenCommandTests: XCTestCase { return ProcessRunResult(status: -1, stdout: "", stderr: String(describing: error), timedOut: false) } - let exitSignal = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .userInitiated).async { - process.waitUntilExit() - exitSignal.signal() - } if let stdinText, let stdinPipe { stdinPipe.fileHandleForWriting.write(Data(stdinText.utf8)) stdinPipe.fileHandleForWriting.closeFile() } - let timedOut = exitSignal.wait(timeout: .now() + timeout) == .timedOut + let timedOut = waitForProcessExit(process, timeout: timeout) == .timedOut if timedOut { process.terminate() - if exitSignal.wait(timeout: .now() + 1) == .timedOut, process.isRunning { + if waitForProcessExit(process, timeout: 1) == .timedOut, process.isRunning { kill(process.processIdentifier, SIGKILL) - _ = exitSignal.wait(timeout: .now() + 1) + _ = waitForProcessExit(process, timeout: 1) } } @@ -3011,14 +3006,9 @@ final class CMUXOpenCommandTests: XCTestCase { private func terminateProcess(_ process: Process) { guard process.isRunning else { return } process.terminate() - let finished = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .utility).async { - process.waitUntilExit() - finished.signal() - } - if finished.wait(timeout: .now() + 1) == .timedOut, process.isRunning { + if waitForProcessExit(process, timeout: 1) == .timedOut, process.isRunning { kill(process.processIdentifier, SIGKILL) - _ = finished.wait(timeout: .now() + 1) + _ = waitForProcessExit(process, timeout: 1) } } diff --git a/cmuxTests/CampfireHookNotificationTests.swift b/cmuxTests/CampfireHookNotificationTests.swift index 6ccd2e577b2a..9daae401d437 100644 --- a/cmuxTests/CampfireHookNotificationTests.swift +++ b/cmuxTests/CampfireHookNotificationTests.swift @@ -307,18 +307,13 @@ struct CampfireHookNotificationTests { stdinPipe.fileHandleForWriting.write(Data(standardInput.utf8)) try? stdinPipe.fileHandleForWriting.close() - let exitSignal = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .userInitiated).async { - process.waitUntilExit() - exitSignal.signal() - } - let timedOut = exitSignal.wait(timeout: .now() + timeout) == .timedOut + let timedOut = waitForProcessExit(process, timeout: timeout) == .timedOut if timedOut { process.terminate() - if exitSignal.wait(timeout: .now() + 1) == .timedOut { + if waitForProcessExit(process, timeout: 1) == .timedOut { kill(process.processIdentifier, SIGKILL) - _ = exitSignal.wait(timeout: .now() + 1) + _ = waitForProcessExit(process, timeout: 1) } } diff --git a/cmuxTests/CodexTerminalErrorNotificationTests.swift b/cmuxTests/CodexTerminalErrorNotificationTests.swift index 32c2466333a0..ffef2620d534 100644 --- a/cmuxTests/CodexTerminalErrorNotificationTests.swift +++ b/cmuxTests/CodexTerminalErrorNotificationTests.swift @@ -264,17 +264,12 @@ private struct CodexTerminalErrorProcess { stdin.fileHandleForWriting.write(Data(standardInput.utf8)) try? stdin.fileHandleForWriting.close() - let finished = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .userInitiated).async { - process.waitUntilExit() - finished.signal() - } - let timedOut = finished.wait(timeout: .now() + timeout) == .timedOut + let timedOut = waitForProcessExit(process, timeout: timeout) == .timedOut if timedOut { process.terminate() - if finished.wait(timeout: .now() + 1) == .timedOut { + if waitForProcessExit(process, timeout: 1) == .timedOut { Darwin.kill(process.processIdentifier, SIGKILL) - guard finished.wait(timeout: .now() + 1) == .success else { + guard waitForProcessExit(process, timeout: 1) == .success else { return Result( status: -1, stdout: "", diff --git a/cmuxTests/FishShellIntegrationTests.swift b/cmuxTests/FishShellIntegrationTests.swift index c4a1e8374e90..8b159d8b3db3 100644 --- a/cmuxTests/FishShellIntegrationTests.swift +++ b/cmuxTests/FishShellIntegrationTests.swift @@ -491,16 +491,11 @@ struct FishShellIntegrationTests { return ProcessRunResult(status: -1, stdout: "", stderr: String(describing: error), timedOut: false) } - let exitSignal = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .userInitiated).async { - process.waitUntilExit() - exitSignal.signal() - } - let timedOut = exitSignal.wait(timeout: .now() + timeout) == .timedOut + let timedOut = waitForProcessExit(process, timeout: timeout) == .timedOut if timedOut { process.terminate() - _ = exitSignal.wait(timeout: .now() + 1) + _ = waitForProcessExit(process, timeout: 1) } return ProcessRunResult( diff --git a/cmuxTests/GhosttyConfigTests.swift b/cmuxTests/GhosttyConfigTests.swift index 2616e243f331..9eeea7f47deb 100644 --- a/cmuxTests/GhosttyConfigTests.swift +++ b/cmuxTests/GhosttyConfigTests.swift @@ -1397,16 +1397,11 @@ final class GhosttyConfigTests: XCTestCase { ) } - let exitSignal = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .userInitiated).async { - process.waitUntilExit() - exitSignal.signal() - } - let timedOut = exitSignal.wait(timeout: .now() + timeout) == .timedOut + let timedOut = waitForProcessExit(process, timeout: timeout) == .timedOut if timedOut { process.terminate() - _ = exitSignal.wait(timeout: .now() + 1) + _ = waitForProcessExit(process, timeout: 1) } let stdout = String( diff --git a/cmuxTests/OpenCodeHookRegressionTests.swift b/cmuxTests/OpenCodeHookRegressionTests.swift index 621650e4ad1d..3a59cb205eba 100644 --- a/cmuxTests/OpenCodeHookRegressionTests.swift +++ b/cmuxTests/OpenCodeHookRegressionTests.swift @@ -227,15 +227,10 @@ const fs = require("node:fs"); } catch { return ProcessRunResult(status: -1, stdout: "", stderr: String(describing: error), timedOut: false) } - let exitSignal = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .userInitiated).async { - process.waitUntilExit() - exitSignal.signal() - } - let timedOut = exitSignal.wait(timeout: .now() + timeout) == .timedOut + let timedOut = waitForProcessExit(process, timeout: timeout) == .timedOut if timedOut { process.terminate() - _ = exitSignal.wait(timeout: .now() + 1) + _ = waitForProcessExit(process, timeout: 1) } return ProcessRunResult( status: process.terminationStatus, diff --git a/cmuxTests/ProcessExitWait.swift b/cmuxTests/ProcessExitWait.swift new file mode 100644 index 000000000000..c7d3f8bac32e --- /dev/null +++ b/cmuxTests/ProcessExitWait.swift @@ -0,0 +1,21 @@ +import Foundation + +/// Waits for `process` to exit on the calling thread, polling `isRunning`. +/// +/// Returns `.success` once the process has exited, `.timedOut` if it is still +/// running at the deadline. Use this instead of queueing a blocking +/// `waitUntilExit()` on `DispatchQueue.global()` and waiting on a semaphore: +/// every queued waiter pins a pool thread until its child exits, waiters for +/// children that never exit accumulate across a test batch, and once the pool +/// is exhausted a new waiter never starts, so a child that exited normally is +/// reported as a timeout. +func waitForProcessExit(_ process: Process, timeout: TimeInterval) -> DispatchTimeoutResult { + let deadline = Date().addingTimeInterval(timeout) + while process.isRunning { + if Date() >= deadline { + return .timedOut + } + usleep(5_000) + } + return .success +} diff --git a/cmuxTests/SSHDeepSleepReattachTests.swift b/cmuxTests/SSHDeepSleepReattachTests.swift index 8a311d2f467c..08569696acd9 100644 --- a/cmuxTests/SSHDeepSleepReattachTests.swift +++ b/cmuxTests/SSHDeepSleepReattachTests.swift @@ -541,15 +541,10 @@ struct SSHDeepSleepReattachTests { } catch { return ProcessRunResult(status: -1, stderr: String(describing: error), timedOut: false) } - let exitSignal = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .userInitiated).async { - process.waitUntilExit() - exitSignal.signal() - } - let timedOut = exitSignal.wait(timeout: .now() + 5) == .timedOut + let timedOut = waitForProcessExit(process, timeout: 5) == .timedOut if timedOut { process.terminate() - _ = exitSignal.wait(timeout: .now() + 1) + _ = waitForProcessExit(process, timeout: 1) } let stderr = String(data: stderrPipe.fileHandleForReading.readDataToEndOfFile(), encoding: .utf8) ?? "" return ProcessRunResult(status: process.terminationStatus, stderr: stderr, timedOut: timedOut) diff --git a/cmuxTests/SSHRemoteCWDRegressionTests.swift b/cmuxTests/SSHRemoteCWDRegressionTests.swift index 4a4a8c7b68db..59b646e40102 100644 --- a/cmuxTests/SSHRemoteCWDRegressionTests.swift +++ b/cmuxTests/SSHRemoteCWDRegressionTests.swift @@ -45,15 +45,10 @@ struct SSHRemoteCWDRegressionTests { return ProcessRunResult(status: -1, stdout: "", stderr: String(describing: error), timedOut: false) } - let exitSignal = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .userInitiated).async { - process.waitUntilExit() - exitSignal.signal() - } - let timedOut = exitSignal.wait(timeout: .now() + timeout) == .timedOut + let timedOut = waitForProcessExit(process, timeout: timeout) == .timedOut if timedOut { process.terminate() - _ = exitSignal.wait(timeout: .now() + 1) + _ = waitForProcessExit(process, timeout: 1) } return ProcessRunResult( status: process.terminationStatus, diff --git a/cmuxTests/WorkspaceRemoteConnectionTests.swift b/cmuxTests/WorkspaceRemoteConnectionTests.swift index 890cff7d7318..646be5bdcddd 100644 --- a/cmuxTests/WorkspaceRemoteConnectionTests.swift +++ b/cmuxTests/WorkspaceRemoteConnectionTests.swift @@ -159,16 +159,11 @@ final class WorkspaceRemoteConnectionTests: XCTestCase { ) } - let exitSignal = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .userInitiated).async { - process.waitUntilExit() - exitSignal.signal() - } - let timedOut = exitSignal.wait(timeout: .now() + timeout) == .timedOut + let timedOut = waitForProcessExit(process, timeout: timeout) == .timedOut if timedOut { process.terminate() - _ = exitSignal.wait(timeout: .now() + 1) + _ = waitForProcessExit(process, timeout: 1) } let stdout = String(data: stdoutPipe.fileHandleForReading.readDataToEndOfFile(), encoding: .utf8) ?? "" @@ -5065,15 +5060,10 @@ final class CLINotifyProcessIntegrationTests: XCTestCase { process.standardError = stderrPipe try process.run() - let exitSignal = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .userInitiated).async { - process.waitUntilExit() - exitSignal.signal() - } defer { if process.isRunning { process.terminate() - _ = exitSignal.wait(timeout: .now() + 1) + _ = waitForProcessExit(process, timeout: 1) } _ = stdoutPipe.fileHandleForReading.readDataToEndOfFile() _ = stderrPipe.fileHandleForReading.readDataToEndOfFile() @@ -5165,15 +5155,10 @@ final class CLINotifyProcessIntegrationTests: XCTestCase { process.standardError = stderrPipe try process.run() - let exitSignal = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .userInitiated).async { - process.waitUntilExit() - exitSignal.signal() - } defer { if process.isRunning { process.terminate() - _ = exitSignal.wait(timeout: .now() + 1) + _ = waitForProcessExit(process, timeout: 1) } _ = stdoutPipe.fileHandleForReading.readDataToEndOfFile() _ = stderrPipe.fileHandleForReading.readDataToEndOfFile() @@ -5352,11 +5337,6 @@ final class CLINotifyProcessIntegrationTests: XCTestCase { } } - let exitSignal = DispatchSemaphore(value: 0) - DispatchQueue.global(qos: .userInitiated).async { - process.waitUntilExit() - exitSignal.signal() - } XCTAssertTrue( waitForProcess(process, toHoldOpenFile: transcriptURL.path, timeout: 2), @@ -5374,10 +5354,10 @@ final class CLINotifyProcessIntegrationTests: XCTestCase { try appendHandle.close() let serverTimedOut = serverHandled.wait(timeout: .now() + 5) == .timedOut - let timedOut = exitSignal.wait(timeout: .now() + 5) == .timedOut + let timedOut = waitForProcessExit(process, timeout: 5) == .timedOut if timedOut { process.terminate() - _ = exitSignal.wait(timeout: .now() + 1) + _ = waitForProcessExit(process, timeout: 1) } let stdout = String(data: stdoutPipe.fileHandleForReading.readDataToEndOfFile(), encoding: .utf8) ?? "" let stderr = String(data: stderrPipe.fileHandleForReading.readDataToEndOfFile(), encoding: .utf8) ?? "" From 21cabfb80151aa5d384c39ca8702f03bee6aa313 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Wed, 23 Sep 2026 16:59:31 -0700 Subject: [PATCH 86/92] test: wait for the key-window change instead of a fixed 50 ms testSenderRelativeSidebarActionKeysItsOriginatingWindowBeforeMutation gave AppKit one fixed 50 ms run-loop spin to move key status after makeKeyAndOrderFront. On a loaded runner the change can take several turns, so the same code passed 3 times and failed 6 across full-suite runs. Wait for the transition with a 5 s deadline instead. Also normalize project.pbxproj ordering for ProcessExitWait.swift. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit be5fa50ee0bb5c0cf320f7ef18ec9829837d53ae) --- cmux.xcodeproj/project.pbxproj | 6 +++--- cmuxTests/AppDelegateShortcutRoutingTests.swift | 11 ++++++----- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/cmux.xcodeproj/project.pbxproj b/cmux.xcodeproj/project.pbxproj index 8bb12ee13fd0..a27644b2faf6 100644 --- a/cmux.xcodeproj/project.pbxproj +++ b/cmux.xcodeproj/project.pbxproj @@ -2658,6 +2658,7 @@ C0DEA771000000000000000D /* PricingPlansScreen.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0DEA771000000000000000E /* PricingPlansScreen.swift */; }; C0DEA771000000000000000B /* ProBadgeStyle.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0DEA771000000000000000C /* ProBadgeStyle.swift */; }; F0ACC0DE000000000000000D /* ProcessDetectedResumeIndexes.swift in Sources */ = {isa = PBXBuildFile; fileRef = F0ACC0DE000000000000000E /* ProcessDetectedResumeIndexes.swift */; }; + AC0FCEFB6665CF134634D60B /* ProcessExitWait.swift in Sources */ = {isa = PBXBuildFile; fileRef = E06E7588C05AC6A8659CFA51 /* ProcessExitWait.swift */; }; C47110010000000000000001 /* ProcessPipeReadCrashRegressionTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C47110010000000000000002 /* ProcessPipeReadCrashRegressionTests.swift */; }; 0A327C88E531C7E407CEF58C /* ProcessSnapshotMeasurement.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4587EDAC6B63EF4846FB24A6 /* ProcessSnapshotMeasurement.swift */; }; D75280010000000000000001 /* ProfilingSignposts.swift in Sources */ = {isa = PBXBuildFile; fileRef = D75280010000000000000002 /* ProfilingSignposts.swift */; }; @@ -3299,7 +3300,6 @@ FCB4880428BE4BB3990BCD51 /* SSHRemoteCommandCLIIntegrationTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = BB6498DB903042C8B9EB34C5 /* SSHRemoteCommandCLIIntegrationTests.swift */; }; D7AB1204800000000000000A /* SSHRemoteCWDRegressionTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7AB1204800000000000000B /* SSHRemoteCWDRegressionTests.swift */; }; FF16E760F7BC847FDBB97489 /* SSHStartupCommandTestSupport.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4575F4BB7F52CF88AB6D0022 /* SSHStartupCommandTestSupport.swift */; }; - AC0FCEFB6665CF134634D60B /* ProcessExitWait.swift in Sources */ = {isa = PBXBuildFile; fileRef = E06E7588C05AC6A8659CFA51 /* ProcessExitWait.swift */; }; D6B29560DEE34B08842A143A /* SSHStartupManualReconnectTests+Fixtures.swift in Sources */ = {isa = PBXBuildFile; fileRef = 37B04BC7D56945E59781D11F /* SSHStartupManualReconnectTests+Fixtures.swift */; }; EF70B7123200D1FB6F03DB26 /* SSHStartupManualReconnectTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4F73A5C0BB93507E18261385 /* SSHStartupManualReconnectTests.swift */; }; F6355600A1B2C3D4E5F60718 /* SSHStartupSignalLifecycleTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = F6355601A1B2C3D4E5F60718 /* SSHStartupSignalLifecycleTests.swift */; }; @@ -6816,6 +6816,7 @@ C0DEA771000000000000000E /* PricingPlansScreen.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PricingPlansScreen.swift; sourceTree = ""; }; C0DEA771000000000000000C /* ProBadgeStyle.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProBadgeStyle.swift; sourceTree = ""; }; F0ACC0DE000000000000000E /* ProcessDetectedResumeIndexes.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProcessDetectedResumeIndexes.swift; sourceTree = ""; }; + E06E7588C05AC6A8659CFA51 /* ProcessExitWait.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProcessExitWait.swift; sourceTree = ""; }; C47110010000000000000002 /* ProcessPipeReadCrashRegressionTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProcessPipeReadCrashRegressionTests.swift; sourceTree = ""; }; 4587EDAC6B63EF4846FB24A6 /* ProcessSnapshotMeasurement.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "ProcessSnapshotMeasurement.swift"; sourceTree = ""; }; D75280010000000000000002 /* ProfilingSignposts.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProfilingSignposts.swift; sourceTree = ""; }; @@ -7445,7 +7446,6 @@ BB6498DB903042C8B9EB34C5 /* SSHRemoteCommandCLIIntegrationTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SSHRemoteCommandCLIIntegrationTests.swift; sourceTree = ""; }; D7AB1204800000000000000B /* SSHRemoteCWDRegressionTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SSHRemoteCWDRegressionTests.swift; sourceTree = ""; }; 4575F4BB7F52CF88AB6D0022 /* SSHStartupCommandTestSupport.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SSHStartupCommandTestSupport.swift; sourceTree = ""; }; - E06E7588C05AC6A8659CFA51 /* ProcessExitWait.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProcessExitWait.swift; sourceTree = ""; }; 37B04BC7D56945E59781D11F /* SSHStartupManualReconnectTests+Fixtures.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "SSHStartupManualReconnectTests+Fixtures.swift"; sourceTree = ""; }; 4F73A5C0BB93507E18261385 /* SSHStartupManualReconnectTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SSHStartupManualReconnectTests.swift; sourceTree = ""; }; F6355601A1B2C3D4E5F60718 /* SSHStartupSignalLifecycleTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SSHStartupSignalLifecycleTests.swift; sourceTree = ""; }; @@ -17002,6 +17002,7 @@ F83620010000000000000001 /* PortScannerTTYFreshnessTests.swift in Sources */, B8E9500A0000000000000001 /* PostHogAnalyticsPropertiesTests.swift in Sources */, DE71CE00000000000000000A /* PresenceHeartbeatClientTests.swift in Sources */, + AC0FCEFB6665CF134634D60B /* ProcessExitWait.swift in Sources */, C47110010000000000000001 /* ProcessPipeReadCrashRegressionTests.swift in Sources */, 0A327C88E531C7E407CEF58C /* ProcessSnapshotMeasurement.swift in Sources */, A64610020000000000000001 /* QuitConfirmationAlertPresenterTests.swift in Sources */, @@ -17186,7 +17187,6 @@ FCB4880428BE4BB3990BCD51 /* SSHRemoteCommandCLIIntegrationTests.swift in Sources */, D7AB1204800000000000000A /* SSHRemoteCWDRegressionTests.swift in Sources */, FF16E760F7BC847FDBB97489 /* SSHStartupCommandTestSupport.swift in Sources */, - AC0FCEFB6665CF134634D60B /* ProcessExitWait.swift in Sources */, D6B29560DEE34B08842A143A /* SSHStartupManualReconnectTests+Fixtures.swift in Sources */, EF70B7123200D1FB6F03DB26 /* SSHStartupManualReconnectTests.swift in Sources */, F6355600A1B2C3D4E5F60718 /* SSHStartupSignalLifecycleTests.swift in Sources */, diff --git a/cmuxTests/AppDelegateShortcutRoutingTests.swift b/cmuxTests/AppDelegateShortcutRoutingTests.swift index fcfb3c5f7375..30ab6a12ed11 100644 --- a/cmuxTests/AppDelegateShortcutRoutingTests.swift +++ b/cmuxTests/AppDelegateShortcutRoutingTests.swift @@ -1146,17 +1146,18 @@ final class AppDelegateShortcutRoutingTests: XCTestCase { originatingWindow.orderFront(nil) previouslyFocusedWindow.makeKeyAndOrderFront(nil) - RunLoop.main.run(until: Date(timeIntervalSinceNow: 0.05)) - - XCTAssertTrue(appDelegate.shortcutRoutingKeyWindow === previouslyFocusedWindow) + // A key-window change lands on a later run-loop turn, and a loaded + // runner can take several; wait for it instead of a fixed 50 ms. + XCTAssertTrue( + waitForCondition(timeout: 5) { appDelegate.shortcutRoutingKeyWindow === previouslyFocusedWindow } + ) XCTAssertTrue( appDelegate.toggleSidebarInActiveMainWindow(preferredWindow: originatingWindow) ) - RunLoop.main.run(until: Date(timeIntervalSinceNow: 0.05)) XCTAssertTrue( - appDelegate.shortcutRoutingKeyWindow === originatingWindow, + waitForCondition(timeout: 5) { appDelegate.shortcutRoutingKeyWindow === originatingWindow }, "An in-window action must request key status for its originating window before mutating window state" ) XCTAssertEqual( From baa7f27fbf5063e1c88d1e1f784de9e17c57c45b Mon Sep 17 00:00:00 2001 From: Leo Li Date: Wed, 23 Sep 2026 16:26:30 -0700 Subject: [PATCH 87/92] test: scale the SSH retry-budget harness deadline with its attempts foregroundAuthenticatedAttachUsesConfiguredRetryBudget runs the full 20-attempt fallback budget through fake ssh/cmux/sleep, which takes about 4.5 s per case on an idle runner, against a fixed 5 s process deadline. Under shard load the harness SIGTERMed the script after 19 sleeps (status 143). The deadline now scales with the attempts the case expects; the assertions are unchanged. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit d2126d9891037e40782aa7b7152653eca72427a2) --- cmuxTests/SSHDeepSleepReattachTests.swift | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/cmuxTests/SSHDeepSleepReattachTests.swift b/cmuxTests/SSHDeepSleepReattachTests.swift index 08569696acd9..f323911b601d 100644 --- a/cmuxTests/SSHDeepSleepReattachTests.swift +++ b/cmuxTests/SSHDeepSleepReattachTests.swift @@ -361,7 +361,12 @@ struct SSHDeepSleepReattachTests { sessionID: "ssh-test-session", foregroundAuth: Self.foregroundAuth() ).replacingOccurrences(of: "/usr/bin/ssh", with: fakeSSH.path), - environment: environment + environment: environment, + // Every attach attempt spawns uuidgen, the fake ssh, cmux and + // sleep, so the full budget costs ~4.5 s on an idle runner and + // more under shard load. The deadline only bounds a hang; it + // scales with the attempts the case expects to run. + timeout: max(5, Double(expectedSleepCount + 1) * 2) ) #expect(!result.timedOut, Comment(rawValue: result.stderr)) @@ -527,7 +532,11 @@ struct SSHDeepSleepReattachTests { try (lines.joined(separator: "\n") + "\n").write(to: url, atomically: true, encoding: .utf8) } - private static func runProcess(command: String, environment: [String: String]) -> ProcessRunResult { + private static func runProcess( + command: String, + environment: [String: String], + timeout: TimeInterval = 5 + ) -> ProcessRunResult { let process = Process() let stderrPipe = Pipe() process.executableURL = URL(fileURLWithPath: "/bin/sh") @@ -541,7 +550,7 @@ struct SSHDeepSleepReattachTests { } catch { return ProcessRunResult(status: -1, stderr: String(describing: error), timedOut: false) } - let timedOut = waitForProcessExit(process, timeout: 5) == .timedOut + let timedOut = waitForProcessExit(process, timeout: timeout) == .timedOut if timedOut { process.terminate() _ = waitForProcessExit(process, timeout: 1) From 0fa1f555c52a1aa0c3afa372eb0c4bfd1f6f22c9 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Wed, 23 Sep 2026 16:50:46 -0700 Subject: [PATCH 88/92] Wait for config reload fanout by suspending, not spinning the run loop #10564 moved reload surface fanout from RunLoop.main.perform onto TerminalConfigurationApplyScheduler, which yields through MainActorDeferredActionScheduler's Task { @MainActor }. The four reload cases are synchronous main-actor tests that wait with a nested RunLoop.main.run, which cannot run main-actor tasks while the test job holds the main queue. The fanout never finished, so completions, the reload notification, and the queued transaction never arrived. Make the cases async and wait with waitWhileSuspended. Each first settles any in-flight reload. An idle full reload now commits synchronously, so the staged-appearance case holds one transaction open to exercise the queued path it describes. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 6733553819cee9b9258bbeb3b46cda736ef04b3a) --- ...pDelegateEqualizeSplitsShortcutTests.swift | 112 ++++++++++++------ 1 file changed, 77 insertions(+), 35 deletions(-) diff --git a/cmuxTests/AppDelegateEqualizeSplitsShortcutTests.swift b/cmuxTests/AppDelegateEqualizeSplitsShortcutTests.swift index 0aa94215199d..ff618dfa0ccb 100644 --- a/cmuxTests/AppDelegateEqualizeSplitsShortcutTests.swift +++ b/cmuxTests/AppDelegateEqualizeSplitsShortcutTests.swift @@ -367,6 +367,30 @@ private func waitWhileSuspended( } } +/// Finishes any configuration reload still fanning out to surfaces. +/// +/// A reload commits synchronously but completes after its bounded surface +/// fanout, which `TerminalConfigurationApplyScheduler` drives through +/// main-actor tasks. A nested `RunLoop.main.run` inside a synchronous +/// main-actor test cannot execute those tasks, so reload cases must suspend +/// instead. A reload issued here merges with any queued one, and its +/// completion runs only after the shared coordinator has finished them. +@MainActor +private func settleConfigurationReload( + _ app: GhosttyApp +) async { + let settled = expectation( + description: "earlier configuration reload settled" + ) + app.reloadConfiguration( + source: "test.settleConfigurationReload", + reloadSettingsFromFile: false + ) { + settled.fulfill() + } + await waitWhileSuspended(for: [settled], timeout: 5) +} + @MainActor private extension TabManager { @discardableResult @@ -5499,11 +5523,20 @@ final class AppDelegateEqualizeSplitsShortcutTests { @Test func testFullConfigurationReloadStagesAppearanceUntilConfigurationCommit() - throws { + async throws { #if DEBUG let app = GhosttyApp.shared + await settleConfigurationReload(app) let originalProfile = GhosttyStartupAppearancePreviewState.profile + // An idle reload commits synchronously. Hold one transaction in its + // surface fanout so the staged reload below is queued behind it. It + // reloads the unchanged configuration, so the appearance captured + // next is still the one to restore. + app.reloadConfiguration( + source: "test.stageAppearance.active", + reloadSettingsFromFile: false + ) let originalBackgroundHex = app.defaultBackgroundColor.hexString() let targetProfile: GhosttyStartupAppearancePreviewProfile = @@ -5528,33 +5561,6 @@ final class AppDelegateEqualizeSplitsShortcutTests { ) { _ in reloadCompleted.fulfill() } - defer { - NotificationCenter.default.removeObserver(observer) - GhosttyStartupAppearancePreviewState.profile = - originalProfile - GhosttyConfig.invalidateLoadCache() - - let restoreCompleted = expectation( - description: "original appearance restored" - ) - let restoreObserver = - NotificationCenter.default.addObserver( - forName: .ghosttyConfigDidReload, - object: nil, - queue: .main - ) { _ in - restoreCompleted.fulfill() - } - app.reloadConfiguration( - source: "test.restoreStagedAppearance", - reloadSettingsFromFile: false - ) - wait(for: [restoreCompleted], timeout: 5) - NotificationCenter.default.removeObserver( - restoreObserver - ) - withExtendedLifetime(retainedPanels) {} - } GhosttyStartupAppearancePreviewState.profile = targetProfile @@ -5568,14 +5574,36 @@ final class AppDelegateEqualizeSplitsShortcutTests { XCTAssertEqual( app.defaultBackgroundColor.hexString(), originalBackgroundHex, - "A pending full reload must not publish its new background before the matching Ghostty config commits" + "A full reload queued behind an active transaction must not publish its new background before its own Ghostty config commits" + ) + await waitWhileSuspended( + for: [reloadCompleted], + timeout: 5 ) - wait(for: [reloadCompleted], timeout: 5) XCTAssertNotEqual( app.defaultBackgroundColor.hexString(), originalBackgroundHex, "The staged appearance must publish when the full configuration commits" ) + + NotificationCenter.default.removeObserver(observer) + GhosttyStartupAppearancePreviewState.profile = + originalProfile + GhosttyConfig.invalidateLoadCache() + let restoreCompleted = expectation( + description: "original appearance restored" + ) + app.reloadConfiguration( + source: "test.restoreStagedAppearance", + reloadSettingsFromFile: false + ) { + restoreCompleted.fulfill() + } + await waitWhileSuspended( + for: [restoreCompleted], + timeout: 5 + ) + withExtendedLifetime(retainedPanels) {} #else throw XCTSkip("Startup appearance previews require DEBUG") #endif @@ -5583,9 +5611,10 @@ final class AppDelegateEqualizeSplitsShortcutTests { @Test func testConfigurationReloadRemainsActiveUntilAsyncReconciliationCompletes() - throws { + async throws { #if DEBUG let app = GhosttyApp.shared + await settleConfigurationReload(app) let retainedPanels = (0..<16).map { _ in TerminalPanel( workspaceId: UUID(), @@ -5607,7 +5636,10 @@ final class AppDelegateEqualizeSplitsShortcutTests { app.isConfigurationReloadActive, "Appearance synchronization must stay deferred while incremental reconciliation is pending" ) - wait(for: [reloadCompleted], timeout: 5) + await waitWhileSuspended( + for: [reloadCompleted], + timeout: 5 + ) XCTAssertFalse(app.isConfigurationReloadActive) withExtendedLifetime(retainedPanels) {} #else @@ -5701,9 +5733,10 @@ final class AppDelegateEqualizeSplitsShortcutTests { @Test func testConfigurationReloadQueuesRequestDuringAsyncReconciliation() - throws { + async throws { #if DEBUG let app = GhosttyApp.shared + await settleConfigurationReload(app) let retainedPanels = (0..<16).map { _ in TerminalPanel( workspaceId: UUID(), @@ -5742,7 +5775,7 @@ final class AppDelegateEqualizeSplitsShortcutTests { secondCompletionGeneration, "A request queued during reconciliation must not report success against the active transaction" ) - wait( + await waitWhileSuspended( for: [ firstReloadCompleted, secondReloadCompleted @@ -6227,11 +6260,12 @@ final class AppDelegateEqualizeSplitsShortcutTests { } @Test - func testGhosttyAppConfigUpdateWaitsForFontBarrier() { + func testGhosttyAppConfigUpdateWaitsForFontBarrier() async { guard let appDelegate = AppDelegate.shared else { XCTFail("Expected AppDelegate.shared") return } + await settleConfigurationReload(GhosttyApp.shared) let manager = TabManager() guard let workspace = manager.selectedWorkspace, let panelId = workspace.focusedPanelId, @@ -6280,12 +6314,16 @@ final class AppDelegateEqualizeSplitsShortcutTests { XCTAssertEqual(applyAttemptCount, 2) var didUpdateGhosttyAppConfig = false + let configUpdated = expectation( + description: "ghostty app config update published" + ) let observer = NotificationCenter.default.addObserver( forName: .ghosttyConfigDidReload, object: nil, queue: .main ) { _ in didUpdateGhosttyAppConfig = true + configUpdated.fulfill() } defer { NotificationCenter.default.removeObserver(observer) @@ -6305,6 +6343,10 @@ final class AppDelegateEqualizeSplitsShortcutTests { scheduler.fire(at: 2) } XCTAssertEqual(applyAttemptCount, 3) + // Releasing the barrier commits the app configuration; the reload + // notification is published after its bounded surface fanout, which + // runs on later main-actor turns. + await waitWhileSuspended(for: [configUpdated], timeout: 5) XCTAssertTrue(didUpdateGhosttyAppConfig) } From 6846508542adc15002785d589a6de3b06892b00f Mon Sep 17 00:00:00 2001 From: Leo Li Date: Wed, 23 Sep 2026 18:45:35 -0700 Subject: [PATCH 89/92] test: prove the font barrier holds the reload, not that it has not run yet With the reload fanout on main-actor tasks, the notification arrives only after later turns, so asserting it had not fired right after reloadConfiguration proved nothing: an unblocked reload also reads false at that instant. The case now asserts the coordinator is parked in .waitingForFontWork, that five main-actor yields leave it there, and that the commit callback has not run, before releasing the barrier. Adds a DEBUG-only GhosttyApp.debugConfigurationReloadPhase for that. Raised by CodeRabbit on #14006. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 38d9a9c58948ac4869a2a8eb303c4d5624526dd7) --- Sources/GhosttyTerminalView.swift | 10 ++++++++ ...pDelegateEqualizeSplitsShortcutTests.swift | 24 ++++++++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/Sources/GhosttyTerminalView.swift b/Sources/GhosttyTerminalView.swift index fb8e02df974b..519a026ce6d1 100644 --- a/Sources/GhosttyTerminalView.swift +++ b/Sources/GhosttyTerminalView.swift @@ -2206,6 +2206,16 @@ class GhosttyApp { configurationReloadCoordinator.isReloadActive } +#if DEBUG + /// Where the app-scoped reload transaction is, so tests can assert that a + /// reload is held at the font-work barrier instead of inferring it from a + /// notification that has not arrived yet. + @MainActor + var debugConfigurationReloadPhase: TerminalConfigurationReloadPhase { + configurationReloadCoordinator.phase + } +#endif + @MainActor func terminalFontConfigurationSnapshot() -> WorkspaceTerminalFontConfigurationSnapshot { diff --git a/cmuxTests/AppDelegateEqualizeSplitsShortcutTests.swift b/cmuxTests/AppDelegateEqualizeSplitsShortcutTests.swift index ff618dfa0ccb..fd1ac54840c7 100644 --- a/cmuxTests/AppDelegateEqualizeSplitsShortcutTests.swift +++ b/cmuxTests/AppDelegateEqualizeSplitsShortcutTests.swift @@ -6329,10 +6329,32 @@ final class AppDelegateEqualizeSplitsShortcutTests { NotificationCenter.default.removeObserver(observer) } + var didCommitGhosttyAppConfig = false GhosttyApp.shared.reloadConfiguration( soft: true, source: "test.fontBarrier", - reloadSettingsFromFile: false + reloadSettingsFromFile: false, + commitCompletion: { _ in didCommitGhosttyAppConfig = true } + ) + // Held, not merely not yet run: the transaction is parked at the + // font-work barrier, and giving the main actor turns does not move it. + // Only releasing the barrier below may. +#if DEBUG + XCTAssertEqual( + GhosttyApp.shared.debugConfigurationReloadPhase, + .waitingForFontWork, + "The app config update must wait at the font-work barrier" + ) + for _ in 0..<5 { await Task.yield() } + XCTAssertEqual( + GhosttyApp.shared.debugConfigurationReloadPhase, + .waitingForFontWork, + "Main-actor turns must not release the font-work barrier" + ) +#endif + XCTAssertFalse( + didCommitGhosttyAppConfig, + "The app config must not commit before font work finishes" ) XCTAssertFalse( didUpdateGhosttyAppConfig, From fe8c900832b56dccd4aecc176f2d0b63dc9f675d Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 23 Sep 2026 19:26:00 -0700 Subject: [PATCH 90/92] test: settle earlier config reloads before the font-size reset checks testConfiguredWorkspaceTerminalFontSizeResetRestoresEverySplit failed in the #13151 full-ci shard with twelve issues: four surfaces, each still 3 points below the configured size after Cmd+0 was accepted. An earlier case in the same process left a configuration reload fanning out. That reload holds the app-wide font-size work barrier (GhosttyTerminalView reload transaction), and the arbiter retains font mutations issued behind the barrier until reconciliation finishes, by design. The reset was accepted and queued, so the synchronous checks ran before it applied. Settle any in-flight reload while suspended (the helper from #14057) before issuing the reset, and assert no reload is active. The reset assertions are unchanged: every split, the dock terminal, and the split created after the zoom must still reset immediately. Co-Authored-By: Claude Opus 5.5 --- .../AppDelegateEqualizeSplitsShortcutTests.swift | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/cmuxTests/AppDelegateEqualizeSplitsShortcutTests.swift b/cmuxTests/AppDelegateEqualizeSplitsShortcutTests.swift index fd1ac54840c7..30ca5afc7cdb 100644 --- a/cmuxTests/AppDelegateEqualizeSplitsShortcutTests.swift +++ b/cmuxTests/AppDelegateEqualizeSplitsShortcutTests.swift @@ -7631,7 +7631,16 @@ final class AppDelegateEqualizeSplitsShortcutTests { } @Test - func testConfiguredWorkspaceTerminalFontSizeResetRestoresEverySplit() { + func testConfiguredWorkspaceTerminalFontSizeResetRestoresEverySplit() async { + // A reload an earlier case left fanning out holds the app-wide + // font-size barrier, and the arbiter queues a reset issued behind it + // until the barrier lifts, which is after these synchronous checks. + // Settle it while suspended so the reset below applies immediately. + await settleConfigurationReload(GhosttyApp.shared) + XCTAssertFalse( + GhosttyApp.shared.isConfigurationReloadActive, + "A configuration reload in flight would queue the reset past these checks" + ) withTemporaryShortcut(action: .resetWorkspaceTerminalFontSize) { guard let appDelegate = AppDelegate.shared else { XCTFail("Expected AppDelegate.shared") From ad02f4e2111fd0ef325dfc36c6aa5e35b670216f Mon Sep 17 00:00:00 2001 From: Leo Li Date: Wed, 23 Sep 2026 19:07:27 -0700 Subject: [PATCH 91/92] test: report the geometry behind a too-small Cloud failure card The card-layout expectation in "Cloud failure controls stay above native surfaces" fails in the full app-host shard with only `overlay.frame.width > 100 -> false`. Record the card, source terminal, content reference and window frames so the failing run names which one was small. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 34993029e220371a7da6d15a98affb5e9302c84b) --- cmuxTests/SurfacePaneFactoryFocusTests.swift | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/cmuxTests/SurfacePaneFactoryFocusTests.swift b/cmuxTests/SurfacePaneFactoryFocusTests.swift index 3c2cef923e6b..7b4992002a1f 100644 --- a/cmuxTests/SurfacePaneFactoryFocusTests.swift +++ b/cmuxTests/SurfacePaneFactoryFocusTests.swift @@ -262,7 +262,18 @@ import SwiftUI overlay.layoutSubtreeIfNeeded() await Task.yield() } - #expect(overlay.frame.width > 100 && overlay.frame.height > 50) + #expect( + overlay.frame.width > 100 && overlay.frame.height > 50, + """ + card=\(overlay.frame) inContainer=\(overlay.superview === target.container) \ + source=\(target.container.convert(source.hostedView.bounds, from: source.hostedView)) \ + sourceVisible=\(source.hostedView.visibleRect) sourceHidden=\(source.hostedView.isHiddenOrHasHiddenAncestor) \ + sourceWindowMatches=\(source.hostedView.window === window) \ + reference=\(target.container.convert(target.reference.bounds, from: target.reference)) \ + window=\(window.frame) visible=\(window.isVisible) key=\(window.isKeyWindow) \ + mainWindows=\(NSApp.windows.filter { $0.identifier?.rawValue.hasPrefix("cmux.main.") == true }.map { "\($0.frame)" }) + """ + ) let terminalFrame = target.container.convert(source.hostedView.bounds, from: source.hostedView) #expect(abs(overlay.frame.midX - terminalFrame.midX) < 2) #expect(abs(overlay.frame.midY - terminalFrame.midY) < 2) From 054bd4b1094d24f2f9730e60dc400db4ba9234b0 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Wed, 23 Sep 2026 19:49:47 -0700 Subject: [PATCH 92/92] fix: import workspace liveness type --- Sources/CodexTurnRestoreIntentPolicy.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/Sources/CodexTurnRestoreIntentPolicy.swift b/Sources/CodexTurnRestoreIntentPolicy.swift index 2c79917337f3..d5d1df19d9e9 100644 --- a/Sources/CodexTurnRestoreIntentPolicy.swift +++ b/Sources/CodexTurnRestoreIntentPolicy.swift @@ -1,3 +1,4 @@ +import CmuxWorkspaces import Foundation /// Preserves one Codex restore intent after a stale turn owner exits.