diff --git a/web/app/api/vm/[id]/attach-endpoint/route.ts b/web/app/api/vm/[id]/attach-endpoint/route.ts index 59b5f46c57d5..40f508c409e8 100644 --- a/web/app/api/vm/[id]/attach-endpoint/route.ts +++ b/web/app/api/vm/[id]/attach-endpoint/route.ts @@ -3,9 +3,11 @@ import { preconnectFreestyle } from "../../../../../services/vms/drivers/freesty import { jsonResponse, resolveVmRouteAccountScope, + runAfterResponse, withAuthedVmApiRoute, } from "../../../../../services/vms/routeHelpers"; import { setSpanAttributes } from "../../../../../services/telemetry"; +import { VmTimingRecorder } from "../../../../../services/vms/timings"; import { runVmRoute } from "../../../../../services/vms/routeWorkflow"; import { openAttachEndpoint, openVmCmuxRemote } from "../../../../../services/vms/workflows"; import { @@ -27,7 +29,19 @@ export async function POST( "/api/vm/[id]/attach-endpoint", { "cmux.vm.operation": "open_attach" }, "/api/vm/[id]/attach-endpoint failed", - async ({ user, span }) => { + async ({ user, span, authDurationMs, routeStartedAtMs, setResponseFinalizer }) => { + const timing = new VmTimingRecorder(span, "open_attach", { startedAt: routeStartedAtMs }); + timing.record("auth", authDurationMs); + setResponseFinalizer((response) => { + timing.finish({ status: response.status }); + // Per-stage timings travel with the response, as on create, so a + // client or a bench run sees where an attach spent its time. + try { + response.headers.set("Server-Timing", timing.serverTimingHeader()); + } catch { + // Immutable headers on a passthrough Response: the span still has them. + } + }); const { id } = await params; const body = await parseLenientObjectBody(request); const requireDaemon = body.requireDaemon === true || body.require_daemon === true; @@ -72,6 +86,10 @@ export async function POST( deviceFingerprint, clientCapabilities, callerPlanId: account.entitlements.planId, + timing, + // The attach usage event and the address backfill are written + // after the response has left. + defer: runAfterResponse, }), { request }); if (!run.ok) return run.response; return jsonResponse(run.value); diff --git a/web/app/api/vm/route.ts b/web/app/api/vm/route.ts index ee4d8acf85cd..607866096c9d 100644 --- a/web/app/api/vm/route.ts +++ b/web/app/api/vm/route.ts @@ -36,9 +36,14 @@ import { vmFreeAccessWindowDays, } from "../../../services/vms/entitlements"; import { + findVmImageManifestEntry, inferVmProviderForImage, resolveVmImage, + vmImageEntryEpoch, + type VmImageSelection, } from "../../../services/vms/images/resolver"; +import { createAttachBlock } from "../../../services/vms/attachContract"; +import { orderedDeferSink } from "../../../services/vms/defer"; import { reportVmImageConfigError, isVmImageKind, @@ -69,6 +74,7 @@ import { annotateVmRequestBilling } from "../../../services/vms/requestContext"; import { createVm, listUserVms, + type VmEntry, } from "../../../services/vms/workflows"; import { recordSpanError, setSpanAttributes } from "../../../services/telemetry"; import { @@ -292,29 +298,55 @@ export async function POST(request: Request): Promise { imageSize: imageSelection.size ?? undefined, modelPlane, timing, + imageEpoch: vmImageEntryEpoch(imageSelection.manifestEntry), + // Usage-event rows are written after the response has left, in + // lifecycle order (requested before created). + defer: orderedDeferSink(runAfterResponse), }), { request, onError: createErrorResponders(entitlements), }); if (!run.ok) return run.response; const created = run.value; - setSpanAttributes(span, { "cmux.vm.id": created.providerVmId }); - return jsonResponse({ - id: created.providerVmId, - provider: created.provider, - image: created.image, - imageVersion: created.imageVersion, - kind: vmImageKindFor(created.provider, created.image), - ...(imageSelection.size ? { size: imageSelection.size } : {}), - createdAt: created.createdAt, - capabilities: vmCapabilitiesFor(created.provider), - displayName: created.displayName, - slug: created.slug, - }); + const payload = createResponseBody(created, imageSelection); + setSpanAttributes(span, { "cmux.vm.id": created.providerVmId, "cmux.vm.attach_block": "attach" in payload }); + return jsonResponse(payload); }, ); } +/** + * The create response: the machine shape `GET /api/vm` lists plus what a + * client needs to dial the daemon straight away. `address` is the same object + * the GET routes return; `attach` is the block a client dials from without a + * status GET or an attach-endpoint call, absent when the row holds no private + * address or the image is outside the manifest (clients feature-detect on it + * and fall back to attach-endpoint). + */ +function createResponseBody(created: VmEntry, imageSelection: VmImageSelection): Record { + // An idempotent replay answers with the row's own image, which may not be + // the one this request resolved. + const manifestEntry = imageSelection.manifestEntry?.imageId === created.image + ? imageSelection.manifestEntry + : findVmImageManifestEntry(created.provider, created.image, imageSelection.kind); + const attach = createAttachBlock({ entry: created, manifestEntry }); + return { + id: created.providerVmId, + provider: created.provider, + image: created.image, + imageVersion: created.imageVersion, + kind: vmImageKindFor(created.provider, created.image), + ...(imageSelection.size ? { size: imageSelection.size } : {}), + createdAt: created.createdAt, + capabilities: vmCapabilitiesFor(created.provider), + displayName: created.displayName, + slug: created.slug, + status: created.status, + address: { ipv4: created.addressIpv4 ?? null, ipv6: created.addressIpv6 ?? null }, + ...(attach ? { attach } : {}), + }; +} + /** * How a create request's optional flags meet the resolved provider's capabilities. * diff --git a/web/scripts/cloud-vm/bench-vm-startup.mjs b/web/scripts/cloud-vm/bench-vm-startup.mjs index 937e6d9fc6d4..1a470efbfe77 100644 --- a/web/scripts/cloud-vm/bench-vm-startup.mjs +++ b/web/scripts/cloud-vm/bench-vm-startup.mjs @@ -50,7 +50,8 @@ const CREATE_TIMEOUT_MS = 630_000; const ATTACH_BUDGET_MS = 180_000; const requireFromWeb = createRequire(path.join(webDir, "package.json")); -const { StackServerApp } = await import(pathToFileURL(requireFromWeb.resolve("@stackframe/js")).href); +// The Stack SDK the app itself uses (app/lib/stack.ts), resolved like smoke-vm-api.mjs does. +const { StackServerApp } = await import(pathToFileURL(requireFromWeb.resolve("@hexclave/js")).href); // ESM-only package (no require entry): resolved from this script's own tree. const { Freestyle, FreestyleApiError } = await import("freestyle"); @@ -251,6 +252,9 @@ async function attachUntilReady(vmId, stage) { return { [`${stage}Ms`]: elapsedMs(startedAt), [`${stage}Attempts`]: attempts, + // The attach route's own per-stage timings (access check, provider + // probe, provider attach, lease), when the backend sends them. + [`${stage}Stages`]: parseServerTiming(response.headers.get("server-timing")), [`${stage}TrustedCarrier`]: body.trustedCarrier === true, [`${stage}RouteFamily`]: typeof body.route === "string" ? (body.route.includes("[") ? "ipv6" : "ipv4") : null, [`${stage}DaemonCommit`]: body.daemonBuild?.commit ?? null, @@ -328,6 +332,9 @@ async function runTrial(trial) { trial.vmId = vmId; trial.imageVersion = created.imageVersion ?? null; trial.size = created.size?.name ?? null; + // Whether the create response carried the attach block a client can dial + // from without this attach-endpoint round trip (absent on older backends). + trial.createAttachBlock = created.attach ? { route: created.attach.route, trustedCarrier: created.attach.trustedCarrier, guestToolsBaked: created.attach.guestToolsBaked } : null; Object.assign(trial, await attachUntilReady(vmId, "attach")); // Create plus the attach-endpoint's own time: the route and lease exist, // but the link, the terminal and the shell prompt come after this point @@ -881,6 +888,8 @@ function emitReport({ results, listMs, startedAt, runError, cleanup }) { stages: summarizeFields(measured, ["createMs", "attachMs", "createToAttachReadyMs", "warmAttachMs", "execMs", "edgeReadyMs", "pauseMs", "resumeAttachMs", "destroyMs"]), attachAttempts: summarizeFields(measured.map((trial) => ({ attempts: trial.attachAttempts?.length })), ["attempts"]).attempts, createServerTiming: summarizeStages(measured.map((trial) => trial.createStages)), + attachServerTiming: summarizeStages(measured.map((trial) => trial.attachStages)), + warmAttachServerTiming: summarizeStages(measured.map((trial) => trial.warmAttachStages)), results, }; if (measured.length > 0) { diff --git a/web/scripts/devbox-image-common.ts b/web/scripts/devbox-image-common.ts index 8719f46245cb..e6417e684ddb 100644 --- a/web/scripts/devbox-image-common.ts +++ b/web/scripts/devbox-image-common.ts @@ -23,6 +23,7 @@ import { CMUX_TUI_SESSION, cmuxTuiAsDaemonUser, cmuxTuiLayoutSelector, cmuxTuiRu import { DEVBOX_WORK_HOME, DEVBOX_WORK_USER } from "../services/vms/images/workUser"; import { VM_IMAGE_SIZES, VM_IMAGE_SIZE_NAMES, vmImageSizeRank, type VmImageSizeName } from "../services/vms/images/sizes"; import { DEVBOX_HOSTNAME, DEVBOX_HOSTNAME_LOOPBACK, DEVBOX_PROVIDER_HOSTNAME } from "../services/vms/images/identity"; +import { vmImageEntryEpoch } from "../services/vms/images/resolver"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); export const webRoot = path.resolve(__dirname, ".."); @@ -1347,10 +1348,12 @@ export function upgradeDevboxSourceRecords( return { manifest: { schemaVersion: manifest.schemaVersion, images }, upgraded, skipped }; } -/** The epoch an entry was baked at: the field, or the `cmux devbox epoch ` prefix every bake writes into `notes`. */ -export function manifestEntryEpoch(entry: Pick): string | undefined { - return entry.epoch ?? /cmux devbox epoch (\S+)/.exec(entry.notes ?? "")?.[1]; -} +/** + * The epoch an entry was baked at. One implementation: the runtime resolver + * owns it (the create response's attach block gates on the same reading), the + * bake and promotion scripts use it through this name. + */ +export const manifestEntryEpoch: (entry: Pick) => string | undefined = vmImageEntryEpoch; /** * The invariant that makes the checked-in manifest describe the machine diff --git a/web/services/vms/attachContract.ts b/web/services/vms/attachContract.ts new file mode 100644 index 000000000000..2f6c714be740 --- /dev/null +++ b/web/services/vms/attachContract.ts @@ -0,0 +1,70 @@ +import { CMUX_TUI_SESSION } from "./drivers/cmuxTuiDaemon"; +import { freestyleCmuxRemoteRoute } from "./drivers/freestyle"; +import { + GUEST_TOOLS_BAKED_EPOCH, + TRUSTED_CARRIER_EPOCH, + imageEpochAtLeast, + vmImageEntryEpoch, + type VmImageManifestEntry, +} from "./images/resolver"; + +/** + * What a client needs to dial a machine's daemon straight from the create + * response, before any attach call: the transport and route, whether the + * daemon's cloud listener grants carrier authentication, the daemon build the + * image carries, and whether the guest tools are baked (no heal execs on + * attach). `readiness` is always `"dial"`: the daemon may still be starting, + * and the Noise handshake is the readiness proof. + * + * `attach-endpoint` keeps its own shape as the repair and reconnect path. + */ +export type VmAttachBlock = { + readonly transport: "cmux-remote"; + readonly route: string; + readonly session: string; + readonly trustedCarrier: boolean; + readonly daemonBuild: { + readonly commit: string | null; + readonly remoteProtocol: null; + readonly version: null; + }; + readonly guestToolsBaked: boolean; + readonly readiness: "dial"; +}; + +export type VmAttachEntry = { + readonly addressIpv4: string | null; + readonly addressIpv6: string | null; + /** The epoch stamped on the row at create; null on older rows (the manifest entry answers then). */ + readonly imageEpoch?: string | null; + readonly providerVmId?: string; +}; + +/** + * The attach block for a created machine, derived from the row and the + * checked-in manifest alone (no provider round trip). Null when the row holds + * no private address (the daemon is unreachable by address) or the image is + * not a manifest entry (nothing is known about its daemon). + * + * The route follows the driver's rule: IPv4 first because only the v4 path is + * reliable over the WireGuard tunnel, `[ipv6]` bracketed otherwise. + */ +export function createAttachBlock(input: { + readonly entry: VmAttachEntry; + readonly manifestEntry: VmImageManifestEntry | null; +}): VmAttachBlock | null { + const { entry, manifestEntry } = input; + const ipv4 = entry.addressIpv4?.trim() || undefined; + const ipv6 = entry.addressIpv6?.trim() || undefined; + if (!manifestEntry || (!ipv4 && !ipv6)) return null; + const epoch = entry.imageEpoch ?? vmImageEntryEpoch(manifestEntry); + return { + transport: "cmux-remote", + route: freestyleCmuxRemoteRoute({ vpcs: [{ ipv4, ipv6 }] }, entry.providerVmId ?? "unknown"), + session: CMUX_TUI_SESSION, + trustedCarrier: imageEpochAtLeast(epoch, TRUSTED_CARRIER_EPOCH), + daemonBuild: { commit: manifestEntry.cmuxTuiCommit ?? null, remoteProtocol: null, version: null }, + guestToolsBaked: imageEpochAtLeast(epoch, GUEST_TOOLS_BAKED_EPOCH), + readiness: "dial", + }; +} diff --git a/web/services/vms/defer.ts b/web/services/vms/defer.ts new file mode 100644 index 000000000000..12643d944dbb --- /dev/null +++ b/web/services/vms/defer.ts @@ -0,0 +1,32 @@ +/** + * Work a route may run once its response has left (usage-event rows, + * metadata backfills): the route supplies `runAfterResponse`, so the writes + * still happen but the client does not wait for them. Workflows given no + * sink run the work inline, as they always did. + */ +export type VmDeferSink = (work: () => Promise) => void; + +/** + * A sink whose units run in the order they were handed in, each after the + * previous one settles, so deferred ledger rows keep their lifecycle order + * (`vm.create.requested` before `vm.created`) whatever the scheduler does + * with the callbacks it is given (Next's `after()` queue is concurrent; the + * detached fallback starts everything at once). A unit still starts only + * when the scheduler invokes it, and a failed unit never blocks the next. + */ +export function orderedDeferSink(schedule: VmDeferSink): VmDeferSink { + let tail: Promise = Promise.resolve(); + return (work) => { + const previous = tail; + let start: () => void = () => undefined; + const started = new Promise((resolve) => { + start = resolve; + }); + const run = started.then(() => previous).then(work); + tail = run.catch(() => undefined); + schedule(() => { + start(); + return run; + }); + }; +} diff --git a/web/services/vms/drivers/freestyle.ts b/web/services/vms/drivers/freestyle.ts index 6e62ee65278d..d18134275297 100644 --- a/web/services/vms/drivers/freestyle.ts +++ b/web/services/vms/drivers/freestyle.ts @@ -1750,8 +1750,7 @@ export class FreestyleProvider implements VMProvider { private async installGuestCliFiles(vm: Vm, vmId: string, promptIdentity?: GuestPromptIdentity): Promise { const temporaryPath = `${GUEST_CMUX_SHIM_PATH}.tmp-${randomBytes(12).toString("hex")}`; try { - await this.execOrThrow(vm, vmId, "mkdir -p /usr/local/libexec", 5_000); - await vm.fs.writeTextFile(temporaryPath, GUEST_CMUX_SHIM, { mode: 0o755 }); + await this.uploadGuestShim(vm, vmId, temporaryPath); const result = await vm.exec({ command: `${guestBrowserInstallCommand()} && chmod 0755 '${temporaryPath}' && mv -f '${temporaryPath}' '${GUEST_CMUX_SHIM_PATH}' && ${guestCliDistributionCommand()}` + (promptIdentity ? ` && ${guestPromptInstallCommand(promptIdentity)}` : ""), @@ -1768,6 +1767,22 @@ export class FreestyleProvider implements VMProvider { } } + /** + * The bake creates the shim's directory (`/usr/local/libexec`, the + * opencode launcher step), so the upload goes straight in; an image from + * before that step rejects it once, and the directory is created and the + * upload retried instead of paying a mkdir round trip on every create. + */ + private async uploadGuestShim(vm: Vm, vmId: string, temporaryPath: string): Promise { + try { + await vm.fs.writeTextFile(temporaryPath, GUEST_CMUX_SHIM, { mode: 0o755 }); + } catch (error) { + if (!isMissingGuestDirectoryError(error)) throw error; + await this.execOrThrow(vm, vmId, "mkdir -p /usr/local/libexec", 5_000); + await vm.fs.writeTextFile(temporaryPath, GUEST_CMUX_SHIM, { mode: 0o755 }); + } + } + private async execResult(vm: Vm, command: string, timeoutMs = EXEC_DEFAULT_TIMEOUT_MS): Promise { try { const r = await vm.exec({ command, timeoutMs, linuxUser: GUEST_LINUX_USER }); @@ -1794,6 +1809,14 @@ export class FreestyleProvider implements VMProvider { } } +/** A filesystem write refused because the target directory does not exist in the guest. */ +function isMissingGuestDirectoryError(error: unknown): boolean { + if (error instanceof FreestyleApiError) { + return error.status === 404 || /no such file|not found|enoent/i.test(error.message); + } + return error instanceof Error && /no such file|enoent/i.test(error.message); +} + /** The resources a machine of `memoryMb` is sold with (see entitlements.ts). */ export function freestyleTargetResources( memoryMb: number, diff --git a/web/services/vms/images/resolver.ts b/web/services/vms/images/resolver.ts index 0c6f9f1891a2..81c2300e4f03 100644 --- a/web/services/vms/images/resolver.ts +++ b/web/services/vms/images/resolver.ts @@ -59,8 +59,13 @@ export type VmImageManifestEntry = { /** The shape this snapshot boots at. Entries without one are size-less (pre-ladder bakes). */ readonly size?: VmImageManifestSize; readonly cmuxdRemoteCommit: string; + /** The cmux-tui build baked in the daemon user's home; absent on images that installed it at create time. */ + readonly cmuxTuiCommit?: string; + readonly cmuxTuiSha256?: string; /** The cmux commit whose devbox definition produced this image. */ readonly repoCommit?: string; + /** The Dockerfile's CMUX_IMAGE_EPOCH at bake time; older entries carry it in `notes` only (see vmImageEntryEpoch). */ + readonly epoch?: string; readonly builtAt: string; readonly builderScriptVersion: string; readonly agentToolResolvedVersions?: Record; @@ -68,6 +73,43 @@ export type VmImageManifestEntry = { readonly notes?: string; }; +/** + * The first devbox epoch whose daemon serves the trusted cloud listener: a + * client on the owner's private network dials `remote connect --carrier` with + * no enrollment. Every current manifest default is at or past it. + */ +export const TRUSTED_CARRIER_EPOCH = "2026-09-10-r1"; + +/** + * The first devbox epoch that bakes the guest tools the drivers used to + * install by exec on every create and re-check on every attach (the guest + * `cmux` shim, the guest CLI distribution, the browser openers, the resource + * reporter). Nothing in the manifest reaches it until that bake is promoted, + * so the drivers keep installing until then. + */ +export const GUEST_TOOLS_BAKED_EPOCH = "2026-09-21-r1"; + +/** The epoch an entry was baked at: the field, or the `cmux devbox epoch ` prefix every bake writes into `notes`. */ +export function vmImageEntryEpoch(entry: Pick | null | undefined): string | undefined { + if (!entry) return undefined; + return entry.epoch ?? /cmux devbox epoch (\S+)/.exec(entry.notes ?? "")?.[1]; +} + +const IMAGE_EPOCH_PATTERN = /^(\d{4}-\d{2}-\d{2})-r(\d+)$/; + +/** + * Whether `epoch` is at or past `floor`: the date first, then the numeric + * revision (`r10` is newer than `r9`). An unknown or malformed epoch never + * satisfies a floor, so a gate on it fails closed. + */ +export function imageEpochAtLeast(epoch: string | null | undefined, floor: string): boolean { + const candidate = epoch ? IMAGE_EPOCH_PATTERN.exec(epoch) : null; + const bound = IMAGE_EPOCH_PATTERN.exec(floor); + if (!candidate || !bound) return false; + if (candidate[1] !== bound[1]) return candidate[1]! > bound[1]!; + return Number(candidate[2]) >= Number(bound[2]); +} + export type VmImageSelection = { readonly provider: ProviderId; readonly image: string; diff --git a/web/services/vms/timings.ts b/web/services/vms/timings.ts index 40ba0e4936fd..8500e5750566 100644 --- a/web/services/vms/timings.ts +++ b/web/services/vms/timings.ts @@ -17,6 +17,12 @@ export type VmTimingStage = | "mark_running" | "mark_base_running" | "usage_events" + // Attach (open_attach): ownership and plan checks, the optional provider + // status probe, the provider's attach, and the lease ledger write. + | "access_check" + | "preflight_probe" + | "provider_attach" + | "lease" | "total"; export type VmTimingSink = { diff --git a/web/services/vms/workflows.ts b/web/services/vms/workflows.ts index 5e207ca027b2..8cd62f2a5847 100644 --- a/web/services/vms/workflows.ts +++ b/web/services/vms/workflows.ts @@ -114,6 +114,8 @@ import { type VmResizeReservation, } from "./repository"; import { measureVmEffect, type VmTimingSink } from "./timings"; +import type { VmDeferSink } from "./defer"; +import { findVmImageManifestEntry, vmImageEntryEpoch } from "./images/resolver"; import { guestPromptInstallCommand, vmPromptIdentity } from "./guestPrompt"; export { @@ -159,6 +161,12 @@ export type VmEntry = { /** The machine's address on its owner's private network, when it has one. */ readonly addressIpv4: string | null; readonly addressIpv6: string | null; + /** + * The devbox epoch the machine's image was baked at: stamped on the row at + * create, read from the manifest for rows created before the stamp. Null + * for an image the manifest does not list. + */ + readonly imageEpoch: string | null; }; export type BaseVmEntry = VmEntry & { @@ -579,6 +587,9 @@ type CreateVmInput = { */ readonly modelPlane?: VmModelPlaneProvisioner; readonly timing?: VmTimingSink; + /** The manifest epoch of the resolved image, stamped on the row so attach gates never re-resolve it. */ + readonly imageEpoch?: string | null; + readonly defer?: VmDeferSink; }; function createVmBeginInput(input: CreateVmInput): CreateVmInput { @@ -746,7 +757,10 @@ export function createVm(input: CreateVmInput): Effect.Effect @@ -3384,11 +3398,13 @@ export function openVmCmuxRemote(input: { readonly clientCapabilities?: readonly string[]; /** Caller's CURRENT billing plan; the free access window applies to cmux-tui attaches too. */ readonly callerPlanId?: string | null; + readonly timing?: VmTimingSink; + readonly defer?: VmDeferSink; }) { return Effect.gen(function* () { const repo = yield* VmRepository; const providers = yield* VmProviderGateway; - const vm = yield* requireAccessibleUserVm(input); + const vm = yield* measureVmEffect(input.timing, "access_check", requireAccessibleUserVm(input)); const supportedTransports = providers.attachTransports?.(vm.provider); if (supportedTransports && !supportedTransports.includes("cmux-remote")) { return yield* Effect.fail(new VmAttachTransportUnsupportedError({ @@ -3407,15 +3423,22 @@ export function openVmCmuxRemote(input: { }), ); } - yield* preflightResumeIfSuspended( + // A row the control plane wrote seconds ago (a create, a resume, a + // rename) describes a machine that cannot have been idle-paused since: + // Cloud machines run with idle pausing disabled, and the row is updated + // on every control-plane transition. Trusting it saves the provider + // status round trip on the attach that follows a create; the re-probe + // after a failed attach still wakes a machine paused out of band. + const trustedRow = vm.status === "running" && Date.now() - vm.updatedAt.getTime() < VM_ROW_TRUST_WINDOW_MS; + yield* measureVmEffect(input.timing, "preflight_probe", preflightResumeIfSuspended( repo, providers, vm, input.providerVmId, "attach", - { forceProviderProbe: true, maxActiveVms: input.maxActiveVms }, - ); - const endpoint = yield* withResumeOnSuspendedAfterFailure( + { forceProviderProbe: !trustedRow, maxActiveVms: input.maxActiveVms }, + )); + const endpoint = yield* measureVmEffect(input.timing, "provider_attach", withResumeOnSuspendedAfterFailure( repo, providers, vm, @@ -3428,8 +3451,10 @@ export function openVmCmuxRemote(input: { providerMetadata: vm.providerMetadata, }), input.maxActiveVms, - ); - yield* repo.recordLease({ + )); + // The lease is what sign-out revocation finds, so it is written before + // the endpoint is handed out. + yield* measureVmEffect(input.timing, "lease", repo.recordLease({ vmId: vm.id, userId: input.userId, kind: "preview", @@ -3444,34 +3469,53 @@ export function openVmCmuxRemote(input: { : Effect.void; return cleanup.pipe(Effect.andThen(Effect.fail(err))); }), - ); - yield* repo.recordUsageEvent({ - userId: input.userId, - billingTeamId: vm.billingTeamId, - billingPlanId: vm.billingPlanId, - vmId: vm.id, - eventType: "vm.attach", - provider: vm.provider, - imageId: vm.imageId, - metadata: { transport: "cmux-remote", invited: false, trustedCarrier: endpoint.trustedCarrier }, - }).pipe(Effect.catchAll(() => Effect.void)); - // Backfill: machines created before address recording learn their private - // address on first attach, so "Copy IP Address" appears for them too. - const learned = endpoint.networkAddresses; - if (learned && repo.mergeProviderMetadata) { - const metadata = vm.providerMetadata ?? {}; - const patch = { - ...(learned.ipv4 && metadata["networkIpv4"] !== learned.ipv4 ? { networkIpv4: learned.ipv4 } : {}), - ...(learned.ipv6 && metadata["networkIpv6"] !== learned.ipv6 ? { networkIpv6: learned.ipv6 } : {}), - }; - if (Object.keys(patch).length) { - yield* repo.mergeProviderMetadata({ id: vm.id, patch }).pipe(Effect.catchAll(() => Effect.void)); - } - } + )); + // The attach usage event and the address backfill are bookkeeping the + // response does not wait for. + yield* deferOrRun(input.defer, Effect.all([ + repo.recordUsageEvent({ + userId: input.userId, + billingTeamId: vm.billingTeamId, + billingPlanId: vm.billingPlanId, + vmId: vm.id, + eventType: "vm.attach", + provider: vm.provider, + imageId: vm.imageId, + metadata: { transport: "cmux-remote", invited: false, trustedCarrier: endpoint.trustedCarrier }, + }).pipe(Effect.catchAll(() => Effect.void)), + backfillLearnedAddresses(repo, vm, endpoint.networkAddresses), + ], { discard: true })); return endpoint; }); } +/** + * How long a `running` row is trusted without a provider status probe + * (see openVmCmuxRemote). Long enough to cover the attach that follows a + * create or a resume, short enough that a machine paused from the provider + * console is still noticed on the next open. + */ +const VM_ROW_TRUST_WINDOW_MS = 120_000; + +/** + * Backfill: machines created before address recording learn their private + * address on first attach, so "Copy IP Address" appears for them too. + */ +function backfillLearnedAddresses( + repo: VmRepositoryShape, + vm: CloudVmRow, + learned: { readonly ipv4?: string; readonly ipv6?: string } | undefined, +): Effect.Effect { + if (!learned || !repo.mergeProviderMetadata) return Effect.void; + const metadata = vm.providerMetadata ?? {}; + const patch = { + ...(learned.ipv4 && metadata["networkIpv4"] !== learned.ipv4 ? { networkIpv4: learned.ipv4 } : {}), + ...(learned.ipv6 && metadata["networkIpv6"] !== learned.ipv6 ? { networkIpv6: learned.ipv6 } : {}), + }; + if (!Object.keys(patch).length) return Effect.void; + return repo.mergeProviderMetadata({ id: vm.id, patch }).pipe(Effect.catchAll(() => Effect.void)); +} + export function approveVmCmuxRemoteEnrollment(input: { readonly userId: string; readonly billingTeamId?: string | null; @@ -4028,6 +4072,7 @@ function recordCreateRequestedEvents( readonly imageVersion?: string | null; readonly idempotencyKey?: string; readonly timing?: VmTimingSink; + readonly defer?: VmDeferSink; }, requestedVm: CloudVmRow, creditReservation: VmCreateCreditReservation, @@ -4035,7 +4080,7 @@ function recordCreateRequestedEvents( return measureVmEffect( input.timing, "usage_events", - repo.recordUsageEvents([ + deferOrRun(input.defer, repo.recordUsageEvents([ ...(creditReservation.kind === "none" ? [] : [creditUsageEvent(requestedVm, "vm.create.credit.reserved", creditReservation)]), @@ -4052,7 +4097,7 @@ function recordCreateRequestedEvents( imageVersion: input.imageVersion ?? null, }, }, - ]).pipe(Effect.catchAll(() => Effect.void)), + ])), ); } @@ -4068,13 +4113,14 @@ function recordCreateSuccessEvents( readonly persistentHome?: boolean; readonly perMachineHome?: boolean; readonly imageSize?: CreateOptions["imageSize"]; + readonly defer?: VmDeferSink; }, running: CloudVmRow, ) { return measureVmEffect( input.timing, "usage_events", - repo.recordUsageEvents([ + deferOrRun(input.defer, repo.recordUsageEvents([ { userId: running.userId, billingTeamId: running.billingTeamId, @@ -4095,10 +4141,24 @@ function recordCreateSuccessEvents( ...(input.perMachineHome !== undefined ? { perMachineHome: input.perMachineHome } : {}), }, }, - ]).pipe(Effect.catchAll(() => Effect.void)), + ])), ); } +/** + * Best-effort bookkeeping: run `effect` inline, or hand it to the caller's + * defer sink so it runs after the response. Failures are swallowed either + * way (the repository logs them); the sink's own error handling covers the + * deferred case. + */ +function deferOrRun(defer: VmDeferSink | undefined, effect: Effect.Effect): Effect.Effect { + const guarded = effect.pipe(Effect.catchAll(() => Effect.void)); + if (!defer) return guarded; + return Effect.sync(() => { + defer(() => Effect.runPromise(guarded)); + }); +} + function recordCreateFailureEvent( repo: VmRepositoryShape, input: { @@ -4266,6 +4326,7 @@ function vmEntryFromRow(row: CloudVmRow): VmEntry { const metadata = row.providerMetadata ?? {}; const addressIpv4 = metadata["networkIpv4"]; const addressIpv6 = metadata["networkIpv6"]; + const stampedEpoch = metadata["imageEpoch"]; return { providerVmId: row.providerVmId, provider: row.provider, @@ -4277,6 +4338,9 @@ function vmEntryFromRow(row: CloudVmRow): VmEntry { slug: row.slug ?? null, addressIpv4: typeof addressIpv4 === "string" && addressIpv4 ? addressIpv4 : null, addressIpv6: typeof addressIpv6 === "string" && addressIpv6 ? addressIpv6 : null, + imageEpoch: typeof stampedEpoch === "string" && stampedEpoch + ? stampedEpoch + : vmImageEntryEpoch(findVmImageManifestEntry(row.provider, row.imageId)) ?? null, }; } diff --git a/web/tests/bun-test.d.ts b/web/tests/bun-test.d.ts index 553fa9bd73e4..8a064fca3741 100644 --- a/web/tests/bun-test.d.ts +++ b/web/tests/bun-test.d.ts @@ -50,6 +50,7 @@ declare module "bun:test" { objectContaining: (value: unknown) => unknown; arrayContaining: (value: readonly unknown[]) => unknown; stringContaining: (value: string) => unknown; + any: (constructor: unknown) => unknown; }; export const mock: Mock; export const setSystemTime: (time?: Date | number) => void; diff --git a/web/tests/vm-attach-contract.test.ts b/web/tests/vm-attach-contract.test.ts new file mode 100644 index 000000000000..f07bc341e543 --- /dev/null +++ b/web/tests/vm-attach-contract.test.ts @@ -0,0 +1,135 @@ +import { describe, expect, test } from "bun:test"; +import { createAttachBlock } from "../services/vms/attachContract"; +import { freestyleCmuxRemoteRoute } from "../services/vms/drivers/freestyle"; +import { + GUEST_TOOLS_BAKED_EPOCH, + TRUSTED_CARRIER_EPOCH, + imageEpochAtLeast, + listVmImageKindDefaults, + listVmImageManifestEntries, + vmImageEntryEpoch, + type VmImageManifestEntry, +} from "../services/vms/images/resolver"; + +// The create response tells the app where to dial before any attach call: +// the machine's private address and the attach block (transport, route, +// carrier trust, daemon build, guest-tools state) are derived from the row +// and the checked-in manifest alone, never from a provider round trip. + +function manifestEntry(overrides: Partial = {}): VmImageManifestEntry { + return { + provider: "freestyle", + version: "freestyle-cmux-devbox-test", + imageId: "sh-0000000000000000000000000000test", + envVar: "FREESTYLE_SANDBOX_SNAPSHOT", + kind: "desktop", + cmuxdRemoteCommit: "none-cmux-tui", + builtAt: "2026-09-10T00:00:00.000Z", + builderScriptVersion: "0".repeat(64), + validationStatus: "passed", + epoch: "2026-09-10-r2", + cmuxTuiCommit: "f3b652d8dc0000000000000000000000000000ab", + cmuxTuiSha256: "1".repeat(64), + ...overrides, + }; +} + +describe("image epochs", () => { + test("imageEpochAtLeast orders by date, then by numeric revision", () => { + expect(imageEpochAtLeast("2026-09-10-r2", "2026-09-10-r1")).toBe(true); + expect(imageEpochAtLeast("2026-09-10-r1", "2026-09-10-r1")).toBe(true); + expect(imageEpochAtLeast("2026-09-10-r1", "2026-09-10-r2")).toBe(false); + // r10 is newer than r9: the revision is a number, not a string. + expect(imageEpochAtLeast("2026-09-10-r10", "2026-09-10-r9")).toBe(true); + expect(imageEpochAtLeast("2026-09-11-r1", "2026-09-10-r9")).toBe(true); + expect(imageEpochAtLeast("2026-09-09-r9", "2026-09-10-r1")).toBe(false); + // An unknown or malformed epoch never satisfies a floor. + expect(imageEpochAtLeast(undefined, "2026-09-10-r1")).toBe(false); + expect(imageEpochAtLeast(null, "2026-09-10-r1")).toBe(false); + expect(imageEpochAtLeast("devbox-2026", "2026-09-10-r1")).toBe(false); + expect(imageEpochAtLeast("2026-09-10-r1", "not-an-epoch")).toBe(false); + }); + + test("vmImageEntryEpoch reads the field, then the notes prefix every bake writes", () => { + expect(vmImageEntryEpoch(manifestEntry({ epoch: "2026-09-10-r2" }))).toBe("2026-09-10-r2"); + expect(vmImageEntryEpoch(manifestEntry({ epoch: undefined, notes: "cmux devbox epoch 2026-09-01-r3 Devbox on Freestyle." }))).toBe("2026-09-01-r3"); + expect(vmImageEntryEpoch(manifestEntry({ epoch: undefined, notes: "no epoch here" }))).toBeUndefined(); + expect(vmImageEntryEpoch(manifestEntry({ epoch: undefined, notes: undefined }))).toBeUndefined(); + }); + + test("no current manifest entry reads as guest-tools baked", () => { + // The gate flips only with the promotion of a bake at GUEST_TOOLS_BAKED_EPOCH; + // until then the drivers keep installing the guest tools on every create. + expect(GUEST_TOOLS_BAKED_EPOCH).toBe("2026-09-21-r1"); + const entries = listVmImageManifestEntries(); + expect(entries.length).toBeGreaterThan(0); + for (const entry of entries) { + expect(imageEpochAtLeast(vmImageEntryEpoch(entry), GUEST_TOOLS_BAKED_EPOCH)).toBe(false); + } + }); + + test("every manifest default serves a trusted-carrier daemon", () => { + expect(TRUSTED_CARRIER_EPOCH).toBe("2026-09-10-r1"); + for (const kind of ["desktop", "base"] as const) { + const defaults = listVmImageKindDefaults("freestyle", kind); + expect(defaults.length).toBeGreaterThan(0); + for (const entry of defaults) { + expect(imageEpochAtLeast(vmImageEntryEpoch(entry), TRUSTED_CARRIER_EPOCH)).toBe(true); + expect(entry.cmuxTuiCommit).toMatch(/^[0-9a-f]{40}$/); + } + } + }); +}); + +describe("createAttachBlock", () => { + test("dials the IPv4 address first and brackets IPv6, exactly like the driver", () => { + const entry = manifestEntry(); + const dual = createAttachBlock({ + entry: { addressIpv4: "10.16.0.7", addressIpv6: "fd00:4::7", imageEpoch: null }, + manifestEntry: entry, + }); + expect(dual).toEqual({ + transport: "cmux-remote", + route: "ws://10.16.0.7:1337/v1/link", + session: "cloud", + trustedCarrier: true, + daemonBuild: { commit: entry.cmuxTuiCommit ?? null, remoteProtocol: null, version: null }, + guestToolsBaked: false, + readiness: "dial", + }); + expect(dual?.route).toBe(freestyleCmuxRemoteRoute({ vpcs: [{ ipv4: "10.16.0.7", ipv6: "fd00:4::7" }] }, "vm-test")); + + const v6 = createAttachBlock({ + entry: { addressIpv4: null, addressIpv6: "fd00:4::7", imageEpoch: null }, + manifestEntry: entry, + }); + expect(v6?.route).toBe("ws://[fd00:4::7]:1337/v1/link"); + expect(v6?.route).toBe(freestyleCmuxRemoteRoute({ vpcs: [{ ipv6: "fd00:4::7" }] }, "vm-test")); + }); + + test("is absent without a private address or a manifest entry", () => { + expect(createAttachBlock({ + entry: { addressIpv4: null, addressIpv6: null, imageEpoch: "2026-09-10-r2" }, + manifestEntry: manifestEntry(), + })).toBeNull(); + expect(createAttachBlock({ + entry: { addressIpv4: "10.16.0.7", addressIpv6: null, imageEpoch: "2026-09-10-r2" }, + manifestEntry: null, + })).toBeNull(); + }); + + test("carrier trust and guest-tools state follow the epoch; the row's stamped epoch wins", () => { + const old = createAttachBlock({ + entry: { addressIpv4: "10.16.0.7", addressIpv6: null, imageEpoch: null }, + manifestEntry: manifestEntry({ epoch: undefined, notes: "cmux devbox epoch 2026-09-01-r3", cmuxTuiCommit: undefined }), + }); + expect(old).toMatchObject({ trustedCarrier: false, guestToolsBaked: false, daemonBuild: { commit: null } }); + + const baked = createAttachBlock({ + entry: { addressIpv4: "10.16.0.7", addressIpv6: null, imageEpoch: GUEST_TOOLS_BAKED_EPOCH }, + // The manifest lookup says older, but the row was stamped at create. + manifestEntry: manifestEntry({ epoch: "2026-09-10-r2" }), + }); + expect(baked).toMatchObject({ trustedCarrier: true, guestToolsBaked: true, readiness: "dial" }); + }); +}); diff --git a/web/tests/vm-defer-sink.test.ts b/web/tests/vm-defer-sink.test.ts new file mode 100644 index 000000000000..4f8be83cbb4e --- /dev/null +++ b/web/tests/vm-defer-sink.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, test } from "bun:test"; +import { orderedDeferSink } from "../services/vms/defer"; + +// Deferred ledger writes (vm.create.requested, then vm.created) must land in +// lifecycle order even though the platform runs after-response callbacks +// concurrently, and a unit must not start before the scheduler invokes it. +describe("orderedDeferSink", () => { + test("runs units in hand-in order even when the scheduler starts them out of order", async () => { + const scheduled: Array<() => Promise> = []; + const sink = orderedDeferSink((work) => { + scheduled.push(work); + }); + const events: string[] = []; + // The first unit parks inside its work until the test releases it. That + // makes the ordering observable without a timer: while the first is + // parked, a second unit that was started earlier must still be waiting. + let releaseFirst: () => void = () => undefined; + const firstReleased = new Promise((resolve) => { + releaseFirst = resolve; + }); + let markFirstRunning: () => void = () => undefined; + const firstRunning = new Promise((resolve) => { + markFirstRunning = resolve; + }); + sink(async () => { + events.push("requested:start"); + markFirstRunning(); + await firstReleased; + events.push("requested"); + }); + sink(async () => { + events.push("created"); + }); + expect(scheduled).toHaveLength(2); + await Promise.resolve(); + // Nothing runs until the scheduler says so. + expect(events).toEqual([]); + + // The scheduler starts the second unit before the first. + const second = scheduled[1]!(); + const first = scheduled[0]!(); + await firstRunning; + // The first is parked mid-work; the second, started earlier, waits for it. + expect(events).toEqual(["requested:start"]); + releaseFirst(); + await Promise.all([first, second]); + expect(events).toEqual(["requested:start", "requested", "created"]); + }); + + test("a failed unit surfaces to the scheduler and does not block the next", async () => { + const outcomes: string[] = []; + const settled: Array> = []; + const sink = orderedDeferSink((work) => { + settled.push(work().then( + () => { + outcomes.push("ok"); + }, + () => { + outcomes.push("failed"); + }, + )); + }); + const events: string[] = []; + sink(async () => { + throw new Error("db down"); + }); + sink(async () => { + events.push("after"); + }); + expect(settled).toHaveLength(2); + await Promise.all(settled); + expect(events).toEqual(["after"]); + expect(outcomes).toEqual(["failed", "ok"]); + }); +}); diff --git a/web/tests/vm-freestyle-provider.test.ts b/web/tests/vm-freestyle-provider.test.ts index a7448569b436..23452aab1ab4 100644 --- a/web/tests/vm-freestyle-provider.test.ts +++ b/web/tests/vm-freestyle-provider.test.ts @@ -493,6 +493,37 @@ describe("FreestyleProvider create with edge rules", () => { expect(JSON.stringify(fake.writes)).not.toContain("crt_secret-token"); }); + test("uploads the guest adapter without a separate libexec mkdir exec", async () => { + // The bake creates /usr/local/libexec, so the create path pays one upload + // and one install exec, not a round trip to create a directory that exists. + const fake = fakeFreestyle({ probeExit: 0 }); + await providerWith(fake).create({ image: "sh-devbox", network: { id: "vpc_1" } }); + expect(fake.writes).toHaveLength(1); + expect(fake.execs.filter((command) => /^\s*mkdir -p \/usr\/local\/libexec\s*$/.test(command))).toEqual([]); + expect(fake.deletes).toEqual([]); + }); + + test("creates the libexec directory once and retries the upload when it is missing", async () => { + // An image from before the directory was baked rejects the upload; the + // driver heals with one mkdir and one retry instead of failing the create. + const fake = fakeFreestyle({ probeExit: 0 }); + const vm = (fake.client.vms as unknown as { ref: () => { fs: { writeTextFile: (path: string, content: string) => Promise } } }).ref(); + let uploads = 0; + vm.fs.writeTextFile = async (path: string, content: string) => { + uploads += 1; + fake.writes.push({ path, content }); + if (uploads === 1) throw new FreestyleApiError(404, { code: "NOT_FOUND", message: "no such file or directory: /usr/local/libexec" }, "/fs/write"); + }; + const handle = await providerWith(fake).create({ image: "sh-devbox", network: { id: "vpc_1" } }); + expect(handle.providerVmId).toBe(VM_ID); + expect(uploads).toBe(2); + const mkdirAt = fake.execs.findIndex((command) => /^\s*mkdir -p \/usr\/local\/libexec\s*$/.test(command)); + const installAt = fake.execs.findIndex((command) => command.includes("mv -f") && command.includes("/usr/local/libexec/cmux-cloud-adapter'")); + expect(fake.execs.filter((command) => /^\s*mkdir -p \/usr\/local\/libexec\s*$/.test(command))).toHaveLength(1); + expect(installAt).toBeGreaterThan(mkdirAt); + expect(fake.deletes).toEqual([]); + }); + test("omits the tls block and the probe when no rules are given", async () => { const fake = fakeFreestyle({ probeExit: 1 }); await providerWith(fake).create({ image: "sh-devbox" }); diff --git a/web/tests/vm-image-manifest.test.ts b/web/tests/vm-image-manifest.test.ts index bb7191170b6a..8c780375b0fa 100644 --- a/web/tests/vm-image-manifest.test.ts +++ b/web/tests/vm-image-manifest.test.ts @@ -23,6 +23,7 @@ import { type DevboxImageManifest, type DevboxManifestEntry, } from "../scripts/devbox-image-common"; +import { vmImageEntryEpoch } from "../services/vms/images/resolver"; // The checked-in image manifest (services/vms/images/manifest.json) is the // source of truth for the image Cloud VM users get: the resolver serves the @@ -111,6 +112,8 @@ describe("devboxSourceDriftProblems", () => { // an epoch bump is the exact case this catches. const legacy = passedEntry({ kind: "base", defaultForKind: true, notes: "cmux devbox epoch 1999-01-01-r1 Devbox on Freestyle." }); expect(manifestEntryEpoch(legacy)).toBe("1999-01-01-r1"); + // One implementation: the runtime resolver owns it, the bake scripts import it. + expect(manifestEntryEpoch).toBe(vmImageEntryEpoch); expect(devboxSourceDriftProblems(manifestOf(legacy))).toHaveLength(1); expect(devboxSourceDriftProblems(manifestOf(passedEntry({ kind: "base", defaultForKind: true, notes: `cmux devbox epoch ${epoch}` })))).toEqual([]); expect(manifestEntryEpoch(passedEntry({ notes: "no epoch here" }))).toBeUndefined(); diff --git a/web/tests/vm-route-auth.test.ts b/web/tests/vm-route-auth.test.ts index 3497d939eb71..5b36e54eb7a6 100644 --- a/web/tests/vm-route-auth.test.ts +++ b/web/tests/vm-route-auth.test.ts @@ -19,6 +19,8 @@ type ManifestTestEntry = { kind?: string; defaultForKind?: boolean; size?: { name: string }; + epoch?: string; + cmuxTuiCommit?: string; }; function manifestDefault(kind: "desktop" | "base"): ManifestTestEntry { return (manifestJson.images as ManifestTestEntry[]).find((entry) => @@ -471,7 +473,14 @@ describe("VM REST auth", () => { providerVmId: "provider-vm-1", provider: "freestyle", image: "snapshot-test", + imageVersion: null, + status: "running", createdAt: 1_777_000_000_000, + displayName: null, + slug: null, + addressIpv4: null, + addressIpv6: null, + imageEpoch: null, }); const response = await POST( @@ -483,10 +492,13 @@ describe("VM REST auth", () => { ); expect(response.status).toBe(200); + // A machine without a private address (or outside the manifest) carries + // no attach block: the client falls back to GET /api/vm/[id] + attach-endpoint. expect(await response.json()).toEqual({ id: "provider-vm-1", provider: "freestyle", image: "snapshot-test", + imageVersion: null, kind: "base", createdAt: 1_777_000_000_000, capabilities: { @@ -501,7 +513,15 @@ describe("VM REST auth", () => { persistentHome: false, attachTransports: ["cmux-remote"], }, + displayName: null, + slug: null, + status: "running", + address: { ipv4: null, ipv6: null }, }); + // Per-stage timings ride on the response so a client can see where a + // create spent its time without a tracing backend. + expect(response.headers.get("Server-Timing")).toMatch(/\bauth;dur=/); + expect(response.headers.get("Server-Timing")).toMatch(/\btotal;dur=/); expect(createVm).toHaveBeenCalledWith(expect.objectContaining({ userId: "user-1", billingCustomerType: "team", @@ -513,11 +533,66 @@ describe("VM REST auth", () => { imageVersion: null, idempotencyKey: "idem-1", memoryMb: 8192, + // Telemetry writes run after the response has left. + defer: expect.any(Function), })); expect(listTeams).not.toHaveBeenCalled(); expect(runVmWorkflow).toHaveBeenCalled(); }); + test("create response carries the private address and the attach route for a manifest image", async () => { + // The app dials the daemon straight from this response (no status GET, no + // attach-endpoint on the happy path), so the block must be complete and + // derived from the row plus the manifest alone. + getUser.mockResolvedValue(authedStackUser()); + runVmWorkflow.mockResolvedValue({ + providerVmId: "provider-vm-attach", + provider: "freestyle", + image: MANIFEST_DESKTOP_DEFAULT.imageId, + imageVersion: MANIFEST_DESKTOP_DEFAULT.version, + status: "running", + createdAt: 1_777_000_000_000, + displayName: null, + slug: "giddy-cherry-emu", + addressIpv4: "10.16.0.7", + addressIpv6: "fd00:4::7", + imageEpoch: null, + }); + + const response = await POST( + new Request("https://cmux.test/api/vm", { + method: "POST", + headers: { origin: "https://cmux.test" }, + body: JSON.stringify({ provider: "freestyle", kind: "desktop" }), + }), + ); + + expect(response.status).toBe(200); + const payload = await response.json(); + expect(payload).toMatchObject({ + id: "provider-vm-attach", + kind: "desktop", + status: "running", + slug: "giddy-cherry-emu", + address: { ipv4: "10.16.0.7", ipv6: "fd00:4::7" }, + attach: { + transport: "cmux-remote", + route: "ws://10.16.0.7:1337/v1/link", + session: "cloud", + trustedCarrier: true, + daemonBuild: { commit: MANIFEST_DESKTOP_DEFAULT.cmuxTuiCommit, remoteProtocol: null, version: null }, + guestToolsBaked: false, + readiness: "dial", + }, + }); + expect(payload).not.toHaveProperty("claimed"); + expect(createVm).toHaveBeenCalledWith(expect.objectContaining({ + image: MANIFEST_DESKTOP_DEFAULT.imageId, + imageEpoch: MANIFEST_DESKTOP_DEFAULT.epoch, + defer: expect.any(Function), + })); + }); + test("rejects an unknown `kind` on create and base open before touching workflows", async () => { getUser.mockResolvedValue(authedStackUser()); @@ -1776,7 +1851,7 @@ describe("VM REST auth", () => { context, ); expect(response.status).toBe(200); - expect(openVmCmuxRemote).toHaveBeenCalledWith({ + expect(openVmCmuxRemote).toHaveBeenCalledWith(expect.objectContaining({ userId: "user-1", billingTeamId: "team-1", teamIds: ["team-1"], @@ -1785,8 +1860,15 @@ describe("VM REST auth", () => { clientCapabilities: ["direct-ws-user-agent"], callerPlanId: "pro", maxActiveVms: 50, - }); + // Stage timings reach the span and the response; the attach usage + // event and address backfill are written after the response has left. + timing: expect.objectContaining({ record: expect.any(Function) }), + defer: expect.any(Function), + })); expect(openAttachEndpoint).not.toHaveBeenCalled(); + // The attach route reports its stages the same way the create route does. + expect(response.headers.get("Server-Timing")).toMatch(/\bauth;dur=/); + expect(response.headers.get("Server-Timing")).toMatch(/\btotal;dur=/); const payload = await response.json(); expect(payload.transport).toBe("cmux-remote"); expect(payload.trustedCarrier).toBe(true); diff --git a/web/tests/vm-workflows.test.ts b/web/tests/vm-workflows.test.ts index 78812173198a..345f17d50ba6 100644 --- a/web/tests/vm-workflows.test.ts +++ b/web/tests/vm-workflows.test.ts @@ -41,6 +41,7 @@ import { } from "../services/vms/errors"; import { accountDeletionUserHash } from "../services/account/deletionLock"; import { isVmAttachTransportUnsupportedError } from "../services/vms/errors"; +import type { VmTimingStage } from "../services/vms/timings"; import { VM_DISK_MB_MAX, VM_RESOURCE_RESIZE_PENDING_METADATA_KEY, @@ -2162,6 +2163,8 @@ describe("VM Effect workflows", () => { billingTeamId: "team-workflow-remote-stale-running", providerVmId: "provider-vm-remote-stale-running", status: "running", + // Old enough that the row is not trusted: the provider is probed first. + updatedAt: new Date(Date.now() - 10 * 60_000), }); const usageEvents: RecordedUsageEvent[] = []; const leases: RecordedLease[] = []; @@ -2231,6 +2234,7 @@ describe("VM Effect workflows", () => { userId: "user-workflow-remote-probe-fail", providerVmId: "provider-vm-remote-probe-fail", status: "running", + updatedAt: new Date(Date.now() - 10 * 60_000), }); const repo = testWorkflowRepo({ vm }); const probeError = providerOperationError("getStatus", "provider status unavailable"); @@ -2270,6 +2274,8 @@ describe("VM Effect workflows", () => { userId: "user-workflow-remote-destroyed", providerVmId: "provider-vm-remote-destroyed", status: "running", + // Old enough that the row is not trusted: the provider is probed first. + updatedAt: new Date(Date.now() - 10 * 60_000), }); const observedStatuses: ObservedStatusUpdate[] = []; const repo = testWorkflowRepo({ vm, observedStatuses }); @@ -2312,6 +2318,186 @@ describe("VM Effect workflows", () => { ]); }); + test("openVmCmuxRemote trusts a row updated within 120 s and still fails closed when the attach and the probe both fail", async () => { + // A machine the control plane touched seconds ago (a create, a resume) + // cannot have been idle-paused since (Cloud machines run with idle pausing + // disabled), so the attach dials the daemon without a status round trip. + // The re-probe after a failed attach keeps a provider-paused machine + // recoverable. + const fresh = testCloudVmRow({ + id: "00000000-0000-4000-8000-000000000149", + userId: "user-workflow-remote-fresh", + providerVmId: "provider-vm-remote-fresh", + status: "running", + updatedAt: new Date(Date.now() - 5_000), + }); + const leases: RecordedLease[] = []; + const repo = testWorkflowRepo({ vm: fresh, leases }); + const endpoint = { + transport: "cmux-remote" as const, + route: "ws://10.0.0.5:1337/v1/link", + token: "remote-token", + expiresAtUnix: Math.floor(Date.now() / 1000) + 300, + session: "cloud", + trustedCarrier: true, + }; + const calls: string[] = []; + const provider: VmProviderGatewayShape = { + ...unusedProviderGateway(), + getStatus: () => + Effect.sync(() => { + calls.push("getStatus"); + return "paused" as const; + }), + resume: () => + Effect.sync(() => { + calls.push("resume"); + return testVmHandle({ providerVmId: "provider-vm-remote-fresh" }); + }), + openCmuxRemote: () => + Effect.sync(() => { + calls.push("openCmuxRemote"); + return endpoint; + }), + }; + + const result = await Effect.runPromise( + openVmCmuxRemote({ + userId: "user-workflow-remote-fresh", + providerVmId: "provider-vm-remote-fresh", + }).pipe(Effect.provide(workflowLayer(repo, provider))), + ); + expect(result).toEqual(endpoint); + expect(calls).toEqual(["openCmuxRemote"]); + expect(leases).toHaveLength(1); + + // The attach fails and the provider cannot be probed either: the attach + // error surfaces unchanged and nothing is minted or recorded. + const attachError = providerOperationError("openCmuxRemote", "daemon unreachable"); + const probeError = providerOperationError("getStatus", "provider status unavailable"); + const failedLeases: RecordedLease[] = []; + const failedCalls: string[] = []; + const failing: VmProviderGatewayShape = { + ...unusedProviderGateway(), + getStatus: () => + Effect.suspend(() => { + failedCalls.push("getStatus"); + return Effect.fail(probeError); + }), + resume: () => Effect.fail(providerOperationError("resume", "should not resume")), + openCmuxRemote: () => + Effect.suspend(() => { + failedCalls.push("openCmuxRemote"); + return Effect.fail(attachError); + }), + }; + const error = await Effect.runPromise( + openVmCmuxRemote({ + userId: "user-workflow-remote-fresh", + providerVmId: "provider-vm-remote-fresh", + }).pipe(Effect.flip, Effect.provide(workflowLayer(testWorkflowRepo({ vm: fresh, leases: failedLeases }), failing))), + ); + expect(error).toBe(attachError); + expect(failedCalls).toEqual(["openCmuxRemote", "getStatus"]); + expect(failedLeases).toHaveLength(0); + }); + + test("openVmCmuxRemote records its stages on the timing sink", async () => { + const vm = testCloudVmRow({ + id: "00000000-0000-4000-8000-00000000014a", + userId: "user-workflow-remote-timing", + providerVmId: "provider-vm-remote-timing", + status: "running", + updatedAt: new Date(Date.now() - 10 * 60_000), + }); + const stages: VmTimingStage[] = []; + const timing = { + record: (stage: VmTimingStage, durationMs: number) => { + expect(durationMs).toBeGreaterThanOrEqual(0); + stages.push(stage); + }, + }; + const provider: VmProviderGatewayShape = { + ...unusedProviderGateway(), + getStatus: () => Effect.succeed("running" as const), + openCmuxRemote: () => + Effect.succeed({ + transport: "cmux-remote" as const, + route: "ws://10.0.0.5:1337/v1/link", + token: "remote-token", + expiresAtUnix: Math.floor(Date.now() / 1000) + 300, + session: "cloud", + trustedCarrier: true, + }), + }; + + await Effect.runPromise( + openVmCmuxRemote({ + userId: "user-workflow-remote-timing", + providerVmId: "provider-vm-remote-timing", + timing, + }).pipe(Effect.provide(workflowLayer(testWorkflowRepo({ vm }), provider))), + ); + expect(stages).toEqual(["access_check", "preflight_probe", "provider_attach", "lease"]); + }); + + test("openVmCmuxRemote records the lease before returning and defers the usage event and address backfill", async () => { + // The lease is what sign-out revocation finds, so it is written before + // the endpoint is handed out; the attach usage event and the learned + // address are bookkeeping the response does not wait for. + const vm = testCloudVmRow({ + id: "00000000-0000-4000-8000-00000000014b", + userId: "user-workflow-remote-defer", + providerVmId: "provider-vm-remote-defer", + status: "running", + providerMetadata: {}, + }); + const usageEvents: RecordedUsageEvent[] = []; + const leases: RecordedLease[] = []; + const patches: Array>> = []; + const repo: VmRepositoryShape = { + ...testWorkflowRepo({ vm, usageEvents, leases }), + mergeProviderMetadata: ({ patch }) => + Effect.sync(() => { + patches.push(patch); + }), + }; + const endpoint = { + transport: "cmux-remote" as const, + route: "ws://10.0.0.5:1337/v1/link", + token: "remote-token", + expiresAtUnix: Math.floor(Date.now() / 1000) + 300, + session: "cloud", + trustedCarrier: true, + networkAddresses: { ipv4: "10.0.0.5", ipv6: "fd00:4::5" }, + }; + const provider: VmProviderGatewayShape = { + ...unusedProviderGateway(), + getStatus: () => Effect.succeed("running" as const), + openCmuxRemote: () => Effect.succeed(endpoint), + }; + const deferred: Array<() => Promise> = []; + + const result = await Effect.runPromise( + openVmCmuxRemote({ + userId: "user-workflow-remote-defer", + providerVmId: "provider-vm-remote-defer", + defer: (work) => { + deferred.push(work); + }, + }).pipe(Effect.provide(workflowLayer(repo, provider))), + ); + expect(result).toEqual(endpoint); + expect(leases).toHaveLength(1); + expect(usageEvents).toEqual([]); + expect(patches).toEqual([]); + expect(deferred.length).toBeGreaterThan(0); + + for (const work of deferred) await work(); + expect(usageEvents.map((event) => event.eventType)).toEqual(["vm.attach"]); + expect(patches).toEqual([{ networkIpv4: "10.0.0.5", networkIpv6: "fd00:4::5" }]); + }); + test("openAttachEndpoint fails when resumed status persistence fails", async () => { const vm = testCloudVmRow({ id: "00000000-0000-4000-8000-000000000110", @@ -2790,6 +2976,51 @@ describe("VM Effect workflows", () => { expect(result.displayName).toBe("Build box"); }); + test("create hands its usage events to the defer sink and stamps the image epoch on the row", async () => { + // The requested/created ledger rows are analytics, not lifecycle state: + // with a sink the create returns before they are written and the route + // runs them after the response. Without a sink they are written inline. + const requested = testCloudVmRow({ slug: "calm-heron" }); + let runningMetadata: Record | undefined; + const usageEvents: RecordedUsageEvent[] = []; + const repo: VmRepositoryShape = { + ...testWorkflowRepo({ vm: requested, usageEvents }), + beginCreate: () => Effect.succeed({ inserted: true, vm: requested }), + markCreateRunning: (input) => + Effect.sync(() => { + runningMetadata = input.providerMetadata; + return { ...requested, status: "running" as const, providerVmId: "deferred-vm", providerMetadata: input.providerMetadata ?? {} }; + }), + }; + const provider: VmProviderGatewayShape = { + ...unusedProviderGateway(), + create: () => Effect.succeed(testVmHandle({ providerVmId: "deferred-vm", providerMetadata: { networkIpv4: "10.0.0.9" } })), + }; + const deferred: Array<() => Promise> = []; + + const result = await Effect.runPromise(createVm({ + userId: "user-workflow-usage-events", + billingCustomerType: "team", + billingTeamId: "user-workflow-usage-events", + billingPlanId: "free", maxActiveVms: 1, provider: "freestyle", image: requested.imageId, + imageEpoch: "2026-09-10-r2", + defer: (work) => { + deferred.push(work); + }, + }).pipe(Effect.provide(workflowLayer(repo, provider)))); + + expect(result.providerVmId).toBe("deferred-vm"); + expect(result.imageEpoch).toBe("2026-09-10-r2"); + expect(result.addressIpv4).toBe("10.0.0.9"); + // The epoch rides with the provider's own metadata, nothing is dropped. + expect(runningMetadata).toEqual({ networkIpv4: "10.0.0.9", imageEpoch: "2026-09-10-r2" }); + expect(usageEvents).toEqual([]); + expect(deferred).toHaveLength(2); + + for (const work of deferred) await work(); + expect(usageEvents.map((event) => event.eventType)).toEqual(["vm.create.requested", "vm.created"]); + }); + dbTest("create reserves the display name atomically and an idempotent replay preserves it", async () => { if (!sql) throw new Error("test database not initialized"); await sql`truncate cloud_vm_billing_grants, cloud_vm_usage_events, cloud_vm_leases, cloud_vms restart identity cascade`;