Cloud create response carries the attach route; attach route timing and probe skip

web/app/api/vm/[id]/attach-endpoint/route.ts

@@ -3,9 +3,11 @@ import { preconnectFreestyle } from "../../../../../services/vms/drivers/freesty
 import {
   jsonResponse,
   resolveVmRouteAccountScope,
+  runAfterResponse,
   withAuthedVmApiRoute,
 } from "../../../../../services/vms/routeHelpers";
 import { setSpanAttributes } from "../../../../../services/telemetry";
+import { VmTimingRecorder } from "../../../../../services/vms/timings";
 import { runVmRoute } from "../../../../../services/vms/routeWorkflow";
 import { openAttachEndpoint, openVmCmuxRemote } from "../../../../../services/vms/workflows";
 import {
@@ -27,7 +29,19 @@ export async function POST(
     "/api/vm/[id]/attach-endpoint",
     { "cmux.vm.operation": "open_attach" },
     "/api/vm/[id]/attach-endpoint failed",
-    async ({ user, span }) => {
+    async ({ user, span, authDurationMs, routeStartedAtMs, setResponseFinalizer }) => {
+      const timing = new VmTimingRecorder(span, "open_attach", { startedAt: routeStartedAtMs });
+      timing.record("auth", authDurationMs);
+      setResponseFinalizer((response) => {
+        timing.finish({ status: response.status });
+        // Per-stage timings travel with the response, as on create, so a
+        // client or a bench run sees where an attach spent its time.
+        try {
+          response.headers.set("Server-Timing", timing.serverTimingHeader());
+        } catch {
+          // Immutable headers on a passthrough Response: the span still has them.
+        }
+      });
       const { id } = await params;
       const body = await parseLenientObjectBody(request);
       const requireDaemon = body.requireDaemon === true || body.require_daemon === true;
@@ -72,6 +86,10 @@ export async function POST(
           deviceFingerprint,
           clientCapabilities,
           callerPlanId: account.entitlements.planId,
+          timing,
+          // The attach usage event and the address backfill are written
+          // after the response has left.
+          defer: runAfterResponse,
         }), { request });
         if (!run.ok) return run.response;
         return jsonResponse(run.value);

web/app/api/vm/route.ts

@@ -36,9 +36,14 @@ import {
   vmFreeAccessWindowDays,
 } from "../../../services/vms/entitlements";
 import {
+  findVmImageManifestEntry,
   inferVmProviderForImage,
   resolveVmImage,
+  vmImageEntryEpoch,
+  type VmImageSelection,
 } from "../../../services/vms/images/resolver";
+import { createAttachBlock } from "../../../services/vms/attachContract";
+import { orderedDeferSink } from "../../../services/vms/defer";
 import {
   reportVmImageConfigError,
   isVmImageKind,
@@ -69,6 +74,7 @@ import { annotateVmRequestBilling } from "../../../services/vms/requestContext";
 import {
   createVm,
   listUserVms,
+  type VmEntry,
 } from "../../../services/vms/workflows";
 import { recordSpanError, setSpanAttributes } from "../../../services/telemetry";
 import {
@@ -292,29 +298,55 @@ export async function POST(request: Request): Promise<Response> {
         imageSize: imageSelection.size ?? undefined,
         modelPlane,
         timing,
+        imageEpoch: vmImageEntryEpoch(imageSelection.manifestEntry),
+        // Usage-event rows are written after the response has left, in
+        // lifecycle order (requested before created).
+        defer: orderedDeferSink(runAfterResponse),
       }), {
         request,
         onError: createErrorResponders(entitlements),
       });
       if (!run.ok) return run.response;
       const created = run.value;
-      setSpanAttributes(span, { "cmux.vm.id": created.providerVmId });
-      return jsonResponse({
-        id: created.providerVmId,
-        provider: created.provider,
-        image: created.image,
-        imageVersion: created.imageVersion,
-        kind: vmImageKindFor(created.provider, created.image),
-        ...(imageSelection.size ? { size: imageSelection.size } : {}),
-        createdAt: created.createdAt,
-        capabilities: vmCapabilitiesFor(created.provider),
-        displayName: created.displayName,
-        slug: created.slug,
-      });
+      const payload = createResponseBody(created, imageSelection);
+      setSpanAttributes(span, { "cmux.vm.id": created.providerVmId, "cmux.vm.attach_block": "attach" in payload });
+      return jsonResponse(payload);
     },
   );
 }
 
+/**
+ * The create response: the machine shape `GET /api/vm` lists plus what a
+ * client needs to dial the daemon straight away. `address` is the same object
+ * the GET routes return; `attach` is the block a client dials from without a
+ * status GET or an attach-endpoint call, absent when the row holds no private
+ * address or the image is outside the manifest (clients feature-detect on it
+ * and fall back to attach-endpoint).
+ */
+function createResponseBody(created: VmEntry, imageSelection: VmImageSelection): Record<string, unknown> {
+  // An idempotent replay answers with the row's own image, which may not be
+  // the one this request resolved.
+  const manifestEntry = imageSelection.manifestEntry?.imageId === created.image
+    ? imageSelection.manifestEntry
+    : findVmImageManifestEntry(created.provider, created.image, imageSelection.kind);
+  const attach = createAttachBlock({ entry: created, manifestEntry });
+  return {
+    id: created.providerVmId,
+    provider: created.provider,
+    image: created.image,
+    imageVersion: created.imageVersion,
+    kind: vmImageKindFor(created.provider, created.image),
+    ...(imageSelection.size ? { size: imageSelection.size } : {}),
+    createdAt: created.createdAt,
+    capabilities: vmCapabilitiesFor(created.provider),
+    displayName: created.displayName,
+    slug: created.slug,
+    status: created.status,
+    address: { ipv4: created.addressIpv4 ?? null, ipv6: created.addressIpv6 ?? null },
+    ...(attach ? { attach } : {}),
+  };
+}
+
 /**
  * How a create request's optional flags meet the resolved provider's capabilities.
  *

web/scripts/cloud-vm/bench-vm-startup.mjs

@@ -50,7 +50,8 @@ const CREATE_TIMEOUT_MS = 630_000;
 const ATTACH_BUDGET_MS = 180_000;
 
 const requireFromWeb = createRequire(path.join(webDir, "package.json"));
-const { StackServerApp } = await import(pathToFileURL(requireFromWeb.resolve("@stackframe/js")).href);
+// The Stack SDK the app itself uses (app/lib/stack.ts), resolved like smoke-vm-api.mjs does.
+const { StackServerApp } = await import(pathToFileURL(requireFromWeb.resolve("@hexclave/js")).href);
 // ESM-only package (no require entry): resolved from this script's own tree.
 const { Freestyle, FreestyleApiError } = await import("freestyle");
 
@@ -251,6 +252,9 @@ async function attachUntilReady(vmId, stage) {
       return {
         [`${stage}Ms`]: elapsedMs(startedAt),
         [`${stage}Attempts`]: attempts,
+        // The attach route's own per-stage timings (access check, provider
+        // probe, provider attach, lease), when the backend sends them.
+        [`${stage}Stages`]: parseServerTiming(response.headers.get("server-timing")),
         [`${stage}TrustedCarrier`]: body.trustedCarrier === true,
         [`${stage}RouteFamily`]: typeof body.route === "string" ? (body.route.includes("[") ? "ipv6" : "ipv4") : null,
         [`${stage}DaemonCommit`]: body.daemonBuild?.commit ?? null,
@@ -328,6 +332,9 @@ async function runTrial(trial) {
   trial.vmId = vmId;
   trial.imageVersion = created.imageVersion ?? null;
   trial.size = created.size?.name ?? null;
+  // Whether the create response carried the attach block a client can dial
+  // from without this attach-endpoint round trip (absent on older backends).
+  trial.createAttachBlock = created.attach ? { route: created.attach.route, trustedCarrier: created.attach.trustedCarrier, guestToolsBaked: created.attach.guestToolsBaked } : null;
   Object.assign(trial, await attachUntilReady(vmId, "attach"));
   // Create plus the attach-endpoint's own time: the route and lease exist,
   // but the link, the terminal and the shell prompt come after this point
@@ -881,6 +888,8 @@ function emitReport({ results, listMs, startedAt, runError, cleanup }) {
     stages: summarizeFields(measured, ["createMs", "attachMs", "createToAttachReadyMs", "warmAttachMs", "execMs", "edgeReadyMs", "pauseMs", "resumeAttachMs", "destroyMs"]),
     attachAttempts: summarizeFields(measured.map((trial) => ({ attempts: trial.attachAttempts?.length })), ["attempts"]).attempts,
     createServerTiming: summarizeStages(measured.map((trial) => trial.createStages)),
+    attachServerTiming: summarizeStages(measured.map((trial) => trial.attachStages)),
+    warmAttachServerTiming: summarizeStages(measured.map((trial) => trial.warmAttachStages)),
     results,
   };
   if (measured.length > 0) {

web/scripts/devbox-image-common.ts

@@ -23,6 +23,7 @@ import { CMUX_TUI_SESSION, cmuxTuiAsDaemonUser, cmuxTuiLayoutSelector, cmuxTuiRu
 import { DEVBOX_WORK_HOME, DEVBOX_WORK_USER } from "../services/vms/images/workUser";
 import { VM_IMAGE_SIZES, VM_IMAGE_SIZE_NAMES, vmImageSizeRank, type VmImageSizeName } from "../services/vms/images/sizes";
 import { DEVBOX_HOSTNAME, DEVBOX_HOSTNAME_LOOPBACK, DEVBOX_PROVIDER_HOSTNAME } from "../services/vms/images/identity";
+import { vmImageEntryEpoch } from "../services/vms/images/resolver";
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 export const webRoot = path.resolve(__dirname, "..");
@@ -1347,10 +1348,12 @@ export function upgradeDevboxSourceRecords(
   return { manifest: { schemaVersion: manifest.schemaVersion, images }, upgraded, skipped };
 }
 
-/** The epoch an entry was baked at: the field, or the `cmux devbox epoch <x>` prefix every bake writes into `notes`. */
-export function manifestEntryEpoch(entry: Pick<DevboxManifestEntry, "epoch" | "notes">): string | undefined {
-  return entry.epoch ?? /cmux devbox epoch (\S+)/.exec(entry.notes ?? "")?.[1];
-}
+/**
+ * The epoch an entry was baked at. One implementation: the runtime resolver
+ * owns it (the create response's attach block gates on the same reading), the
+ * bake and promotion scripts use it through this name.
+ */
+export const manifestEntryEpoch: (entry: Pick<DevboxManifestEntry, "epoch" | "notes">) => string | undefined = vmImageEntryEpoch;
 
 /**
  * The invariant that makes the checked-in manifest describe the machine

web/services/vms/attachContract.ts

@@ -0,0 +1,70 @@
+import { CMUX_TUI_SESSION } from "./drivers/cmuxTuiDaemon";
+import { freestyleCmuxRemoteRoute } from "./drivers/freestyle";
+import {
+  GUEST_TOOLS_BAKED_EPOCH,
+  TRUSTED_CARRIER_EPOCH,
+  imageEpochAtLeast,
+  vmImageEntryEpoch,
+  type VmImageManifestEntry,
+} from "./images/resolver";
+
+/**
+ * What a client needs to dial a machine's daemon straight from the create
+ * response, before any attach call: the transport and route, whether the
+ * daemon's cloud listener grants carrier authentication, the daemon build the
+ * image carries, and whether the guest tools are baked (no heal execs on
+ * attach). `readiness` is always `"dial"`: the daemon may still be starting,
+ * and the Noise handshake is the readiness proof.
+ *
+ * `attach-endpoint` keeps its own shape as the repair and reconnect path.
+ */
+export type VmAttachBlock = {
+  readonly transport: "cmux-remote";
+  readonly route: string;
+  readonly session: string;
+  readonly trustedCarrier: boolean;
+  readonly daemonBuild: {
+    readonly commit: string | null;
+    readonly remoteProtocol: null;
+    readonly version: null;
+  };
+  readonly guestToolsBaked: boolean;
+  readonly readiness: "dial";
+};
+
+export type VmAttachEntry = {
+  readonly addressIpv4: string | null;
+  readonly addressIpv6: string | null;
+  /** The epoch stamped on the row at create; null on older rows (the manifest entry answers then). */
+  readonly imageEpoch?: string | null;
+  readonly providerVmId?: string;
+};
+
+/**
+ * The attach block for a created machine, derived from the row and the
+ * checked-in manifest alone (no provider round trip). Null when the row holds
+ * no private address (the daemon is unreachable by address) or the image is
+ * not a manifest entry (nothing is known about its daemon).
+ *
+ * The route follows the driver's rule: IPv4 first because only the v4 path is
+ * reliable over the WireGuard tunnel, `[ipv6]` bracketed otherwise.
+ */
+export function createAttachBlock(input: {
+  readonly entry: VmAttachEntry;
+  readonly manifestEntry: VmImageManifestEntry | null;
+}): VmAttachBlock | null {
+  const { entry, manifestEntry } = input;
+  const ipv4 = entry.addressIpv4?.trim() || undefined;
+  const ipv6 = entry.addressIpv6?.trim() || undefined;
+  if (!manifestEntry || (!ipv4 && !ipv6)) return null;
+  const epoch = entry.imageEpoch ?? vmImageEntryEpoch(manifestEntry);
+  return {
+    transport: "cmux-remote",
+    route: freestyleCmuxRemoteRoute({ vpcs: [{ ipv4, ipv6 }] }, entry.providerVmId ?? "unknown"),
+    session: CMUX_TUI_SESSION,
+    trustedCarrier: imageEpochAtLeast(epoch, TRUSTED_CARRIER_EPOCH),
+    daemonBuild: { commit: manifestEntry.cmuxTuiCommit ?? null, remoteProtocol: null, version: null },
+    guestToolsBaked: imageEpochAtLeast(epoch, GUEST_TOOLS_BAKED_EPOCH),
+    readiness: "dial",
+  };
+}

web/services/vms/defer.ts

@@ -0,0 +1,32 @@
+/**
+ * Work a route may run once its response has left (usage-event rows,
+ * metadata backfills): the route supplies `runAfterResponse`, so the writes
+ * still happen but the client does not wait for them. Workflows given no
+ * sink run the work inline, as they always did.
+ */
+export type VmDeferSink = (work: () => Promise<void>) => void;
+
+/**
+ * A sink whose units run in the order they were handed in, each after the
+ * previous one settles, so deferred ledger rows keep their lifecycle order
+ * (`vm.create.requested` before `vm.created`) whatever the scheduler does
+ * with the callbacks it is given (Next's `after()` queue is concurrent; the
+ * detached fallback starts everything at once). A unit still starts only
+ * when the scheduler invokes it, and a failed unit never blocks the next.
+ */
+export function orderedDeferSink(schedule: VmDeferSink): VmDeferSink {
+  let tail: Promise<void> = Promise.resolve();
+  return (work) => {
+    const previous = tail;
+    let start: () => void = () => undefined;
+    const started = new Promise<void>((resolve) => {
+      start = resolve;
+    });
+    const run = started.then(() => previous).then(work);
+    tail = run.catch(() => undefined);
+    schedule(() => {
+      start();
+      return run;
+    });
+  };
+}

web/services/vms/drivers/freestyle.ts

@@ -1750,8 +1750,7 @@ export class FreestyleProvider implements VMProvider {
   private async installGuestCliFiles(vm: Vm, vmId: string, promptIdentity?: GuestPromptIdentity): Promise<void> {
     const temporaryPath = `${GUEST_CMUX_SHIM_PATH}.tmp-${randomBytes(12).toString("hex")}`;
     try {
-      await this.execOrThrow(vm, vmId, "mkdir -p /usr/local/libexec", 5_000);
-      await vm.fs.writeTextFile(temporaryPath, GUEST_CMUX_SHIM, { mode: 0o755 });
+      await this.uploadGuestShim(vm, vmId, temporaryPath);
       const result = await vm.exec({
         command: `${guestBrowserInstallCommand()} && chmod 0755 '${temporaryPath}' && mv -f '${temporaryPath}' '${GUEST_CMUX_SHIM_PATH}' && ${guestCliDistributionCommand()}`
           + (promptIdentity ? ` && ${guestPromptInstallCommand(promptIdentity)}` : ""),
@@ -1768,6 +1767,22 @@ export class FreestyleProvider implements VMProvider {
     }
   }
 
+  /**
+   * The bake creates the shim's directory (`/usr/local/libexec`, the
+   * opencode launcher step), so the upload goes straight in; an image from
+   * before that step rejects it once, and the directory is created and the
+   * upload retried instead of paying a mkdir round trip on every create.
+   */
+  private async uploadGuestShim(vm: Vm, vmId: string, temporaryPath: string): Promise<void> {
+    try {
+      await vm.fs.writeTextFile(temporaryPath, GUEST_CMUX_SHIM, { mode: 0o755 });
+    } catch (error) {
+      if (!isMissingGuestDirectoryError(error)) throw error;
+      await this.execOrThrow(vm, vmId, "mkdir -p /usr/local/libexec", 5_000);
+      await vm.fs.writeTextFile(temporaryPath, GUEST_CMUX_SHIM, { mode: 0o755 });
+    }
+  }
+
   private async execResult(vm: Vm, command: string, timeoutMs = EXEC_DEFAULT_TIMEOUT_MS): Promise<ExecResult | null> {
     try {
       const r = await vm.exec({ command, timeoutMs, linuxUser: GUEST_LINUX_USER });
@@ -1794,6 +1809,14 @@ export class FreestyleProvider implements VMProvider {
   }
 }
 
+/** A filesystem write refused because the target directory does not exist in the guest. */
+function isMissingGuestDirectoryError(error: unknown): boolean {
+  if (error instanceof FreestyleApiError) {
+    return error.status === 404 || /no such file|not found|enoent/i.test(error.message);
+  }
+  return error instanceof Error && /no such file|enoent/i.test(error.message);
+}
+
 /** The resources a machine of `memoryMb` is sold with (see entitlements.ts). */
 export function freestyleTargetResources(
   memoryMb: number,