fix(cloud): eliminate first-machine connection cold start

web/app/api/vm/route.ts

@@ -92,6 +92,7 @@ import { getGoVmUsage, GO_SAVED_VM_LIMIT } from "../../../services/vms/goUsage";
 // stuck-provisioning alert. The plan allows more (app/v1/responses/route.ts
 // uses 1800).
 export const maxDuration = 600;
+const VM_CREATE_ADMISSION_BUDGET_MS = 200;
 
 export async function GET(request: Request): Promise<Response> {
   return withAuthedVmApiRoute(
@@ -100,6 +101,11 @@ export async function GET(request: Request): Promise<Response> {
     { "cmux.vm.operation": "list" },
     "/api/vm GET failed",
     async ({ user, span }) => {
+      // List traffic is the natural idle-path signal from the Cloud sidebar.
+      // Start warming only after auth so an unauthenticated poll cannot open a
+      // provider socket or database pool on every fresh function instance.
+      void preconnectCloudDb();
+      runAfterResponse(() => preconnectFreestyle());
       let billingTeamId: string | null = null;
       let listEntitlements: ReturnType<typeof resolveVmEntitlements> | null = null;
       const requestedBillingTeamId = requestedVmTeamIdFromRequest(request);
@@ -234,9 +240,10 @@ export async function GET(request: Request): Promise<Response> {
 }
 
 export async function POST(request: Request): Promise<Response> {
-  // Warm the Freestyle and database connections while the caller is being verified.
-  preconnectFreestyle();
-  preconnectCloudDb();
+  // Database warming is an optimization only: its driver may wait on a
+  // provider-controlled connect deadline, so it must never delay validation or
+  // turn an authenticated create into an unbounded database health check.
+  void preconnectCloudDb();
   return withAuthedVmApiRoute(
     request,
     "/api/vm",
@@ -245,7 +252,31 @@ export async function POST(request: Request): Promise<Response> {
     async ({ user: initialUser, span, authDurationMs, routeStartedAtMs, setResponseFinalizer }) => {
       const timing = new VmTimingRecorder(span, "create", { startedAt: routeStartedAtMs });
       timing.record("auth", authDurationMs);
+      // Start the provider probe only after authentication. It is shared by
+      // concurrent creates, so the first authenticated request pays the cold
+      // connection once and unauthenticated traffic cannot consume provider
+      // capacity. The await below is bounded by the probe's own timeout.
+      const warmupStartedAt = performance.now();
+      const freestyleWarmup = preconnectFreestyle();
+      const connectionInitDuration = freestyleWarmup.then(
+        () => ({ durationMs: performance.now() - warmupStartedAt, endedAtMs: Date.now() }),
+      );
+      let admissionRecorded = false;
+      let admissionStartedAt = performance.now();
+      /** Records request validation even when it exits before provisioning. */
+      const recordAdmission = () => {
+        if (admissionRecorded) return;
+        admissionRecorded = true;
+        const durationMs = performance.now() - admissionStartedAt;
+        timing.record("admission", durationMs);
+        setSpanAttributes(span, {
+          "cmux.vm.admission_ms": Math.round(durationMs * 100) / 100,
+          "cmux.vm.admission_budget_ms": VM_CREATE_ADMISSION_BUDGET_MS,
+          "cmux.vm.admission_within_budget": durationMs <= VM_CREATE_ADMISSION_BUDGET_MS,
+        });
+      };
       setResponseFinalizer((response) => {
+        recordAdmission();
         timing.finish({ status: response.status });
         // Per-stage timings travel with the response too, so a client or a
         // smoke run sees where a create spent its time without Axiom.
@@ -294,6 +325,19 @@ export async function POST(request: Request): Promise<Response> {
         "cmux.idempotency_key_set": !!idempotencyKey,
       });
 
+      // Only Freestyle creation needs this probe. Other providers must not
+      // wait behind an unrelated connection check, while the Freestyle path
+      // still overlaps the probe with authentication and request parsing.
+      if (provider === "freestyle") {
+        const connectionInit = await connectionInitDuration;
+        timing.record("connection_init", connectionInit.durationMs, { endedAtMs: connectionInit.endedAtMs });
+      }
+      // Admission starts after provider-specific connection readiness. Its
+      // budget describes only request validation; the durable begin_create
+      // phase is recorded inside the workflow and remains authoritative.
+      admissionStartedAt = performance.now();
+      recordAdmission();
+
       // Wire the machine to coderouter inside the workflow: the route token
       // is bound to the VM row id, so provisioning runs after the row exists
       // and before the provider call, and a failure fails the create.

web/db/client.ts

@@ -300,12 +300,13 @@ function coalesceWarm(run: () => Promise<void>): () => Promise<void> {
  * first success a no-op for the life of the process, so calling it ahead of
  * auth cannot amplify an unauthenticated burst beyond one pooled connection.
  */
-export function preconnectCloudDb(): void {
+export function preconnectCloudDb(): Promise<void> {
   try {
     cloudDb();
-    void globalForDb.__cmuxCloudDb?.warm().catch(() => undefined);
+    return globalForDb.__cmuxCloudDb?.warm().catch(() => undefined) ?? Promise.resolve();
   } catch {
     // No database configured (tests, offline builds): the first query reports it.
+    return Promise.resolve();
   }
 }
 

web/scripts/cloud-vm/bench-vm-startup.mjs

@@ -15,13 +15,14 @@ import { pathToFileURL } from "node:url";
 import { elapsedMs, formatSummary, ownerNetworkSlug, parseServerTiming, pollBoundedFetch, providerCredentialsFromEnv, summarizeFields, summarizeStages } from "./benchStats.mjs";
 import { loadTargetEnv, optionValue, parseWebDirAndTarget, requireEnvKeys, runVercel } from "./projects.mjs";
 
-const usage = "Usage: bench-vm-startup.mjs [web-dir] <staging|production> [--trials N] [--concurrency K] [--url https://preview.example] [--allow-preview] [--allow-any-url] [--skip-pause] [--skip-exec] [--edge-check] [--edge-alias <host>] [--label <text>] [--out <file.json>]";
+const usage = "Usage: bench-vm-startup.mjs [web-dir] <staging|production> [--trials N] [--concurrency K] [--url https://preview.example] [--allow-preview] [--allow-any-url] [--skip-initial-list] [--skip-pause] [--skip-exec] [--edge-check] [--edge-alias <host>] [--label <text>] [--out <file.json>]";
 const { webDir, target, project, rest } = parseWebDirAndTarget(process.argv.slice(2), usage);
 const trials = positiveInteger(optionValue(rest, "--trials") ?? "3", "--trials");
 const concurrency = Math.min(positiveInteger(optionValue(rest, "--concurrency") ?? "1", "--concurrency"), trials);
 const targetUrl = resolveTargetUrl(project, rest);
 const skipPause = rest.includes("--skip-pause");
 const skipExec = rest.includes("--skip-exec");
+const skipInitialList = rest.includes("--skip-initial-list");
 // Full-feature readiness: poll the model-plane edge alias from inside the
 // guest until the coderouter reflection route answers, so the report can
 // separate "terminal usable" from "agents can reach their credentials". The
@@ -48,6 +49,10 @@ const REQUEST_TIMEOUT_MS = 330_000;
 const CREATE_TIMEOUT_MS = 630_000;
 // Bounds when a new attach attempt may start; it never cuts a request short.
 const ATTACH_BUDGET_MS = 180_000;
+// The durable row should become visible immediately after the create response;
+// keep this separate from provider/daemon readiness so a lagging list index is
+// visible instead of being folded into attach time.
+const ROW_READY_BUDGET_MS = 60_000;
 
 const requireFromWeb = createRequire(path.join(webDir, "package.json"));
 const { StackServerApp } = await import(pathToFileURL(requireFromWeb.resolve("@hexclave/js")).href);
@@ -223,7 +228,7 @@ function json(text) {
 
 const vmUrl = (vmId, tail = "") => `${targetUrl}/api/vm/${encodeURIComponent(vmId)}${tail}`;
 
-/** Attach until the daemon answers; a 502 with `retryable` is the documented not-ready contract. */
+/** Request endpoint metadata until the attach contract is issued; this does not dial the daemon. */
 async function attachUntilReady(vmId, stage) {
   const startedAt = performance.now();
   const attempts = [];
@@ -266,6 +271,34 @@ async function attachUntilReady(vmId, stage) {
   }
 }
 
+/** Reads the authoritative VM row after create; the detail endpoint is the completion signal. */
+async function readAuthoritativeRow(vmId) {
+  const startedAt = performance.now();
+  if (interrupted) throw new Error(`VM row for ${vmId} was interrupted before readiness`);
+  const response = await fetchTimed(
+    vmUrl(vmId),
+    { headers: authHeaders },
+    Math.min(REQUEST_TIMEOUT_MS, ROW_READY_BUDGET_MS),
+  );
+  const body = json(response.text);
+  const attempt = { status: response.status, ms: response.ms };
+  if (response.status !== 200) {
+    throw new Error(`VM row for ${vmId} was not readable after create: ${response.status} ${response.text.slice(0, 300)}`);
+  }
+  if (body.status === "failed") {
+    throw new Error(`VM row for ${vmId} became failed while waiting for readiness`);
+  }
+  if (body.id !== vmId || body.status !== "running") {
+    throw new Error(`VM row for ${vmId} was not running after create: ${response.text.slice(0, 300)}`);
+  }
+  return {
+    rowReadyMs: elapsedMs(startedAt),
+    rowReadyStatus: body.status,
+    rowReadyAttempts: [attempt],
+    rowReadySource: "detail",
+  };
+}
+
 /** Time from the first probe until the edge alias answers with an HTTP status (any status proves injection). */
 async function edgeReady(vmId) {
   const startedAt = performance.now();
@@ -297,6 +330,7 @@ async function runTrial(trial) {
   const { index } = trial;
   const idempotencyKey = `bench-${suffix}-${index}`;
   const createRequestedAt = Date.now();
+  const createStartedAt = performance.now();
   let create;
   try {
     create = await fetchTimed(`${targetUrl}/api/vm`, {
@@ -313,6 +347,7 @@ async function runTrial(trial) {
   trial.createMs = create.ms;
   trial.createStatus = create.status;
   trial.createTraceId = create.headers.get("x-cmux-trace-id");
+  trial.createVercelId = create.headers.get("x-vercel-id");
   trial.createStages = parseServerTiming(create.headers.get("server-timing"));
   if (create.status >= 500) {
     // A 5xx (a gateway or function timeout above all) can end the invocation
@@ -328,12 +363,13 @@ async function runTrial(trial) {
   trial.vmId = vmId;
   trial.imageVersion = created.imageVersion ?? null;
   trial.size = created.size?.name ?? null;
-  Object.assign(trial, await attachUntilReady(vmId, "attach"));
-  // Create plus the attach-endpoint's own time: the route and lease exist,
-  // but the link, the terminal and the shell prompt come after this point
-  // (bench-private-link.ts measures those), so this is attach readiness,
-  // not a usable terminal.
-  trial.createToAttachReadyMs = trial.createMs + trial.attachMs;
+  Object.assign(trial, await readAuthoritativeRow(vmId));
+  trial.createToRowReadyMs = elapsedMs(createStartedAt);
+  Object.assign(trial, await attachUntilReady(vmId, "attachEndpoint"));
+  // The authoritative row read is serialized before endpoint issuance. Use
+  // one monotonic origin so totals include both phases; endpoint issuance is
+  // not a terminal-ready claim (bench-private-link.ts measures the prompt).
+  trial.createToAttachEndpointMs = elapsedMs(createStartedAt);
   Object.assign(trial, await attachUntilReady(vmId, "warmAttach"));
   if (!skipExec) {
     const exec = await fetchTimed(vmUrl(vmId, "/exec"), {
@@ -739,7 +775,17 @@ async function mintSessionHeaders(expiresInMillis) {
   const session = await withTimeout(user.createSession({ expiresInMillis, isImpersonation: true }), STACK_TIMEOUT_MS, "Stack createSession");
   const tokens = await withTimeout(session.getTokens(), STACK_TIMEOUT_MS, "Stack getTokens");
   if (!tokens.accessToken || !tokens.refreshToken) throw new Error("Stack did not return bench session tokens");
-  return { authorization: `Bearer ${tokens.accessToken}`, "x-stack-refresh-token": tokens.refreshToken };
+  // Vercel SSO-protected unaliased deployments accept this header when the
+  // operator supplies the project-scoped automation bypass secret. Keep it
+  // scoped to a non-canonical target and never include it in reports/logs.
+  const canonicalHost = new URL(project.url).host;
+  const targetHost = new URL(targetUrl).host;
+  const bypassSecret = targetHost !== canonicalHost ? process.env.VERCEL_AUTOMATION_BYPASS_SECRET?.trim() : undefined;
+  return {
+    authorization: `Bearer ${tokens.accessToken}`,
+    "x-stack-refresh-token": tokens.refreshToken,
+    ...(bypassSecret ? { "x-vercel-protection-bypass": bypassSecret } : {}),
+  };
 }
 
 /** Everything teardown learned, so the report can say what really happened. */
@@ -878,8 +924,8 @@ function emitReport({ results, listMs, startedAt, runError, cleanup }) {
     totalMs: startedAt === null ? null : elapsedMs(startedAt),
     succeeded: ok.length,
     failed: results.length - ok.length,
-    stages: summarizeFields(measured, ["createMs", "attachMs", "createToAttachReadyMs", "warmAttachMs", "execMs", "edgeReadyMs", "pauseMs", "resumeAttachMs", "destroyMs"]),
-    attachAttempts: summarizeFields(measured.map((trial) => ({ attempts: trial.attachAttempts?.length })), ["attempts"]).attempts,
+    stages: summarizeFields(measured, ["createMs", "rowReadyMs", "createToRowReadyMs", "attachEndpointMs", "createToAttachEndpointMs", "warmAttachMs", "execMs", "edgeReadyMs", "pauseMs", "resumeAttachMs", "destroyMs"]),
+    attachAttempts: summarizeFields(measured.map((trial) => ({ attempts: trial.attachEndpointAttempts?.length })), ["attempts"]).attempts,
     createServerTiming: summarizeStages(measured.map((trial) => trial.createStages)),
     results,
   };
@@ -911,13 +957,15 @@ try {
   // request), exec, pause and destroy requests and the edge probe, in
   // ceil(trials / concurrency) rounds; capped at a day. Cleanup mints its
   // own fresh session, so it does not depend on this one.
-  const trialWorstMs = CREATE_TIMEOUT_MS + 3 * (ATTACH_BUDGET_MS + REQUEST_TIMEOUT_MS) + 3 * REQUEST_TIMEOUT_MS + EDGE_BUDGET_MS + REQUEST_TIMEOUT_MS;
+  const trialWorstMs = CREATE_TIMEOUT_MS + ROW_READY_BUDGET_MS + 3 * (ATTACH_BUDGET_MS + REQUEST_TIMEOUT_MS) + 3 * REQUEST_TIMEOUT_MS + EDGE_BUDGET_MS + REQUEST_TIMEOUT_MS;
   const sessionMs = Math.min(24 * 60 * 60 * 1000, 30 * 60 * 1000 + Math.ceil(trials / concurrency) * trialWorstMs);
   authHeaders = await mintSessionHeaders(sessionMs);
 
-  const list = await fetchTimed(`${targetUrl}/api/vm`, { headers: authHeaders });
-  if (list.status !== 200) throw new Error(`authenticated GET /api/vm expected 200, got ${list.status}: ${list.text.slice(0, 200)}`);
-  listMs = list.ms;
+  if (!skipInitialList) {
+    const list = await fetchTimed(`${targetUrl}/api/vm`, { headers: authHeaders });
+    if (list.status !== 200) throw new Error(`authenticated GET /api/vm expected 200, got ${list.status}: ${list.text.slice(0, 200)}`);
+    listMs = list.ms;
+  }
 
   startedAt = performance.now();
   results = await runBatches();