fix(e2e): reap connect after dashboard forward handoff

test/e2e/live/dashboard-connect-handoff.ts

@@ -0,0 +1,208 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import type { ChildProcess } from "node:child_process";
+
+import type { ArtifactSink } from "../fixtures/artifacts.ts";
+import {
+  type ChildProcessProgress,
+  spawnObservedChild,
+} from "../fixtures/observed-child-process.ts";
+import { REPO_ROOT } from "../fixtures/paths.ts";
+import { resolveLiveE2eWorkloadSourceEnv } from "../fixtures/workload-source-env.ts";
+import { dashboardRemoteBindConnectStarted } from "./dashboard-remote-bind-env.ts";
+
+const CONNECT_CAPTURE_LIMIT_BYTES = 1024 * 1024;
+const CONNECT_STOP_GRACE_MS = 5_000;
+
+export interface DashboardConnectHandoffResult {
+  readonly exitCode: number | null;
+  readonly proof: "command-completed" | "forward-started";
+  readonly signal: NodeJS.Signals | null;
+  readonly stderr: string;
+  readonly stdout: string;
+}
+
+export interface DashboardConnectHandoffOptions {
+  readonly artifacts: ArtifactSink;
+  readonly command?: readonly [string, ...string[]];
+  readonly env: NodeJS.ProcessEnv;
+  readonly progress: ChildProcessProgress;
+  readonly sandboxName: string;
+  readonly signal?: AbortSignal;
+  readonly stopGraceMs?: number;
+  readonly timeoutMs: number;
+  readonly dashboardPort: string;
+}
+
+function signalChild(child: ChildProcess, signal: NodeJS.Signals): void {
+  try {
+    child.kill(signal);
+  } catch {
+    // The child may have exited between the proof callback and cleanup.
+  }
+}
+
+function signalChildGroup(child: ChildProcess, signal: NodeJS.Signals): void {
+  try {
+    if (child.pid !== undefined) {
+      process.kill(-child.pid, signal);
+      return;
+    }
+  } catch {
+    // Fall back to the group leader when the process group is already gone.
+  }
+  signalChild(child, signal);
+}
+
+function appendCaptured(current: string, chunk: string): string {
+  const next = current + chunk;
+  if (Buffer.byteLength(next, "utf8") > CONNECT_CAPTURE_LIMIT_BYTES) {
+    throw new Error("dashboard connect output exceeded the 1 MiB capture limit");
+  }
+  return next;
+}
+
+/**
+ * Observe ordinary interactive `connect` until it either finishes normally or
+ * proves that forward recovery completed. A proof stops only the connect group
+ * leader first: NemoClaw forwards SIGTERM to its attached OpenShell shell,
+ * while a correctly backgrounded dashboard forward has already detached its
+ * descriptors and remains available for the caller's independent health check.
+ */
+export async function runDashboardConnectUntilForwardHandoff(
+  options: DashboardConnectHandoffOptions,
+): Promise<DashboardConnectHandoffResult> {
+  if (!Number.isFinite(options.timeoutMs) || options.timeoutMs <= 0) {
+    throw new RangeError("dashboard connect handoff timeout must be a positive finite value");
+  }
+  const stopGraceMs = options.stopGraceMs ?? CONNECT_STOP_GRACE_MS;
+  if (!Number.isFinite(stopGraceMs) || stopGraceMs <= 0) {
+    throw new RangeError("dashboard connect stop grace must be a positive finite value");
+  }
+
+  const [command, ...args] = options.command ?? ["nemoclaw", options.sandboxName, "connect"];
+  const child = spawnObservedChild(command, args, {
+    activityLabel: "command: dashboard-remote-bind-connect",
+    progress: options.progress,
+    spawn: {
+      cwd: REPO_ROOT,
+      detached: true,
+      env: resolveLiveE2eWorkloadSourceEnv({ ...options.env }),
+      stdio: ["ignore", "pipe", "pipe"],
+    },
+  });
+
+  let stdout = "";
+  let stderr = "";
+  let forwardProof = false;
+  let proofStopRequested = false;
+  let deadlineExpired = false;
+  let aborted = false;
+  let cleanupEscalated = false;
+  let captureError: Error | null = null;
+  let forceKillTimer: NodeJS.Timeout | undefined;
+
+  const scheduleForcedCleanup = (): void => {
+    if (forceKillTimer) return;
+    forceKillTimer = setTimeout(() => {
+      cleanupEscalated = true;
+      signalChildGroup(child, "SIGKILL");
+    }, stopGraceMs);
+  };
+  const terminateGroup = (): void => {
+    signalChildGroup(child, "SIGTERM");
+    scheduleForcedCleanup();
+  };
+  const requestProofStop = (): void => {
+    if (proofStopRequested) return;
+    proofStopRequested = true;
+    signalChild(child, "SIGTERM");
+    scheduleForcedCleanup();
+  };
+  const inspectProof = (): void => {
+    if (forwardProof || captureError) return;
+    forwardProof = dashboardRemoteBindConnectStarted(
+      { exitCode: null, stdout, stderr },
+      options.sandboxName,
+      options.dashboardPort,
+    );
+    if (forwardProof) requestProofStop();
+  };
+  const capture = (stream: "stdout" | "stderr", chunk: Buffer | string): void => {
+    if (captureError) return;
+    try {
+      if (stream === "stdout") stdout = appendCaptured(stdout, chunk.toString());
+      else stderr = appendCaptured(stderr, chunk.toString());
+      inspectProof();
+    } catch (error) {
+      captureError = error instanceof Error ? error : new Error(String(error));
+      terminateGroup();
+    }
+  };
+  child.stdout?.on("data", (chunk: Buffer | string) => capture("stdout", chunk));
+  child.stderr?.on("data", (chunk: Buffer | string) => capture("stderr", chunk));
+
+  const deadline = setTimeout(() => {
+    deadlineExpired = true;
+    terminateGroup();
+  }, options.timeoutMs);
+  const abort = (): void => {
+    aborted = true;
+    terminateGroup();
+  };
+  if (options.signal?.aborted) abort();
+  else options.signal?.addEventListener("abort", abort, { once: true });
+
+  let spawnError: Error | null = null;
+  child.once("error", (error) => {
+    spawnError = error;
+  });
+  const { exitCode, signal } = await new Promise<{
+    exitCode: number | null;
+    signal: NodeJS.Signals | null;
+  }>((resolve) => {
+    child.once("close", (code, closeSignal) => resolve({ exitCode: code, signal: closeSignal }));
+  });
+  clearTimeout(deadline);
+  if (forceKillTimer) clearTimeout(forceKillTimer);
+  options.signal?.removeEventListener("abort", abort);
+
+  const artifactBase = "dashboard-connect-handoff";
+  const artifactPaths = {
+    stdout: await options.artifacts.writeText(`${artifactBase}.stdout.txt`, stdout),
+    stderr: await options.artifacts.writeText(`${artifactBase}.stderr.txt`, stderr),
+  };
+  await options.artifacts.writeJson(`${artifactBase}.result.json`, {
+    command: [command, ...args],
+    exitCode,
+    signal,
+    deadlineExpired,
+    cleanupEscalated,
+    forwardProof,
+    proofStopRequested,
+    stdout: artifactPaths.stdout,
+    stderr: artifactPaths.stderr,
+  });
+
+  if (spawnError) throw spawnError;
+  if (captureError) throw captureError;
+  if (aborted) throw new Error("dashboard connect handoff was cancelled");
+  if (deadlineExpired) {
+    throw new Error("dashboard connect did not complete or prove forward handoff within budget");
+  }
+  if (forwardProof) {
+    if (cleanupEscalated) {
+      throw new Error(
+        "dashboard connect retained captured descriptors after forward proof and required forced cleanup",
+      );
+    }
+    return { exitCode, proof: "forward-started", signal, stderr, stdout };
+  }
+  if (exitCode === 0) {
+    return { exitCode, proof: "command-completed", signal, stderr, stdout };
+  }
+  throw new Error(
+    `dashboard connect exited before proving forward handoff (exit ${exitCode ?? "unknown"}${signal ? `, signal ${signal}` : ""})`,
+  );
+}

test/e2e/live/dashboard-remote-bind-env.ts

@@ -41,3 +41,8 @@ export function dashboardRemoteBindConnectStarted(
           output.includes(`sandbox ${sandboxName}`))))
   );
 }
+
+export function dashboardForwardIsRunning(forwardLine: string): boolean {
+  const columns = forwardLine.trim().split(/\s+/u);
+  return columns.length === 5 && columns[4] === "running";
+}

test/e2e/live/dashboard-remote-bind.test.ts

@@ -9,9 +9,10 @@ import { sandboxAccessEnv, trustedSandboxShellScript } from "../fixtures/clients
 import { expect, test } from "../fixtures/e2e-test.ts";
 import { requireHostedInferenceConfig } from "../fixtures/hosted-inference.ts";
 import { CLI_ENTRYPOINT, REPO_ROOT } from "../fixtures/paths.ts";
+import { runDashboardConnectUntilForwardHandoff } from "./dashboard-connect-handoff.ts";
 import {
   buildDashboardRemoteBindEnv,
-  dashboardRemoteBindConnectStarted,
+  dashboardForwardIsRunning,
 } from "./dashboard-remote-bind-env.ts";
 import { parseJsonFromText } from "./json-envelope.ts";
 
@@ -183,15 +184,19 @@ runDashboardRemoteBindTest(
       timeoutMs: 30_000,
     });
 
-    const connect = await host.nemoclaw([sandboxName, "connect"], {
-      artifactName: "dashboard-remote-bind-connect",
+    const connect = await runDashboardConnectUntilForwardHandoff({
+      artifacts,
+      dashboardPort,
       env: testEnv(),
+      progress,
+      sandboxName,
+      signal: cleanup.currentSignal(),
       timeoutMs: 120_000,
     });
     expect(
-      dashboardRemoteBindConnectStarted(connect, sandboxName, dashboardPort),
-      `nemoclaw connect did not complete or print background-forward proof\nstdout:\n${connect.stdout}\nstderr:\n${connect.stderr}`,
-    ).toBe(true);
+      connect.proof,
+      "nemoclaw connect did not complete or print background-forward proof; see the dashboard-connect-handoff.stdout.txt and dashboard-connect-handoff.stderr.txt artifacts",
+    ).toBe("forward-started");
 
     progress.phase("verify all-interface dashboard forward");
     const forwardList = await sandbox.openshell(["forward", "list"], {
@@ -207,6 +212,10 @@ runDashboardRemoteBindTest(
       forwardLine,
       `No OpenShell forward found for ${sandboxName} on ${dashboardPort}`,
     ).not.toBe("");
+    expect(
+      dashboardForwardIsRunning(forwardLine),
+      `Dashboard forward is not running after connect handoff: ${forwardLine}`,
+    ).toBe(true);
     expect(
       bindsLoopback(forwardLine, dashboardPort),
       `Dashboard forward is still localhost-only; expected an all-interface bind: ${forwardLine}`,
@@ -216,6 +225,30 @@ runDashboardRemoteBindTest(
       `Could not prove dashboard forward uses 0.0.0.0:${dashboardPort}: ${forwardLine}`,
     ).toBe(true);
 
+    const forwardReachable = await host.command(
+      process.execPath,
+      [
+        "-e",
+        [
+          'const net = require("node:net");',
+          "const socket = net.connect({ host: '127.0.0.1', port: Number(process.argv[1]) });",
+          "const deadline = setTimeout(() => { socket.destroy(); process.exit(1); }, 5000);",
+          "socket.once('connect', () => { clearTimeout(deadline); socket.destroy(); process.exit(0); });",
+          "socket.once('error', () => { clearTimeout(deadline); process.exit(1); });",
+        ].join("\n"),
+        dashboardPort,
+      ],
+      {
+        artifactName: "dashboard-remote-bind-post-handoff-reachability",
+        env: testEnv(),
+        timeoutMs: 10_000,
+      },
+    );
+    expect(
+      forwardReachable.exitCode,
+      `Dashboard forward is unreachable after connect handoff\n${resultText(forwardReachable)}`,
+    ).toBe(0);
+
     progress.phase("audit exposed dashboard controls");
     const audit = await sandbox.execShell(
       sandboxName,