diff --git a/test/e2e/live/dashboard-connect-handoff.ts b/test/e2e/live/dashboard-connect-handoff.ts new file mode 100644 index 00000000000..e05dafd0cc1 --- /dev/null +++ b/test/e2e/live/dashboard-connect-handoff.ts @@ -0,0 +1,208 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { ChildProcess } from "node:child_process"; + +import type { ArtifactSink } from "../fixtures/artifacts.ts"; +import { + type ChildProcessProgress, + spawnObservedChild, +} from "../fixtures/observed-child-process.ts"; +import { REPO_ROOT } from "../fixtures/paths.ts"; +import { resolveLiveE2eWorkloadSourceEnv } from "../fixtures/workload-source-env.ts"; +import { dashboardRemoteBindConnectStarted } from "./dashboard-remote-bind-env.ts"; + +const CONNECT_CAPTURE_LIMIT_BYTES = 1024 * 1024; +const CONNECT_STOP_GRACE_MS = 5_000; + +export interface DashboardConnectHandoffResult { + readonly exitCode: number | null; + readonly proof: "command-completed" | "forward-started"; + readonly signal: NodeJS.Signals | null; + readonly stderr: string; + readonly stdout: string; +} + +export interface DashboardConnectHandoffOptions { + readonly artifacts: ArtifactSink; + readonly command?: readonly [string, ...string[]]; + readonly env: NodeJS.ProcessEnv; + readonly progress: ChildProcessProgress; + readonly sandboxName: string; + readonly signal?: AbortSignal; + readonly stopGraceMs?: number; + readonly timeoutMs: number; + readonly dashboardPort: string; +} + +function signalChild(child: ChildProcess, signal: NodeJS.Signals): void { + try { + child.kill(signal); + } catch { + // The child may have exited between the proof callback and cleanup. + } +} + +function signalChildGroup(child: ChildProcess, signal: NodeJS.Signals): void { + try { + if (child.pid !== undefined) { + process.kill(-child.pid, signal); + return; + } + } catch { + // Fall back to the group leader when the process group is already gone. + } + signalChild(child, signal); +} + +function appendCaptured(current: string, chunk: string): string { + const next = current + chunk; + if (Buffer.byteLength(next, "utf8") > CONNECT_CAPTURE_LIMIT_BYTES) { + throw new Error("dashboard connect output exceeded the 1 MiB capture limit"); + } + return next; +} + +/** + * Observe ordinary interactive `connect` until it either finishes normally or + * proves that forward recovery completed. A proof stops only the connect group + * leader first: NemoClaw forwards SIGTERM to its attached OpenShell shell, + * while a correctly backgrounded dashboard forward has already detached its + * descriptors and remains available for the caller's independent health check. + */ +export async function runDashboardConnectUntilForwardHandoff( + options: DashboardConnectHandoffOptions, +): Promise { + if (!Number.isFinite(options.timeoutMs) || options.timeoutMs <= 0) { + throw new RangeError("dashboard connect handoff timeout must be a positive finite value"); + } + const stopGraceMs = options.stopGraceMs ?? CONNECT_STOP_GRACE_MS; + if (!Number.isFinite(stopGraceMs) || stopGraceMs <= 0) { + throw new RangeError("dashboard connect stop grace must be a positive finite value"); + } + + const [command, ...args] = options.command ?? ["nemoclaw", options.sandboxName, "connect"]; + const child = spawnObservedChild(command, args, { + activityLabel: "command: dashboard-remote-bind-connect", + progress: options.progress, + spawn: { + cwd: REPO_ROOT, + detached: true, + env: resolveLiveE2eWorkloadSourceEnv({ ...options.env }), + stdio: ["ignore", "pipe", "pipe"], + }, + }); + + let stdout = ""; + let stderr = ""; + let forwardProof = false; + let proofStopRequested = false; + let deadlineExpired = false; + let aborted = false; + let cleanupEscalated = false; + let captureError: Error | null = null; + let forceKillTimer: NodeJS.Timeout | undefined; + + const scheduleForcedCleanup = (): void => { + if (forceKillTimer) return; + forceKillTimer = setTimeout(() => { + cleanupEscalated = true; + signalChildGroup(child, "SIGKILL"); + }, stopGraceMs); + }; + const terminateGroup = (): void => { + signalChildGroup(child, "SIGTERM"); + scheduleForcedCleanup(); + }; + const requestProofStop = (): void => { + if (proofStopRequested) return; + proofStopRequested = true; + signalChild(child, "SIGTERM"); + scheduleForcedCleanup(); + }; + const inspectProof = (): void => { + if (forwardProof || captureError) return; + forwardProof = dashboardRemoteBindConnectStarted( + { exitCode: null, stdout, stderr }, + options.sandboxName, + options.dashboardPort, + ); + if (forwardProof) requestProofStop(); + }; + const capture = (stream: "stdout" | "stderr", chunk: Buffer | string): void => { + if (captureError) return; + try { + if (stream === "stdout") stdout = appendCaptured(stdout, chunk.toString()); + else stderr = appendCaptured(stderr, chunk.toString()); + inspectProof(); + } catch (error) { + captureError = error instanceof Error ? error : new Error(String(error)); + terminateGroup(); + } + }; + child.stdout?.on("data", (chunk: Buffer | string) => capture("stdout", chunk)); + child.stderr?.on("data", (chunk: Buffer | string) => capture("stderr", chunk)); + + const deadline = setTimeout(() => { + deadlineExpired = true; + terminateGroup(); + }, options.timeoutMs); + const abort = (): void => { + aborted = true; + terminateGroup(); + }; + if (options.signal?.aborted) abort(); + else options.signal?.addEventListener("abort", abort, { once: true }); + + let spawnError: Error | null = null; + child.once("error", (error) => { + spawnError = error; + }); + const { exitCode, signal } = await new Promise<{ + exitCode: number | null; + signal: NodeJS.Signals | null; + }>((resolve) => { + child.once("close", (code, closeSignal) => resolve({ exitCode: code, signal: closeSignal })); + }); + clearTimeout(deadline); + if (forceKillTimer) clearTimeout(forceKillTimer); + options.signal?.removeEventListener("abort", abort); + + const artifactBase = "dashboard-connect-handoff"; + const artifactPaths = { + stdout: await options.artifacts.writeText(`${artifactBase}.stdout.txt`, stdout), + stderr: await options.artifacts.writeText(`${artifactBase}.stderr.txt`, stderr), + }; + await options.artifacts.writeJson(`${artifactBase}.result.json`, { + command: [command, ...args], + exitCode, + signal, + deadlineExpired, + cleanupEscalated, + forwardProof, + proofStopRequested, + stdout: artifactPaths.stdout, + stderr: artifactPaths.stderr, + }); + + if (spawnError) throw spawnError; + if (captureError) throw captureError; + if (aborted) throw new Error("dashboard connect handoff was cancelled"); + if (deadlineExpired) { + throw new Error("dashboard connect did not complete or prove forward handoff within budget"); + } + if (forwardProof) { + if (cleanupEscalated) { + throw new Error( + "dashboard connect retained captured descriptors after forward proof and required forced cleanup", + ); + } + return { exitCode, proof: "forward-started", signal, stderr, stdout }; + } + if (exitCode === 0) { + return { exitCode, proof: "command-completed", signal, stderr, stdout }; + } + throw new Error( + `dashboard connect exited before proving forward handoff (exit ${exitCode ?? "unknown"}${signal ? `, signal ${signal}` : ""})`, + ); +} diff --git a/test/e2e/live/dashboard-remote-bind-env.ts b/test/e2e/live/dashboard-remote-bind-env.ts index 415ab072dfc..518be062ebe 100644 --- a/test/e2e/live/dashboard-remote-bind-env.ts +++ b/test/e2e/live/dashboard-remote-bind-env.ts @@ -41,3 +41,8 @@ export function dashboardRemoteBindConnectStarted( output.includes(`sandbox ${sandboxName}`)))) ); } + +export function dashboardForwardIsRunning(forwardLine: string): boolean { + const columns = forwardLine.trim().split(/\s+/u); + return columns.length === 5 && columns[4] === "running"; +} diff --git a/test/e2e/live/dashboard-remote-bind.test.ts b/test/e2e/live/dashboard-remote-bind.test.ts index ac4c55c5627..c8651fdfdb3 100644 --- a/test/e2e/live/dashboard-remote-bind.test.ts +++ b/test/e2e/live/dashboard-remote-bind.test.ts @@ -9,9 +9,10 @@ import { sandboxAccessEnv, trustedSandboxShellScript } from "../fixtures/clients import { expect, test } from "../fixtures/e2e-test.ts"; import { requireHostedInferenceConfig } from "../fixtures/hosted-inference.ts"; import { CLI_ENTRYPOINT, REPO_ROOT } from "../fixtures/paths.ts"; +import { runDashboardConnectUntilForwardHandoff } from "./dashboard-connect-handoff.ts"; import { buildDashboardRemoteBindEnv, - dashboardRemoteBindConnectStarted, + dashboardForwardIsRunning, } from "./dashboard-remote-bind-env.ts"; import { parseJsonFromText } from "./json-envelope.ts"; @@ -183,15 +184,19 @@ runDashboardRemoteBindTest( timeoutMs: 30_000, }); - const connect = await host.nemoclaw([sandboxName, "connect"], { - artifactName: "dashboard-remote-bind-connect", + const connect = await runDashboardConnectUntilForwardHandoff({ + artifacts, + dashboardPort, env: testEnv(), + progress, + sandboxName, + signal: cleanup.currentSignal(), timeoutMs: 120_000, }); expect( - dashboardRemoteBindConnectStarted(connect, sandboxName, dashboardPort), - `nemoclaw connect did not complete or print background-forward proof\nstdout:\n${connect.stdout}\nstderr:\n${connect.stderr}`, - ).toBe(true); + connect.proof, + "nemoclaw connect did not complete or print background-forward proof; see the dashboard-connect-handoff.stdout.txt and dashboard-connect-handoff.stderr.txt artifacts", + ).toBe("forward-started"); progress.phase("verify all-interface dashboard forward"); const forwardList = await sandbox.openshell(["forward", "list"], { @@ -207,6 +212,10 @@ runDashboardRemoteBindTest( forwardLine, `No OpenShell forward found for ${sandboxName} on ${dashboardPort}`, ).not.toBe(""); + expect( + dashboardForwardIsRunning(forwardLine), + `Dashboard forward is not running after connect handoff: ${forwardLine}`, + ).toBe(true); expect( bindsLoopback(forwardLine, dashboardPort), `Dashboard forward is still localhost-only; expected an all-interface bind: ${forwardLine}`, @@ -216,6 +225,30 @@ runDashboardRemoteBindTest( `Could not prove dashboard forward uses 0.0.0.0:${dashboardPort}: ${forwardLine}`, ).toBe(true); + const forwardReachable = await host.command( + process.execPath, + [ + "-e", + [ + 'const net = require("node:net");', + "const socket = net.connect({ host: '127.0.0.1', port: Number(process.argv[1]) });", + "const deadline = setTimeout(() => { socket.destroy(); process.exit(1); }, 5000);", + "socket.once('connect', () => { clearTimeout(deadline); socket.destroy(); process.exit(0); });", + "socket.once('error', () => { clearTimeout(deadline); process.exit(1); });", + ].join("\n"), + dashboardPort, + ], + { + artifactName: "dashboard-remote-bind-post-handoff-reachability", + env: testEnv(), + timeoutMs: 10_000, + }, + ); + expect( + forwardReachable.exitCode, + `Dashboard forward is unreachable after connect handoff\n${resultText(forwardReachable)}`, + ).toBe(0); + progress.phase("audit exposed dashboard controls"); const audit = await sandbox.execShell( sandboxName, diff --git a/test/e2e/support/dashboard-connect-handoff.test.ts b/test/e2e/support/dashboard-connect-handoff.test.ts new file mode 100644 index 00000000000..f259301ec46 --- /dev/null +++ b/test/e2e/support/dashboard-connect-handoff.test.ts @@ -0,0 +1,185 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { expect, test } from "../fixtures/e2e-test.ts"; +import { runDashboardConnectUntilForwardHandoff } from "../live/dashboard-connect-handoff.ts"; + +const SANDBOX_NAME = "e2e-dashboard-bind"; +const DASHBOARD_PORT = "18789"; + +function processExists(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +async function waitForProcessExit(pid: number): Promise { + const deadline = Date.now() + 2_000; + while (processExists(pid) && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 20)); + } +} + +async function stopFixtureProcess(pid: number): Promise { + try { + process.kill(pid, "SIGTERM"); + } catch { + // The forward may have already exited. + } + await waitForProcessExit(pid); + expect(processExists(pid)).toBe(false); +} + +test("accepts a normally completed connect when the forward is already healthy", async ({ + artifacts, + progress, +}) => { + const result = await runDashboardConnectUntilForwardHandoff({ + artifacts, + command: [process.execPath, "-e", "process.exit(0)"], + dashboardPort: DASHBOARD_PORT, + env: process.env, + progress, + sandboxName: SANDBOX_NAME, + timeoutMs: 2_000, + }); + + expect(result).toMatchObject({ exitCode: 0, proof: "command-completed", signal: null }); +}); + +test("rejects invalid handoff budgets before spawning connect", async ({ artifacts, progress }) => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-handoff-budget-")); + const marker = path.join(directory, "spawned"); + const base = { + artifacts, + command: [ + process.execPath, + "-e", + 'require("node:fs").writeFileSync(process.argv[1], "1")', + marker, + ] as const, + dashboardPort: DASHBOARD_PORT, + env: process.env, + progress, + sandboxName: SANDBOX_NAME, + }; + + try { + await expect(runDashboardConnectUntilForwardHandoff({ ...base, timeoutMs: 0 })).rejects.toThrow( + /timeout must be a positive finite value/, + ); + await expect( + runDashboardConnectUntilForwardHandoff({ + ...base, + stopGraceMs: Number.POSITIVE_INFINITY, + timeoutMs: 2_000, + }), + ).rejects.toThrow(/stop grace must be a positive finite value/); + expect(fs.existsSync(marker)).toBe(false); + } finally { + fs.rmSync(directory, { force: true, recursive: true }); + } +}); + +test("reaps interactive connect after missing-forward proof while its detached forward survives", async ({ + artifacts, + progress, +}) => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-connect-handoff-")); + const pidFile = path.join(directory, "forward.pid"); + let forwardPid = Number.NaN; + try { + const script = [ + 'const fs = require("node:fs");', + 'const { spawn } = require("node:child_process");', + 'const forward = spawn(process.execPath, ["-e", "setInterval(() => undefined, 1000)"], { detached: true, stdio: "ignore" });', + "forward.unref();", + "try { fs.writeFileSync(process.argv[1], String(forward.pid)); } catch (error) { forward.kill('SIGTERM'); throw error; }", + `process.stdout.write(${JSON.stringify( + `Dashboard port forward to '${SANDBOX_NAME}' is missing or dead.\nRe-establishing...\n\u001B[32m✓\u001B[0m Dashboard port forward re-established.\n`, + )});`, + "process.on('SIGTERM', () => process.exit(0));", + "setInterval(() => undefined, 1000);", + ].join("\n"); + const result = await runDashboardConnectUntilForwardHandoff({ + artifacts, + command: [process.execPath, "-e", script, pidFile], + dashboardPort: DASHBOARD_PORT, + env: process.env, + progress, + sandboxName: SANDBOX_NAME, + timeoutMs: 2_000, + }); + + forwardPid = Number(fs.readFileSync(pidFile, "utf8")); + expect(result.proof).toBe("forward-started"); + expect(result.stdout).toContain("Dashboard port forward re-established."); + expect(processExists(forwardPid)).toBe(true); + } finally { + const cleanupPid = Number.isInteger(forwardPid) + ? forwardPid + : Number(fs.existsSync(pidFile) ? fs.readFileSync(pidFile, "utf8") : Number.NaN); + try { + expect( + Number.isInteger(cleanupPid) && cleanupPid > 0, + "fixture forward PID is unavailable; detached cleanup cannot be proven", + ).toBe(true); + await stopFixtureProcess(cleanupPid); + } finally { + fs.rmSync(directory, { force: true, recursive: true }); + } + } +}); + +test("fails when an attached descendant retains captured stdio after forward proof", async ({ + artifacts, + progress, +}) => { + const script = [ + 'const { spawn } = require("node:child_process");', + 'spawn(process.execPath, ["-e", "setInterval(() => undefined, 1000)"], { stdio: "inherit" });', + `process.stdout.write(${JSON.stringify( + "\u001B[32m✓\u001B[0m Dashboard port forward re-established.\n", + )});`, + "process.on('SIGTERM', () => process.exit(0));", + "setInterval(() => undefined, 1000);", + ].join("\n"); + + await expect( + runDashboardConnectUntilForwardHandoff({ + artifacts, + command: [process.execPath, "-e", script], + dashboardPort: DASHBOARD_PORT, + env: process.env, + progress, + sandboxName: SANDBOX_NAME, + stopGraceMs: 100, + timeoutMs: 2_000, + }), + ).rejects.toThrow(/retained captured descriptors/); +}); + +test("fails within budget and reaps a connect process that never proves handoff", async ({ + artifacts, + progress, +}) => { + await expect( + runDashboardConnectUntilForwardHandoff({ + artifacts, + command: [process.execPath, "-e", "setInterval(() => undefined, 1000)"], + dashboardPort: DASHBOARD_PORT, + env: process.env, + progress, + sandboxName: SANDBOX_NAME, + stopGraceMs: 100, + timeoutMs: 100, + }), + ).rejects.toThrow(/did not complete or prove forward handoff within budget/); +}); diff --git a/test/e2e/support/dashboard-remote-bind-env.test.ts b/test/e2e/support/dashboard-remote-bind-env.test.ts index 8d7a89b2279..56e2678c001 100644 --- a/test/e2e/support/dashboard-remote-bind-env.test.ts +++ b/test/e2e/support/dashboard-remote-bind-env.test.ts @@ -6,6 +6,7 @@ import { describe, expect, it } from "vitest"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import { buildDashboardRemoteBindEnv, + dashboardForwardIsRunning, dashboardRemoteBindConnectStarted, } from "../live/dashboard-remote-bind-env.ts"; @@ -32,7 +33,7 @@ describe("dashboard remote-bind E2E environment", () => { expect(env.NEMOCLAW_DASHBOARD_BIND).toBe("0.0.0.0"); }); - it("accepts recovery proof when connect has no numeric exit code", () => { + it("accepts recovery proof while connect remains interactive", () => { expect( dashboardRemoteBindConnectStarted( { @@ -59,4 +60,12 @@ describe("dashboard remote-bind E2E environment", () => { ), ).toBe(false); }); + + it.each([ + ["e2e-dashboard-bind 0.0.0.0 18789 4242 running", true], + ["e2e-dashboard-bind 0.0.0.0 18789 4242 not running", false], + ["e2e-dashboard-bind 0.0.0.0 18789 4242 stopped", false], + ])("recognizes only the exact running forward status: %s", (forwardLine, expected) => { + expect(dashboardForwardIsRunning(forwardLine)).toBe(expected); + }); }); diff --git a/tools/e2e/check-semantic-phases.mts b/tools/e2e/check-semantic-phases.mts index a7795771ccf..4198fc1e209 100644 --- a/tools/e2e/check-semantic-phases.mts +++ b/tools/e2e/check-semantic-phases.mts @@ -411,6 +411,10 @@ const OBSERVED_CHILD_PROGRESS_POLICIES = new Map