From 2545d0883f3946aa8df653b6a495b51748c41713 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Thu, 10 Sep 2026 18:54:25 -0400 Subject: [PATCH 1/6] fix(e2e): respect Launchable gateway ownership and retain command evidence Signed-off-by: Julie Yaunches --- test/e2e-runtime/brev-launchable-e2e.test.ts | 90 +++++++++- test/e2e/README.md | 20 +++ test/e2e/fixtures/artifacts.ts | 6 +- test/e2e/fixtures/full-e2e-gateway.ts | 46 +++++ test/e2e/fixtures/shell-probe.ts | 35 +++- test/e2e/live/full-e2e.test.ts | 39 +++-- test/e2e/mock-parity.json | 1 + test/e2e/support/e2e-redaction-entry.test.ts | 111 +++++++++++- test/e2e/support/full-e2e-gateway.test.ts | 168 +++++++++++++++++++ test/helpers/brev-launchable-e2e-fixture.ts | 29 +++- tools/e2e/brev-launchable-e2e.sh | 8 +- 11 files changed, 527 insertions(+), 26 deletions(-) create mode 100644 test/e2e/fixtures/full-e2e-gateway.ts create mode 100644 test/e2e/support/full-e2e-gateway.test.ts diff --git a/test/e2e-runtime/brev-launchable-e2e.test.ts b/test/e2e-runtime/brev-launchable-e2e.test.ts index 8d609250055..1fec0c2a62a 100644 --- a/test/e2e-runtime/brev-launchable-e2e.test.ts +++ b/test/e2e-runtime/brev-launchable-e2e.test.ts @@ -29,6 +29,88 @@ function identitySmokeEnv(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { } describe("focused staging Brev Launchable lane", () => { + it("reports missing command evidence from an older baked suite without changing its result (#9851)", () => { + const { env, workDir } = fixture({ omitCommandEvidence: true }); + expect(run(env).status).toBe(0); + expect(fs.readFileSync(path.join(workDir, "full-e2e.log"), "utf8")).not.toContain( + "NEMOCLAW_E2E_COMMAND ", + ); + expect(fs.readFileSync(path.join(workDir, "lane.log"), "utf8")).toContain( + "Completed command metadata unavailable: the guest emitted no command records", + ); + }); + + it("retains completed command timestamps when onboarding fails (#9851)", () => { + const { env, workDir } = fixture({ e2eFails: true }); + expect(run(env).status).not.toBe(0); + const log = fs.readFileSync(path.join(workDir, "full-e2e.log"), "utf8"); + const record = JSON.parse( + log + .split("\n") + .find((line) => line.startsWith("NEMOCLAW_E2E_COMMAND "))! + .slice("NEMOCLAW_E2E_COMMAND ".length), + ); + expect(record).toMatchObject({ + command: ["brev-quickstart", "e2e-staging"], + startedAt: "2026-09-10T18:57:30.000Z", + finishedAt: "2026-09-10T18:57:31.000Z", + durationMs: 1000, + exitCode: 1, + }); + expect(log).not.toContain("nvapi-test-value"); + expect(JSON.parse(fs.readFileSync(path.join(workDir, "cleanup.json"), "utf8"))).toMatchObject({ + status: "ABSENT", + }); + }); + + it.each([ + ["https://127.0.0.1:18080", 18080], + ["https://127.0.0.1:19443", 19443], + ["http://127.0.0.1:18080", 18080], + ["https://[::1]:19443", 19443], + ])("diagnoses the declared gateway at %s (#9851)", (gatewayEndpoint, port) => { + const { env, workDir, calls } = fixture({ + e2eFails: true, + gatewayEndpoint, + }); + expect(run(env).status).not.toBe(0); + expect(fs.readFileSync(calls, "utf8")).toContain(`ss -H -ltnp sport = :${port}`); + expect(fs.readFileSync(path.join(workDir, "lane.log"), "utf8")).toContain( + `declared gateway port: ${port}`, + ); + }); + + it.each([ + "https://untrusted.invalid:18080", + "https://127.0.0.1:99999", + "https://127.0.0.1", + "http://127.0.0.1", + "https://127.0.0.1:1023", + "$(touch /tmp/unsafe)", + ])( + "does not probe a substituted port when the declaration is invalid: %s (#9851)", + (gatewayEndpoint) => { + const { env, workDir, calls } = fixture({ e2eFails: true, gatewayEndpoint }); + expect(run(env).status).not.toBe(0); + expect(fs.readFileSync(calls, "utf8")).not.toMatch(/^ss /m); + expect(JSON.parse(fs.readFileSync(path.join(workDir, "cleanup.json"), "utf8"))).toMatchObject( + { status: "ABSENT" }, + ); + }, + ); + + it("still cleans up when the baked gateway resolver is unavailable (#9851)", () => { + const { env, workDir, calls } = fixture({ e2eFails: true, diagnosticResolverMissing: true }); + expect(run(env).status).not.toBe(0); + expect(fs.readFileSync(calls, "utf8")).not.toMatch(/^ss /m); + expect(fs.readFileSync(path.join(workDir, "lane.log"), "utf8")).toContain( + "Full E2E failure diagnostic declared gateway listener: status 1", + ); + expect(JSON.parse(fs.readFileSync(path.join(workDir, "cleanup.json"), "utf8"))).toMatchObject({ + status: "ABSENT", + }); + }); + it("keeps the staging SSH wrapper outside the full E2E deadline", () => { const source = fs.readFileSync( path.resolve(import.meta.dirname, "../../tools/e2e/brev-launchable-e2e.sh"), @@ -687,7 +769,9 @@ describe("focused staging Brev Launchable lane", () => { expect(laneLog).toContain("[REDACTED PRIVATE KEY]"); expect(laneLog).toContain("[REDACTED LONG LINE]"); expect(laneLog).toContain("Full E2E failure diagnostic gateway lifecycle: status 0; output:"); - expect(laneLog).toContain("Full E2E failure diagnostic port 8080 listener: status 0; output:"); + expect(laneLog).toContain( + "Full E2E failure diagnostic declared gateway listener: status 0; output:", + ); const commands = fs.readFileSync(calls, "utf8"); expect(commands.indexOf("ssh full-e2e diagnostic platform state")).toBeLessThan( commands.indexOf("ssh full-e2e diagnostic gateway lifecycle"), @@ -783,7 +867,7 @@ describe("focused staging Brev Launchable lane", () => { ["listener presence: present", "listener owner: unavailable"], ], ])( - "classifies port 8080 listener evidence with %s (#6409)", + "classifies declared gateway listener evidence with %s (#6409)", (_name, listenerOutput, expectedEvidence) => { const { env, workDir } = fixture({ e2eFails: true, @@ -850,7 +934,7 @@ describe("focused staging Brev Launchable lane", () => { "Full E2E failure diagnostic platform state: not run; output: diagnostic budget exhausted", ); expect(laneLog).toContain( - "Full E2E failure diagnostic port 8080 listener: not run; output: diagnostic budget exhausted", + "Full E2E failure diagnostic declared gateway listener: not run; output: diagnostic budget exhausted", ); const commands = fs.readFileSync(calls, "utf8"); expect(commands).not.toContain("ssh full-e2e diagnostic platform state"); diff --git a/test/e2e/README.md b/test/e2e/README.md index 4ec7ed2c489..b488327610d 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -1110,6 +1110,22 @@ phase artifact created before exit. A preparation failure can produce no artifact. A later early failure can retain only `lane.log`. A successful job contains `launchable-e2e.json`, `full-e2e.log`, and `cleanup.json`; `cleanup.json` exists only after the job confirms workspace absence. +The preinstalled suite resolves its gateway name and port from the external +gateway declaration before registering cleanup. It removes its sandbox but +does not remove the platform gateway registration or service. Source-install +runs retain their test-owned gateway cleanup. +The Launchable controller enables `NEMOCLAW_E2E_COMMAND_EVIDENCE=1` to retain +completed command records in `full-e2e.log`. Each `NEMOCLAW_E2E_COMMAND` JSON +line contains redacted argv, UTC start and finish timestamps, duration, exit +status, signal, and timeout state. Spawn failures also emit a record. Commands +that explicitly disable artifact persistence emit none. Output bodies remain +in guest artifacts; this stream does not export them. Oversized command argv +is omitted with `commandOmitted: "size-limit"`. Abrupt guest or transport loss +can leave no completion record for an active command. Older baked suites may +emit no records; the controller reports that absence rather than inferring +command times from phase reports. +The preinstalled suite does not run the source-install cold-onboarding budget +and does not declare that budget as tested coverage. When the preinstalled full E2E fails after SSH succeeds, the job attempts to append bounded, redacted host state and fixed lifecycle classifications to `lane.log` before cleanup. On the host, the SSH command reads the system journal @@ -1119,6 +1135,10 @@ GitHub-hosted runner or `lane.log`. If a probe fails or the shared budget expires, `lane.log` records that result and cleanup continues. The diagnostic phase is read-only, uses one 30-second budget, and does not retry the failed E2E or repair the workspace. +The listener diagnostic uses the baked suite's gateway resolver, including its +declaration path, loopback endpoint parsing, port checks, and conflict checks. +An unavailable resolver or rejected declaration leaves that probe failed; +it does not substitute port 8080 or prevent workspace cleanup. Manual ordinary and full runs exclude the Jetson nvmap and DGX Spark llama.cpp jobs unless their independent opt-in flags are `true`. diff --git a/test/e2e/fixtures/artifacts.ts b/test/e2e/fixtures/artifacts.ts index 833f4f823d3..a5a74c06fdb 100644 --- a/test/e2e/fixtures/artifacts.ts +++ b/test/e2e/fixtures/artifacts.ts @@ -131,10 +131,14 @@ export class ArtifactSink { async writeText(relativePath: string, text: string): Promise { const target = this.pathFor(relativePath); await fs.mkdir(path.dirname(target), { recursive: true }); - await fs.writeFile(target, redactString(text, this.redactionValues), "utf8"); + await fs.writeFile(target, this.redact(text), "utf8"); return target; } + redact(text: string): string { + return redactString(text, this.redactionValues); + } + async writeJson(relativePath: string, value: unknown): Promise { return this.writeText(relativePath, `${JSON.stringify(value, null, 2)}\n`); } diff --git a/test/e2e/fixtures/full-e2e-gateway.ts b/test/e2e/fixtures/full-e2e-gateway.ts new file mode 100644 index 00000000000..36873aaeb12 --- /dev/null +++ b/test/e2e/fixtures/full-e2e-gateway.ts @@ -0,0 +1,46 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { DEFAULT_GATEWAY_PORT, parsePort } from "../../../src/lib/core/ports.ts"; +import { resolveGatewayName } from "../../../src/lib/onboard/gateway-binding/identity.ts"; +import { loadGatewayManagementDeclaration } from "../../../src/lib/onboard/gateway-management.ts"; + +/** Both eager cleanup and registered teardown must respect the same gateway owner. */ +export async function withOwnedFullE2eGateway( + gateway: { owned: boolean }, + cleanup: () => unknown, +): Promise { + if (gateway.owned) await cleanup(); +} + +/** Resolve the platform declaration before the test can register destructive cleanup. */ +export function fullE2eGateway(preinstalled: boolean, env: NodeJS.ProcessEnv = process.env) { + if (!preinstalled) { + const port = parsePort("NEMOCLAW_GATEWAY_PORT", DEFAULT_GATEWAY_PORT, env); + return { owned: true, env: { OPENSHELL_GATEWAY: resolveGatewayName(port) } }; + } + const declarationPath = + env.NEMOCLAW_GATEWAY_MANAGEMENT ?? "/etc/nemoclaw/gateway-management.json"; + const loaded = loadGatewayManagementDeclaration({ + env: { ...env, NEMOCLAW_GATEWAY_MANAGEMENT: declarationPath }, + }); + if (!loaded.ok) throw new Error(`Launchable gateway declaration: ${loaded.reason}`); + if (loaded.declaration?.mode !== "externally-supervised" || !loaded.declaration.endpoint) { + throw new Error("The preinstalled Launchable requires an externally supervised gateway"); + } + const endpoint = new URL(loaded.declaration.endpoint); + const port = parsePort("NEMOCLAW_GATEWAY_PORT", DEFAULT_GATEWAY_PORT, { + NEMOCLAW_GATEWAY_PORT: endpoint.port || (endpoint.protocol === "https:" ? "443" : "80"), + }); + if (parsePort("NEMOCLAW_GATEWAY_PORT", port, env) !== port) { + throw new Error("Launchable gateway port conflicts with its declaration"); + } + return { + owned: false, + env: { + OPENSHELL_GATEWAY: resolveGatewayName(port), + NEMOCLAW_GATEWAY_MANAGEMENT: declarationPath, + NEMOCLAW_GATEWAY_PORT: String(port), + }, + }; +} diff --git a/test/e2e/fixtures/shell-probe.ts b/test/e2e/fixtures/shell-probe.ts index f996ca38e0c..ca65e5e93d6 100644 --- a/test/e2e/fixtures/shell-probe.ts +++ b/test/e2e/fixtures/shell-probe.ts @@ -97,6 +97,8 @@ export function resolveLiveE2eWorkloadSourceEnv(input: NodeJS.ProcessEnv): NodeJ export interface ShellProbeResult { command: string[]; + startedAt?: string; + finishedAt?: string; /** Wall-clock command duration, persisted for CI bottleneck analysis. */ durationMs?: number; exitCode: number | null; @@ -258,6 +260,28 @@ export class ShellProbe { result: Omit, ): Promise => { if (options.persistArtifacts === false) return { stdout: "", stderr: "", result: "" }; + if (process.env.NEMOCLAW_E2E_COMMAND_EVIDENCE === "1") { + // Preserve completed command metadata through the existing remote log transport. + // Output bodies stay in redacted guest artifacts, outside this metadata-only stream. + const record = { + schemaVersion: 1, + artifactName: this.artifacts.redact(activityName).slice(0, 256), + command: result.command.map((argument) => this.artifacts.redact(argument)), + startedAt: result.startedAt, + finishedAt: result.finishedAt, + durationMs: result.durationMs, + exitCode: result.exitCode, + signal: result.signal, + timedOut: result.timedOut, + }; + let encoded = this.artifacts.redact(JSON.stringify(record)); + if (Buffer.byteLength(encoded) > 65_536) { + encoded = this.artifacts.redact( + JSON.stringify({ ...record, command: [], commandOmitted: "size-limit" }), + ); + } + process.stderr.write(`NEMOCLAW_E2E_COMMAND ${encoded}\n`); + } return { stdout: await this.artifacts.writeText(`${artifactBase}.stdout.txt`, result.stdout), stderr: await this.artifacts.writeText(`${artifactBase}.stderr.txt`, result.stderr), @@ -308,13 +332,18 @@ export class ShellProbe { const redactedStdout = renderCapturedText(stdout); const redactedStderr = renderCapturedText(stderr); - const durationMs = Date.now() - startedAtMs; + const finishedAtMs = Date.now(); + const timing = { + startedAt: new Date(startedAtMs).toISOString(), + finishedAt: new Date(finishedAtMs).toISOString(), + durationMs: finishedAtMs - startedAtMs, + }; if (supervised.spawnError) { const redactedMessage = redactProbeText(errorMessage(supervised.spawnError)); const stderrWithError = [redactedStderr, redactedMessage].filter(Boolean).join("\n"); await writeArtifacts({ command: redactedCommand, - durationMs, + ...timing, exitCode: null, signal: null, timedOut: supervised.timedOut, @@ -326,7 +355,7 @@ export class ShellProbe { const result: Omit = { command: redactedCommand, - durationMs, + ...timing, exitCode: supervised.exitCode, signal: supervised.signal, timedOut: supervised.timedOut, diff --git a/test/e2e/live/full-e2e.test.ts b/test/e2e/live/full-e2e.test.ts index 9e638bc72f3..d4b96c46437 100644 --- a/test/e2e/live/full-e2e.test.ts +++ b/test/e2e/live/full-e2e.test.ts @@ -56,6 +56,7 @@ import { runOpenClawLaunchReadinessLeaseTurns } from "./launch-agent-turn.ts"; import { bindApprovedPrBaseForBaseImageComparison } from "./pr-base-comparison.ts"; import { FULL_E2E_TEST_TIMEOUT_MS } from "../../../tools/e2e/full-e2e-timeout-contract.mts"; import { parseOpenClawJsonDocuments } from "../../../src/lib/openclaw/agent-json-provenance.ts"; +import { fullE2eGateway, withOwnedFullE2eGateway } from "../fixtures/full-e2e-gateway.ts"; const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-full"; const FULL_E2E_TARGET_ID = process.env.E2E_TARGET_ID ?? "full-e2e"; @@ -73,6 +74,7 @@ const AUTHORITATIVE_LOCAL_BASE_BUILD_OUTPUT = "Building OpenClaw sandbox base image locally because no compatible published base image was found."; const MEASURE_COLD_ONBOARD = !USE_PREINSTALLED_LAUNCHABLE && process.env.E2E_TARGET_ID === "full-e2e"; +let gateway: ReturnType; interface ColdOnboardCapture { outputEvents: ShellProbeOutputEvent[]; @@ -95,7 +97,7 @@ function env(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { NEMOCLAW_NON_INTERACTIVE: "1", NEMOCLAW_RECREATE_SANDBOX: "1", NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, - OPENSHELL_GATEWAY: "nemoclaw", + ...gateway.env, ...securityPostureModeEnv(), ...extra, }; @@ -311,13 +313,15 @@ async function cleanup(host: HostCliClient, sandbox: SandboxClient): Promise undefined); - await sandbox - .openshell(["gateway", "destroy", "-g", "nemoclaw"], { - artifactName: "cleanup-openshell-gateway-destroy", - env: env(), - timeoutMs: 60_000, - }) - .catch(() => undefined); + await withOwnedFullE2eGateway(gateway, () => + sandbox + .openshell(["gateway", "destroy", "-g", gateway.env.OPENSHELL_GATEWAY], { + artifactName: "cleanup-openshell-gateway-destroy", + env: env(), + timeoutMs: 60_000, + }) + .catch(() => undefined), + ); } function readAndDeleteTraceWindow(traceFile: string, traceDirectory: string): OnboardTraceWindow { @@ -528,6 +532,7 @@ test( secrets, skip, }) => { + gateway = fullE2eGateway(USE_PREINSTALLED_LAUNCHABLE); const hosted = requireHostedInferenceConfig( secrets, process.env, @@ -553,7 +558,9 @@ test( USE_PREINSTALLED_LAUNCHABLE ? "the baked Launchable completes onboarding without installing from source" : "install.sh --non-interactive completes onboarding", - "cold onboarding stays within the checked-in full-E2E performance budgets", + ...(MEASURE_COLD_ONBOARD + ? ["cold onboarding stays within the checked-in full-E2E performance budgets"] + : []), "nemoclaw and openshell are installed and usable", "sandbox appears in list/status and has policy/inference configuration", "direct hosted inference and sandbox inference.local both respond", @@ -579,12 +586,14 @@ test( }); !USE_PREINSTALLED_LAUNCHABLE && lifecycle.trackInstallerGatewayUserService(); - cleanupRegistry.trackGateway(host, "nemoclaw", { - artifactName: "cleanup-openshell-gateway-destroy", - env: env(), - redactionValues: [hosted.apiKey], - timeoutMs: 60_000, - }); + await withOwnedFullE2eGateway(gateway, () => + cleanupRegistry.trackGateway(host, gateway.env.OPENSHELL_GATEWAY, { + artifactName: "cleanup-openshell-gateway-destroy", + env: env(), + redactionValues: [hosted.apiKey], + timeoutMs: 60_000, + }), + ); cleanupRegistry.trackDisposable(`delete OpenShell sandbox ${SANDBOX_NAME}`, () => sandbox.cleanupSandbox(SANDBOX_NAME, { artifactName: "cleanup-openshell-sandbox-delete", diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index 362a9181d79..4885afca436 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -167,6 +167,7 @@ "test/runtime/sandbox/sandbox-provisioning.test.ts", "test/e2e/support/hosted-inference.test.ts", "test/e2e/support/full-e2e-inference-probe.test.ts", + "test/e2e/support/full-e2e-gateway.test.ts", "test/e2e/support/launch-agent-turn-provider-availability.test.ts", "test/e2e/support/launch-agent-turn.test.ts", "test/e2e/support/openclaw-agent-output.test.ts", diff --git a/test/e2e/support/e2e-redaction-entry.test.ts b/test/e2e/support/e2e-redaction-entry.test.ts index 6b4dcdceb05..166c043e74f 100644 --- a/test/e2e/support/e2e-redaction-entry.test.ts +++ b/test/e2e/support/e2e-redaction-entry.test.ts @@ -19,7 +19,7 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { ArtifactSink } from "../fixtures/artifacts.ts"; import { startTestProgress } from "../fixtures/progress.ts"; @@ -35,7 +35,116 @@ function supportProgress() { ); } +async function captureCommandEvidence(outcome: string) { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "e2e-command-evidence-")); + const secret = "sink-only-command-secret"; + const artifacts = new ArtifactSink(directory, [secret]); + const progress = supportProgress(); + const writes: string[] = []; + const stderr = vi.spyOn(process.stderr, "write").mockImplementation((chunk) => { + writes.push(String(chunk)); + return true; + }); + vi.stubEnv("NEMOCLAW_E2E_COMMAND_EVIDENCE", outcome === "not-requested" ? "" : "1"); + try { + const probe = new ShellProbe({ + artifacts, + progress, + redact: redactString, + signal: new AbortController().signal, + }); + let failed = false; + await probe + .run( + trustedShellCommand({ + command: outcome === "spawn-error" ? "/nonexistent/e2e-command" : process.execPath, + args: [ + "-e", + outcome === "timeout" + ? "setInterval(() => {}, 1000)" + : `console.log('private-output-body'); process.exit(${outcome === "failure" ? 7 : 0})`, + secret, + ...(outcome === "oversized" ? ["a".repeat(70_000)] : []), + ], + reason: "verify timestamped command evidence and redaction", + }), + { + artifactName: "command-evidence", + timeoutMs: outcome === "timeout" ? 100 : 5000, + persistArtifacts: outcome !== "disabled", + }, + ) + .catch(() => { + failed = true; + }); + const lines = writes.filter((line) => line.startsWith("NEMOCLAW_E2E_COMMAND ")); + const result = await fs + .readFile(path.join(directory, "shell/command-evidence.result.json"), "utf8") + .catch(() => null); + return { lines, result, secret, failed }; + } finally { + stderr.mockRestore(); + vi.unstubAllEnvs(); + progress.stop(); + await fs.rm(directory, { recursive: true, force: true }); + } +} + describe("fixture redaction entry point", () => { + it.each([ + { outcome: "success", exitCode: 0, timedOut: false, failed: false, commandOmitted: undefined }, + { outcome: "failure", exitCode: 7, timedOut: false, failed: false, commandOmitted: undefined }, + { + outcome: "timeout", + exitCode: null, + timedOut: true, + failed: false, + commandOmitted: undefined, + }, + { + outcome: "spawn-error", + exitCode: null, + timedOut: false, + failed: true, + commandOmitted: undefined, + }, + { + outcome: "oversized", + exitCode: 0, + timedOut: false, + failed: false, + commandOmitted: "size-limit", + }, + ])( + "retains redacted UTC command metadata for $outcome without publishing output bodies", + async ({ outcome, exitCode, timedOut, failed, commandOmitted }) => { + const evidence = await captureCommandEvidence(outcome); + expect(evidence.failed).toBe(failed); + expect(evidence.lines).toHaveLength(1); + const record = JSON.parse(evidence.lines[0]!.slice("NEMOCLAW_E2E_COMMAND ".length)); + expect(record.schemaVersion).toBe(1); + expect(record.startedAt).toMatch(/^\d{4}-\d{2}-\d{2}T.*Z$/); + expect(Date.parse(record.finishedAt) - Date.parse(record.startedAt)).toBe(record.durationMs); + expect(record.exitCode).toBe(exitCode); + expect(record.timedOut).toBe(timedOut); + expect(evidence.lines[0]).not.toContain(evidence.secret); + expect(record).not.toHaveProperty("stdout"); + expect(record).not.toHaveProperty("stderr"); + expect(Buffer.byteLength(evidence.lines[0]!)).toBeLessThan(65_600); + expect(record.commandOmitted).toBe(commandOmitted); + expect(evidence.result).not.toBeNull(); + expect(evidence.result).not.toContain(evidence.secret); + }, + ); + it.each(["disabled", "not-requested"])( + "emits no command metadata when evidence is %s", + async (outcome) => { + const evidence = await captureCommandEvidence(outcome); + expect(evidence.failed).toBe(false); + expect(evidence.lines).toEqual([]); + }, + ); + it("recognizes pass env names only at exact or underscore-delimited boundaries", () => { expect( ["PASS", "PASSWD", "CUSTOM_PASS", "CUSTOM_PASSWD"].every((key) => diff --git a/test/e2e/support/full-e2e-gateway.test.ts b/test/e2e/support/full-e2e-gateway.test.ts new file mode 100644 index 00000000000..38bea586efb --- /dev/null +++ b/test/e2e/support/full-e2e-gateway.test.ts @@ -0,0 +1,168 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { fullE2eGateway } from "../fixtures/full-e2e-gateway.ts"; + +const directories: string[] = []; +afterEach(() => { + for (const directory of directories.splice(0)) fs.rmSync(directory, { recursive: true }); + vi.unstubAllEnvs(); +}); + +const captured = vi.hoisted(() => ({ test: vi.fn() })); +vi.mock("../fixtures/e2e-test.ts", async () => ({ + expect: (await import("vitest")).expect, + test: captured.test, +})); +vi.mock("../fixtures/runtime-provider.ts", () => ({ + ensureConfiguredRuntimeProviderAvailable: vi.fn(), +})); +vi.mock("../fixtures/hosted-inference.ts", () => ({ + requireHostedInferenceConfig: () => ({ + apiKey: "fixture-key", + endpointUrl: "https://example.com/v1", + model: "fixture-model", + env: {}, + }), + buildHostedInferenceModelsProbe: vi.fn(), + stagePortableHostedInferenceDescriptor: vi.fn(), +})); +function declaration( + value: unknown = { + version: 1, + mode: "externally-supervised", + endpoint: "https://127.0.0.1:18080", + stateDir: "/var/lib/brev/openshell-gateway", + supervisor: { + kind: "systemd-system", + serviceName: "openshell-gateway.service", + execPath: "/usr/local/bin/openshell-gateway", + }, + requiredCapabilities: ["gateway.health", "sandbox.create", "sandbox.exec"], + }, + endpoint?: string, +) { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "full-e2e-gateway-")); + directories.push(directory); + const file = path.join(directory, "gateway.json"); + fs.writeFileSync( + file, + JSON.stringify(endpoint === undefined ? value : { ...(value as object), endpoint }), + ); + return { NEMOCLAW_GATEWAY_MANAGEMENT: file }; +} + +describe("full E2E gateway ownership", () => { + it.each([true, false])( + "respects gateway cleanup ownership in the live suite (preinstalled=%s) (#9851)", + async (preinstalled) => { + vi.resetModules(); + captured.test.mockClear(); + vi.stubEnv( + "NEMOCLAW_E2E_SETUP_MODE", + preinstalled ? "preinstalled-launchable" : "source-install", + ); + vi.stubEnv("E2E_TARGET_ID", "staging-brev-launchable"); + vi.stubEnv("NEMOCLAW_EXPERIMENTAL_PROFILE", ""); + vi.stubEnv("NEMOCLAW_GATEWAY_MANAGEMENT", declaration().NEMOCLAW_GATEWAY_MANAGEMENT); + const result = { exitCode: 0, stdout: "", stderr: "", timedOut: false, signal: null }; + const host = { + command: vi.fn(async (command: string) => ({ + ...result, + exitCode: command === "brev-quickstart" || command === "bash" ? 42 : 0, + })), + }; + const sandbox = { openshell: vi.fn(async () => result) }; + const cleanup = { trackGateway: vi.fn(), trackDisposable: vi.fn(), trackSandbox: vi.fn() }; + const lifecycle = { trackInstallerGatewayUserService: vi.fn() }; + const declare = vi.fn(); + await import("../live/full-e2e.test.ts"); + const run = captured.test.mock.calls[0]![2] as (input: object) => Promise; + await expect( + run({ + host, + sandbox, + cleanup, + lifecycle, + artifacts: { target: { declare } }, + progress: { phase: vi.fn() }, + }), + ).rejects.toThrow(); + expect(host.command.mock.calls.map((call) => call[0])).toContain( + preinstalled ? "brev-quickstart" : "bash", + ); + expect(cleanup.trackGateway).toHaveBeenCalledTimes(preinstalled ? 0 : 1); + expect(lifecycle.trackInstallerGatewayUserService).toHaveBeenCalledTimes( + preinstalled ? 0 : 1, + ); + const calls = sandbox.openshell.mock.calls as unknown as [ + string[], + { env: NodeJS.ProcessEnv }, + ][]; + expect(calls.some(([args]) => args[0] === "gateway")).toBe(!preinstalled); + expect(calls[0]![1].env.OPENSHELL_GATEWAY).toBe(preinstalled ? "nemoclaw-18080" : "nemoclaw"); + expect(declare.mock.calls[0]![0].contracts).not.toContain( + "cold onboarding stays within the checked-in full-E2E performance budgets", + ); + }, + 20_000, + ); + + it.each(["https://127.0.0.1:18080", "http://127.0.0.1:18080", "https://[::1]:18080"])( + "targets the declared Launchable gateway at %s without cleanup ownership (#9851)", + (endpoint) => { + const env = declaration(undefined, endpoint); + expect(fullE2eGateway(true, env)).toEqual({ + owned: false, + env: { + ...env, + NEMOCLAW_GATEWAY_PORT: "18080", + OPENSHELL_GATEWAY: "nemoclaw-18080", + }, + }); + }, + ); + it.each(["https://127.0.0.1", "http://127.0.0.1", "https://127.0.0.1:1023"])( + "preserves the CLI port restriction for the declared endpoint %s (#9851)", + (endpoint) => { + expect(() => fullE2eGateway(true, declaration(undefined, endpoint))).toThrow("Invalid port"); + }, + ); + it("retains source-install gateway ownership and respects its selected port (#9851)", () => { + expect(fullE2eGateway(false, {})).toEqual({ + owned: true, + env: { OPENSHELL_GATEWAY: "nemoclaw" }, + }); + expect(fullE2eGateway(false, { NEMOCLAW_GATEWAY_PORT: "19090" })).toEqual({ + owned: true, + env: { OPENSHELL_GATEWAY: "nemoclaw-19090" }, + }); + }); + it("refuses conflicting Launchable ports before cleanup can be registered (#9851)", () => { + expect(() => fullE2eGateway(true, { ...declaration(), NEMOCLAW_GATEWAY_PORT: "8080" })).toThrow( + "conflicts with its declaration", + ); + }); + it("rejects malformed explicit port overrides before registering cleanup (#9851)", () => { + expect(() => + fullE2eGateway(true, { ...declaration(), NEMOCLAW_GATEWAY_PORT: "0x46a0" }), + ).toThrow("Invalid port"); + }); + it("refuses absent or malformed Launchable declarations (#9851)", () => { + expect(() => + fullE2eGateway(true, { NEMOCLAW_GATEWAY_MANAGEMENT: "/nonexistent/gateway.json" }), + ).toThrow("could not be read"); + expect(() => fullE2eGateway(true, declaration({ version: 99 }))).toThrow( + "unsupported gateway-management contract version", + ); + }); + it("does not interpret a managed declaration as Launchable cleanup authority (#9851)", () => { + expect(() => + fullE2eGateway(true, declaration({ version: 1, mode: "nemoclaw-managed" })), + ).toThrow("externally supervised gateway"); + }); +}); diff --git a/test/helpers/brev-launchable-e2e-fixture.ts b/test/helpers/brev-launchable-e2e-fixture.ts index 7d8f940aaa3..06bc19c241f 100644 --- a/test/helpers/brev-launchable-e2e-fixture.ts +++ b/test/helpers/brev-launchable-e2e-fixture.ts @@ -45,13 +45,16 @@ export function fixture( createAppearsAfterRefresh?: number; createStatus?: number; deleteFails?: boolean; + diagnosticResolverMissing?: boolean; e2eDiagnosticTimesOut?: boolean; e2eFails?: boolean; gatewayChildJournal?: string; gatewayExecStart?: string; imageRepositorySha?: string; listenerOutput?: string; + gatewayEndpoint?: string; missingProvisionReceipt?: boolean; + omitCommandEvidence?: boolean; omitReceiptField?: "imageName" | "imageRepositorySha" | "project"; platformDiagnosticFails?: boolean; provisionImageRepositorySha?: string; @@ -89,6 +92,21 @@ export function fixture( fs.mkdirSync(bin); fs.mkdirSync(workDir); fs.writeFileSync(timeoutBlock, "block\n"); + fs.writeFileSync( + path.join(root, "gateway.json"), + JSON.stringify({ + version: 1, + mode: "externally-supervised", + endpoint: options.gatewayEndpoint ?? "https://127.0.0.1:18080", + stateDir: "/var/lib/brev/openshell-gateway", + supervisor: { + kind: "systemd-system", + serviceName: "openshell-gateway.service", + execPath: "/usr/local/bin/openshell-gateway", + }, + requiredCapabilities: ["gateway.health", "sandbox.create", "sandbox.exec"], + }), + ); executable( path.join(bin, "timeout"), @@ -168,6 +186,7 @@ exec ${JSON.stringify(REAL_STAT)} "$@" path.join(bin, "ss"), `#!/usr/bin/env bash set -euo pipefail +printf 'ss %s\n' "$*" >> "$FAKE_CALLS" printf '%s\n' "$FAKE_LISTENER_OUTPUT" `, ); @@ -454,7 +473,8 @@ case "$remote" in exit $? ;; *"ss -H -ltnp"*) probe_options_present "$@" - printf 'ssh full-e2e diagnostic port 8080 listener\n' >> "$FAKE_CALLS" + printf 'ssh full-e2e diagnostic declared gateway listener\n' >> "$FAKE_CALLS" + remote="\${remote/\\/opt\\/nemoclaw-image\\/NemoClaw/$FAKE_REPO_ROOT}" bash -c "$remote" exit $? ;; "bash -s") ;; @@ -469,6 +489,10 @@ grep -q 'NEMOCLAW_SOURCE_PATH=/opt/nemoclaw-image/NemoClaw' <<<"$script" grep -q 'runtime-overrides.json' <<<"$script" printf 'ssh preinstalled full-e2e.test.ts\\n' >> "$FAKE_CALLS" printf 'remote output contains %s\\n' "$NVIDIA_INFERENCE_API_KEY" +grep -q 'NEMOCLAW_E2E_COMMAND_EVIDENCE=1' <<<"$script" +if [ "$FAKE_OMIT_COMMAND_EVIDENCE" != 1 ]; then +printf 'NEMOCLAW_E2E_COMMAND {"schemaVersion":1,"command":["brev-quickstart","e2e-staging"],"startedAt":"2026-09-10T18:57:30.000Z","finishedAt":"2026-09-10T18:57:31.000Z","durationMs":1000,"exitCode":1,"signal":null,"timedOut":false}\\n' +fi [ "$FAKE_E2E_FAILS" != 1 ] || exit 7 printf 'NEMOCLAW_FULL_E2E_PASSED\\n' `, @@ -476,6 +500,9 @@ printf 'NEMOCLAW_FULL_E2E_PASSED\\n' const env: NodeJS.ProcessEnv = { ...process.env, + NEMOCLAW_GATEWAY_MANAGEMENT: path.join(root, "gateway.json"), + FAKE_REPO_ROOT: options.diagnosticResolverMissing ? root : REPO_ROOT, + FAKE_OMIT_COMMAND_EVIDENCE: options.omitCommandEvidence ? "1" : "0", PATH: `${bin}:${process.env.PATH ?? ""}`, BREV_DELETE_TIMEOUT_SECONDS: "5", BREV_READY_TIMEOUT_SECONDS: "5", diff --git a/tools/e2e/brev-launchable-e2e.sh b/tools/e2e/brev-launchable-e2e.sh index 1709f75a034..62059f6310f 100755 --- a/tools/e2e/brev-launchable-e2e.sh +++ b/tools/e2e/brev-launchable-e2e.sh @@ -304,8 +304,8 @@ capture_full_e2e_failure_diagnostics() { # shellcheck disable=SC2016 run_budgeted_diagnostic_probe "$deadline" diagnostic_output diagnostic_status \ ssh "${SSH_PROBE_OPTIONS[@]}" "$INSTANCE_NAME" \ - 'set -eu; listeners=$(sudo -n ss -H -ltnp "sport = :8080"); if [ -z "$listeners" ]; then printf "listener presence: absent\n"; else printf "listener presence: present\n"; pids=; if parsed_pids=$(printf "%s\n" "$listeners" | awk "function reject(){bad=1;exit} { marker=\"users:(\"; at=index(\$0,marker); if(!at) reject(); s=substr(\$0,at+length(marker)); sub(/[[:space:]]+\$/, \"\", s); parsed=0; while(match(s,/^\\(\"[^\"]*\",pid=[0-9]+,fd=[0-9]+\\)/)){ tuple=substr(s,1,RLENGTH); pid=tuple; sub(/^.*\",pid=/,\"\",pid); sub(/,fd=.*/,\"\",pid); seen[pid]=1; total++; parsed++; s=substr(s,RLENGTH+1); if(s==\")\"){s=\"\";break} if(substr(s,1,1)!=\",\") reject(); s=substr(s,2) } if(!parsed||s!=\"\") reject() } END{if(bad||!total) exit 1; for(pid in seen) print pid}"); then pids=$parsed_pids; fi; gateway_cgroup=$(sudo -n systemctl show --no-pager --property=ControlGroup --value openshell-gateway.service); if [ -z "$pids" ] || [ -z "$gateway_cgroup" ]; then printf "listener owner: unavailable\n"; else gateway_owner=0; other_owner=0; unavailable_owner=0; for pid in $pids; do if cgroup=$(sudo -n cat "/proc/$pid/cgroup" 2>/dev/null); then if printf "%s\n" "$cgroup" | awk -F: -v wanted="$gateway_cgroup" "\$3 == wanted || (wanted != \"/\" && index(\$3, wanted \"/\") == 1) { found=1 } END { exit !found }"; then gateway_owner=1; else other_owner=1; fi; else unavailable_owner=1; fi; done; if [ "$unavailable_owner" -eq 1 ]; then printf "listener owner: unavailable\n"; elif [ "$gateway_owner" -eq 1 ] && [ "$other_owner" -eq 1 ]; then printf "listener owner: mixed\n"; elif [ "$gateway_owner" -eq 1 ]; then printf "listener owner: openshell-gateway\n"; elif [ "$other_owner" -eq 1 ]; then printf "listener owner: unexpected\n"; else printf "listener owner: unavailable\n"; fi; fi; fi' - report_full_e2e_failure_diagnostic "port 8080 listener" "$diagnostic_status" "$diagnostic_output" + 'set -eu; cd /opt/nemoclaw-image/NemoClaw; port=$(node --import tsx -e "console.log(require(\"./test/e2e/fixtures/full-e2e-gateway.ts\").fullE2eGateway(true).env.NEMOCLAW_GATEWAY_PORT)"); printf "declared gateway port: %s\n" "$port"; listeners=$(sudo -n ss -H -ltnp "sport = :$port"); if [ -z "$listeners" ]; then printf "listener presence: absent\n"; else printf "listener presence: present\n"; pids=; if parsed_pids=$(printf "%s\n" "$listeners" | awk "function reject(){bad=1;exit} { marker=\"users:(\"; at=index(\$0,marker); if(!at) reject(); s=substr(\$0,at+length(marker)); sub(/[[:space:]]+\$/, \"\", s); parsed=0; while(match(s,/^\\(\"[^\"]*\",pid=[0-9]+,fd=[0-9]+\\)/)){ tuple=substr(s,1,RLENGTH); pid=tuple; sub(/^.*\",pid=/,\"\",pid); sub(/,fd=.*/,\"\",pid); seen[pid]=1; total++; parsed++; s=substr(s,RLENGTH+1); if(s==\")\"){s=\"\";break} if(substr(s,1,1)!=\",\") reject(); s=substr(s,2) } if(!parsed||s!=\"\") reject() } END{if(bad||!total) exit 1; for(pid in seen) print pid}"); then pids=$parsed_pids; fi; gateway_cgroup=$(sudo -n systemctl show --no-pager --property=ControlGroup --value openshell-gateway.service); if [ -z "$pids" ] || [ -z "$gateway_cgroup" ]; then printf "listener owner: unavailable\n"; else gateway_owner=0; other_owner=0; unavailable_owner=0; for pid in $pids; do if cgroup=$(sudo -n cat "/proc/$pid/cgroup" 2>/dev/null); then if printf "%s\n" "$cgroup" | awk -F: -v wanted="$gateway_cgroup" "\$3 == wanted || (wanted != \"/\" && index(\$3, wanted \"/\") == 1) { found=1 } END { exit !found }"; then gateway_owner=1; else other_owner=1; fi; else unavailable_owner=1; fi; done; if [ "$unavailable_owner" -eq 1 ]; then printf "listener owner: unavailable\n"; elif [ "$gateway_owner" -eq 1 ] && [ "$other_owner" -eq 1 ]; then printf "listener owner: mixed\n"; elif [ "$gateway_owner" -eq 1 ]; then printf "listener owner: openshell-gateway\n"; elif [ "$other_owner" -eq 1 ]; then printf "listener owner: unexpected\n"; else printf "listener owner: unavailable\n"; fi; fi; fi' + report_full_e2e_failure_diagnostic "declared gateway listener" "$diagnostic_status" "$diagnostic_output" } report_probe() { @@ -995,6 +995,7 @@ cd "$NEMOCLAW_SOURCE_PATH" test -x ./node_modules/.bin/vitest export CI=true GITHUB_ACTIONS=true E2E_TARGET_ID=staging-brev-launchable export NEMOCLAW_E2E_SETUP_MODE=preinstalled-launchable NEMOCLAW_RUN_LIVE_E2E=1 +export NEMOCLAW_E2E_COMMAND_EVIDENCE=1 export NEMOCLAW_MODEL="$(node /usr/local/lib/nemoclaw/launchable-config.mjs /usr/local/share/nemoclaw/launchable-agents.json openclaw cloudModel)" export NEMOCLAW_SANDBOX_NAME=e2e-staging ./node_modules/.bin/vitest run --project e2e-live test/e2e/live/full-e2e.test.ts --silent=false --reporter=default @@ -1016,6 +1017,9 @@ Path(target).write_bytes(Path(source).read_bytes().replace(secret.encode(), b"[R Path(source).unlink(missing_ok=True) PY raw_log="" +if ! grep -q '^NEMOCLAW_E2E_COMMAND ' "$WORK_DIR/full-e2e.log"; then + log "Completed command metadata unavailable: the guest emitted no command records" +fi if [ "$e2e_status" -ne 0 ] || ! grep -q '^NEMOCLAW_FULL_E2E_PASSED$' "$WORK_DIR/full-e2e.log"; then jq '.fullE2e = "failed" | .validation.fullE2E = "failed"' \ "$WORK_DIR/launchable-e2e.json" >"$WORK_DIR/launchable-e2e.tmp" From a545c56be76a445536e500e39591507114470081 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Thu, 10 Sep 2026 19:36:05 -0400 Subject: [PATCH 2/6] test(e2e): align gateway diagnostics with declared listener Signed-off-by: Julie Yaunches --- .../brev-launchable-gateway-diagnostics.test.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/e2e-runtime/brev-launchable-gateway-diagnostics.test.ts b/test/e2e-runtime/brev-launchable-gateway-diagnostics.test.ts index b514e4d6e6f..314363b5286 100644 --- a/test/e2e-runtime/brev-launchable-gateway-diagnostics.test.ts +++ b/test/e2e-runtime/brev-launchable-gateway-diagnostics.test.ts @@ -338,7 +338,10 @@ describe("focused staging Brev Launchable failure diagnostics", () => { expect(laneLog).toContain("[REDACTED PRIVATE KEY]"); expect(laneLog).toContain("[REDACTED LONG LINE]"); expect(laneLog).toContain("Full E2E failure diagnostic gateway lifecycle: status 0; output:"); - expect(laneLog).toContain("Full E2E failure diagnostic port 8080 listener: status 0; output:"); + expect(laneLog).toContain( + "Full E2E failure diagnostic declared gateway listener: status 0; output:", + ); + expect(laneLog).toContain("declared gateway port: 18080"); const commands = fs.readFileSync(calls, "utf8"); expect(commands.indexOf("ssh full-e2e diagnostic platform state")).toBeLessThan( commands.indexOf("ssh full-e2e diagnostic gateway lifecycle"), @@ -434,7 +437,7 @@ describe("focused staging Brev Launchable failure diagnostics", () => { ["listener presence: present", "listener owner: unavailable"], ], ])( - "classifies port 8080 listener evidence with %s (#6409)", + "classifies declared gateway listener evidence with %s (#6409)", (_name, listenerOutput, expectedEvidence) => { const { env, workDir } = fixture({ e2eFails: true, @@ -501,7 +504,7 @@ describe("focused staging Brev Launchable failure diagnostics", () => { "Full E2E failure diagnostic platform state: not run; output: diagnostic budget exhausted", ); expect(laneLog).toContain( - "Full E2E failure diagnostic port 8080 listener: not run; output: diagnostic budget exhausted", + "Full E2E failure diagnostic declared gateway listener: not run; output: diagnostic budget exhausted", ); const commands = fs.readFileSync(calls, "utf8"); expect(commands).not.toContain("ssh full-e2e diagnostic platform state"); From a4f8f6214203382225058e7de457482d4ae701de Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Thu, 10 Sep 2026 23:52:36 -0400 Subject: [PATCH 3/6] test(e2e): preserve source-install budget coverage Signed-off-by: Julie Yaunches --- test/e2e/support/full-e2e-gateway.test.ts | 34 ++++++++++++++++------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/test/e2e/support/full-e2e-gateway.test.ts b/test/e2e/support/full-e2e-gateway.test.ts index 38bea586efb..be17c6c4538 100644 --- a/test/e2e/support/full-e2e-gateway.test.ts +++ b/test/e2e/support/full-e2e-gateway.test.ts @@ -8,7 +8,9 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { fullE2eGateway } from "../fixtures/full-e2e-gateway.ts"; const directories: string[] = []; -afterEach(() => { +const disposables: (() => Promise)[] = []; +afterEach(async () => { + for (const dispose of disposables.splice(0).reverse()) await dispose(); for (const directory of directories.splice(0)) fs.rmSync(directory, { recursive: true }); vi.unstubAllEnvs(); }); @@ -57,16 +59,20 @@ function declaration( } describe("full E2E gateway ownership", () => { - it.each([true, false])( - "respects gateway cleanup ownership in the live suite (preinstalled=%s) (#9851)", - async (preinstalled) => { + it.each([ + { preinstalled: true, targetId: "staging-brev-launchable", measuresColdOnboard: false }, + { preinstalled: false, targetId: "staging-brev-launchable", measuresColdOnboard: false }, + { preinstalled: false, targetId: "full-e2e", measuresColdOnboard: true }, + ])( + "respects cleanup and budget contracts for $targetId (preinstalled=$preinstalled) (#9851)", + async ({ preinstalled, targetId, measuresColdOnboard }) => { vi.resetModules(); captured.test.mockClear(); vi.stubEnv( "NEMOCLAW_E2E_SETUP_MODE", preinstalled ? "preinstalled-launchable" : "source-install", ); - vi.stubEnv("E2E_TARGET_ID", "staging-brev-launchable"); + vi.stubEnv("E2E_TARGET_ID", targetId); vi.stubEnv("NEMOCLAW_EXPERIMENTAL_PROFILE", ""); vi.stubEnv("NEMOCLAW_GATEWAY_MANAGEMENT", declaration().NEMOCLAW_GATEWAY_MANAGEMENT); const result = { exitCode: 0, stdout: "", stderr: "", timedOut: false, signal: null }; @@ -76,8 +82,14 @@ describe("full E2E gateway ownership", () => { exitCode: command === "brev-quickstart" || command === "bash" ? 42 : 0, })), }; - const sandbox = { openshell: vi.fn(async () => result) }; - const cleanup = { trackGateway: vi.fn(), trackDisposable: vi.fn(), trackSandbox: vi.fn() }; + const sandbox = { openshell: vi.fn(async () => result), cleanupSandbox: vi.fn() }; + const cleanup = { + trackGateway: vi.fn(), + trackDisposable: vi.fn((_name: string, dispose: () => Promise) => { + disposables.push(dispose); + }), + trackSandbox: vi.fn(), + }; const lifecycle = { trackInstallerGatewayUserService: vi.fn() }; const declare = vi.fn(); await import("../live/full-e2e.test.ts"); @@ -105,9 +117,11 @@ describe("full E2E gateway ownership", () => { ][]; expect(calls.some(([args]) => args[0] === "gateway")).toBe(!preinstalled); expect(calls[0]![1].env.OPENSHELL_GATEWAY).toBe(preinstalled ? "nemoclaw-18080" : "nemoclaw"); - expect(declare.mock.calls[0]![0].contracts).not.toContain( - "cold onboarding stays within the checked-in full-E2E performance budgets", - ); + expect( + declare.mock.calls[0]![0].contracts.includes( + "cold onboarding stays within the checked-in full-E2E performance budgets", + ), + ).toBe(measuresColdOnboard); }, 20_000, ); From caa309de9fe9da85dec596c8cf114781e4ade21b Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 11 Sep 2026 00:13:24 -0400 Subject: [PATCH 4/6] test(e2e): assert Launchable onboarding gateway environment Signed-off-by: Julie Yaunches --- test/e2e/support/full-e2e-gateway.test.ts | 26 +++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/test/e2e/support/full-e2e-gateway.test.ts b/test/e2e/support/full-e2e-gateway.test.ts index be17c6c4538..7aa000b2b5b 100644 --- a/test/e2e/support/full-e2e-gateway.test.ts +++ b/test/e2e/support/full-e2e-gateway.test.ts @@ -77,10 +77,16 @@ describe("full E2E gateway ownership", () => { vi.stubEnv("NEMOCLAW_GATEWAY_MANAGEMENT", declaration().NEMOCLAW_GATEWAY_MANAGEMENT); const result = { exitCode: 0, stdout: "", stderr: "", timedOut: false, signal: null }; const host = { - command: vi.fn(async (command: string) => ({ - ...result, - exitCode: command === "brev-quickstart" || command === "bash" ? 42 : 0, - })), + command: vi.fn( + async ( + command: string, + _args?: readonly string[], + _options?: { env?: NodeJS.ProcessEnv }, + ) => ({ + ...result, + exitCode: command === "brev-quickstart" || command === "bash" ? 42 : 0, + }), + ), }; const sandbox = { openshell: vi.fn(async () => result), cleanupSandbox: vi.fn() }; const cleanup = { @@ -107,6 +113,18 @@ describe("full E2E gateway ownership", () => { expect(host.command.mock.calls.map((call) => call[0])).toContain( preinstalled ? "brev-quickstart" : "bash", ); + const install = host.command.mock.calls.find( + ([command]) => command === (preinstalled ? "brev-quickstart" : "bash"), + ); + expect(install?.[2]?.env).toMatchObject({ + OPENSHELL_GATEWAY: preinstalled ? "nemoclaw-18080" : "nemoclaw", + ...(preinstalled + ? { + NEMOCLAW_GATEWAY_PORT: "18080", + NEMOCLAW_GATEWAY_MANAGEMENT: process.env.NEMOCLAW_GATEWAY_MANAGEMENT, + } + : {}), + }); expect(cleanup.trackGateway).toHaveBeenCalledTimes(preinstalled ? 0 : 1); expect(lifecycle.trackInstallerGatewayUserService).toHaveBeenCalledTimes( preinstalled ? 0 : 1, From cb6de7f50f4fbbeafa7535c683ea21f4f02649f3 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 11 Sep 2026 01:15:51 -0400 Subject: [PATCH 5/6] test(e2e): prove Launchable command evidence transport Signed-off-by: Julie Yaunches --- test/e2e-runtime/brev-launchable-e2e.test.ts | 26 ++++++++++ test/helpers/brev-launchable-e2e-fixture.ts | 51 ++++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/test/e2e-runtime/brev-launchable-e2e.test.ts b/test/e2e-runtime/brev-launchable-e2e.test.ts index 1fec0c2a62a..0eb203cdd09 100644 --- a/test/e2e-runtime/brev-launchable-e2e.test.ts +++ b/test/e2e-runtime/brev-launchable-e2e.test.ts @@ -29,6 +29,32 @@ function identitySmokeEnv(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { } describe("focused staging Brev Launchable lane", () => { + it("retains real guest ShellProbe evidence through Vitest and SSH capture (#9851)", () => { + const { env, workDir } = fixture({ realCommandEvidence: true }); + const startedAt = Date.now(); + const result = run(env); + const finishedAt = Date.now(); + const log = fs.readFileSync(path.join(workDir, "full-e2e.log"), "utf8"); + expect(result.status, log).toBe(0); + const records = log + .split("\n") + .filter((line) => line.startsWith("NEMOCLAW_E2E_COMMAND ")) + .map((line) => JSON.parse(line.slice("NEMOCLAW_E2E_COMMAND ".length))); + expect(records).toHaveLength(1); + expect(records[0]).toMatchObject({ schemaVersion: 1, exitCode: 0, timedOut: false }); + expect(records[0].command).toContain("guest-command-proof"); + expect(Date.parse(records[0].startedAt)).toBeGreaterThanOrEqual(startedAt); + expect(Date.parse(records[0].finishedAt)).toBeLessThanOrEqual(finishedAt); + expect(Date.parse(records[0].finishedAt) - Date.parse(records[0].startedAt)).toBe( + records[0].durationMs, + ); + expect(log).not.toContain("nvapi-test-value"); + expect(log).not.toContain("guest-private-output\n"); + expect(JSON.parse(fs.readFileSync(path.join(workDir, "cleanup.json"), "utf8"))).toMatchObject({ + status: "ABSENT", + }); + }); + it("reports missing command evidence from an older baked suite without changing its result (#9851)", () => { const { env, workDir } = fixture({ omitCommandEvidence: true }); expect(run(env).status).toBe(0); diff --git a/test/helpers/brev-launchable-e2e-fixture.ts b/test/helpers/brev-launchable-e2e-fixture.ts index 06bc19c241f..682fb2d778a 100644 --- a/test/helpers/brev-launchable-e2e-fixture.ts +++ b/test/helpers/brev-launchable-e2e-fixture.ts @@ -60,6 +60,7 @@ export function fixture( provisionImageRepositorySha?: string; provisionSha?: string; ready?: boolean; + realCommandEvidence?: boolean; receiptSha?: string; refreshError?: string; refreshStatus?: number; @@ -91,6 +92,49 @@ export function fixture( const timeoutBlock = path.join(root, "timeout-block"); fs.mkdirSync(bin); fs.mkdirSync(workDir); + const bakedRoot = path.join(root, "baked"); + if (options.realCommandEvidence) { + fs.mkdirSync(path.join(bakedRoot, "node_modules", ".bin"), { recursive: true }); + fs.mkdirSync(path.join(bakedRoot, "test", "e2e", "live"), { recursive: true }); + fs.writeFileSync( + path.join(bakedRoot, "vitest.config.mts"), + `export default { test: { projects: [{ test: { name: "e2e-live", include: ["test/e2e/live/full-e2e.test.ts"], maxWorkers: 1 } }] } };`, + ); + fs.writeFileSync( + path.join(bakedRoot, "launchable-config.mjs"), + 'console.log("fixture-cloud-model");\n', + ); + fs.writeFileSync( + path.join(bakedRoot, "test", "e2e", "live", "full-e2e.test.ts"), + `import { it, expect } from ${JSON.stringify(path.join(REPO_ROOT, "node_modules/vitest/dist/index.js"))}; +import { ArtifactSink } from ${JSON.stringify(path.join(REPO_ROOT, "test/e2e/fixtures/artifacts.ts"))}; +import { startTestProgress } from ${JSON.stringify(path.join(REPO_ROOT, "test/e2e/fixtures/progress.ts"))}; +import { redactString } from ${JSON.stringify(path.join(REPO_ROOT, "test/e2e/fixtures/redaction.ts"))}; +import { ShellProbe, trustedShellCommand } from ${JSON.stringify(path.join(REPO_ROOT, "test/e2e/fixtures/shell-probe.ts"))}; +it("emits evidence from a completed guest command", async () => { + const progress = startTestProgress("guest command", ["execute command", "verify result"], { logLine: () => undefined }); + try { + const probe = new ShellProbe({ + artifacts: new ArtifactSink(${JSON.stringify(path.join(bakedRoot, "artifacts"))}, [process.env.NVIDIA_INFERENCE_API_KEY]), + progress, redact: redactString, signal: new AbortController().signal, + }); + const result = await probe.run(trustedShellCommand({ + command: process.execPath, + args: ["-e", "console.log('guest-private-output'); process.exit(0)", "guest-command-proof", process.env.NVIDIA_INFERENCE_API_KEY], + reason: "prove completed command evidence reaches the retained controller log", + }), { artifactName: "guest-command-proof" }); + expect(result.exitCode).toBe(0); + } finally { progress.stop(); } +}); +`, + ); + executable( + path.join(bakedRoot, "node_modules", ".bin", "vitest"), + `#!/usr/bin/env bash +exec ${JSON.stringify(process.execPath)} ${JSON.stringify(path.join(REPO_ROOT, "node_modules/vitest/vitest.mjs"))} --root ${JSON.stringify(bakedRoot)} --config ${JSON.stringify(path.join(bakedRoot, "vitest.config.mts"))} "$@" +`, + ); + } fs.writeFileSync(timeoutBlock, "block\n"); fs.writeFileSync( path.join(root, "gateway.json"), @@ -487,6 +531,12 @@ script="$(cat)" grep -q 'NEMOCLAW_E2E_SETUP_MODE=preinstalled-launchable' <<<"$script" grep -q 'NEMOCLAW_SOURCE_PATH=/opt/nemoclaw-image/NemoClaw' <<<"$script" grep -q 'runtime-overrides.json' <<<"$script" +if [ -n "$FAKE_BAKED_ROOT" ]; then + script="\${script/\\/opt\\/nemoclaw-image\\/NemoClaw/$FAKE_BAKED_ROOT}" + script="\${script/\\/etc\\/nemoclaw\\/runtime-overrides.json/$FAKE_BAKED_ROOT/runtime-overrides.json}" + script="\${script/\\/usr\\/local\\/lib\\/nemoclaw\\/launchable-config.mjs/$FAKE_BAKED_ROOT/launchable-config.mjs}" + exec bash -s <<<"$script" +fi printf 'ssh preinstalled full-e2e.test.ts\\n' >> "$FAKE_CALLS" printf 'remote output contains %s\\n' "$NVIDIA_INFERENCE_API_KEY" grep -q 'NEMOCLAW_E2E_COMMAND_EVIDENCE=1' <<<"$script" @@ -502,6 +552,7 @@ printf 'NEMOCLAW_FULL_E2E_PASSED\\n' ...process.env, NEMOCLAW_GATEWAY_MANAGEMENT: path.join(root, "gateway.json"), FAKE_REPO_ROOT: options.diagnosticResolverMissing ? root : REPO_ROOT, + FAKE_BAKED_ROOT: options.realCommandEvidence ? bakedRoot : "", FAKE_OMIT_COMMAND_EVIDENCE: options.omitCommandEvidence ? "1" : "0", PATH: `${bin}:${process.env.PATH ?? ""}`, BREV_DELETE_TIMEOUT_SECONDS: "5", From 7fc3eb895f37972b3e36a3a52a726d6196964836 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Fri, 11 Sep 2026 14:49:32 -0700 Subject: [PATCH 6/6] test(e2e): align Launchable gateway fixtures Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- ...rev-launchable-gateway-diagnostics.test.ts | 28 +++++++++---------- test/e2e/fixtures/full-e2e-gateway.ts | 2 +- test/e2e/support/full-e2e-gateway.test.ts | 13 +++++++++ test/helpers/brev-launchable-e2e-fixture.ts | 2 +- 4 files changed, 29 insertions(+), 16 deletions(-) diff --git a/test/e2e-runtime/brev-launchable-gateway-diagnostics.test.ts b/test/e2e-runtime/brev-launchable-gateway-diagnostics.test.ts index 314363b5286..225f4de9a04 100644 --- a/test/e2e-runtime/brev-launchable-gateway-diagnostics.test.ts +++ b/test/e2e-runtime/brev-launchable-gateway-diagnostics.test.ts @@ -372,67 +372,67 @@ describe("focused staging Brev Launchable failure diagnostics", () => { ["absent", "", ["listener presence: absent"]], [ "expected owner", - 'LISTEN 0 4096 127.0.0.1:8080 0.0.0.0:* users:(("openshell-gateway",pid=98,fd=3))', + 'LISTEN 0 4096 127.0.0.1:18080 0.0.0.0:* users:(("openshell-gateway",pid=98,fd=3))', ["listener presence: present", "listener owner: openshell-gateway"], ], [ "expected owner in a v2 descendant cgroup", - 'LISTEN 0 4096 127.0.0.1:8080 0.0.0.0:* users:(("openshell-gateway",pid=97,fd=3))', + 'LISTEN 0 4096 127.0.0.1:18080 0.0.0.0:* users:(("openshell-gateway",pid=97,fd=3))', ["listener presence: present", "listener owner: openshell-gateway"], ], [ "expected owner in an exact v1 cgroup", - 'LISTEN 0 4096 127.0.0.1:8080 0.0.0.0:* users:(("openshell-gateway",pid=96,fd=3))', + 'LISTEN 0 4096 127.0.0.1:18080 0.0.0.0:* users:(("openshell-gateway",pid=96,fd=3))', ["listener presence: present", "listener owner: openshell-gateway"], ], [ "expected owner in a v1 descendant cgroup", - 'LISTEN 0 4096 127.0.0.1:8080 0.0.0.0:* users:(("openshell-gateway",pid=95,fd=3))', + 'LISTEN 0 4096 127.0.0.1:18080 0.0.0.0:* users:(("openshell-gateway",pid=95,fd=3))', ["listener presence: present", "listener owner: openshell-gateway"], ], [ "mixed owners", [ - 'LISTEN 0 4096 127.0.0.1:8080 0.0.0.0:* users:(("openshell-gateway",pid=98,fd=3))', - 'LISTEN 0 4096 172.18.0.1:8080 0.0.0.0:* users:(("s3cr3t",pid=99,fd=4))', + 'LISTEN 0 4096 127.0.0.1:18080 0.0.0.0:* users:(("openshell-gateway",pid=98,fd=3))', + 'LISTEN 0 4096 172.18.0.1:18080 0.0.0.0:* users:(("s3cr3t",pid=99,fd=4))', ].join("\n"), ["listener presence: present", "listener owner: mixed"], ], [ "mixed owners in one socket record", - 'LISTEN 0 4096 127.0.0.1:8080 0.0.0.0:* users:(("openshell-gateway",pid=98,fd=3),("s3cr3t",pid=99,fd=4))', + 'LISTEN 0 4096 127.0.0.1:18080 0.0.0.0:* users:(("openshell-gateway",pid=98,fd=3),("s3cr3t",pid=99,fd=4))', ["listener presence: present", "listener owner: mixed"], ], [ "unexpected owner", - 'LISTEN 0 4096 127.0.0.1:8080 0.0.0.0:* users:(("openshell-gatew",pid=94,fd=3))', + 'LISTEN 0 4096 127.0.0.1:18080 0.0.0.0:* users:(("openshell-gatew",pid=94,fd=3))', ["listener presence: present", "listener owner: unexpected"], ], [ "unrelated cgroup", - 'LISTEN 0 4096 127.0.0.1:8080 0.0.0.0:* users:(("other-process",pid=93,fd=3))', + 'LISTEN 0 4096 127.0.0.1:18080 0.0.0.0:* users:(("other-process",pid=93,fd=3))', ["listener presence: present", "listener owner: unexpected"], ], [ "owner unavailable", - "LISTEN 0 4096 127.0.0.1:8080 0.0.0.0:*", + "LISTEN 0 4096 127.0.0.1:18080 0.0.0.0:*", ["listener presence: present", "listener owner: unavailable"], ], [ "PID-like text inside a process label", - 'LISTEN 0 4096 127.0.0.1:8080 0.0.0.0:* users:(("s3cr3t,pid=7,fd=8",pid=98,fd=3))', + 'LISTEN 0 4096 127.0.0.1:18080 0.0.0.0:* users:(("s3cr3t,pid=7,fd=8",pid=98,fd=3))', ["listener presence: present", "listener owner: openshell-gateway"], ], [ "an injected owner tuple inside a process label", - 'LISTEN 0 4096 127.0.0.1:8080 0.0.0.0:* users:(("s3cr3t",pid=98,fd=3",pid=99,fd=4))', + 'LISTEN 0 4096 127.0.0.1:18080 0.0.0.0:* users:(("s3cr3t",pid=98,fd=3",pid=99,fd=4))', ["listener presence: present", "listener owner: unavailable"], ], [ "one socket record without owner metadata", [ - 'LISTEN 0 4096 127.0.0.1:8080 0.0.0.0:* users:(("openshell-gateway",pid=98,fd=3))', - "LISTEN 0 4096 172.18.0.1:8080 0.0.0.0:*", + 'LISTEN 0 4096 127.0.0.1:18080 0.0.0.0:* users:(("openshell-gateway",pid=98,fd=3))', + "LISTEN 0 4096 172.18.0.1:18080 0.0.0.0:*", ].join("\n"), ["listener presence: present", "listener owner: unavailable"], ], diff --git a/test/e2e/fixtures/full-e2e-gateway.ts b/test/e2e/fixtures/full-e2e-gateway.ts index 36873aaeb12..c797e7f589e 100644 --- a/test/e2e/fixtures/full-e2e-gateway.ts +++ b/test/e2e/fixtures/full-e2e-gateway.ts @@ -20,7 +20,7 @@ export function fullE2eGateway(preinstalled: boolean, env: NodeJS.ProcessEnv = p return { owned: true, env: { OPENSHELL_GATEWAY: resolveGatewayName(port) } }; } const declarationPath = - env.NEMOCLAW_GATEWAY_MANAGEMENT ?? "/etc/nemoclaw/gateway-management.json"; + env.NEMOCLAW_GATEWAY_MANAGEMENT?.trim() || "/etc/nemoclaw/gateway-management.json"; const loaded = loadGatewayManagementDeclaration({ env: { ...env, NEMOCLAW_GATEWAY_MANAGEMENT: declarationPath }, }); diff --git a/test/e2e/support/full-e2e-gateway.test.ts b/test/e2e/support/full-e2e-gateway.test.ts index 7aa000b2b5b..30251866c3c 100644 --- a/test/e2e/support/full-e2e-gateway.test.ts +++ b/test/e2e/support/full-e2e-gateway.test.ts @@ -12,6 +12,7 @@ const disposables: (() => Promise)[] = []; afterEach(async () => { for (const dispose of disposables.splice(0).reverse()) await dispose(); for (const directory of directories.splice(0)) fs.rmSync(directory, { recursive: true }); + vi.restoreAllMocks(); vi.unstubAllEnvs(); }); @@ -192,6 +193,18 @@ describe("full E2E gateway ownership", () => { "unsupported gateway-management contract version", ); }); + it.each([undefined, "", " "])( + "uses the system declaration path when the configured path is %s (#9851)", + (configuredPath) => { + const readFileSync = vi.spyOn(fs, "readFileSync"); + expect(() => + fullE2eGateway(true, { + NEMOCLAW_GATEWAY_MANAGEMENT: configuredPath, + }), + ).toThrow("declaration file could not be read"); + expect(readFileSync).toHaveBeenCalledWith("/etc/nemoclaw/gateway-management.json", "utf-8"); + }, + ); it("does not interpret a managed declaration as Launchable cleanup authority (#9851)", () => { expect(() => fullE2eGateway(true, declaration({ version: 1, mode: "nemoclaw-managed" })), diff --git a/test/helpers/brev-launchable-e2e-fixture.ts b/test/helpers/brev-launchable-e2e-fixture.ts index 682fb2d778a..a4463fb9ad4 100644 --- a/test/helpers/brev-launchable-e2e-fixture.ts +++ b/test/helpers/brev-launchable-e2e-fixture.ts @@ -589,7 +589,7 @@ printf 'NEMOCLAW_FULL_E2E_PASSED\\n' FAKE_IMAGE_REPOSITORY_SHA: options.imageRepositorySha ?? "b".repeat(40), FAKE_LISTENER_OUTPUT: options.listenerOutput ?? - 'LISTEN 0 4096 127.0.0.1:8080 0.0.0.0:* users:(("s3cr3t",pid=99,fd=3))', + 'LISTEN 0 4096 127.0.0.1:18080 0.0.0.0:* users:(("s3cr3t",pid=99,fd=3))', FAKE_MISSING_PROVISION_RECEIPT: options.missingProvisionReceipt ? "1" : "0", FAKE_OMIT_RECEIPT_FIELD: options.omitReceiptField ?? "", FAKE_PLATFORM_DIAGNOSTIC_FAILS: options.platformDiagnosticFails ? "1" : "0",