fix(e2e): respect Launchable gateway ownership and retain command evidence

test/e2e-runtime/brev-launchable-e2e.test.ts

@@ -29,6 +29,114 @@ function identitySmokeEnv(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
 }
 
 describe("focused staging Brev Launchable lane", () => {
+  it("retains real guest ShellProbe evidence through Vitest and SSH capture (#9851)", () => {
+    const { env, workDir } = fixture({ realCommandEvidence: true });
+    const startedAt = Date.now();
+    const result = run(env);
+    const finishedAt = Date.now();
+    const log = fs.readFileSync(path.join(workDir, "full-e2e.log"), "utf8");
+    expect(result.status, log).toBe(0);
+    const records = log
+      .split("\n")
+      .filter((line) => line.startsWith("NEMOCLAW_E2E_COMMAND "))
+      .map((line) => JSON.parse(line.slice("NEMOCLAW_E2E_COMMAND ".length)));
+    expect(records).toHaveLength(1);
+    expect(records[0]).toMatchObject({ schemaVersion: 1, exitCode: 0, timedOut: false });
+    expect(records[0].command).toContain("guest-command-proof");
+    expect(Date.parse(records[0].startedAt)).toBeGreaterThanOrEqual(startedAt);
+    expect(Date.parse(records[0].finishedAt)).toBeLessThanOrEqual(finishedAt);
+    expect(Date.parse(records[0].finishedAt) - Date.parse(records[0].startedAt)).toBe(
+      records[0].durationMs,
+    );
+    expect(log).not.toContain("nvapi-test-value");
+    expect(log).not.toContain("guest-private-output\n");
+    expect(JSON.parse(fs.readFileSync(path.join(workDir, "cleanup.json"), "utf8"))).toMatchObject({
+      status: "ABSENT",
+    });
+  });
+
+  it("reports missing command evidence from an older baked suite without changing its result (#9851)", () => {
+    const { env, workDir } = fixture({ omitCommandEvidence: true });
+    expect(run(env).status).toBe(0);
+    expect(fs.readFileSync(path.join(workDir, "full-e2e.log"), "utf8")).not.toContain(
+      "NEMOCLAW_E2E_COMMAND ",
+    );
+    expect(fs.readFileSync(path.join(workDir, "lane.log"), "utf8")).toContain(
+      "Completed command metadata unavailable: the guest emitted no command records",
+    );
+  });
+
+  it("retains completed command timestamps when onboarding fails (#9851)", () => {
+    const { env, workDir } = fixture({ e2eFails: true });
+    expect(run(env).status).not.toBe(0);
+    const log = fs.readFileSync(path.join(workDir, "full-e2e.log"), "utf8");
+    const record = JSON.parse(
+      log
+        .split("\n")
+        .find((line) => line.startsWith("NEMOCLAW_E2E_COMMAND "))!
+        .slice("NEMOCLAW_E2E_COMMAND ".length),
+    );
+    expect(record).toMatchObject({
+      command: ["brev-quickstart", "e2e-staging"],
+      startedAt: "2026-09-10T18:57:30.000Z",
+      finishedAt: "2026-09-10T18:57:31.000Z",
+      durationMs: 1000,
+      exitCode: 1,
+    });
+    expect(log).not.toContain("nvapi-test-value");
+    expect(JSON.parse(fs.readFileSync(path.join(workDir, "cleanup.json"), "utf8"))).toMatchObject({
+      status: "ABSENT",
+    });
+  });
+
+  it.each([
+    ["https://127.0.0.1:18080", 18080],
+    ["https://127.0.0.1:19443", 19443],
+    ["http://127.0.0.1:18080", 18080],
+    ["https://[::1]:19443", 19443],
+  ])("diagnoses the declared gateway at %s (#9851)", (gatewayEndpoint, port) => {
+    const { env, workDir, calls } = fixture({
+      e2eFails: true,
+      gatewayEndpoint,
+    });
+    expect(run(env).status).not.toBe(0);
+    expect(fs.readFileSync(calls, "utf8")).toContain(`ss -H -ltnp sport = :${port}`);
+    expect(fs.readFileSync(path.join(workDir, "lane.log"), "utf8")).toContain(
+      `declared gateway port: ${port}`,
+    );
+  });
+
+  it.each([
+    "https://untrusted.invalid:18080",
+    "https://127.0.0.1:99999",
+    "https://127.0.0.1",
+    "http://127.0.0.1",
+    "https://127.0.0.1:1023",
+    "$(touch /tmp/unsafe)",
+  ])(
+    "does not probe a substituted port when the declaration is invalid: %s (#9851)",
+    (gatewayEndpoint) => {
+      const { env, workDir, calls } = fixture({ e2eFails: true, gatewayEndpoint });
+      expect(run(env).status).not.toBe(0);
+      expect(fs.readFileSync(calls, "utf8")).not.toMatch(/^ss /m);
+      expect(JSON.parse(fs.readFileSync(path.join(workDir, "cleanup.json"), "utf8"))).toMatchObject(
+        { status: "ABSENT" },
+      );
+    },
+  );
+
+  it("still cleans up when the baked gateway resolver is unavailable (#9851)", () => {
+    const { env, workDir, calls } = fixture({ e2eFails: true, diagnosticResolverMissing: true });
+    expect(run(env).status).not.toBe(0);
+    expect(fs.readFileSync(calls, "utf8")).not.toMatch(/^ss /m);
+    expect(fs.readFileSync(path.join(workDir, "lane.log"), "utf8")).toContain(
+      "Full E2E failure diagnostic declared gateway listener: status 1",
+    );
+    expect(JSON.parse(fs.readFileSync(path.join(workDir, "cleanup.json"), "utf8"))).toMatchObject({
+      status: "ABSENT",
+    });
+  });
+
   it("keeps the staging SSH wrapper outside the full E2E deadline", () => {
     const source = fs.readFileSync(
       path.resolve(import.meta.dirname, "../../tools/e2e/brev-launchable-e2e.sh"),
@@ -687,7 +795,9 @@ describe("focused staging Brev Launchable lane", () => {
     expect(laneLog).toContain("[REDACTED PRIVATE KEY]");
     expect(laneLog).toContain("[REDACTED LONG LINE]");
     expect(laneLog).toContain("Full E2E failure diagnostic gateway lifecycle: status 0; output:");
-    expect(laneLog).toContain("Full E2E failure diagnostic port 8080 listener: status 0; output:");
+    expect(laneLog).toContain(
+      "Full E2E failure diagnostic declared gateway listener: status 0; output:",
+    );
     const commands = fs.readFileSync(calls, "utf8");
     expect(commands.indexOf("ssh full-e2e diagnostic platform state")).toBeLessThan(
       commands.indexOf("ssh full-e2e diagnostic gateway lifecycle"),
@@ -783,7 +893,7 @@ describe("focused staging Brev Launchable lane", () => {
       ["listener presence: present", "listener owner: unavailable"],
     ],
   ])(
-    "classifies port 8080 listener evidence with %s (#6409)",
+    "classifies declared gateway listener evidence with %s (#6409)",
     (_name, listenerOutput, expectedEvidence) => {
       const { env, workDir } = fixture({
         e2eFails: true,
@@ -850,7 +960,7 @@ describe("focused staging Brev Launchable lane", () => {
       "Full E2E failure diagnostic platform state: not run; output: diagnostic budget exhausted",
     );
     expect(laneLog).toContain(
-      "Full E2E failure diagnostic port 8080 listener: not run; output: diagnostic budget exhausted",
+      "Full E2E failure diagnostic declared gateway listener: not run; output: diagnostic budget exhausted",
     );
     const commands = fs.readFileSync(calls, "utf8");
     expect(commands).not.toContain("ssh full-e2e diagnostic platform state");

test/e2e-runtime/brev-launchable-gateway-diagnostics.test.ts

@@ -338,7 +338,10 @@ describe("focused staging Brev Launchable failure diagnostics", () => {
     expect(laneLog).toContain("[REDACTED PRIVATE KEY]");
     expect(laneLog).toContain("[REDACTED LONG LINE]");
     expect(laneLog).toContain("Full E2E failure diagnostic gateway lifecycle: status 0; output:");
-    expect(laneLog).toContain("Full E2E failure diagnostic port 8080 listener: status 0; output:");
+    expect(laneLog).toContain(
+      "Full E2E failure diagnostic declared gateway listener: status 0; output:",
+    );
+    expect(laneLog).toContain("declared gateway port: 18080");
     const commands = fs.readFileSync(calls, "utf8");
     expect(commands.indexOf("ssh full-e2e diagnostic platform state")).toBeLessThan(
       commands.indexOf("ssh full-e2e diagnostic gateway lifecycle"),
@@ -369,72 +372,72 @@ describe("focused staging Brev Launchable failure diagnostics", () => {
     ["absent", "", ["listener presence: absent"]],
     [
       "expected owner",
-      'LISTEN 0 4096 127.0.0.1:8080 0.0.0.0:* users:(("openshell-gateway",pid=98,fd=3))',
+      'LISTEN 0 4096 127.0.0.1:18080 0.0.0.0:* users:(("openshell-gateway",pid=98,fd=3))',
       ["listener presence: present", "listener owner: openshell-gateway"],
     ],
     [
       "expected owner in a v2 descendant cgroup",
-      'LISTEN 0 4096 127.0.0.1:8080 0.0.0.0:* users:(("openshell-gateway",pid=97,fd=3))',
+      'LISTEN 0 4096 127.0.0.1:18080 0.0.0.0:* users:(("openshell-gateway",pid=97,fd=3))',
       ["listener presence: present", "listener owner: openshell-gateway"],
     ],
     [
       "expected owner in an exact v1 cgroup",
-      'LISTEN 0 4096 127.0.0.1:8080 0.0.0.0:* users:(("openshell-gateway",pid=96,fd=3))',
+      'LISTEN 0 4096 127.0.0.1:18080 0.0.0.0:* users:(("openshell-gateway",pid=96,fd=3))',
       ["listener presence: present", "listener owner: openshell-gateway"],
     ],
     [
       "expected owner in a v1 descendant cgroup",
-      'LISTEN 0 4096 127.0.0.1:8080 0.0.0.0:* users:(("openshell-gateway",pid=95,fd=3))',
+      'LISTEN 0 4096 127.0.0.1:18080 0.0.0.0:* users:(("openshell-gateway",pid=95,fd=3))',
       ["listener presence: present", "listener owner: openshell-gateway"],
     ],
     [
       "mixed owners",
       [
-        'LISTEN 0 4096 127.0.0.1:8080 0.0.0.0:* users:(("openshell-gateway",pid=98,fd=3))',
-        'LISTEN 0 4096 172.18.0.1:8080 0.0.0.0:* users:(("s3cr3t",pid=99,fd=4))',
+        'LISTEN 0 4096 127.0.0.1:18080 0.0.0.0:* users:(("openshell-gateway",pid=98,fd=3))',
+        'LISTEN 0 4096 172.18.0.1:18080 0.0.0.0:* users:(("s3cr3t",pid=99,fd=4))',
       ].join("\n"),
       ["listener presence: present", "listener owner: mixed"],
     ],
     [
       "mixed owners in one socket record",
-      'LISTEN 0 4096 127.0.0.1:8080 0.0.0.0:* users:(("openshell-gateway",pid=98,fd=3),("s3cr3t",pid=99,fd=4))',
+      'LISTEN 0 4096 127.0.0.1:18080 0.0.0.0:* users:(("openshell-gateway",pid=98,fd=3),("s3cr3t",pid=99,fd=4))',
       ["listener presence: present", "listener owner: mixed"],
     ],
     [
       "unexpected owner",
-      'LISTEN 0 4096 127.0.0.1:8080 0.0.0.0:* users:(("openshell-gatew",pid=94,fd=3))',
+      'LISTEN 0 4096 127.0.0.1:18080 0.0.0.0:* users:(("openshell-gatew",pid=94,fd=3))',
       ["listener presence: present", "listener owner: unexpected"],
     ],
     [
       "unrelated cgroup",
-      'LISTEN 0 4096 127.0.0.1:8080 0.0.0.0:* users:(("other-process",pid=93,fd=3))',
+      'LISTEN 0 4096 127.0.0.1:18080 0.0.0.0:* users:(("other-process",pid=93,fd=3))',
       ["listener presence: present", "listener owner: unexpected"],
     ],
     [
       "owner unavailable",
-      "LISTEN 0 4096 127.0.0.1:8080 0.0.0.0:*",
+      "LISTEN 0 4096 127.0.0.1:18080 0.0.0.0:*",
       ["listener presence: present", "listener owner: unavailable"],
     ],
     [
       "PID-like text inside a process label",
-      'LISTEN 0 4096 127.0.0.1:8080 0.0.0.0:* users:(("s3cr3t,pid=7,fd=8",pid=98,fd=3))',
+      'LISTEN 0 4096 127.0.0.1:18080 0.0.0.0:* users:(("s3cr3t,pid=7,fd=8",pid=98,fd=3))',
       ["listener presence: present", "listener owner: openshell-gateway"],
     ],
     [
       "an injected owner tuple inside a process label",
-      'LISTEN 0 4096 127.0.0.1:8080 0.0.0.0:* users:(("s3cr3t",pid=98,fd=3",pid=99,fd=4))',
+      'LISTEN 0 4096 127.0.0.1:18080 0.0.0.0:* users:(("s3cr3t",pid=98,fd=3",pid=99,fd=4))',
       ["listener presence: present", "listener owner: unavailable"],
     ],
     [
       "one socket record without owner metadata",
       [
-        'LISTEN 0 4096 127.0.0.1:8080 0.0.0.0:* users:(("openshell-gateway",pid=98,fd=3))',
-        "LISTEN 0 4096 172.18.0.1:8080 0.0.0.0:*",
+        'LISTEN 0 4096 127.0.0.1:18080 0.0.0.0:* users:(("openshell-gateway",pid=98,fd=3))',
+        "LISTEN 0 4096 172.18.0.1:18080 0.0.0.0:*",
       ].join("\n"),
       ["listener presence: present", "listener owner: unavailable"],
     ],
   ])(
-    "classifies port 8080 listener evidence with %s (#6409)",
+    "classifies declared gateway listener evidence with %s (#6409)",
     (_name, listenerOutput, expectedEvidence) => {
       const { env, workDir } = fixture({
         e2eFails: true,
@@ -501,7 +504,7 @@ describe("focused staging Brev Launchable failure diagnostics", () => {
       "Full E2E failure diagnostic platform state: not run; output: diagnostic budget exhausted",
     );
     expect(laneLog).toContain(
-      "Full E2E failure diagnostic port 8080 listener: not run; output: diagnostic budget exhausted",
+      "Full E2E failure diagnostic declared gateway listener: not run; output: diagnostic budget exhausted",
     );
     const commands = fs.readFileSync(calls, "utf8");
     expect(commands).not.toContain("ssh full-e2e diagnostic platform state");

test/e2e/README.md

@@ -1132,6 +1132,22 @@ phase artifact created before exit. A preparation failure can produce no
 artifact. A later early failure can retain only `lane.log`. A successful job
 contains `launchable-e2e.json`, `full-e2e.log`, and `cleanup.json`;
 `cleanup.json` exists only after the job confirms workspace absence.
+The preinstalled suite resolves its gateway name and port from the external
+gateway declaration before registering cleanup. It removes its sandbox but
+does not remove the platform gateway registration or service. Source-install
+runs retain their test-owned gateway cleanup.
+The Launchable controller enables `NEMOCLAW_E2E_COMMAND_EVIDENCE=1` to retain
+completed command records in `full-e2e.log`. Each `NEMOCLAW_E2E_COMMAND` JSON
+line contains redacted argv, UTC start and finish timestamps, duration, exit
+status, signal, and timeout state. Spawn failures also emit a record. Commands
+that explicitly disable artifact persistence emit none. Output bodies remain
+in guest artifacts; this stream does not export them. Oversized command argv
+is omitted with `commandOmitted: "size-limit"`. Abrupt guest or transport loss
+can leave no completion record for an active command. Older baked suites may
+emit no records; the controller reports that absence rather than inferring
+command times from phase reports.
+The preinstalled suite does not run the source-install cold-onboarding budget
+and does not declare that budget as tested coverage.
 When the preinstalled full E2E fails after SSH succeeds, the job attempts to
 append bounded, redacted host state and fixed lifecycle classifications to
 `lane.log` before cleanup. On the host, the SSH command reads the system journal
@@ -1141,6 +1157,10 @@ GitHub-hosted runner or `lane.log`. If a probe fails or the shared budget
 expires, `lane.log` records that result and cleanup continues. The diagnostic
 phase is read-only, uses one 30-second budget, and does not retry the failed E2E
 or repair the workspace.
+The listener diagnostic uses the baked suite's gateway resolver, including its
+declaration path, loopback endpoint parsing, port checks, and conflict checks.
+An unavailable resolver or rejected declaration leaves that probe failed;
+it does not substitute port 8080 or prevent workspace cleanup.
 
 Manual ordinary and full runs exclude the Jetson nvmap and DGX Spark llama.cpp
 jobs unless their independent opt-in flags are `true`.

test/e2e/fixtures/artifacts.ts

@@ -131,10 +131,14 @@
   async writeText(relativePath: string, text: string): Promise<string> {
     const target = this.pathFor(relativePath);
     await fs.mkdir(path.dirname(target), { recursive: true });
-    await fs.writeFile(target, redactString(text, this.redactionValues), "utf8");
+    await fs.writeFile(target, this.redact(text), "utf8");
     return target;
   }
 
+  redact(text: string): string {
+    return redactString(text, this.redactionValues);
+  }
+
   async writeJson(relativePath: string, value: unknown): Promise<string> {
     return this.writeText(relativePath, `${JSON.stringify(value, null, 2)}\n`);
   }

test/e2e/fixtures/full-e2e-gateway.ts

@@ -0,0 +1,46 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { DEFAULT_GATEWAY_PORT, parsePort } from "../../../src/lib/core/ports.ts";
+import { resolveGatewayName } from "../../../src/lib/onboard/gateway-binding/identity.ts";
+import { loadGatewayManagementDeclaration } from "../../../src/lib/onboard/gateway-management.ts";
+
+/** Both eager cleanup and registered teardown must respect the same gateway owner. */
+export async function withOwnedFullE2eGateway(
+  gateway: { owned: boolean },
+  cleanup: () => unknown,
+): Promise<void> {
+  if (gateway.owned) await cleanup();
+}
+
+/** Resolve the platform declaration before the test can register destructive cleanup. */
+export function fullE2eGateway(preinstalled: boolean, env: NodeJS.ProcessEnv = process.env) {
+  if (!preinstalled) {
+    const port = parsePort("NEMOCLAW_GATEWAY_PORT", DEFAULT_GATEWAY_PORT, env);
+    return { owned: true, env: { OPENSHELL_GATEWAY: resolveGatewayName(port) } };
+  }
+  const declarationPath =
+    env.NEMOCLAW_GATEWAY_MANAGEMENT?.trim() || "/etc/nemoclaw/gateway-management.json";
+  const loaded = loadGatewayManagementDeclaration({
+    env: { ...env, NEMOCLAW_GATEWAY_MANAGEMENT: declarationPath },
+  });
+  if (!loaded.ok) throw new Error(`Launchable gateway declaration: ${loaded.reason}`);
+  if (loaded.declaration?.mode !== "externally-supervised" || !loaded.declaration.endpoint) {
+    throw new Error("The preinstalled Launchable requires an externally supervised gateway");
+  }
+  const endpoint = new URL(loaded.declaration.endpoint);
+  const port = parsePort("NEMOCLAW_GATEWAY_PORT", DEFAULT_GATEWAY_PORT, {
+    NEMOCLAW_GATEWAY_PORT: endpoint.port || (endpoint.protocol === "https:" ? "443" : "80"),
+  });
+  if (parsePort("NEMOCLAW_GATEWAY_PORT", port, env) !== port) {
+    throw new Error("Launchable gateway port conflicts with its declaration");
+  }
+  return {
+    owned: false,
+    env: {
+      OPENSHELL_GATEWAY: resolveGatewayName(port),
+      NEMOCLAW_GATEWAY_MANAGEMENT: declarationPath,
+      NEMOCLAW_GATEWAY_PORT: String(port),
+    },
+  };
+}