test(onboard): qualify Hermes create handoff for #10412

scripts/checks/export-managed-image-failure-diagnostics.ts

@@ -11,6 +11,7 @@ import { isCredentialField } from "../../src/lib/security/credential-filter.ts";
 const EXPORTED_DIAGNOSTIC_FILES = new Set([
   "openshell-gateway-relevant.log",
   "openshell-gateway-tail.log",
+  "openshell-logs.txt",
   "rootfs-console.log",
   "summary.txt",
 ]);

src/lib/onboard/docker-gpu-diagnostic-redaction.ts

@@ -68,6 +68,25 @@ export function discoverDockerGpuDiagnosticSensitiveValues(
     .map(([, value]) => value);
 }
 
+/** Return staged credential values that a failed sandbox can echo into its logs. */
+export function discoverDockerGpuDiagnosticSensitiveValuesFromEnv(
+  env: NodeJS.ProcessEnv,
+): string[] {
+  const extraPlaceholderKeys = new Set(
+    String(env[EXTRA_PLACEHOLDER_KEYS_ENV] ?? "")
+      .split(/[\s,]+/u)
+      .filter(Boolean),
+  );
+  return Object.entries(env)
+    .filter(
+      ([key, value]) =>
+        typeof value === "string" &&
+        Buffer.byteLength(value, "utf8") >= 8 &&
+        (SENSITIVE_ENV_KEY.test(key) || extraPlaceholderKeys.has(key)),
+    )
+    .map(([, value]) => value as string);
+}
+
 /**
  * SOURCE_OF_TRUTH_REVIEW (shared Docker GPU diagnostic redaction; #6110):
  * invalidState: inspect, network, log, or startup-command credentials reach an artifact sink.

src/lib/onboard/sandbox-create-failure.ts

@@ -6,14 +6,27 @@ import os from "node:os";
 import path from "node:path";
 
 import { GATEWAY_PORT } from "../core/ports";
+import { buildSandboxLogsArgs } from "../domain/sandbox/logs";
 import { rejectSymlinksOnPath } from "../state/config-io";
 import { nemoclawStateRoot } from "../state/state-root";
+import {
+  createDockerGpuDiagnosticRedactor,
+  discoverDockerGpuDiagnosticSensitiveValuesFromEnv,
+} from "./docker-gpu-diagnostic-redaction";
 import { resolveGatewayLogPathForPort } from "./gateway/state-dir";
 
 const ANSI_RE = /\x1B(?:\[[0-?]*[ -/]*[@-~]|\][^\x07]*(?:\x07|\x1B\\)|[@-_])/g;
 const UUID_RE = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i;
 const MAX_RELEVANT_LOG_LINES = 120;
 const MAX_GATEWAY_TAIL_LINES = 240;
+const MAX_OPENSHELL_CAPTURE_BYTES = 64 * 1024;
+const MAX_OPENSHELL_CAPTURE_LINES = 120;
+const OPENSHELL_CAPTURE_TIMEOUT_MS = 10_000;
+
+type RunCaptureOpenshell = (
+  args: string[],
+  options?: { ignoreError?: boolean; killProcessTreeOnTimeout?: boolean; timeout?: number },
+) => string;
 
 export type SandboxCreateFailureDiagnostics = {
   dir: string;
@@ -23,6 +36,7 @@ export type SandboxCreateFailureDiagnostics = {
   consoleOutput: string | null;
   copiedConsoleOutput: string | null;
   gatewayTailPath: string | null;
+  openshellLogsPath: string | null;
   backupPath: string | null;
   summaryLines: string[];
 };
@@ -32,6 +46,9 @@ export type SandboxCreateFailureDiagnosticOptions = {
   gatewayPort?: number;
   gatewayLogPath?: string | null;
   gatewayStateDir?: string;
+  gatewayName?: string;
+  runCaptureOpenshell?: RunCaptureOpenshell;
+  env?: NodeJS.ProcessEnv;
   backupPath?: string | null;
   now?: Date;
 };
@@ -154,6 +171,56 @@ function listStateDir(stateDir: string | null): string[] {
   }
 }
 
+function boundedRedactedCapture(output: string, env: NodeJS.ProcessEnv): string[] {
+  const bytes = Buffer.from(stripAnsi(output), "utf8");
+  const offset = Math.max(0, bytes.length - MAX_OPENSHELL_CAPTURE_BYTES);
+  let bounded = bytes.subarray(offset).toString("utf8");
+  if (offset > 0) {
+    const firstCompleteLine = bounded.indexOf("\n");
+    bounded = firstCompleteLine === -1 ? "" : bounded.slice(firstCompleteLine + 1);
+  }
+  return createDockerGpuDiagnosticRedactor(discoverDockerGpuDiagnosticSensitiveValuesFromEnv(env))
+    .redactText(bounded)
+    .split(/\r?\n/u)
+    .filter((line) => line.trim().length > 0)
+    .slice(-MAX_OPENSHELL_CAPTURE_LINES);
+}
+
+function captureOpenShellFailureLogs(
+  dir: string,
+  sandboxName: string,
+  options: SandboxCreateFailureDiagnosticOptions,
+): { path: string | null; summaryLines: string[] } {
+  if (!options.runCaptureOpenshell) return { path: null, summaryLines: [] };
+  try {
+    const lines = boundedRedactedCapture(
+      options.runCaptureOpenshell(
+        buildSandboxLogsArgs(
+          sandboxName,
+          { follow: false, lines: String(MAX_OPENSHELL_CAPTURE_LINES), since: null },
+          options.gatewayName,
+        ),
+        {
+          ignoreError: true,
+          killProcessTreeOnTimeout: true,
+          timeout: OPENSHELL_CAPTURE_TIMEOUT_MS,
+        },
+      ),
+      options.env ?? process.env,
+    );
+    if (lines.length === 0) return { path: null, summaryLines: [] };
+    const filePath = path.join(dir, "openshell-logs.txt");
+    fs.writeFileSync(filePath, `${lines.join("\n")}\n`, { mode: 0o600 });
+    return {
+      path: filePath,
+      summaryLines: lines.slice(-2).map((line) => `sandbox logs: ${line}`),
+    };
+  } catch {
+    // Diagnostics must not replace the original sandbox-create failure.
+    return { path: null, summaryLines: [] };
+  }
+}
+
 export function collectSandboxCreateFailureDiagnostics(
   sandboxName: string,
   options: SandboxCreateFailureDiagnosticOptions = {},
@@ -215,6 +282,7 @@ export function collectSandboxCreateFailureDiagnostics(
   if (gatewayTailPath) {
     fs.writeFileSync(gatewayTailPath, `${gatewayTailLines.join("\n")}\n`, { mode: 0o600 });
   }
+  const openshellLogs = captureOpenShellFailureLogs(dir, sandboxName, options);
   const summaryLines = [
     `created_at=${now.toISOString()}`,
     `sandbox_name=${sandboxName}`,
@@ -224,6 +292,7 @@ export function collectSandboxCreateFailureDiagnostics(
     `state_dir=${stateDir ?? "unknown"}`,
     `console_output=${consoleOutput ?? "unknown"}`,
     `copied_console_output=${copiedConsoleOutput ?? "not-copied"}`,
+    `openshell_logs=${openshellLogs.path ?? "not-written"}`,
     `backup_path=${backupPath ?? "none"}`,
   ];
   if (stateEntries.length > 0) {
@@ -242,8 +311,17 @@ export function collectSandboxCreateFailureDiagnostics(
     consoleOutput,
     copiedConsoleOutput,
     gatewayTailPath,
+    openshellLogsPath: openshellLogs.path,
     backupPath,
-    summaryLines: relevantLines.length > 0 ? relevantLines.slice(-8) : gatewayTailLines.slice(-8),
+    summaryLines:
+      openshellLogs.summaryLines.length > 0
+        ? [
+            ...(relevantLines.length > 0 ? relevantLines : gatewayTailLines).slice(-2),
+            ...openshellLogs.summaryLines,
+          ].slice(-8)
+        : relevantLines.length > 0
+          ? relevantLines.slice(-8)
+          : gatewayTailLines.slice(-8),
   };
 }
 
@@ -256,7 +334,7 @@ export function printSandboxCreateFailureDiagnostics(
 
   console.error(`  Diagnostics saved: ${diagnostics.dir}`);
   if (diagnostics.summaryLines.length > 0) {
-    console.error("  Recent OpenShell gateway failure:");
+    console.error("  Recent OpenShell failure diagnostics:");
     for (const line of diagnostics.summaryLines) {
       console.error(`    ${line}`);
     }

src/lib/onboard/sandbox-fresh-readiness.test.ts

@@ -153,6 +153,8 @@ describe("fresh sandbox executable readiness", () => {
     );
     expect(mocks.printSandboxCreateFailureDiagnostics).toHaveBeenCalledWith("alpha", {
       backupPath: null,
+      gatewayName: "nemoclaw",
+      runCaptureOpenshell: deps.runCaptureOpenshell,
     });
   });
 
@@ -183,6 +185,8 @@ describe("fresh sandbox executable readiness", () => {
     );
     expect(mocks.printSandboxCreateFailureDiagnostics).toHaveBeenCalledWith("alpha", {
       backupPath: null,
+      gatewayName: "nemoclaw",
+      runCaptureOpenshell: deps.runCaptureOpenshell,
     });
   });
 
@@ -244,6 +248,8 @@ describe("fresh sandbox executable readiness", () => {
     );
     expect(mocks.printSandboxCreateFailureDiagnostics).toHaveBeenCalledWith("alpha", {
       backupPath: null,
+      gatewayName: "nemoclaw",
+      runCaptureOpenshell: deps.runCaptureOpenshell,
     });
   });
 });

src/lib/onboard/sandbox-gpu-create-flow.test.ts

@@ -727,17 +727,19 @@ describe("runSandboxGpuCreateFlow native failure and readiness", () => {
       };
     });
     mockExit();
-
     await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow("process.exit:1");
-
-    expect(patch.rollbackManagedStartupAfterCreateFailure).toHaveBeenCalledOnce();
     expect(deps.runOpenshell).not.toHaveBeenCalledWith(
       ["sandbox", "delete", "alpha"],
       expect.anything(),
     );
     expect(mocks.printSandboxCreateFailureDiagnostics).toHaveBeenCalledWith("alpha", {
       backupPath: null,
+      gatewayName: "nemoclaw",
+      runCaptureOpenshell: deps.runCaptureOpenshell,
     });
+    expect(mocks.printSandboxCreateFailureDiagnostics.mock.invocationCallOrder[0]).toBeLessThan(
+      patch.rollbackManagedStartupAfterCreateFailure.mock.invocationCallOrder[0]!,
+    );
     expect(errorOutput()).toContain(
       "NemoClaw left the sandbox in place for inspection and recovery",
     );

src/lib/onboard/sandbox-gpu-create-run-attempt.ts

@@ -447,7 +447,12 @@ export function createSandboxGpuCreateAttemptRunner(
           console.error(
             `  Hermes portable sandbox '${sandboxName}' did not complete receipt-owned creation. Preserve its lifecycle receipt and resume onboarding after correcting the reported failure.`,
           )
-      : printSandboxCreateFailureDiagnostics);
+      : (sandboxName, options) =>
+          printSandboxCreateFailureDiagnostics(sandboxName, {
+            ...options,
+            gatewayName: input.gatewayName,
+            runCaptureOpenshell: deps.runCaptureOpenshell,
+          }));
   if (
     portableLifecycle &&
     (input.gpuRoutePlan === "compatibility-only" ||
@@ -1141,10 +1146,10 @@ export function createSandboxGpuCreateAttemptRunner(
           ...nativeCleanup,
         } as const;
       }
-      await runtimePatch.rollbackManagedStartupAfterCreateFailure();
       printCreateFailureDiagnostics(input.sandboxName, {
         backupPath: input.restoreBackupPath,
       });
+      await runtimePatch.rollbackManagedStartupAfterCreateFailure();
       if (compatibility) runtimePatch.printReadinessFailureIfEnabled();
       else if (expectedRecreatedSandboxId) {
         console.error(

test/inference/managed/managed-image-failure-diagnostics.test.ts

@@ -75,6 +75,10 @@ describe("managed-image failure diagnostic export", () => {
         path.join(diagnosticBundle, "openshell-gateway-relevant.log"),
         `gateway reconnect failed for ${opaqueCanary}\n`,
       );
+      fs.writeFileSync(
+        path.join(diagnosticBundle, "openshell-logs.txt"),
+        `sandbox exited after printing ${opaqueCanary}\n`,
+      );
       fs.writeFileSync(
         path.join(diagnosticBundle, "rootfs-console.log"),
         "managed startup exited before the supervisor reconnected\n",
@@ -93,7 +97,7 @@ describe("managed-image failure diagnostic export", () => {
         sourceRoot,
       });
 
-      expect(result).toMatchObject({ bundles: 1, files: 3 });
+      expect(result).toMatchObject({ bundles: 1, files: 4 });
       const exported = outputText(outputRoot);
       expect(exported).toContain("<REDACTED>");
       expect(exported).toContain("managed startup exited before the supervisor reconnected");
@@ -165,13 +169,11 @@ describe("managed-image failure diagnostic export", () => {
         sourceRoot,
         `2026-07-29T01-02-${String(index).padStart(2, "0")}-000Z-agent`,
       );
-      [
-        "openshell-gateway-relevant.log",
-        "openshell-gateway-tail.log",
-        "summary.txt",
-      ].forEach((name) => {
-        fs.writeFileSync(path.join(diagnosticBundle, name), largeButReadable);
-      });
+      ["openshell-gateway-relevant.log", "openshell-gateway-tail.log", "summary.txt"].forEach(
+        (name) => {
+          fs.writeFileSync(path.join(diagnosticBundle, name), largeButReadable);
+        },
+      );
       fs.writeFileSync(path.join(diagnosticBundle, "rootfs-console.log"), tooLarge);
     }
 

test/onboarding/onboard-sandbox-create-failure.test.ts

@@ -5,7 +5,7 @@ import fs from "node:fs";
 import os from "node:os";
 import path from "node:path";
 
-import { describe, expect, it } from "vitest";
+import { describe, expect, it, vi } from "vitest";
 
 import {
   collectSandboxCreateFailureDiagnostics,
@@ -104,10 +104,12 @@ describe("sandbox create failure diagnostics", () => {
         homeDir,
         backupPath: "/tmp/pre-upgrade-backup",
         now: new Date("2026-05-12T20:35:00.000Z"),
+        runCaptureOpenshell: () => "sandbox startup failed\n",
       });
 
       expect(diagnostics?.dir).toContain(path.join(homeDir, ".nemoclaw", "onboard-failures"));
       expect(messages).toContain(`  Diagnostics saved: ${diagnostics!.dir}`);
+      expect(messages).toContain("  Recent OpenShell failure diagnostics:");
       expect(messages).toContain("  State backup retained: /tmp/pre-upgrade-backup");
     } finally {
       console.error = originalError;
@@ -146,4 +148,68 @@ describe("sandbox create failure diagnostics", () => {
       "gateway_tail=",
     );
   });
+
+  it("captures bounded redacted OpenShell failure evidence before WSL ARM cleanup (#10412)", () => {
+    const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-create-failure-openshell-"));
+    const homeDir = path.join(tmp, "home");
+    const token = "opaque-credential-value-10412";
+    const logLines = [
+      "x".repeat(64 * 1024),
+      ...Array.from({ length: 119 }, (_, index) => `earlier log ${String(index)}`),
+      `custom provider output ${token}`,
+      "[sandbox] /usr/local/bin/hermes: Exec format error",
+    ];
+    const runCaptureOpenshell = vi.fn(
+      (
+        _args: string[],
+        _options?: {
+          ignoreError?: boolean;
+          killProcessTreeOnTimeout?: boolean;
+          timeout?: number;
+        },
+      ) => `${logLines.join("\n")}\n`,
+    );
+
+    const diagnostics = collectSandboxCreateFailureDiagnostics("my-assistant", {
+      gatewayName: "nemoclaw",
+      homeDir,
+      now: new Date("2026-08-26T16:02:46.000Z"),
+      runCaptureOpenshell,
+      env: {
+        CUSTOM_PROVIDER_CREDENTIAL: token,
+        NEMOCLAW_EXTRA_PLACEHOLDER_KEYS: "CUSTOM_PROVIDER_CREDENTIAL",
+      },
+    });
+    const captured = fs.readFileSync(diagnostics!.openshellLogsPath!, "utf8");
+
+    expect({
+      calls: runCaptureOpenshell.mock.calls.map(([args, options]) => ({ args, options })),
+      firstLine: captured.trim().split("\n").at(0),
+      lineCount: captured.trim().split("\n").length,
+      lastLine: captured.trim().split("\n").at(-1),
+      summaryLines: diagnostics?.summaryLines,
+      tokenPresent: captured.includes(token),
+      withinByteLimit: Buffer.byteLength(captured, "utf8") <= 64 * 1024,
+    }).toEqual({
+      calls: [
+        {
+          args: ["logs", "-g", "nemoclaw", "my-assistant", "-n", "120", "--source", "all"],
+          options: {
+            ignoreError: true,
+            killProcessTreeOnTimeout: true,
+            timeout: 10_000,
+          },
+        },
+      ],
+      firstLine: "earlier log 1",
+      lineCount: 120,
+      lastLine: "[sandbox] /usr/local/bin/hermes: Exec format error",
+      summaryLines: [
+        "sandbox logs: custom provider output <REDACTED>",
+        "sandbox logs: [sandbox] /usr/local/bin/hermes: Exec format error",
+      ],
+      tokenPresent: false,
+      withinByteLimit: true,
+    });
+  });
 });