test(onboard): qualify Hermes create handoff for #10412

scripts/checks/export-managed-image-failure-diagnostics.ts

@@ -11,6 +11,7 @@ import { isCredentialField } from "../../src/lib/security/credential-filter.ts";
 const EXPORTED_DIAGNOSTIC_FILES = new Set([
   "openshell-gateway-relevant.log",
   "openshell-gateway-tail.log",
+  "openshell-logs.txt",
   "rootfs-console.log",
   "summary.txt",
 ]);

src/lib/onboard/created-sandbox-failure.test.ts

@@ -25,6 +25,7 @@ function createFailureDeps(
   return {
     classifyCreateFailure: vi.fn(() => ({ kind: "unknown" })),
     printCreateFailureDiagnostics: vi.fn(),
+    rollbackCreateFailure: vi.fn(async () => {}),
     printRecoveryHints: vi.fn(),
     warn: vi.fn(),
     error: vi.fn(),
@@ -49,46 +50,85 @@ function createFailureOptions(
 }
 
 describe("reportSandboxCreateFailure", () => {
-  it("warns and returns (does not exit) when the create is merely incomplete", () => {
+  it("warns and returns (does not exit) when the create is merely incomplete", async () => {
     const deps = createFailureDeps({
       classifyCreateFailure: vi.fn(() => ({ kind: "sandbox_create_incomplete" })),
     });
-    expect(() => reportSandboxCreateFailure(createFailureOptions(), deps)).not.toThrow();
+    await expect(reportSandboxCreateFailure(createFailureOptions(), deps)).resolves.toBeUndefined();
     expect(deps.warn).toHaveBeenCalledWith(
       "  Create stream exited with code 3 after sandbox was created.",
     );
     expect(deps.printCreateFailureDiagnostics).not.toHaveBeenCalled();
+    expect(deps.rollbackCreateFailure).not.toHaveBeenCalled();
     expect(deps.printRecoveryHints).not.toHaveBeenCalled();
     expect(deps.exitProcess).not.toHaveBeenCalled();
   });
 
-  it("prints diagnostics + recovery hints and exits with the create status on a hard failure", () => {
+  it("prints diagnostics + recovery hints and exits with the create status on a hard failure", async () => {
     const deps = createFailureDeps();
-    expect(() =>
+    await expect(
       reportSandboxCreateFailure(
         createFailureOptions({ createStatus: 42, restoreBackupPath: "/tmp/backup" }),
         deps,
       ),
-    ).toThrow(ExitSignal);
+    ).rejects.toThrow(ExitSignal);
     expect(deps.printCreateFailureDiagnostics).toHaveBeenCalledWith("alpha", {
       backupPath: "/tmp/backup",
     });
     expect(deps.printRecoveryHints).toHaveBeenCalledWith("boom", {
       createArgs: ["sandbox", "create", "alpha"],
     });
+    expect(deps.rollbackCreateFailure).toHaveBeenCalledOnce();
     expect(deps.exitProcess).toHaveBeenCalledWith(42);
     expect(deps.warn).not.toHaveBeenCalled();
   });
 
-  it("redacts create output before classification and echoing", () => {
+  it("waits for rollback before recovery hints and exit", async () => {
+    let settleRollback = (): void => {};
+    const rollback = new Promise<void>((resolve) => {
+      settleRollback = resolve;
+    });
+    const deps = createFailureDeps({ rollbackCreateFailure: vi.fn(() => rollback) });
+    const reporting = reportSandboxCreateFailure(createFailureOptions(), deps);
+    await vi.waitFor(() => expect(deps.printCreateFailureDiagnostics).toHaveBeenCalledOnce());
+
+    expect({
+      hints: vi.mocked(deps.printRecoveryHints).mock.calls,
+      exit: vi.mocked(deps.exitProcess).mock.calls,
+    }).toEqual({ hints: [], exit: [] });
+    settleRollback();
+    await expect(reporting).rejects.toThrow(ExitSignal);
+    expect({
+      hints: vi.mocked(deps.printRecoveryHints).mock.calls.length,
+      exit: vi.mocked(deps.exitProcess).mock.calls[0],
+    }).toEqual({ hints: 1, exit: [3] });
+  });
+
+  it("preserves the create status when rollback fails", async () => {
+    const deps = createFailureDeps({
+      rollbackCreateFailure: vi.fn(async () => {
+        throw new Error("rollback unavailable");
+      }),
+    });
+
+    await expect(
+      reportSandboxCreateFailure(createFailureOptions({ createStatus: 42 }), deps),
+    ).rejects.toThrow("exit:42");
+    expect(deps.error).toHaveBeenCalledWith(
+      "  Sandbox failure rollback did not complete: rollback unavailable",
+    );
+    expect(deps.exitProcess).toHaveBeenCalledWith(42);
+  });
+
+  it("redacts create output before classification and echoing", async () => {
     // With output: leading blank + headline + blank + output echo + "Try:" hint = 5 error() calls.
     const withOutput = createFailureDeps();
-    expect(() =>
+    await expect(
       reportSandboxCreateFailure(
         createFailureOptions({ createOutput: "failed with Authorization: Bearer secret-token" }),
         withOutput,
       ),
-    ).toThrow(ExitSignal);
+    ).rejects.toThrow(ExitSignal);
     expect(withOutput.classifyCreateFailure).toHaveBeenCalledWith(
       "failed with Authorization: Bearer secr********",
     );
@@ -104,15 +144,15 @@ describe("reportSandboxCreateFailure", () => {
 
     // Without output: the echo block is skipped, so only 3 error() calls remain.
     const noOutput = createFailureDeps();
-    expect(() =>
+    await expect(
       reportSandboxCreateFailure(createFailureOptions({ createOutput: "" }), noOutput),
-    ).toThrow(ExitSignal);
+    ).rejects.toThrow(ExitSignal);
     expect(noOutput.error).toHaveBeenCalledTimes(3);
     // still exits (createStatus || 1)
     expect(noOutput.exitProcess).toHaveBeenCalledWith(3);
   });
 
-  it("redacts multiple known token formats in create output", () => {
+  it("redacts multiple known token formats in create output", async () => {
     const deps = createFailureDeps();
     const createOutput = [
       "Authorization: Bearer secret-token",
@@ -121,9 +161,9 @@ describe("reportSandboxCreateFailure", () => {
       "aws AKIAABCDEFGHIJKLMNOP", // gitleaks:allow
     ].join("\n");
 
-    expect(() => reportSandboxCreateFailure(createFailureOptions({ createOutput }), deps)).toThrow(
-      ExitSignal,
-    );
+    await expect(
+      reportSandboxCreateFailure(createFailureOptions({ createOutput }), deps),
+    ).rejects.toThrow(ExitSignal);
 
     const echoed = (deps.error as ReturnType<typeof vi.fn>).mock.calls
       .map((call) => String(call[0]))
@@ -141,11 +181,11 @@ describe("reportSandboxCreateFailure", () => {
     expect(hinted).not.toContain("AKIAABCDEFGHIJKLMNOP"); // gitleaks:allow
   });
 
-  it("falls back to exit code 1 when the create status is zero", () => {
+  it("falls back to exit code 1 when the create status is zero", async () => {
     const deps = createFailureDeps();
-    expect(() =>
+    await expect(
       reportSandboxCreateFailure(createFailureOptions({ createStatus: 0 }), deps),
-    ).toThrow(ExitSignal);
+    ).rejects.toThrow(ExitSignal);
     expect(deps.exitProcess).toHaveBeenCalledWith(1);
   });
 });
@@ -317,34 +357,34 @@ describe("reportSandboxReadinessFailure", () => {
     ]);
   });
 
-  it.each([
-    null,
-    "",
-  ])("falls back to a stable terminal readiness gate for missing phase %s", (failurePhase) => {
-    const deps = readinessDeps();
-    expect(() =>
-      reportSandboxReadinessFailure(
-        readinessOptions({
-          readiness: {
-            ready: false,
-            reason: "terminal_failure_phase",
-            failurePhase,
-          },
-        }),
-        deps,
-      ),
-    ).toThrow(ExitSignal);
-    expectReceiptBlock(deps, [
-      "  Sandbox lifecycle receipt:",
-      "    state: created_but_not_ready",
-      "    sandbox: alpha",
-      "    readiness_gate: sandbox_list:terminal_failure",
-      "    readiness_reason: terminal_failure_phase",
-      "    create_stream_status: 0",
-      "    timeout_seconds: 300",
-      "    terminal_resolution: terminal_failure_deleted",
-    ]);
-  });
+  it.each([null, ""])(
+    "falls back to a stable terminal readiness gate for missing phase %s",
+    (failurePhase) => {
+      const deps = readinessDeps();
+      expect(() =>
+        reportSandboxReadinessFailure(
+          readinessOptions({
+            readiness: {
+              ready: false,
+              reason: "terminal_failure_phase",
+              failurePhase,
+            },
+          }),
+          deps,
+        ),
+      ).toThrow(ExitSignal);
+      expectReceiptBlock(deps, [
+        "  Sandbox lifecycle receipt:",
+        "    state: created_but_not_ready",
+        "    sandbox: alpha",
+        "    readiness_gate: sandbox_list:terminal_failure",
+        "    readiness_reason: terminal_failure_phase",
+        "    create_stream_status: 0",
+        "    timeout_seconds: 300",
+        "    terminal_resolution: terminal_failure_deleted",
+      ]);
+    },
+  );
 
   it("preserves a non-zero create-stream status when readiness later fails", () => {
     const deps = readinessDeps();

src/lib/onboard/created-sandbox-failure.ts

@@ -47,6 +47,7 @@ export type SandboxCreateFailureReportOptions = {
 export type SandboxCreateFailureReportDeps = {
   classifyCreateFailure(output: string): { kind: string };
   printCreateFailureDiagnostics(sandboxName: string, options: { backupPath: string | null }): void;
+  rollbackCreateFailure(): Promise<void>;
   printRecoveryHints(output: string, options: { createArgs: readonly string[] }): void;
   warn(message: string): void;
   error(message: string): void;
@@ -59,10 +60,10 @@ export type SandboxCreateFailureReportDeps = {
  * 255) warns and returns so the caller can fall through to the ready-wait loop;
  * any other failure prints diagnostics + recovery hints and exits.
  */
-export function reportSandboxCreateFailure(
+export async function reportSandboxCreateFailure(
   options: SandboxCreateFailureReportOptions,
   deps: SandboxCreateFailureReportDeps,
-): void {
+): Promise<void> {
   const redactedCreateOutput = redact(options.createOutput);
   const failure = deps.classifyCreateFailure(redactedCreateOutput);
   if (failure.kind === "sandbox_create_incomplete") {
@@ -82,12 +83,23 @@ export function reportSandboxCreateFailure(
     deps.error("");
     deps.error(redactedCreateOutput);
   }
-  deps.printCreateFailureDiagnostics(options.sandboxName, {
-    backupPath: options.restoreBackupPath,
-  });
+  try {
+    deps.printCreateFailureDiagnostics(options.sandboxName, {
+      backupPath: options.restoreBackupPath,
+    });
+  } catch {
+    // Diagnostics must not replace the original sandbox-create failure.
+  }
+  try {
+    await deps.rollbackCreateFailure();
+  } catch (error) {
+    deps.error(
+      `  Sandbox failure rollback did not complete: ${redact(error instanceof Error ? error.message : String(error))}`,
+    );
+  }
   deps.error("  Try:  openshell sandbox list        # check gateway state");
   deps.printRecoveryHints(redactedCreateOutput, { createArgs: options.createArgs });
-  return deps.exitProcess(options.createStatus === 0 ? 1 : options.createStatus);
+  deps.exitProcess(options.createStatus === 0 ? 1 : options.createStatus);
 }
 
 export type SandboxReadinessFailureReportOptions = {

src/lib/onboard/docker-gpu-diagnostic-redaction.test.ts

@@ -215,7 +215,7 @@ describe("Docker GPU diagnostic redaction", () => {
       ).toThrow(/__test_exit__/);
 
       const stderr = output.join("\n");
-      expect(stderr).toContain("https://****:****@proxy.example:8443/");
+      expect(stderr).toContain("https://proxy.example:8443/");
       expect(stderr).not.toContain(rawProxy);
       expect(stderr).not.toContain("proxy-user-7a9c");
       expect(stderr).not.toContain("proxy-secret-8b0d");

src/lib/onboard/docker-gpu-diagnostic-redaction.ts

@@ -1,7 +1,7 @@
 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-import { redact, redactFull } from "../security/redact";
+import { redactFullWithUrls } from "../security/redact";
 import type { DockerContainerInspect } from "./docker-gpu-patch-types";
 
 const SENSITIVE_ENV_KEY =
@@ -68,6 +68,25 @@ export function discoverDockerGpuDiagnosticSensitiveValues(
     .map(([, value]) => value);
 }
 
+/** Return staged credential values that a failed sandbox can echo into its logs. */
+export function discoverDockerGpuDiagnosticSensitiveValuesFromEnv(
+  env: NodeJS.ProcessEnv,
+): string[] {
+  const extraPlaceholderKeys = new Set(
+    String(env[EXTRA_PLACEHOLDER_KEYS_ENV] ?? "")
+      .split(/[\s,]+/u)
+      .filter(Boolean),
+  );
+  return Object.entries(env)
+    .filter(
+      ([key, value]) =>
+        typeof value === "string" &&
+        value.length > 0 &&
+        (SENSITIVE_ENV_KEY.test(key) || extraPlaceholderKeys.has(key)),
+    )
+    .map(([, value]) => value as string);
+}
+
 /**
  * SOURCE_OF_TRUTH_REVIEW (shared Docker GPU diagnostic redaction; #6110):
  * invalidState: inspect, network, log, or startup-command credentials reach an artifact sink.
@@ -81,10 +100,7 @@ export function createDockerGpuDiagnosticRedactor(
 ): DockerGpuDiagnosticRedactor {
   const sensitiveValues = new Set([...initialSensitiveValues].filter((value) => value.length > 0));
   const redactText = (text: string): string => {
-    // `redactFull` covers known secret shapes, while `redact` additionally
-    // parses credential-bearing URLs such as proxy values. Apply both before
-    // replacing opaque values learned from Docker inspect records.
-    let redacted = redactFull(redact(text));
+    let redacted = redactFullWithUrls(text);
     for (const value of [...sensitiveValues].sort((left, right) => right.length - left.length)) {
       redacted = redacted.split(value).join("<REDACTED>");
     }