fix(onboard): persist managed Hermes state

src/lib/actions/sandbox/destroy-flow.test.ts

@@ -25,6 +25,23 @@ import {
   resetDestroyModuleCache,
   traceDestroyBoundaryCalls,
 } from "../../../../test/helpers/destroy-flow-test-harness";
+import type { SandboxWorkloadReceipt } from "../../state/registry";
+
+const managedHermesWorkload = {
+  schemaVersion: 1,
+  kind: "managed-image",
+  reference: `ghcr.io/nvidia/nemoclaw/hermes@sha256:${"a".repeat(64)}`,
+  platform: "linux/amd64",
+  release: "v0.0.0",
+  sourceRevision: "a".repeat(40),
+  sourceCohort: "test-cohort",
+  capabilityContractVersion: 1,
+  startupProfileContractVersion: 1,
+  encodedProfile: "e30",
+  startupProfileSha256: "b".repeat(64),
+  credentialProxyReplayRequired: true,
+  shared: true,
+} satisfies SandboxWorkloadReceipt;
 
 describe("destroySandbox flow", () => {
   let exitSpy: MockInstance;
@@ -64,6 +81,71 @@ describe("destroySandbox flow", () => {
     );
   });
 
+  it(
+    "removes the owned managed Hermes state volume after confirmed sandbox deletion",
+    { timeout: 30_000 },
+    async () => {
+      const harness = createDestroyHarness({
+        agent: "hermes",
+        openshellDriver: "docker",
+        workload: managedHermesWorkload,
+        managedHermesStateVolumeCleanupResult: { status: "removed" },
+      });
+
+      await expect(harness.destroySandbox("alpha", { yes: true })).resolves.toBeUndefined();
+
+      expect(harness.removeManagedHermesStateVolumeSpy).toHaveBeenCalledWith({
+        agentName: "hermes",
+        runtimeProviderId: "docker",
+        sandboxName: "alpha",
+        workloadKind: "managed-image",
+      });
+      expect(harness.removeManagedHermesStateVolumeSpy.mock.invocationCallOrder[0]).toBeLessThan(
+        harness.removeSandboxSpy.mock.invocationCallOrder[0],
+      );
+    },
+  );
+
+  it("preserves the registry when owned managed Hermes state-volume cleanup fails", async () => {
+    const harness = createDestroyHarness({
+      agent: "hermes",
+      openshellDriver: "docker",
+      workload: managedHermesWorkload,
+      managedHermesStateVolumeCleanupResult: {
+        status: "failed",
+        detail: "volume is still in use",
+        volumeName: "nemoclaw-hermes-state-v1-alpha",
+      },
+    });
+
+    await expect(harness.destroySandbox("alpha", { yes: true })).rejects.toThrow("process.exit(1)");
+
+    expect(harness.removeSandboxSpy).not.toHaveBeenCalled();
+    expect(harness.errorSpy.mock.calls.map((call) => String(call[0])).join("\n")).toContain(
+      "volume is still in use",
+    );
+  });
+
+  it("leaves a foreign same-name Hermes volume untouched and completes registry cleanup", async () => {
+    const harness = createDestroyHarness({
+      agent: "hermes",
+      openshellDriver: "docker",
+      workload: managedHermesWorkload,
+      managedHermesStateVolumeCleanupResult: {
+        status: "not-owned",
+        detail: "the exact NemoClaw ownership labels are absent or changed",
+        volumeName: "nemoclaw-hermes-state-v1-alpha",
+      },
+    });
+
+    await expect(harness.destroySandbox("alpha", { yes: true })).resolves.toBeUndefined();
+
+    expect(harness.warnSpy.mock.calls.map((call) => String(call[0])).join("\n")).toContain(
+      "Left Docker volume 'nemoclaw-hermes-state-v1-alpha' untouched",
+    );
+    expect(harness.removeSandboxSpy).toHaveBeenCalledWith("alpha");
+  });
+
   it("runs routed teardown under the gateway and host router-port locks (#9098)", async () => {
     const harness = createDestroyHarness({ provider: "nvidia-router" });
 

src/lib/actions/sandbox/destroy.ts

@@ -31,6 +31,7 @@ import {
 } from "../../onboard/runtime-provider/access";
 import {
   emitProviderDetachResidualHint,
+  removeManagedHermesStateVolume,
   SANDBOX_PROVIDER_SUFFIXES,
 } from "../../onboard/sandbox-provider-cleanup";
 import { validateName } from "../../runner";
@@ -612,6 +613,28 @@ async function destroySandboxUnlocked(
   // forcedLocalCleanup — so a forced cleanup of the last registered sandbox does
   // not shut down services for a sandbox we never confirmed deleted (#6046).
   const deleteSucceededOrAlreadyGone = deleteResult.status === 0 || alreadyGone;
+  if (deleteSucceededOrAlreadyGone && sandbox) {
+    const stateVolumeCleanup = removeManagedHermesStateVolume({
+      agentName: sandbox.agent,
+      runtimeProviderId: normalizeRuntimeProviderIdentity(sandbox.openshellDriver),
+      sandboxName,
+      workloadKind: sandbox.workload?.kind ?? "",
+    });
+    if (stateVolumeCleanup.status === "failed") {
+      console.error(
+        `  Sandbox '${sandboxName}' is gone, but its managed Hermes state volume '${stateVolumeCleanup.volumeName}' could not be removed: ${redactDestroyError(stateVolumeCleanup.detail)}`,
+      );
+      console.error("  The sandbox registry entry was preserved so exact cleanup can be retried.");
+      process.exit(1);
+    }
+    if (stateVolumeCleanup.status === "not-owned") {
+      console.warn(
+        `  ${YW}⚠${R} Left Docker volume '${stateVolumeCleanup.volumeName}' untouched because ${stateVolumeCleanup.detail}.`,
+      );
+    } else if (stateVolumeCleanup.status === "removed") {
+      console.log(`  Removed managed Hermes state volume for '${sandboxName}'.`);
+    }
+  }
   const shouldStopHostServices = shouldStopHostServicesAfterDestroy({
     deleteSucceededOrAlreadyGone,
     registeredSandboxCount: registry.listSandboxes().sandboxes.length,

src/lib/adapters/docker/volume.ts

@@ -24,6 +24,13 @@ function normalizeVolumePrefix(prefix: string): string {
   return normalized;
 }
 
+export function dockerVolumeRun(
+  args: readonly string[],
+  opts: DockerRunOptions = {},
+): DockerRunResult {
+  return dockerRun(["volume", ...args], opts);
+}
+
 export function dockerListVolumesByPrefix(
   prefix: string,
   opts: DockerCaptureOptions = {},

src/lib/onboard.ts

@@ -1617,6 +1617,7 @@ async function createSandboxWithBaseImageResolution(
     envMessagingState?.plan.sandboxName === sandboxName ? envMessagingState : undefined;
   const managedWorkloadRuntime = managedWorkloadOnboard.createManagedWorkloadOnboardRuntime({ computePlan, managedWorkloadRebuild, tempManagedRuntime, tempManagedRuntimeCatalog, agentName: requestedAgentName, legacyDockerfilePath, customDockerfilePath: fromDockerfile ?? (preparedBuildContext ? preparedBuildContext.stagedDockerfile : null), rootDir: ROOT, model, provider, preferredInferenceApi, endpointUrl: createIntent?.endpointUrl ?? null, startupProfile: { chatUiUrl, effectiveDashboardPort: effectivePort, manageDashboard, dashboardBindAddress: process.env.NEMOCLAW_DASHBOARD_BIND, wslExposure: requestedAgentName === "openclaw" && isWsl(), hermesDashboardState, webSearch: webSearchConfig, toolDisclosure: effectiveToolDisclosure, hermesToolGateways, messagingPlan: plannedMessagingState?.plan ?? null, dcodeAutoApprovalMode: dcodeAutoApprovalPlan.mode, observabilityEnabled: createIntent?.observabilityEnabled === true, environment: process.env }, note, fallbackBuildEstimate: () => process.env.NEMOCLAW_IGNORE_RUNTIME_RESOURCES === "1" ? null : formatSandboxBuildEstimateNote(assessHost()) }, { resolveAgentInferenceApi: inferenceConfig.resolveAgentInferenceApi, getSandboxInferenceConfig });
   const ensurePreparedSandboxWorkload = () => managedWorkloadOnboard.prepareSandboxWorkloadForPortableLifecycle(managedWorkloadRuntime, sandboxGpuCreateFlow.resolvePortableLifecycleMode(agent));
+  const prepareHermesStateVolumeLifecycle = (workload: Awaited<ReturnType<typeof ensurePreparedSandboxWorkload>>) => managedWorkloadOnboard.createManagedHermesStateVolumeOnboardLifecycle({ agentName: requestedAgentName, runtimeProvider: managedWorkloadRuntime.runtimeProvider, sandboxName, workloadKind: workload.source.kind });
   // #4614: capture default AFTER prune so a stale registry row isn't read as a live sandbox.
   const sandboxWasLiveDefault = liveExists && wasSandboxDefault(registry.getDefault(), sandboxName);
 
@@ -1631,7 +1632,7 @@ async function createSandboxWithBaseImageResolution(
     note,
   });
   const openRecreateJournal = (): recreateJournal.OwnedSandboxRecreateRuntime => recreateJournal.openOnboardRecreateJournal({ target: { sandboxName, gatewayName: GATEWAY_NAME, gatewayPort: GATEWAY_PORT }, agentName: getRequestedSandboxAgentName(agent) || "openclaw", note, observe: (probeTarget) => getSandboxRecreateObservation(probeTarget.sandboxName, probeTarget.gatewayName), intent: { agent: getRequestedSandboxAgentName(agent) || null, fromDockerfile: fromDockerfile ?? null, provider: provider ?? null, model: model ?? null, preferredInferenceApi: preferredInferenceApi ?? null, sandboxGpuConfig: effectiveSandboxGpuConfig ?? null, gatewayName: GATEWAY_NAME, gatewayPort: GATEWAY_PORT, toolDisclosure: effectiveToolDisclosure, dcodeAutoApprovalMode: createIntent?.dcodeAutoApprovalMode ?? null, observabilityEnabled: createIntent?.observabilityEnabled === true, policyTier: createIntent?.policyTier ?? null } });
-  let pendingStateRestoreBackupPath: string | null = null;
+  let pendingStateRestoreBackupPath: string | null = null, preparedSandboxWorkload!: Awaited<ReturnType<typeof ensurePreparedSandboxWorkload>>, hermesStateVolumeLifecycle!: ReturnType<typeof prepareHermesStateVolumeLifecycle>;
   if (!liveExists && existingEntry) ({ runtime: recreateRuntime, backupPath: pendingStateRestoreBackupPath } = recreateProtection.selectJournalBoundPreUpgradeBackup({ runtime: recreateRuntime, openJournal: createIntent?.recreateTransaction ? null : openRecreateJournal, gatewayName: GATEWAY_NAME, gatewayPort: GATEWAY_PORT, readRegistryEntry: () => registry.getSandbox(sandboxName), observe: () => getSandboxRecreateObservation(sandboxName, GATEWAY_NAME) }));
 
   if (liveExists) {
@@ -1857,7 +1858,7 @@ async function createSandboxWithBaseImageResolution(
     }
     // Resolve and validate immutable workload authority before opening a recreate journal or
     // mutating a live sandbox.
-    await ensurePreparedSandboxWorkload();
+    preparedSandboxWorkload = await ensurePreparedSandboxWorkload();
     await hermesApiPortReservationScope.selectAndReserve(hermesApiPortReservationInput);
     if (!createIntent?.recreateTransaction) recreateRuntime = openRecreateJournal();
     if (recreateRuntime.acceptedTarget) {
@@ -1882,6 +1883,7 @@ async function createSandboxWithBaseImageResolution(
       pendingStateRestore = result.backup;
     }
 
+    hermesStateVolumeLifecycle = prepareHermesStateVolumeLifecycle(preparedSandboxWorkload);
     note(`  Deleting and recreating sandbox '${sandboxName}'...`);
 
     if (recreateRuntime.beginDelete() === "source") { runSandboxProviderPreDeleteCleanup(sandboxName, { runOpenshell, redact }); runOpenshell(["sandbox", "delete", "-g", recreateRuntime.journaledGatewayName ?? GATEWAY_NAME, sandboxName], { ignoreError: true }); if (!waitForSandboxRecreateDeleteAbsence(sandboxName, recreateRuntime.journaledGatewayName ?? GATEWAY_NAME, note)) throw new Error(`Cannot continue sandbox '${sandboxName}' recreation: OpenShell did not confirm explicit source absence after delete.`); }
@@ -1891,9 +1893,7 @@ async function createSandboxWithBaseImageResolution(
       hermesApiPortReservationInput,
     );
   }
-  if (!liveExists)
-    await hermesApiPortReservationScope.selectAndReserve(hermesApiPortReservationInput);
-  const preparedSandboxWorkload = await ensurePreparedSandboxWorkload();
+  if (!liveExists) { await hermesApiPortReservationScope.selectAndReserve(hermesApiPortReservationInput); preparedSandboxWorkload = await ensurePreparedSandboxWorkload(); hermesStateVolumeLifecycle = prepareHermesStateVolumeLifecycle(preparedSandboxWorkload); }
   applyExtraProviderReconciliation({
     extraProviders: resolvedCreateIntent.extraProviders,
     staleExtraProviders: resolvedCreateIntent.staleExtraProviders ?? [],
@@ -1906,7 +1906,7 @@ async function createSandboxWithBaseImageResolution(
     launchInput: { agent, observabilityEnabled: createIntent?.observabilityEnabled === true, chatUiUrl, sandboxName, env: process.env, extraPlaceholderKeys: resolvedCreateIntent.extraPlaceholderKeys, getDashboardForwardPort, hermesDashboardState, hermesApiPort: hermesApiPortReservationScope.effectivePort, manageDashboard, openshellShellCommand, openshellArgv },
     plannedMessagingPlan: plannedMessagingState?.plan ?? null,
     gpu: { provider, config: effectiveSandboxGpuConfig, dockerDriverGateway, gatewayPort: GATEWAY_PORT },
-    dependencies: { materializeSandboxCreatePlan: sandboxCreatePlanMaterialization.materializeSandboxCreatePlan, prepareSandboxBuildPatchConfig: sandboxBuildPatchConfig.prepareSandboxBuildPatchConfig },
+    dependencies: { materializeSandboxCreatePlan: (input) => hermesStateVolumeLifecycle.materializeSandboxCreatePlan(input, sandboxCreatePlanMaterialization.materializeSandboxCreatePlan), prepareSandboxBuildPatchConfig: sandboxBuildPatchConfig.prepareSandboxBuildPatchConfig },
   });
   const restoreBackupPath =
     pendingStateRestore?.manifest?.backupPath ?? pendingStateRestoreBackupPath;
@@ -2067,7 +2067,7 @@ async function createSandboxWithBaseImageResolution(
         }),
     },
   );
-  if ("complete" in recreateRuntime) recreateRuntime.complete();
+  hermesStateVolumeLifecycle.commit(); if ("complete" in recreateRuntime) recreateRuntime.complete();
   restoreDefaultAfterRecreate(registry.setDefault, sandboxName, sandboxWasLiveDefault); // #4614: default deferred to finalization
 
   // DNS proxy — run a forwarder in the sandbox pod so the isolated

src/lib/onboard/__test-helpers__/hermes-state-volume.ts

@@ -0,0 +1,51 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { vi } from "vitest";
+
+type VolumeState = { name: string; labels: Record<string, string> } | null;
+
+export function createHermesStateVolumeDockerHarness(initial: VolumeState = null) {
+  let volume = initial;
+  const calls: string[][] = [];
+  const runDocker = vi.fn((args: readonly string[]) => {
+    const argv = [...args];
+    calls.push(argv);
+    switch (argv[0]) {
+      case "inspect":
+        return volume
+          ? {
+              status: 0,
+              stdout: `${JSON.stringify({ Name: volume.name, Labels: volume.labels })}\n`,
+            }
+          : { status: 1, stderr: `Error response from daemon: get ${argv.at(-1)}: no such volume` };
+      case "create": {
+        const labels: Record<string, string> = {};
+        for (let index = 1; index < argv.length - 1; index += 1) {
+          switch (argv[index]) {
+            case "--label": {
+              const [name, ...value] = argv[index + 1]!.split("=");
+              labels[name!] = value.join("=");
+              index += 1;
+              break;
+            }
+          }
+        }
+        volume = { name: argv.at(-1)!, labels };
+        return { status: 0, stdout: `${volume.name}\n` };
+      }
+      case "rm":
+        volume = null;
+        return { status: 0, stdout: `${argv[1]}\n` };
+      default:
+        return { status: 1, stderr: "unexpected Docker command" };
+    }
+  });
+  return {
+    calls,
+    get volume() {
+      return volume;
+    },
+    runDocker,
+  };
+}

src/lib/onboard/docker-gpu-patch-diagnostics.ts

@@ -13,9 +13,9 @@ import { createDockerGpuDiagnosticRedactor } from "./docker-gpu-diagnostic-redac
 import { fullDockerContainerId } from "./docker-gpu-patch-clone";
 import { DOCKER_GPU_PATCH_TIMEOUT_MS } from "./docker-gpu-patch-constants";
 import { getDockerGpuPatchFailureContext } from "./docker-gpu-patch-recreate";
+import { formatDockerContainerState } from "./managed-bootstrap/docker-container-failure-evidence";
 import type {
   DockerContainerInspect,
-  DockerContainerState,
   DockerGpuPatchDeps,
   DockerGpuPatchDiagnostics,
   DockerGpuPatchFailureClassification,
@@ -110,22 +110,6 @@ export function formatDockerInspectNetworkSummary(
   return lines.join("\n");
 }
 
-function describePatchedContainerState(state: DockerContainerState | null): string[] {
-  if (!state) return [];
-  const lines: string[] = [];
-  if (state.Status) lines.push(`patched_container_status=${state.Status}`);
-  if (typeof state.ExitCode === "number") {
-    lines.push(`patched_container_exit_code=${state.ExitCode}`);
-  }
-  if (state.OOMKilled) lines.push("patched_container_oom_killed=true");
-  if (state.Error) lines.push(`patched_container_error=${state.Error}`);
-  if (state.Health?.Status) lines.push(`patched_container_health=${state.Health.Status}`);
-  if (state.FinishedAt && state.FinishedAt !== "0001-01-01T00:00:00Z") {
-    lines.push(`patched_container_finished_at=${state.FinishedAt}`);
-  }
-  return lines;
-}
-
 export function dockerGpuPatchCleanupCommands(sandboxName: string): string[] {
   return [`openshell sandbox delete ${JSON.stringify(sandboxName)}`];
 }
@@ -297,7 +281,9 @@ export function collectDockerGpuPatchDiagnostics(
       summaryLines.push(`sandbox_list_row=${redactor.redactText(snapshot.sandboxListLine)}`);
     }
     summaryLines.push(
-      ...describePatchedContainerState(snapshot.patchedContainerState).map(redactor.redactText),
+      ...formatDockerContainerState(snapshot.patchedContainerState, "patched_container_").map(
+        redactor.redactText,
+      ),
     );
   }
   writeDiagnosticText("summary.txt", summaryLines.join("\n"));

src/lib/onboard/docker-gpu-patch.ts

@@ -29,6 +29,7 @@ export {
 } from "./docker-gpu-patch-clone";
 
 import { collectDockerGpuPatchDiagnostics } from "./docker-gpu-patch-diagnostics";
+import { formatDockerContainerState } from "./managed-bootstrap/docker-container-failure-evidence";
 import {
   getDockerGpuPatchFailureContext,
   recreateOpenShellDockerSandboxContainer,
@@ -467,21 +468,6 @@ const SANDBOX_STARTUP_COMMAND_NOT_FOUND_HINTS: readonly string[] = [
   "Rebuild the sandbox image from the complete Dockerfile and source context for the selected agent and NemoClaw release.",
 ];
 
-function describePatchedContainerState(state: DockerContainerState | null): string[] {
-  if (!state) return [];
-  const lines: string[] = [];
-  if (state.Status) lines.push(`patched_container_status=${state.Status}`);
-  if (typeof state.ExitCode === "number")
-    lines.push(`patched_container_exit_code=${state.ExitCode}`);
-  if (state.OOMKilled) lines.push("patched_container_oom_killed=true");
-  if (state.Error) lines.push(`patched_container_error=${state.Error}`);
-  if (state.Health?.Status) lines.push(`patched_container_health=${state.Health.Status}`);
-  if (state.FinishedAt && state.FinishedAt !== "0001-01-01T00:00:00Z") {
-    lines.push(`patched_container_finished_at=${state.FinishedAt}`);
-  }
-  return lines;
-}
-
 function patchedContainerLooksFailed(state: DockerContainerState | null): boolean {
   if (!state) return false;
   if (state.Dead === true) return true;
@@ -514,7 +500,7 @@ export function classifyDockerGpuPatchFailure(
   const lines: string[] = [];
   if (snapshot.sandboxPhase) lines.push(`sandbox_phase=${snapshot.sandboxPhase}`);
   if (snapshot.sandboxListLine) lines.push(`sandbox_list_row=${snapshot.sandboxListLine}`);
-  lines.push(...describePatchedContainerState(snapshot.patchedContainerState));
+  lines.push(...formatDockerContainerState(snapshot.patchedContainerState, "patched_container_"));
   if (selectedMode) lines.push(`patched_create_option=${selectedMode.label}`);
 
   const containerFailed = patchedContainerLooksFailed(snapshot.patchedContainerState);