perf(e2e): accelerate Hermes rebuild validation

.github/workflows/e2e.yaml

@@ -2265,6 +2265,7 @@ jobs:
       E2E_JOB: "1"
       E2E_TARGET_ID: "rebuild-hermes"
       E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/rebuild-hermes
+      NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
       NEMOCLAW_RUN_LIVE_E2E: "1"
       NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1"
       NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
@@ -2287,17 +2288,33 @@ jobs:
 
       - name: Prepare E2E workspace
         uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28
-        with:
-          build-cli: "false"
+
+      - name: Install OpenShell
+        # Full NemoClaw installer coverage remains in hermes-e2e. Rebuild uses
+        # the checked-out CLI and installs only its external runtime dependency.
+        run: |
+          set -euo pipefail
+          env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN -u NVIDIA_API_KEY -u NVIDIA_INFERENCE_API_KEY -u GITHUB_TOKEN -u GH_TOKEN bash scripts/install-openshell.sh
 
       - name: Run Hermes rebuild live test
-        # Preserves the real
-        # install.sh, Docker/OpenShell, Hermes base-image rebuild, registry,
-        # messaging-placeholder, and backup hygiene boundaries.
+        # Preserves the checked-out CLI, Docker/OpenShell, Hermes base-image
+        # rebuild, registry, messaging-placeholder, and backup hygiene boundaries.
         env:
           NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }}
         run: |
           set -euo pipefail
+          export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH"
+          if command -v openshell >/dev/null 2>&1; then
+            OPENSHELL_BIN="$(command -v openshell)"
+          elif [ -x "$HOME/.local/bin/openshell" ]; then
+            OPENSHELL_BIN="$HOME/.local/bin/openshell"
+          else
+            echo "::error::OpenShell CLI not found after install"
+            ls -la /usr/local/bin/openshell "$HOME/.local/bin/openshell" 2>&1 || true
+            exit 1
+          fi
+          export OPENSHELL_BIN
+          "$OPENSHELL_BIN" --version
           mkdir -p "$E2E_ARTIFACT_DIR"
           baseline_file="$E2E_ARTIFACT_DIR/runner-pressure-baseline.jsonl"
           phase_baselines_file="$E2E_ARTIFACT_DIR/runner-pressure-phase-baselines.jsonl"
@@ -2339,6 +2356,7 @@ jobs:
       E2E_JOB: "1"
       E2E_TARGET_ID: "rebuild-hermes-stale-base"
       E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/rebuild-hermes-stale-base
+      NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
       NEMOCLAW_RUN_LIVE_E2E: "1"
       NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1"
       NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
@@ -2362,8 +2380,13 @@ jobs:
 
       - name: Prepare E2E workspace
         uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28
-        with:
-          build-cli: "false"
+
+      - name: Install OpenShell
+        # Full NemoClaw installer coverage remains in hermes-e2e. Rebuild uses
+        # the checked-out CLI and installs only its external runtime dependency.
+        run: |
+          set -euo pipefail
+          env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN -u NVIDIA_API_KEY -u NVIDIA_INFERENCE_API_KEY -u GITHUB_TOKEN -u GH_TOKEN bash scripts/install-openshell.sh
 
       - name: Run Hermes stale-base rebuild live test
         # Uses NEMOCLAW_HERMES_STALE_BASE_REBUILD_E2E=1, preserving issue #3025's
@@ -2372,6 +2395,18 @@ jobs:
           NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }}
         run: |
           set -euo pipefail
+          export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH"
+          if command -v openshell >/dev/null 2>&1; then
+            OPENSHELL_BIN="$(command -v openshell)"
+          elif [ -x "$HOME/.local/bin/openshell" ]; then
+            OPENSHELL_BIN="$HOME/.local/bin/openshell"
+          else
+            echo "::error::OpenShell CLI not found after install"
+            ls -la /usr/local/bin/openshell "$HOME/.local/bin/openshell" 2>&1 || true
+            exit 1
+          fi
+          export OPENSHELL_BIN
+          "$OPENSHELL_BIN" --version
           mkdir -p "$E2E_ARTIFACT_DIR"
           baseline_file="$E2E_ARTIFACT_DIR/runner-pressure-baseline.jsonl"
           phase_baselines_file="$E2E_ARTIFACT_DIR/runner-pressure-phase-baselines.jsonl"

docs/manage-sandboxes/recover-rebuild-sandboxes.mdx

@@ -140,6 +140,7 @@ The rebuild command preserves manifest-defined Deep Agents state, regenerates `c
 </AgentOnly>
 
 If an archive command preserves at least one state directory, NemoClaw keeps the usable entries and reports the manifest-defined paths that could not be archived.
+If a manifest-declared state file fails, NemoClaw stops before deleting the original sandbox even when it preserved state directories, unless you explicitly pass `--force`.
 If every state directory fails, NemoClaw stops before deleting the original sandbox even when it captured loose files, unless you explicitly pass `--force`.
 
 <Warning>

docs/reference/commands.mdx

@@ -2484,7 +2484,7 @@ $$nemoclaw my-assistant rebuild [--yes|-y|--force] [--verbose|-v] [--tool-disclo
 | Flag | Description |
 |------|-------------|
 | `--yes`, `-y` | Skip the confirmation prompt. |
-| `--force` | Skip the confirmation prompt and continue when no state directory was preserved. NemoClaw restores any captured loose files; after a total failure, it recreates from registry metadata only. |
+| `--force` | Skip the confirmation prompt and continue when no state directory was preserved or a manifest-declared state file failed. NemoClaw restores any captured entries; after a total failure, it recreates from registry metadata only. |
 | `--verbose`, `-v` | Log SSH commands, exit codes, and session state (also enabled by `NEMOCLAW_REBUILD_VERBOSE=1`) |
 | `--tool-disclosure <progressive\|direct>` | Change the model-visible tool catalog during this transactional rebuild. Use this path for sandboxes with managed MCP servers so their providers and adapter state are preserved. |
 | `--dcode-auto-approval <disabled\|thread-opt-in>` | Change the managed Deep Agents Code thread auto-approval capability. `thread-opt-in` is accepted only for managed Deep Agents Code sandboxes and is rejected for other agents or custom images. Enabling prints a warning, and either value requires sandbox recreation. |
@@ -2495,6 +2495,7 @@ Pass `--yes`, `-y`, or `--force` to skip the prompt in scripted workflows.
 
 The sandbox normally must be reachable for the backup step to succeed.
 If an archive command preserves at least one state directory, `rebuild` keeps the captured backup entries and reports the manifest-defined paths that could not be archived.
+If a manifest-declared state file fails, `rebuild` exits before destroying the original sandbox even when it preserved state directories, unless you explicitly pass `--force`.
 If every state directory fails, `rebuild` exits before destroying the original sandbox even when it captured loose files, unless you explicitly pass `--force`.
 With `--force`, NemoClaw preserves any captured loose files in the partial manifest and restores them after recreation.
 If the backup produced nothing usable, it continues from recorded registry metadata without restoring prior sandbox state.
@@ -3505,21 +3506,22 @@ Set them before running `$$nemoclaw onboard`.
 <AgentOnly variant="openclaw">
 
 Set `NEMOCLAW_SANDBOX_BASE_IMAGE_REF` to an OpenClaw sandbox-base tag or digest to override base-image resolution during onboarding.
-Use a release-matched tag or immutable digest for a source-based custom image; NemoClaw resolves the reference and pins a repository digest into a stock-style `ARG BASE_IMAGE` declaration when possible.
+Remote overrides must use the official NVIDIA sandbox-base repository and resolve to an immutable repository digest.
+Local bases are accepted only when NemoClaw builds and pins them during the current operation; a local image reference supplied through this environment variable is rejected because it has no trusted build capability.
 
 </AgentOnly>
 
 <AgentOnly variant="deepagents">
 
 Set `NEMOCLAW_LANGCHAIN_DEEPAGENTS_CODE_SANDBOX_BASE_IMAGE_REF` to a LangChain Deep Agents Code sandbox-base tag or digest to override base-image resolution during onboarding.
-NemoClaw validates the requested image against the manifest-required `deepagents-code` package version before using it.
+NemoClaw requires the official remote repository for environment overrides, then validates the requested image against the manifest-required `deepagents-code` package version before using it. Local bases are accepted only when NemoClaw builds and pins them during the current operation.
 
 </AgentOnly>
 
 <AgentOnly variant="hermes">
 
 Set `NEMOCLAW_HERMES_SANDBOX_BASE_IMAGE_REF` to a Hermes sandbox-base tag or digest to override base-image resolution during onboarding.
-NemoClaw validates the requested image for the required MCP runtime, and the final Hermes image still accepts only the tracked official digest or a repository-built local base.
+NemoClaw requires the official remote repository for environment overrides, validates the requested image for the required MCP runtime, and keeps the final image bound to that trusted base. Local bases are accepted only when NemoClaw builds and pins them during the current operation.
 
 </AgentOnly>
 

src/lib/actions/sandbox/rebuild-agent-base-image-preflight.test.ts

@@ -8,13 +8,15 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 type RebuildFlowHelpersModule = typeof import("./rebuild-flow-helpers");
 type AgentDefsModule = typeof import("../../agent/defs");
 type AgentOnboardModule = typeof import("../../agent/onboard");
+type DockerImageModule = typeof import("../../adapters/docker/image");
 type SandboxBaseImageResolutionMetadata =
   import("../../sandbox-base-image").SandboxBaseImageResolutionMetadata;
 
 const requireDist = createRequire(import.meta.url);
 const rebuildFlowHelpersPath = "./rebuild-flow-helpers.js";
 const agentDefsPath = "../../agent/defs.js";
 const agentOnboardPath = "../../agent/onboard.js";
+const dockerImagePath = "../../adapters/docker/image.js";
 const overrideEnvVar = "NEMOCLAW_HERMES_SANDBOX_BASE_IMAGE_REF";
 
 function loadRebuildFlowHelpers(): RebuildFlowHelpersModule {
@@ -35,6 +37,10 @@ function loadAgentOnboard(): AgentOnboardModule {
   return requireDist(agentOnboardPath);
 }
 
+function loadDockerImage(): DockerImageModule {
+  return requireDist(dockerImagePath);
+}
+
 function makeBail(): (msg: string, code?: number) => never {
   return (msg: string) => {
     throw new Error(`bail: ${msg}`);
@@ -69,7 +75,22 @@ describe("ensureRebuildAgentBaseImage", () => {
             : "hermes:rebuilt",
         built: !options.resolutionHint,
       }));
-    return { agent, ensureAgentBaseImage };
+    const bindLocalAgentBaseImageToPinnedProvenance = vi
+      .spyOn(loadAgentOnboard(), "bindLocalAgentBaseImageToPinnedProvenance")
+      .mockReturnValue(null);
+    const pinAgentSandboxBaseImageRef = vi
+      .spyOn(loadAgentOnboard(), "pinAgentSandboxBaseImageRef")
+      .mockImplementation((_agentName, imageRef) => imageRef);
+    const dockerRmi = vi
+      .spyOn(loadDockerImage(), "dockerRmi")
+      .mockReturnValue({ status: 0 } as never);
+    return {
+      agent,
+      ensureAgentBaseImage,
+      bindLocalAgentBaseImageToPinnedProvenance,
+      pinAgentSandboxBaseImageRef,
+      dockerRmi,
+    };
   }
 
   it("forwards a recorded hint for cache validation without forcing a legacy rebuild (#4680)", () => {
@@ -101,6 +122,27 @@ describe("ensureRebuildAgentBaseImage", () => {
     });
   });
 
+  it("carries the current local-build proof into the recreate preflight", () => {
+    const { ensureAgentBaseImage } = setup();
+    const trustedLocalOverride = {
+      ref: `nemoclaw-hermes-sandbox-base-local:image-${"a".repeat(64)}`,
+      provenance: `${"b".repeat(64)}.${"c".repeat(64)}`,
+    };
+    ensureAgentBaseImage.mockReturnValue({
+      imageTag: trustedLocalOverride.ref,
+      built: true,
+      trustedLocalOverride,
+    });
+    const { ensureRebuildAgentBaseImage } = loadRebuildFlowHelpers();
+
+    expect(ensureRebuildAgentBaseImage("hermes", makeBail())).toEqual({
+      ok: true,
+      imageRef: trustedLocalOverride.ref,
+      overrideEnvVar,
+      trustedLocalOverride,
+    });
+  });
+
   it("reports a forced Hermes base-image failure before rebuild can continue", () => {
     const { ensureAgentBaseImage } = setup();
     ensureAgentBaseImage.mockImplementation(() => {
@@ -140,4 +182,102 @@ describe("ensureRebuildAgentBaseImage", () => {
       forceBaseImageRefresh: true,
     });
   });
+
+  it("preserves resolved provenance with the immutable local recreate handoff (#7144)", () => {
+    const { ensureAgentBaseImage, pinAgentSandboxBaseImageRef, dockerRmi } = setup();
+    const resolutionMetadata = { key: "current-base" } as SandboxBaseImageResolutionMetadata;
+    const platformRef = "ghcr.io/nvidia/nemoclaw/hermes-sandbox-base@sha256:platform";
+    const localRef = "nemoclaw-hermes-sandbox-base-local:image-current";
+    ensureAgentBaseImage.mockReturnValue({
+      imageTag: platformRef,
+      built: false,
+      resolutionMetadata,
+    });
+    pinAgentSandboxBaseImageRef.mockReturnValue(localRef);
+    const { ensureRebuildAgentBaseImage } = loadRebuildFlowHelpers();
+    const exitListenersBefore = process.listenerCount("exit");
+
+    const result = ensureRebuildAgentBaseImage("hermes", makeBail(), { resolutionHint: hint });
+
+    expect(result).toEqual({
+      ok: true,
+      imageRef: localRef,
+      overrideEnvVar,
+      resolutionMetadata,
+      disposeImageRef: expect.any(Function),
+    });
+    expect(pinAgentSandboxBaseImageRef).toHaveBeenCalledWith("hermes", platformRef, {
+      forceLocal: true,
+      temporary: true,
+    });
+    expect(process.listenerCount("exit")).toBe(exitListenersBefore + 1);
+    expect(result.disposeImageRef?.()).toBe(true);
+    expect(result.disposeImageRef?.()).toBe(true);
+    expect(process.listenerCount("exit")).toBe(exitListenersBefore);
+    expect(dockerRmi).toHaveBeenCalledOnce();
+    expect(dockerRmi).toHaveBeenCalledWith(localRef, {
+      ignoreError: true,
+      suppressOutput: true,
+    });
+  });
+
+  it("binds an explicit local override after its immutable recreate handoff (#7144)", () => {
+    vi.stubEnv(overrideEnvVar, "hermes:override");
+    const {
+      agent,
+      ensureAgentBaseImage,
+      bindLocalAgentBaseImageToPinnedProvenance,
+      pinAgentSandboxBaseImageRef,
+    } = setup();
+    const resolutionMetadata = { key: "canonical-base" } as SandboxBaseImageResolutionMetadata;
+    const immutableRef = `nemoclaw-hermes-sandbox-base-local:rebuild-123-${"b".repeat(16)}-image-${"c".repeat(64)}`;
+    ensureAgentBaseImage.mockReturnValue({ imageTag: "hermes:override", built: false });
+    pinAgentSandboxBaseImageRef.mockReturnValue(immutableRef);
+    bindLocalAgentBaseImageToPinnedProvenance.mockReturnValue(resolutionMetadata);
+    const { ensureRebuildAgentBaseImage } = loadRebuildFlowHelpers();
+
+    const result = ensureRebuildAgentBaseImage("hermes", makeBail());
+
+    expect(result).toEqual({
+      ok: true,
+      imageRef: immutableRef,
+      overrideEnvVar,
+      resolutionMetadata,
+      disposeImageRef: expect.any(Function),
+    });
+    expect(ensureAgentBaseImage).toHaveBeenCalledWith(agent, {
+      forceBaseImageRebuild: false,
+    });
+    expect(pinAgentSandboxBaseImageRef).toHaveBeenCalledWith("hermes", "hermes:override", {
+      forceLocal: true,
+      temporary: true,
+    });
+    expect(bindLocalAgentBaseImageToPinnedProvenance).toHaveBeenCalledWith(agent, immutableRef);
+    expect(pinAgentSandboxBaseImageRef.mock.invocationCallOrder[0]).toBeLessThan(
+      bindLocalAgentBaseImageToPinnedProvenance.mock.invocationCallOrder[0],
+    );
+    expect(result.disposeImageRef?.()).toBe(true);
+  });
+
+  it("retains exit cleanup until a failed temporary removal succeeds (#7144)", () => {
+    const { ensureAgentBaseImage, pinAgentSandboxBaseImageRef, dockerRmi } = setup();
+    const platformRef = `ghcr.io/nvidia/nemoclaw/hermes-sandbox-base@sha256:${"a".repeat(64)}`;
+    const localRef = `nemoclaw-hermes-sandbox-base-local:rebuild-123-${"b".repeat(16)}-image-${"c".repeat(64)}`;
+    ensureAgentBaseImage.mockReturnValue({ imageTag: platformRef, built: false });
+    pinAgentSandboxBaseImageRef.mockReturnValue(localRef);
+    dockerRmi
+      .mockReturnValueOnce({ status: 23 } as never)
+      .mockReturnValueOnce({ status: 0 } as never);
+    const { ensureRebuildAgentBaseImage } = loadRebuildFlowHelpers();
+    const exitListenersBefore = process.listenerCount("exit");
+
+    const result = ensureRebuildAgentBaseImage("hermes", makeBail(), { resolutionHint: hint });
+
+    expect(process.listenerCount("exit")).toBe(exitListenersBefore + 1);
+    expect(result.disposeImageRef?.()).toBe(false);
+    expect(process.listenerCount("exit")).toBe(exitListenersBefore + 1);
+    expect(result.disposeImageRef?.()).toBe(true);
+    expect(process.listenerCount("exit")).toBe(exitListenersBefore);
+    expect(dockerRmi).toHaveBeenCalledTimes(2);
+  });
 });

src/lib/actions/sandbox/rebuild-custom-image-preflight.test.ts

@@ -7,6 +7,7 @@ import path from "node:path";
 import { describe, expect, it, vi } from "vitest";
 
 import { ROOT } from "../../runner";
+import type { SandboxBaseImageResolutionMetadata } from "../../sandbox-base-image";
 import {
   preflightRebuildImage,
   type RebuildImagePreflightResult,
@@ -48,6 +49,61 @@ function input(fromDockerfile: string | null) {
 }
 
 describe("preflightRebuildImage", () => {
+  it("carries verified base provenance into the retained managed context (#7144)", async () => {
+    const buildCtx = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-managed-provenance-"));
+    const stagedDockerfile = path.join(buildCtx, "Dockerfile");
+    fs.writeFileSync(stagedDockerfile, "FROM scratch\n");
+    const metadata = {
+      schema: 1,
+      key: "current-base",
+      imageName: "ghcr.io/nvidia/nemoclaw/hermes-sandbox-base",
+      ref: `ghcr.io/nvidia/nemoclaw/hermes-sandbox-base@sha256:${"a".repeat(64)}`,
+      digest: `sha256:${"a".repeat(64)}`,
+      source: "pinned",
+      pinnedRemoteRef: `ghcr.io/nvidia/nemoclaw/hermes-sandbox-base@sha256:${"a".repeat(64)}`,
+      imageId: `sha256:${"b".repeat(64)}`,
+      os: "linux",
+      architecture: "amd64",
+      glibcVersion: "2.41",
+      requireOpenshellSandboxAbi: true,
+      minGlibcVersion: "2.39",
+    } satisfies SandboxBaseImageResolutionMetadata;
+    const prepareDockerfilePatch = vi.fn(async () => ({
+      buildId: "provenance",
+      dashboardRemoteBindPrepared: false,
+      resolvedBaseImage: null,
+    }));
+    const cleanupBuildCtx = vi.fn(() => {
+      fs.rmSync(buildCtx, { recursive: true, force: true });
+      return true;
+    });
+    try {
+      const result = successful(
+        await preflightRebuildImage(
+          { ...input(null), preResolvedBaseImageMetadata: metadata },
+          {
+            stageBuildContext: vi.fn(() => ({
+              buildCtx,
+              stagedDockerfile,
+              cleanupBuildCtx,
+              origin: "generated" as const,
+            })),
+            prepareDockerfilePatch,
+            buildImage: vi.fn(() => ({ status: 0 }) as never),
+            removeImage: vi.fn(() => ({ status: 0 }) as never),
+          },
+        ),
+      );
+
+      expect(prepareDockerfilePatch).toHaveBeenCalledWith(
+        expect.objectContaining({ preResolvedBaseImageMetadata: metadata }),
+      );
+      expect(disposePreparedBuildContext(result.prepared)).toBe(true);
+    } finally {
+      fs.rmSync(buildCtx, { recursive: true, force: true });
+    }
+  });
+
   it("prebuilds the managed OpenClaw image instead of deferring its first build until delete", async () => {
     const buildCtx = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-managed-preflight-"));
     const stagedDockerfile = path.join(buildCtx, "Dockerfile");