feat(sandbox): version staleness detection and rebuild command

.github/workflows/nightly-e2e.yaml

@@ -322,6 +322,64 @@ jobs:
           path: test-sandbox-operations-*.log
           if-no-files-found: ignore
 
+  # ── OpenClaw rebuild upgrade E2E ─────────────────────────────
+  # Reproduces NVBug 6076156: onboard with an older OpenClaw version,
+  # then rebuild to verify workspace state survives the upgrade.
+  rebuild-openclaw-e2e:
+    if: github.repository == 'NVIDIA/NemoClaw'
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Run OpenClaw rebuild upgrade E2E test
+        env:
+          NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
+          NEMOCLAW_NON_INTERACTIVE: "1"
+          NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
+          NEMOCLAW_SANDBOX_NAME: "e2e-rebuild-oc"
+          NEMOCLAW_RECREATE_SANDBOX: "1"
+          GITHUB_TOKEN: ${{ github.token }}
+        run: bash test/e2e/test-rebuild-openclaw.sh
+
+      - name: Upload install log on failure
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: rebuild-openclaw-install-log
+          path: /tmp/nemoclaw-e2e-install.log
+          if-no-files-found: ignore
+
+  # ── Hermes rebuild upgrade E2E ──────────────────────────────
+  # Same upgrade scenario as OpenClaw but for Hermes Agent.
+  rebuild-hermes-e2e:
+    if: github.repository == 'NVIDIA/NemoClaw'
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Run Hermes rebuild upgrade E2E test
+        env:
+          NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
+          NEMOCLAW_NON_INTERACTIVE: "1"
+          NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
+          NEMOCLAW_SANDBOX_NAME: "e2e-rebuild-hm"
+          NEMOCLAW_RECREATE_SANDBOX: "1"
+          NEMOCLAW_AGENT: "hermes"
+          GITHUB_TOKEN: ${{ github.token }}
+        run: bash test/e2e/test-rebuild-hermes.sh
+
+      - name: Upload install log on failure
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: rebuild-hermes-install-log
+          path: /tmp/nemoclaw-e2e-install.log
+          if-no-files-found: ignore
+
   # ── GPU E2E (Ollama local inference) ──────────────────────────
   # Enable by setting repository variable GPU_E2E_ENABLED=true
   # (Settings → Secrets and variables → Actions → Variables)
@@ -374,8 +432,8 @@ jobs:
 
   notify-on-failure:
     runs-on: ubuntu-latest
-    needs: [cloud-e2e, cloud-experimental-e2e, messaging-providers-e2e, sandbox-survival-e2e, hermes-e2e, skip-permissions-e2e, sandbox-operations-e2e, gpu-e2e]
-    if: ${{ always() && (needs.cloud-e2e.result == 'failure' || needs.cloud-experimental-e2e.result == 'failure' || needs.messaging-providers-e2e.result == 'failure' || needs.sandbox-survival-e2e.result == 'failure' || needs.hermes-e2e.result == 'failure' || needs.skip-permissions-e2e.result == 'failure' || needs.sandbox-operations-e2e.result == 'failure' || needs.gpu-e2e.result == 'failure') }}
+    needs: [cloud-e2e, cloud-experimental-e2e, messaging-providers-e2e, sandbox-survival-e2e, hermes-e2e, skip-permissions-e2e, sandbox-operations-e2e, rebuild-openclaw-e2e, rebuild-hermes-e2e, gpu-e2e]
+    if: ${{ always() && (needs.cloud-e2e.result == 'failure' || needs.cloud-experimental-e2e.result == 'failure' || needs.messaging-providers-e2e.result == 'failure' || needs.sandbox-survival-e2e.result == 'failure' || needs.hermes-e2e.result == 'failure' || needs.skip-permissions-e2e.result == 'failure' || needs.sandbox-operations-e2e.result == 'failure' || needs.rebuild-openclaw-e2e.result == 'failure' || needs.rebuild-hermes-e2e.result == 'failure' || needs.gpu-e2e.result == 'failure') }}
     permissions:
       issues: write
     steps:

agents/hermes/manifest.yaml

@@ -15,6 +15,8 @@ homepage: "https://github.com/NousResearch/hermes-agent"
 # ── Binary & process ────────────────────────────────────────────
 install_method: curl  # curl install.sh | bash
 binary_path: /usr/local/bin/hermes
+version_command: "hermes --version"
+expected_version: "2026.4.8"
 gateway_command: "hermes gateway run"
 
 # ── Health probe ────────────────────────────────────────────────

agents/openclaw/manifest.yaml

@@ -18,6 +18,8 @@ homepage: "https://openclaw.ai"
 # ── Binary & process ────────────────────────────────────────────
 install_method: npm  # npm install -g openclaw@<version>
 binary_path: /usr/local/bin/openclaw
+version_command: "openclaw --version"
+expected_version: "2026.4.2"
 gateway_command: "openclaw gateway run"
 
 # ── Health probe ────────────────────────────────────────────────

nemoclaw/src/blueprint/state.ts

@@ -15,6 +15,8 @@ export interface NemoClawState {
   hostBackupPath: string | null;
   createdAt: string | null;
   updatedAt: string;
+  lastRebuildAt: string | null;
+  lastRebuildBackupPath: string | null;
 }
 
 let stateDirCreated = false;
@@ -41,6 +43,8 @@ function blankState(): NemoClawState {
     hostBackupPath: null,
     createdAt: null,
     updatedAt: new Date().toISOString(),
+    lastRebuildAt: null,
+    lastRebuildBackupPath: null,
   };
 }
 

nemoclaw/src/commands/slash.ts

@@ -80,6 +80,13 @@ function slashStatus(): PluginCommandResult {
     lines.push("", `Rollback snapshot: ${state.migrationSnapshot}`);
   }
 
+  if (state.lastRebuildAt) {
+    lines.push("", `Last rebuild: ${state.lastRebuildAt}`);
+    if (state.lastRebuildBackupPath) {
+      lines.push(`Rebuild backup: ${state.lastRebuildBackupPath}`);
+    }
+  }
+
   return { text: lines.join("\n") };
 }
 

scripts/install.sh

@@ -1203,6 +1203,28 @@ main() {
   install_nemoclaw
   verify_nemoclaw
 
+  # Pre-upgrade safety: back up all sandbox state before onboarding (which may
+  # upgrade OpenShell). If the upgrade destroys sandbox contents, the backups
+  # in ~/.nemoclaw/rebuild-backups/ let the user recover via `nemoclaw <name> rebuild`.
+  # Check the registry file directly to avoid shelling out to nemoclaw (which
+  # may be a stub in test environments).
+  local _reg_file="${HOME}/.nemoclaw/sandboxes.json"
+  if [ -f "$_reg_file" ] && command_exists nemoclaw && command_exists openshell; then
+    local _has_sandboxes
+    _has_sandboxes="$(python3 -c "
+import json, sys
+try:
+    d = json.load(open(sys.argv[1]))
+    print(len(d.get('sandboxes', {})))
+except Exception:
+    print(0)
+" "$_reg_file" 2>/dev/null || echo 0)"
+    if [ "$_has_sandboxes" -gt 0 ]; then
+      info "Backing up $_has_sandboxes sandbox(es) before upgrade…"
+      nemoclaw backup-all 2>&1 || warn "Pre-upgrade backup failed (non-fatal). Continuing."
+    fi
+  fi
+
   step 3 "Onboarding"
   if command_exists nemoclaw; then
     if run_installer_host_preflight; then

src/lib/agent-defs.ts

@@ -41,6 +41,8 @@ export interface AgentDefinition {
   description?: string;
   display_name?: string;
   binary_path?: string;
+  version_command?: string;
+  expected_version?: string;
   gateway_command?: string;
   device_pairing?: boolean;
   phone_home_hosts?: string[];
@@ -57,6 +59,8 @@ export interface AgentDefinition {
   readonly forwardPort: number;
   readonly configPaths: AgentConfigPaths;
   readonly stateDirs: string[];
+  readonly versionCommand: string;
+  readonly expectedVersion: string | null;
   readonly hasDevicePairing: boolean;
   readonly phoneHomeHosts: string[];
   readonly messagingPlatforms: string[];
@@ -148,6 +152,14 @@ export function loadAgent(name: string): AgentDefinition {
       return (raw.state_dirs as string[]) || [];
     },
 
+    get versionCommand(): string {
+      return (raw.version_command as string) || `${raw.binary_path || "unknown"} --version`;
+    },
+
+    get expectedVersion(): string | null {
+      return (raw.expected_version as string) || null;
+    },
+
     get hasDevicePairing(): boolean {
       return raw.device_pairing === true;
     },

src/lib/credential-filter.test.ts

@@ -0,0 +1,132 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { describe, expect, it, beforeEach, afterEach } from "vitest";
+import { mkdtempSync, writeFileSync, readFileSync, existsSync, mkdirSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { rmSync } from "node:fs";
+
+import {
+  isCredentialField,
+  stripCredentials,
+  sanitizeConfigFile,
+  isSensitiveFile,
+  CREDENTIAL_SENSITIVE_BASENAMES,
+} from "./credential-filter.js";
+
+describe("isCredentialField", () => {
+  it("matches explicit field names", () => {
+    expect(isCredentialField("apiKey")).toBe(true);
+    expect(isCredentialField("api_key")).toBe(true);
+    expect(isCredentialField("token")).toBe(true);
+    expect(isCredentialField("secret")).toBe(true);
+    expect(isCredentialField("password")).toBe(true);
+    expect(isCredentialField("resolvedKey")).toBe(true);
+  });
+
+  it("matches pattern-based names", () => {
+    expect(isCredentialField("accessToken")).toBe(true);
+    expect(isCredentialField("refreshToken")).toBe(true);
+    expect(isCredentialField("clientSecret")).toBe(true);
+    expect(isCredentialField("bearerToken")).toBe(true);
+    expect(isCredentialField("privateKey")).toBe(true);
+    expect(isCredentialField("sessionToken")).toBe(true);
+  });
+
+  it("does not match safe field names", () => {
+    expect(isCredentialField("name")).toBe(false);
+    expect(isCredentialField("model")).toBe(false);
+    expect(isCredentialField("provider")).toBe(false);
+    expect(isCredentialField("endpoint")).toBe(false);
+    expect(isCredentialField("version")).toBe(false);
+  });
+});
+
+describe("stripCredentials", () => {
+  it("strips top-level credential fields", () => {
+    const input = { model: "gpt-4", apiKey: "sk-123", name: "test" };
+    const result = stripCredentials(input) as Record<string, unknown>;
+    expect(result.model).toBe("gpt-4");
+    expect(result.apiKey).toBe("[STRIPPED_BY_MIGRATION]");
+    expect(result.name).toBe("test");
+  });
+
+  it("strips nested credential fields", () => {
+    const input = { providers: { openai: { apiKey: "sk-123", model: "gpt-4" } } };
+    const result = stripCredentials(input) as Record<string, unknown>;
+    const providers = result.providers as Record<string, unknown>;
+    const openai = providers.openai as Record<string, unknown>;
+    expect(openai.apiKey).toBe("[STRIPPED_BY_MIGRATION]");
+    expect(openai.model).toBe("gpt-4");
+  });
+
+  it("strips credentials in arrays", () => {
+    const input = { items: [{ token: "abc" }, { name: "safe" }] };
+    const result = stripCredentials(input) as Record<string, unknown>;
+    const items = result.items as Array<Record<string, unknown>>;
+    expect(items[0].token).toBe("[STRIPPED_BY_MIGRATION]");
+    expect(items[1].name).toBe("safe");
+  });
+
+  it("handles null and primitives", () => {
+    expect(stripCredentials(null)).toBeNull();
+    expect(stripCredentials(undefined)).toBeUndefined();
+    expect(stripCredentials("hello")).toBe("hello");
+    expect(stripCredentials(42)).toBe(42);
+  });
+});
+
+describe("sanitizeConfigFile", () => {
+  let tmpDir: string;
+
+  beforeEach(() => {
+    tmpDir = mkdtempSync(join(tmpdir(), "cred-filter-test-"));
+  });
+
+  afterEach(() => {
+    rmSync(tmpDir, { recursive: true, force: true });
+  });
+
+  it("strips credentials and removes gateway section", () => {
+    const configPath = join(tmpDir, "openclaw.json");
+    writeFileSync(configPath, JSON.stringify({
+      model: "gpt-4",
+      apiKey: "sk-secret",
+      gateway: { port: 8080, authToken: "gw-token" },
+    }));
+
+    sanitizeConfigFile(configPath);
+
+    const result = JSON.parse(readFileSync(configPath, "utf-8"));
+    expect(result.model).toBe("gpt-4");
+    expect(result.apiKey).toBe("[STRIPPED_BY_MIGRATION]");
+    expect(result.gateway).toBeUndefined();
+  });
+
+  it("skips non-existent files", () => {
+    sanitizeConfigFile(join(tmpDir, "nonexistent.json"));
+    // Should not throw
+  });
+
+  it("skips invalid JSON", () => {
+    const configPath = join(tmpDir, "bad.json");
+    writeFileSync(configPath, "not json at all");
+    sanitizeConfigFile(configPath);
+    // Should not throw, file unchanged
+    expect(readFileSync(configPath, "utf-8")).toBe("not json at all");
+  });
+});
+
+describe("isSensitiveFile", () => {
+  it("detects auth-profiles.json", () => {
+    expect(isSensitiveFile("auth-profiles.json")).toBe(true);
+    expect(isSensitiveFile("Auth-Profiles.json")).toBe(true);
+  });
+
+  it("does not flag normal files", () => {
+    expect(isSensitiveFile("openclaw.json")).toBe(false);
+    expect(isSensitiveFile("config.yaml")).toBe(false);
+    expect(isSensitiveFile("SOUL.md")).toBe(false);
+  });
+});