fix(shields): re-confirm auto-restore lock after settle to prevent config-hash drift (#4663)

docs/reference/commands-nemohermes.mdx

@@ -1893,6 +1893,7 @@ These flags change defaults for commands that manage existing sandboxes.
 | `NEMOCLAW_CLEANUP_GATEWAY` | `1`, `true`, or `yes` to enable; `0`, `false`, or `no` to disable | Sets the default for whether `nemohermes <name> destroy` removes the shared gateway when destroying the last sandbox. Command-line `--cleanup-gateway` and `--no-cleanup-gateway` still take precedence. |
 | `NEMOCLAW_DISABLE_INFERENCE_ROUTE_REPAIR` | `1` to enable | Skips the automatic DNS-proxy repair for stale `inference.local` routes during `nemohermes <name> connect` and `nemohermes <name> connect --probe-only`. Use only as a troubleshooting escape hatch. |
 | `NEMOCLAW_SHIELDS_ACCEPT_LEGACY_BASELINE` | `1` to opt in | Allows advanced immutable-config verification to trust the current on-disk bytes for older or partial content baselines. Use only after you have rebuilt or manually inspected the sandbox state and accepted that the baseline is operator-approved. |
+| `NEMOCLAW_SHIELDS_SETTLE_MS` | milliseconds (default `750`, clamped to `0`–`10000`) | Settle window NemoClaw waits after re-applying a config lockdown (during shields auto-restore and `nemohermes <name> shields up` drift remediation) before re-confirming the lock still holds. Detects when an in-sandbox reconciler changes config file permissions after lockdown and re-applies the lock; if NemoClaw cannot re-confirm the lock within the retry budget, shields stay down. This narrows the window in which a reconciler can revert permissions rather than eliminating it — the best-effort `chattr +i` immutable bit remains the only fully durable lock. Raise it on hosts where the gateway settles slowly. |
 
 ### Legacy `nemohermes setup`
 

docs/reference/commands.mdx

@@ -2233,6 +2233,7 @@ These flags change defaults for commands that manage existing sandboxes.
 | `NEMOCLAW_CLEANUP_GATEWAY` | `1`, `true`, or `yes` to enable; `0`, `false`, or `no` to disable | Sets the default for whether `$$nemoclaw <name> destroy` removes the shared gateway when destroying the last sandbox. Command-line `--cleanup-gateway` and `--no-cleanup-gateway` still take precedence. |
 | `NEMOCLAW_DISABLE_INFERENCE_ROUTE_REPAIR` | `1` to enable | Skips the automatic DNS-proxy repair for stale `inference.local` routes during `$$nemoclaw <name> connect` and `$$nemoclaw <name> connect --probe-only`. Use only as a troubleshooting escape hatch. |
 | `NEMOCLAW_SHIELDS_ACCEPT_LEGACY_BASELINE` | `1` to opt in | Allows advanced immutable-config verification to trust the current on-disk bytes for older or partial content baselines. Use only after you have rebuilt or manually inspected the sandbox state and accepted that the baseline is operator-approved. |
+| `NEMOCLAW_SHIELDS_SETTLE_MS` | milliseconds (default `750`, clamped to `0`–`10000`) | Settle window NemoClaw waits after re-applying a config lockdown (during shields auto-restore and `$$nemoclaw <name> shields up` drift remediation) before re-confirming the lock still holds. Detects when an in-sandbox reconciler changes config file permissions after lockdown and re-applies the lock; if NemoClaw cannot re-confirm the lock within the retry budget, shields stay down. This narrows the window in which a reconciler can revert permissions rather than eliminating it — the best-effort `chattr +i` immutable bit remains the only fully durable lock. Raise it on hosts where the gateway settles slowly. |
 
 <AgentOnly variant="openclaw">
 ### Remote Deployment

src/lib/shields/index.ts

@@ -42,6 +42,7 @@ const {
 }: typeof import("./permissive-runtime") = require("./permissive-runtime");
 const { cleanupTempDir } = require("../onboard/temp-files");
 const { verifyShieldsLockState }: typeof import("./verify-lock") = require("./verify-lock");
+const { relockAndReconfirm }: typeof import("./relock-reconfirm") = require("./relock-reconfirm");
 const {
   parseSha256Output,
   isHashVerificationIssue,
@@ -794,12 +795,18 @@ function rollbackShieldsDown(
   let rollbackChattrApplied: boolean | null = null;
   let rollbackFileHashes: { [path: string]: string } | null = null;
   if (rollbackResult.status === 0) {
-    try {
-      const lockResult = lockAgentConfig(sandboxName, target);
-      rollbackChattrApplied = lockResult.chattrApplied;
-      rollbackFileHashes = lockResult.fileHashes;
-    } catch {
-      console.error("  Warning: Rollback re-lock could not be verified. Check config manually.");
+    // Re-confirm after the settle window so a reconciler revert cannot leave
+    // the rolled-back config DRIFTED — same fail-closed treatment as the
+    // auto-restore path. Leaves the hashes null (→ "manual intervention"
+    // below) when the lock will not re-confirm.
+    const relock = relockAndReconfirm(() => lockAgentConfig(sandboxName, target));
+    if (relock.ok && relock.lastResult) {
+      rollbackChattrApplied = relock.lastResult.chattrApplied;
+      rollbackFileHashes = relock.lastResult.fileHashes;
+    } else {
+      console.error(
+        "  Warning: Rollback re-lock could not be re-confirmed. Check config manually.",
+      );
     }
   } else {
     console.error("  Warning: Policy restore failed during rollback.");
@@ -850,17 +857,23 @@ function activateLockdownFromSnapshot(
   }
 
   const target = resolveAgentConfig(sandboxName);
-  try {
-    const lockResult = lockAgentConfig(sandboxName, target);
+  // Re-confirm the lock after a settle window. This restore feeds the
+  // auto-restore inline recovery and the `shields up` snapshot path, both of
+  // which mark shields UP on this result — so a reconciler revert here would
+  // otherwise leave the same DRIFTED state #4663 is about. relockAndReconfirm
+  // fails closed (ok:false) when the lock will not hold past the settle window.
+  const relock = relockAndReconfirm(() => lockAgentConfig(sandboxName, target));
+  if (!relock.ok || !relock.lastResult) {
     return {
-      ok: true,
-      chattrApplied: lockResult.chattrApplied,
-      fileHashes: lockResult.fileHashes,
+      ok: false,
+      error: relock.error ?? "config re-lock did not re-confirm after the settle window",
     };
-  } catch (error) {
-    const message = error instanceof Error ? error.message : String(error);
-    return { ok: false, error: message };
   }
+  return {
+    ok: true,
+    chattrApplied: relock.lastResult.chattrApplied,
+    fileHashes: relock.lastResult.fileHashes,
+  };
 }
 
 function recoverExpiredAutoRestoreInline(
@@ -1256,15 +1269,20 @@ function shieldsUp(sandboxName: string, opts: { throwOnError?: boolean } = {}):
     // both cases re-applying the lock rewrites perms and captures a
     // fresh, complete seal.
     console.log(`  Lockdown drifted — re-applying lock for ${sandboxName}...`);
-    let lockResult: { chattrApplied: boolean; fileHashes: { [path: string]: string } };
-    try {
-      lockResult = lockAgentConfig(sandboxName, target);
-    } catch (err) {
-      const message = err instanceof Error ? err.message : String(err);
+    // #4663: re-confirm the lock held after the in-sandbox reconciler settles,
+    // re-applying if it reverts perms. A single re-apply here was also being
+    // reverted on DGX Station / DGX Spark, leaving the sandbox DRIFTED. This
+    // narrows (does not close) the revert window; the chattr +i immutable bit
+    // applied inside lockAgentConfig is the only fully durable defense.
+    const relock = relockAndReconfirm(() => lockAgentConfig(sandboxName, target));
+    if (!relock.ok || !relock.lastResult) {
+      const message = relock.error ?? "Config re-lock did not re-confirm after settle window";
       console.error(`  ERROR: ${message}`);
       console.error("  Config remains drifted — manual intervention required.");
       return failShieldsCommand(message, opts.throwOnError);
     }
+    const lockResult: { chattrApplied: boolean; fileHashes: { [path: string]: string } } =
+      relock.lastResult;
     saveShieldsState(sandboxName, {
       shieldsDown: false,
       chattrApplied: lockResult.chattrApplied,

src/lib/shields/relock-reconfirm.test.ts

@@ -0,0 +1,158 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+import { relockAndReconfirm, resolveSettleMs } from "./relock-reconfirm";
+
+const sealedHashes = {
+  "/sandbox/.openclaw/openclaw.json":
+    "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
+  "/sandbox/.openclaw/.config-hash":
+    "fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210",
+};
+
+function okResult() {
+  return { chattrApplied: true, fileHashes: sealedHashes };
+}
+
+describe("relockAndReconfirm", () => {
+  it("returns ok with the re-confirmed result when the lock always succeeds", () => {
+    const lock = vi.fn(() => okResult());
+    const sleep = vi.fn();
+
+    const result = relockAndReconfirm(lock, { sleep, settleMs: 5 });
+
+    expect(result.ok).toBe(true);
+    expect(result.attempts).toBe(1);
+    expect(result.lastResult).toEqual(okResult());
+    // One apply + one re-confirm.
+    expect(lock).toHaveBeenCalledTimes(2);
+    expect(sleep).toHaveBeenCalledTimes(1);
+    expect(sleep).toHaveBeenCalledWith(5);
+  });
+
+  it("fails closed (bounded) when the re-confirm always throws after a clean apply", () => {
+    // Apply succeeds every time, but the reconciler reverts before each
+    // re-confirm so every re-confirm throws.
+    const lock = vi
+      .fn()
+      .mockImplementationOnce(() => okResult()) // attempt 1 apply
+      .mockImplementationOnce(() => {
+        throw new Error("drift");
+      }) // attempt 1 re-confirm
+      .mockImplementationOnce(() => okResult()) // attempt 2 apply
+      .mockImplementationOnce(() => {
+        throw new Error("drift");
+      }) // attempt 2 re-confirm
+      .mockImplementationOnce(() => okResult()) // attempt 3 apply
+      .mockImplementationOnce(() => {
+        throw new Error("drift");
+      }); // attempt 3 re-confirm
+    const sleep = vi.fn();
+
+    const result = relockAndReconfirm(lock, { sleep, settleMs: 0, maxAttempts: 3 });
+
+    expect(result.ok).toBe(false);
+    expect(result.attempts).toBe(3);
+    expect(result.lastResult).toBeNull();
+    expect(result.error).toBe("drift");
+    // Bounded: 2 calls per attempt × 3 attempts.
+    expect(lock).toHaveBeenCalledTimes(6);
+    expect(sleep).toHaveBeenCalledTimes(3);
+  });
+
+  it("retries the whole cycle when the lock reverts once then holds", () => {
+    const lock = vi
+      .fn()
+      .mockImplementationOnce(() => okResult()) // attempt 1 apply
+      .mockImplementationOnce(() => {
+        throw new Error("reverted during settle");
+      }) // attempt 1 re-confirm — reverted
+      .mockImplementationOnce(() => okResult()) // attempt 2 apply
+      .mockImplementationOnce(() => okResult()); // attempt 2 re-confirm — holds
+    const sleep = vi.fn();
+
+    const result = relockAndReconfirm(lock, { sleep, settleMs: 0, maxAttempts: 3 });
+
+    expect(result.ok).toBe(true);
+    expect(result.attempts).toBe(2);
+    expect(result.lastResult).toEqual(okResult());
+    expect(lock).toHaveBeenCalledTimes(4);
+    expect(sleep).toHaveBeenCalledTimes(2);
+  });
+
+  it("fails immediately on attempt 1 when the first apply throws, never sleeping", () => {
+    const lock = vi.fn(() => {
+      throw new Error("cannot apply lock");
+    });
+    const sleep = vi.fn();
+
+    const result = relockAndReconfirm(lock, { sleep, settleMs: 0, maxAttempts: 3 });
+
+    expect(result.ok).toBe(false);
+    expect(result.attempts).toBe(1);
+    expect(result.lastResult).toBeNull();
+    expect(result.error).toBe("cannot apply lock");
+    expect(lock).toHaveBeenCalledTimes(1);
+    expect(sleep).not.toHaveBeenCalled();
+  });
+});
+
+describe("resolveSettleMs", () => {
+  const originalVitest = process.env.VITEST;
+  const originalNodeEnv = process.env.NODE_ENV;
+  const originalSettle = process.env.NEMOCLAW_SHIELDS_SETTLE_MS;
+
+  afterEach(() => {
+    if (originalVitest === undefined) delete process.env.VITEST;
+    else process.env.VITEST = originalVitest;
+    if (originalNodeEnv === undefined) delete process.env.NODE_ENV;
+    else process.env.NODE_ENV = originalNodeEnv;
+    if (originalSettle === undefined) delete process.env.NEMOCLAW_SHIELDS_SETTLE_MS;
+    else process.env.NEMOCLAW_SHIELDS_SETTLE_MS = originalSettle;
+  });
+
+  it("returns 0 under test (VITEST=true) so suites do not block", () => {
+    process.env.VITEST = "true";
+    expect(resolveSettleMs()).toBe(0);
+  });
+
+  it("applies the real settle window when only NODE_ENV=test (VITEST is the sole test signal)", () => {
+    // Security: NODE_ENV=test must NOT collapse the durability wait to 0. Only
+    // Vitest (VITEST=true) is treated as a test runtime; a production
+    // deployment that happens to run with NODE_ENV=test keeps the real settle.
+    delete process.env.VITEST;
+    process.env.NODE_ENV = "test";
+    delete process.env.NEMOCLAW_SHIELDS_SETTLE_MS;
+    expect(resolveSettleMs()).toBe(750);
+  });
+
+  it("defaults to 750ms outside test when env is unset", () => {
+    delete process.env.VITEST;
+    process.env.NODE_ENV = "production";
+    delete process.env.NEMOCLAW_SHIELDS_SETTLE_MS;
+    expect(resolveSettleMs()).toBe(750);
+  });
+
+  it("clamps an over-range env value to the upper bound", () => {
+    delete process.env.VITEST;
+    process.env.NODE_ENV = "production";
+    process.env.NEMOCLAW_SHIELDS_SETTLE_MS = "999999";
+    expect(resolveSettleMs()).toBe(10_000);
+  });
+
+  it("clamps a negative env value to 0", () => {
+    delete process.env.VITEST;
+    process.env.NODE_ENV = "production";
+    process.env.NEMOCLAW_SHIELDS_SETTLE_MS = "-500";
+    expect(resolveSettleMs()).toBe(0);
+  });
+
+  it("honours a valid in-range env value", () => {
+    delete process.env.VITEST;
+    process.env.NODE_ENV = "production";
+    process.env.NEMOCLAW_SHIELDS_SETTLE_MS = "1500";
+    expect(resolveSettleMs()).toBe(1500);
+  });
+});