fix(shields): recover completed auto-restore locks

src/lib/shields/index.ts

@@ -63,7 +63,10 @@ const {
   timerMarkerPath,
   readTimerMarker,
   clearTimerMarker,
+  clearTimerMarkerGeneration,
+  sameTimerMarkerGeneration,
   hasExactTimerAuthorizationProof,
+  isShieldsTimerDeadlineAbandoned,
   isProcessAlive,
   readProcessStartIdentity,
   processInspectionDeadlineAfter,
@@ -490,23 +493,6 @@ type ShieldsDownTransition = {
 
 const transitionPollBuffer = new Int32Array(new SharedArrayBuffer(4));
 
-function sameTimerMarkerGeneration(current: TimerMarker | null, expected: TimerMarker): boolean {
-  return (
-    current?.pid === expected.pid &&
-    current.sandboxName === expected.sandboxName &&
-    current.snapshotPath === expected.snapshotPath &&
-    current.restoreAt === expected.restoreAt &&
-    current.processToken === expected.processToken &&
-    current.timerProcessStartIdentity === expected.timerProcessStartIdentity &&
-    current.allowLegacyHermesProtocol === expected.allowLegacyHermesProtocol &&
-    current.agentName === expected.agentName &&
-    current.configPath === expected.configPath &&
-    current.configDir === expected.configDir &&
-    current.leaseOwnerPid === expected.leaseOwnerPid &&
-    current.leaseOwnerStartIdentity === expected.leaseOwnerStartIdentity
-  );
-}
-
 function assertTimerMarkerGeneration(sandboxName: string, expected: TimerMarker): void {
   if (!sameTimerMarkerGeneration(readTimerMarker(sandboxName), expected)) {
     throw new Error("Auto-restore authority changed before Shields transition takeover");
@@ -2330,6 +2316,18 @@ function inspectExpiredAutoRestoreMarker(sandboxName: string): TimerMarker | nul
   return isExactLiveFutureTimerAuthority(marker) ? null : marker;
 }
 
+function inspectCompletedAbandonedAutoRestoreMarker(
+  sandboxName: string,
+): (TimerMarker & { processToken: string }) | null {
+  const state = loadShieldsState(sandboxName);
+  if (state._isCorrupt || state.shieldsDown !== false) return null;
+  const marker = readTimerMarker(sandboxName);
+  if (!marker?.processToken || !/^[0-9a-f]{32}$/.test(marker.processToken)) return null;
+  if (!isShieldsTimerDeadlineAbandoned(sandboxName, STATE_DIR)) return null;
+  if (readShieldsDownTransition(sandboxName, marker.processToken)) return null;
+  return marker as TimerMarker & { processToken: string };
+}
+
 function inspectExpiredAutoRestoreTakeover(
   sandboxName: string,
   marker = inspectExpiredAutoRestoreMarker(sandboxName),
@@ -2463,6 +2461,42 @@ function withExpiredAutoRestoreDeadlineFence<T>(
   fallbackTakeoverToken?: string,
   assertCommandAvailable?: () => void,
 ): T {
+  const completedMarker = inspectCompletedAbandonedAutoRestoreMarker(sandboxName);
+  if (completedMarker) {
+    const assertCompletedAuthority = () => {
+      const currentState = loadShieldsState(sandboxName);
+      if (
+        currentState._isCorrupt ||
+        currentState.shieldsDown !== false ||
+        readShieldsDownTransition(sandboxName, completedMarker.processToken) ||
+        !sameTimerMarkerGeneration(readTimerMarker(sandboxName), completedMarker) ||
+        !isShieldsTimerDeadlineAbandoned(sandboxName, STATE_DIR)
+      ) {
+        throw new Error("Completed auto-restore recovery authority changed");
+      }
+    };
+    return withMcpLifecycleDeadlineFenceSync(
+      sandboxName,
+      completedMarker.processToken,
+      () => {
+        assertCompletedAuthority();
+        assertCommandAvailable?.();
+        return operation(false);
+      },
+      {
+        stateDir: STATE_DIR,
+        throwOnCommittedContainment: true,
+        completedAutoRestoreRecovery: {
+          ownerPid: completedMarker.pid,
+          assertAuthority: assertCompletedAuthority,
+        },
+        onReleased: () => {
+          const result = clearTimerMarkerGeneration(sandboxName, completedMarker);
+          if (result.warning) console.warn(`  ${result.warning}`);
+        },
+      },
+    );
+  }
   const expiredMarker = inspectExpiredAutoRestoreMarker(sandboxName);
   const takeover = inspectExpiredAutoRestoreTakeover(sandboxName, expiredMarker);
   const runWithHostLock = (callback: () => T) =>

src/lib/shields/openclaw-transition.test.ts

@@ -502,11 +502,121 @@ describe("OpenClaw shields flow rollback and recovery", () => {
     };
   }
 
+  function writeCompletedAutoRestoreOrphanFixture() {
+    const sandboxName = "openclaw";
+    const processToken = "c".repeat(32);
+    const timerPid = 2_147_483_647;
+    const stateDir = path.join(tmpDir, ".nemoclaw", "state");
+    const markerPath = path.join(stateDir, `shields-timer-${sandboxName}.json`);
+    const lifecycleLock = requireSource(
+      "../state/mcp-lifecycle-lock.js"
+    ) as typeof import("../state/mcp-lifecycle-lock.js");
+    const lockPath = lifecycleLock.getMcpLifecycleLockPath(sandboxName, stateDir);
+    const deadlinePath = `${lockPath}.deadline`;
+    const containmentPath = `${lockPath}.containment`;
+    const timerOwner = {
+      version: 1,
+      sandboxName,
+      pid: timerPid,
+      processIdentity: "dead-auto-restore-owner",
+      hostIdentity: lifecycleLock.readMcpLockHostIdentity(),
+      pidNamespaceIdentity: lifecycleLock.readMcpLockPidNamespaceIdentity(),
+      shieldsTakeoverToken: processToken,
+      token: "orphaned-auto-restore-main",
+      acquiredAt: new Date(Date.now() - 60_000).toISOString(),
+    };
+
+    fs.mkdirSync(path.dirname(lockPath), { recursive: true });
+    fs.writeFileSync(
+      path.join(stateDir, `shields-${sandboxName}.json`),
+      JSON.stringify({
+        shieldsDown: false,
+        shieldsDownAt: null,
+        shieldsDownTimeout: null,
+        shieldsDownReason: null,
+        shieldsDownPolicy: null,
+        fileHashes: { "/sandbox/.openclaw/openclaw.json": "a".repeat(64) },
+        updatedAt: new Date(Date.now() - 60_000).toISOString(),
+      }),
+    );
+    fs.writeFileSync(
+      markerPath,
+      JSON.stringify({
+        pid: timerPid,
+        sandboxName,
+        snapshotPath: path.join(stateDir, "completed-auto-restore-snapshot.yaml"),
+        restoreAt: new Date(Date.now() - 60_000).toISOString(),
+        processToken,
+        timerProcessStartIdentity: "dead-auto-restore-timer",
+      }),
+    );
+    fs.writeFileSync(lockPath, JSON.stringify(timerOwner));
+    fs.writeFileSync(
+      deadlinePath,
+      JSON.stringify({ ...timerOwner, token: "orphaned-auto-restore-deadline" }),
+    );
+    const mainStat = fs.statSync(lockPath);
+    fs.writeFileSync(
+      containmentPath,
+      JSON.stringify({
+        ...timerOwner,
+        pid: timerPid - 1,
+        processIdentity: "dead-containment-owner",
+        token: "completed-auto-restore-containment",
+        containmentReason: `The sandbox mutation owner PID ${String(
+          timerPid,
+        )} was already gone before takeover; contained generation ${String(mainStat.dev)}:${String(
+          mainStat.ino,
+        )}:${timerOwner.token}`,
+      }),
+    );
+    return { containmentPath, deadlinePath, lockPath, markerPath };
+  }
+
   beforeEach(() => {
     tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "shields-openclaw-recovery-"));
     vi.stubEnv("HOME", tmpDir);
   });
 
+  it("retires an orphaned completed auto-restore generation and containment (#10094)", {
+    timeout: 15_000,
+  }, () => {
+    const harness = createHarness();
+    const paths = writeCompletedAutoRestoreOrphanFixture();
+    const status = () =>
+      harness.shieldsStatus("openclaw", true, {
+        verifyLockState: () => ({ ok: true, issues: [] }),
+        verifyStateLockPlan: () => [],
+      });
+
+    status();
+    status();
+
+    expect(harness.logSpy).toHaveBeenCalledWith("  Shields: UP (lockdown active)");
+    expect(Object.values(paths).map((file) => fs.existsSync(file))).toEqual([
+      false,
+      false,
+      false,
+      false,
+    ]);
+  });
+
+  it("preserves completed containment for a live ambiguous timer PID", () => {
+    const harness = createHarness();
+    const paths = writeCompletedAutoRestoreOrphanFixture();
+    const marker = JSON.parse(fs.readFileSync(paths.markerPath, "utf8"));
+    marker.pid = process.pid;
+    delete marker.timerProcessStartIdentity;
+    fs.writeFileSync(paths.markerPath, JSON.stringify(marker));
+
+    expect(() => harness.shieldsStatus("openclaw")).toThrow("containment is active");
+    expect([paths.lockPath, paths.containmentPath, paths.markerPath].map(fs.existsSync)).toEqual([
+      true,
+      true,
+      true,
+    ]);
+  });
+
   afterEach(() => {
     vi.restoreAllMocks();
     vi.unstubAllEnvs();

src/lib/shields/timer-control.test.ts

@@ -1,9 +1,19 @@
 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
 import { performance } from "node:perf_hooks";
 import { afterEach, describe, expect, it, vi } from "vitest";
-import { processInspectionDeadlineAfter, processInspectionDeadlineReached } from "./timer-control";
+import {
+  clearTimerMarkerGeneration,
+  processInspectionDeadlineAfter,
+  processInspectionDeadlineReached,
+  readTimerMarker,
+  timerMarkerPath,
+  type TimerMarker,
+} from "./timer-control";
 
 describe("process inspection deadlines", () => {
   afterEach(() => {
@@ -23,3 +33,49 @@ describe("process inspection deadlines", () => {
     expect(processInspectionDeadlineReached(deadline)).toBe(true);
   });
 });
+
+describe("timer marker generation cleanup", () => {
+  afterEach(() => {
+    vi.unstubAllEnvs();
+  });
+
+  function marker(sandboxName: string, processToken: string): TimerMarker {
+    return {
+      pid: process.pid,
+      sandboxName,
+      snapshotPath: "/tmp/snapshot.yaml",
+      restoreAt: "2026-01-01T00:00:00.000Z",
+      processToken,
+    };
+  }
+
+  it("removes only the expected completed timer marker", () => {
+    const home = fs.mkdtempSync(path.join(os.tmpdir(), "timer-marker-cleanup-"));
+    vi.stubEnv("HOME", home);
+    const expected = marker("alpha", "a".repeat(32));
+    const markerPath = timerMarkerPath("alpha");
+    fs.mkdirSync(path.dirname(markerPath), { recursive: true });
+    fs.writeFileSync(markerPath, JSON.stringify(expected));
+
+    expect(clearTimerMarkerGeneration("alpha", expected)).toEqual({ cleared: true });
+    expect(readTimerMarker("alpha")).toBeNull();
+    fs.rmSync(home, { recursive: true, force: true });
+  });
+
+  it("restores a replacement marker instead of deleting it", () => {
+    const home = fs.mkdtempSync(path.join(os.tmpdir(), "timer-marker-race-"));
+    vi.stubEnv("HOME", home);
+    const expected = marker("alpha", "a".repeat(32));
+    const replacement = marker("alpha", "b".repeat(32));
+    const markerPath = timerMarkerPath("alpha");
+    fs.mkdirSync(path.dirname(markerPath), { recursive: true });
+    fs.writeFileSync(markerPath, JSON.stringify(replacement));
+
+    expect(clearTimerMarkerGeneration("alpha", expected)).toMatchObject({
+      cleared: false,
+      warning: expect.stringContaining("authority changed"),
+    });
+    expect(readTimerMarker("alpha")).toEqual(replacement);
+    fs.rmSync(home, { recursive: true, force: true });
+  });
+});

src/lib/shields/timer-control.ts

@@ -8,7 +8,9 @@ import path from "node:path";
 import { performance } from "node:perf_hooks";
 
 import {
+  isShieldsTimerDeadlineAbandoned,
   readShieldsTimerMarker,
+  readShieldsTimerMarkerFile,
   readShieldsTimerTakeoverToken,
   readTimerProcessStartIdentity,
   type ShieldsTimerMarker,
@@ -236,6 +238,67 @@ function clearTimerMarker(sandboxName: string): ClearTimerMarkerResult {
   }
 }
 
+function sameTimerMarkerGeneration(
+  current: ShieldsTimerMarker | null,
+  expected: ShieldsTimerMarker,
+): boolean {
+  return (
+    current?.pid === expected.pid &&
+    current.sandboxName === expected.sandboxName &&
+    current.snapshotPath === expected.snapshotPath &&
+    current.restoreAt === expected.restoreAt &&
+    current.processToken === expected.processToken &&
+    current.timerProcessStartIdentity === expected.timerProcessStartIdentity &&
+    current.allowLegacyHermesProtocol === expected.allowLegacyHermesProtocol &&
+    current.agentName === expected.agentName &&
+    current.configPath === expected.configPath &&
+    current.configDir === expected.configDir &&
+    current.leaseOwnerPid === expected.leaseOwnerPid &&
+    current.leaseOwnerStartIdentity === expected.leaseOwnerStartIdentity
+  );
+}
+
+function clearTimerMarkerGeneration(
+  sandboxName: string,
+  expected: ShieldsTimerMarker,
+): ClearTimerMarkerResult {
+  const markerPath = timerMarkerPath(sandboxName);
+  const quarantinePath = `${markerPath}.completed-${String(process.pid)}-${Date.now().toString(16)}`;
+  try {
+    fs.renameSync(markerPath, quarantinePath);
+  } catch (error) {
+    const errno = error as NodeJS.ErrnoException;
+    if (errno.code === "ENOENT") return { cleared: false };
+    return {
+      cleared: false,
+      warning: `Failed to claim shields timer marker '${markerPath}': ${errno.message}`,
+    };
+  }
+
+  if (sameTimerMarkerGeneration(readShieldsTimerMarkerFile(quarantinePath), expected)) {
+    fs.unlinkSync(quarantinePath);
+    removeTimerAuthorizationProof(sandboxName, expected.processToken);
+    const directoryFd = fs.openSync(path.dirname(markerPath), "r");
+    try {
+      fs.fsyncSync(directoryFd);
+    } finally {
+      fs.closeSync(directoryFd);
+    }
+    return { cleared: true };
+  }
+
+  try {
+    fs.linkSync(quarantinePath, markerPath);
+    fs.unlinkSync(quarantinePath);
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
+  }
+  return {
+    cleared: false,
+    warning: `Shields timer authority changed while retiring completed marker '${markerPath}'`,
+  };
+}
+
 function readProcessState(pid: number, deadline = processInspectionDeadline()): string | null {
   if (!Number.isInteger(pid) || pid <= 0) return null;
   if (remainingProcessInspectionTimeout(deadline) === null) return null;
@@ -410,7 +473,9 @@ function killTimer(sandboxName: string): KillTimerResult {
 export type { ClearTimerMarkerResult, KillTimerResult, ShieldsTimerMarker as TimerMarker };
 export {
   clearTimerMarker,
+  clearTimerMarkerGeneration,
   hasExactTimerAuthorizationProof,
+  isShieldsTimerDeadlineAbandoned,
   isProcessAlive,
   killTimer,
   processInspectionDeadlineAfter,
@@ -423,6 +488,7 @@ export {
   timerAuthoritySha256,
   timerAuthorizationProofPath,
   timerMarkerPath,
+  sameTimerMarkerGeneration,
   verifyTimerMarkerIdentity,
   writeTimerAuthorizationProofForMarker,
 };