fix(sandbox): recover relaunches a restart-killed gateway supervisor

docs/manage-sandboxes/lifecycle.mdx

@@ -232,6 +232,10 @@ If the gateway is already healthy, `recover` exits after the probe and does not
 For built-in OpenClaw sandboxes, recovery sends an authenticated request through registry-scoped privileged direct-container control.
 The request reaches either the root PID 1 supervisor in a direct root-entrypoint container or the installed root controller in an OpenShell-managed container.
 It does not fall back to ordinary `openshell sandbox exec` or a manual relaunch inside the sandbox.
+For a local Docker-driver sandbox whose container still uses the legacy keepalive startup, `recover` can transactionally recreate the registered container with a credential-free managed startup command.
+NemoClaw keeps the previous container available until the managed controller proves the supervisor topology, gateway health, and settle check, and attempts to restore it if that proof fails.
+The recreation preserves mounted sandbox state, but a committed swap does not retain changes stored only in the previous container's writable layer.
+If the gateway still does not recover, for example when the sandbox needs credentials that only re-onboarding can restore, recreate the runtime with `$$nemoclaw <name> rebuild --yes`, or destroy and re-onboard the sandbox.
 Refer to [`$$nemoclaw <name> recover`](../reference/commands#$$nemoclaw-name-recover) for details.
 
 Use `gateway restart` when you intentionally need a supported OpenClaw gateway to reload runtime configuration or plugins.
@@ -257,6 +261,10 @@ If the gateway is already healthy, `recover` exits after the probe and does not
 For built-in Hermes sandboxes, recovery sends an authenticated request through registry-scoped privileged direct-container control.
 The request reaches either the root PID 1 supervisor in a direct root-entrypoint container or the installed root controller in an OpenShell-managed container.
 It does not fall back to ordinary `openshell sandbox exec` or a manual relaunch inside the sandbox.
+For a local Docker-driver sandbox whose container still uses the legacy keepalive startup, `recover` can transactionally recreate the registered container with a credential-free managed startup command.
+NemoClaw keeps the previous container available until the managed controller proves the supervisor topology, gateway health, and settle check, and attempts to restore it if that proof fails.
+The recreation preserves mounted sandbox state, but a committed swap does not retain changes stored only in the previous container's writable layer.
+If the gateway still does not recover, for example when the sandbox needs credentials that only re-onboarding can restore, recreate the runtime with `$$nemoclaw <name> rebuild --yes`, or destroy and re-onboard the sandbox.
 Refer to [`$$nemoclaw <name> recover`](../reference/commands#$$nemoclaw-name-recover) for details.
 
 Use `gateway restart` when you intentionally need a supported Hermes gateway to reload runtime configuration or plugins.

docs/reference/commands.mdx

@@ -1033,6 +1033,9 @@ The host selects the controller from the live container topology.
 In a direct root-entrypoint container, the request reaches the root PID 1 supervisor.
 In an OpenShell-managed container, the request enters the root-owned mode `0500` managed controller through a sanitized root exec while OpenShell remains PID 1.
 It does not use ordinary `openshell sandbox exec` or an in-sandbox manual relaunch as a fallback.
+When the root-owned managed controller attests two unchanged zero-supervisor process scans with a stable PID 1 and reports `SUPERVISOR_NOT_RUNNING`, a local Docker-driver sandbox with the legacy keepalive startup can enter a transactional container recreation.
+The recreation uses a credential-free managed startup command, pins the registered container identity, retains the previous container for rollback, and commits only after managed gateway health and the settle check pass.
+The recreation preserves mounted sandbox state, but a committed swap does not retain changes stored only in the previous container's writable layer.
 It is idempotent.
 When `recover` repairs a stopped built-in OpenClaw or Hermes gateway, it retries only when stdout is empty and stderr is exactly one `SUPERVISOR_BUSY` or `SUPERVISOR_UNAVAILABLE` line, with at most three controller attempts.
 Other controller failures stop immediately.
@@ -1107,7 +1110,7 @@ Run `$$nemoclaw <name> shields down` before a Hermes config or inference change;
 
 <AgentOnly variant="openclaw,hermes">
 
-The command can fail at these layers: unsupported agent, privileged control unavailable, secret-boundary refusal, unsafe config path, config hash mismatch when a strict hash is available, launch failure, health timeout, or forward recovery failure.
+The command can fail at these layers: unsupported agent, privileged control unavailable, supervisor not running, secret-boundary refusal, unsafe config path, config hash mismatch when a strict hash is available, launch failure, health timeout, or forward recovery failure.
 An older direct-container image without the matching supervisor or managed controller helper reports `privileged control unavailable` and requires `$$nemoclaw <name> rebuild --yes`.
 Ordinary OpenShell exec and manual in-sandbox relaunch are not fallback paths.
 Terminal agents do not have a gateway runtime and fail as unsupported.
@@ -3135,7 +3138,9 @@ Keep the OpenShell gateway on loopback and use `NEMOCLAW_DASHBOARD_BIND` when yo
 `NEMOCLAW_DASHBOARD_BIND` controls the dashboard or API port forward bind address.
 Outside WSL, the forward stays on `127.0.0.1` (loopback only) by default.
 On WSL, NemoClaw binds the host-side forward on all interfaces so the Windows host can reach it, while the ready summary continues to print a loopback dashboard URL.
-Set `NEMOCLAW_DASHBOARD_BIND=0.0.0.0` before `$$nemoclaw onboard` to prepare the sandbox for remote exposure and bind the forward on all interfaces. Use the same setting for later `$$nemoclaw <sandbox> connect` calls. A sandbox created without this opt-in must be recreated with `NEMOCLAW_DASHBOARD_BIND=0.0.0.0 $$nemoclaw onboard --recreate-sandbox` before a remote-bind connect is allowed.
+On non-WSL SSH-deployed hosts, set `NEMOCLAW_DASHBOARD_BIND=0.0.0.0` before `$$nemoclaw onboard` to prepare the sandbox for remote exposure and bind the forward on all interfaces.
+Use the same setting for later `$$nemoclaw <sandbox> connect` calls.
+A sandbox created without this opt-in must be recreated with `NEMOCLAW_DASHBOARD_BIND=0.0.0.0 $$nemoclaw onboard --recreate-sandbox` before a remote-bind connect is allowed.
 Only `0.0.0.0` enables the remote bind; onboarding rejects any other non-empty value.
 </AgentOnly>
 
@@ -3509,6 +3514,7 @@ The following flags change defaults for commands that manage existing sandboxes.
 | `NEMOCLAW_CLEANUP_GATEWAY` | `1`, `true`, or `yes` to enable; `0`, `false`, or `no` to disable | Overrides the platform default (macOS unattended: cleanup; Linux/Windows: preserve) for whether `$$nemoclaw <name> destroy` removes the shared gateway when destroying the last sandbox. Command-line `--cleanup-gateway` and `--no-cleanup-gateway` still take precedence. |
 | `NEMOCLAW_CONFIRM_LEGACY_MANAGED_RECREATE` | Exact JSON array of sandbox names | Confirms to the installer that the exact listed set of pre-fingerprint OpenClaw or Hermes sandboxes used NemoClaw-managed images, allowing recovery onto the current managed image. The normalized names must exactly match the installer's printed array. Set it only after verifying every named sandbox. Recorded custom-image evidence remains blocked. |
 | `NEMOCLAW_DISABLE_INFERENCE_ROUTE_REPAIR` | `1` to enable | Skips the automatic DNS-proxy repair for stale `inference.local` routes during `$$nemoclaw <name> connect` and `$$nemoclaw <name> connect --probe-only`. Use only as a troubleshooting escape hatch. |
+| `NEMOCLAW_DISABLE_SUPERVISOR_RELAUNCH` | `1` to enable | Skips the automatic trusted container recreation during `$$nemoclaw <name> recover` when two managed scans find no supervisor while PID 1 remains stable. Use only as a troubleshooting escape hatch; recovery then falls back to the rebuild or re-onboard guidance. |
 | `NEMOCLAW_SHIELDS_ACCEPT_LEGACY_BASELINE` | `1` to opt in | Allows advanced immutable-config verification to trust the current on-disk bytes for older or partial content baselines. Use only after you have rebuilt or manually inspected the sandbox state and accepted that the baseline is operator-approved. |
 | `NEMOCLAW_SHIELDS_SETTLE_MS` | milliseconds (default `750`, clamped to `0` to `10000`) | Settle window NemoClaw waits after re-applying a config lockdown (during shields auto-restore and `$$nemoclaw <name> shields up` drift remediation) before re-confirming the lock still holds. Detects when an in-sandbox reconciler changes config file permissions after lockdown and re-applies the lock; if NemoClaw cannot re-confirm the lock within the retry budget, shields stay down. This narrows the window in which a reconciler can revert permissions rather than eliminating it. The best-effort `chattr +i` immutable bit remains the only fully durable lock. Raise it on hosts where the gateway settles slowly. |
 | `NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP` | Exactly `1` to opt in (`true`, `yes`, `0` are not accepted) | Applies to standalone `$$nemoclaw backup-all` runs. Skips running sandboxes whose in-sandbox SSH endpoint does not answer. It does not relax the installer's strict pre-upgrade backup, which still aborts if any registered sandbox is skipped or fails. Any uncommitted state since the last successful backup is not included in the skipped backup. |

docs/reference/troubleshooting.mdx

@@ -931,8 +931,11 @@ $$nemoclaw <name> status
 
 When `recover` repairs a stopped built-in gateway, NemoClaw automatically retries only when stdout is empty and stderr is exactly one `SUPERVISOR_BUSY` or `SUPERVISOR_UNAVAILABLE` line, with at most three controller attempts.
 It does not retry other status or output combinations.
+`SUPERVISOR_NOT_RUNNING` is a separate result that requires two zero-supervisor scans with a stable PID 1 and does not enter that retry loop.
+On a supported local Docker-driver sandbox with the legacy keepalive startup, it can authorize a container-identity-pinned recreation that commits only after managed health and settle checks pass.
+To bypass that trusted recreation while troubleshooting, run `NEMOCLAW_DISABLE_SUPERVISOR_RELAUNCH=1 $$nemoclaw <name> recover`; NemoClaw leaves the container unchanged and returns rebuild or re-onboard guidance.
 If that bounded retry is exhausted, or if `gateway restart` reports `SUPERVISOR_BUSY`, wait for the active request to finish and retry the command.
-If the error mentions `SUPERVISOR_REBUILD_REQUIRED`, `SUPERVISOR_UNAVAILABLE`, a missing `nemoclaw-gateway-control` helper, or a missing managed controller, the sandbox image may predate the current lifecycle contract.
+If the error mentions `SUPERVISOR_NOT_RUNNING` and trusted recreation could not proceed, `SUPERVISOR_REBUILD_REQUIRED`, `SUPERVISOR_UNAVAILABLE`, a missing `nemoclaw-gateway-control` helper, or a missing managed controller, the sandbox image may predate the current lifecycle contract.
 The managed controller also refuses an ambiguous or changing process tree instead of guessing which same-UID process is the gateway.
 Rebuild the image before retrying:
 

scripts/managed-gateway-control.py

@@ -748,18 +748,20 @@ def _sandbox_uid() -> int:
         raise ControlError("SUPERVISOR_UNAVAILABLE") from exc
 
 
-def _discover_supervisor(reader: ProcReader) -> ProcessIdentity:
-    pid1 = reader.capture(1)
-    if not _is_openshell(pid1):
-        raise ControlError("SUPERVISOR_UNAVAILABLE")
-    sandbox_uid = _sandbox_uid()
+def _supervisor_candidates(
+    reader: ProcReader, pid1: ProcessIdentity, sandbox_uid: int
+) -> tuple[list[ProcessIdentity], bool]:
     matches: list[ProcessIdentity] = []
+    inconclusive = False
     for pid in reader.pids():
         if pid == 1:
             continue
         try:
             identity = reader.capture(pid)
-        except (ControlError, FileNotFoundError, ProcessLookupError, PermissionError):
+        except (FileNotFoundError, ProcessLookupError):
+            continue
+        except (ControlError, PermissionError):
+            inconclusive = True
             continue
         if (
             _is_nemoclaw_start(identity, sandbox_uid)
@@ -771,6 +773,37 @@ def _discover_supervisor(reader: ProcReader) -> ProcessIdentity:
             matches.append(identity)
             if len(matches) > 1:
                 break
+    return matches, inconclusive
+
+
+def _discover_supervisor(reader: ProcReader) -> ProcessIdentity:
+    pid1 = reader.capture(1)
+    if not _is_openshell(pid1):
+        raise ControlError("SUPERVISOR_UNAVAILABLE")
+    sandbox_uid = _sandbox_uid()
+    matches, inconclusive = _supervisor_candidates(reader, pid1, sandbox_uid)
+    if inconclusive:
+        raise ControlError("SUPERVISOR_UNAVAILABLE")
+    if len(matches) == 0:
+        # A zero-match scan is the only absence signal that may authorize the
+        # host to recreate a legacy Docker container with its managed startup
+        # command. Re-scan the complete process table and pin PID 1 around both
+        # observations so ambiguity, process churn, and supervisor startup
+        # races remain generic unavailability rather than destructive-recovery
+        # authorization.
+        between_pid1 = reader.capture(1)
+        second_matches, second_inconclusive = _supervisor_candidates(
+            reader, pid1, sandbox_uid
+        )
+        after_pid1 = reader.capture(1)
+        if (
+            between_pid1.stable_key() == pid1.stable_key()
+            and after_pid1.stable_key() == pid1.stable_key()
+            and not second_inconclusive
+            and len(second_matches) == 0
+        ):
+            raise ControlError("SUPERVISOR_NOT_RUNNING")
+        raise ControlError("SUPERVISOR_UNAVAILABLE")
     if len(matches) != 1:
         raise ControlError("SUPERVISOR_UNAVAILABLE")
     current_pid1 = reader.capture(1)

src/lib/actions/sandbox/forward-recovery.ts

@@ -32,6 +32,11 @@ type SandboxPortDeps = {
   getSessionAgent?: (sandboxName?: string) => SandboxPortAgent;
 };
 
+type SandboxForwardRecoveryOptions = {
+  afterSuccess?: () => boolean;
+  beforeStart?: () => boolean;
+};
+
 function isValidPort(value: unknown): value is number {
   return typeof value === "number" && Number.isInteger(value) && value >= 1 && value <= 65535;
 }
@@ -62,7 +67,10 @@ export function resolveSandboxDashboardPort(
  * Returns true when `forward start` succeeded and a follow-up probe
  * confirms the new entry is running, false otherwise.
  */
-export function ensureSandboxPortForward(sandboxName: string): boolean {
+export function ensureSandboxPortForward(
+  sandboxName: string,
+  options: SandboxForwardRecoveryOptions = {},
+): boolean {
   const port = resolveSandboxDashboardPort(sandboxName);
   const remoteBindRequested = isRemoteDashboardBindRequested(process.env.NEMOCLAW_DASHBOARD_BIND);
   const allInterfaceBindRequired = remoteBindRequested || isWsl();
@@ -79,9 +87,11 @@ export function ensureSandboxPortForward(sandboxName: string): boolean {
     forwardTarget: allInterfaceBindRequired ? `0.0.0.0:${port}` : String(port),
     forceRestart: remoteBindRequested,
     expectedBind: allInterfaceBindRequired ? "0.0.0.0" : "127.0.0.1",
-    beforeStart: remoteBindRequested
-      ? () => registry.getSandbox(sandboxName)?.dashboardRemoteBindPrepared === true
-      : undefined,
+    afterSuccess: options.afterSuccess,
+    beforeStart: () =>
+      (!remoteBindRequested ||
+        registry.getSandbox(sandboxName)?.dashboardRemoteBindPrepared === true) &&
+      (options.beforeStart?.() ?? true),
   });
 }
 
@@ -133,20 +143,36 @@ export function ensureSandboxPortForwardForPort(
   sandboxName: string,
   port: number,
   options: {
+    afterSuccess?: () => boolean;
     forwardTarget?: string;
     forceRestart?: boolean;
     expectedBind?: string;
     beforeStart?: () => boolean;
   } = {},
 ): boolean {
   const {
+    afterSuccess = () => true,
     forwardTarget = String(port),
     forceRestart = false,
     expectedBind,
     beforeStart = () => true,
   } = options;
+  const acceptSuccessfulForward = () => {
+    let accepted = false;
+    try {
+      accepted = afterSuccess();
+    } catch {
+      accepted = false;
+    }
+    if (accepted) return true;
+    runOpenshell(["forward", "stop", String(port), sandboxName], {
+      ignoreError: true,
+      stdio: "ignore",
+    });
+    return false;
+  };
   let forwardHealth = isSandboxPortForwardHealthy(sandboxName, port, expectedBind);
-  if (forwardHealth === true && !forceRestart) return true;
+  if (forwardHealth === true && !forceRestart) return acceptSuccessfulForward();
   if (forwardHealth === "occupied") return false;
   const configuredWaitMs = Number(process.env.NEMOCLAW_FORWARD_RECOVERY_WAIT_MS ?? "3000");
   const waitMs = Number.isFinite(configuredWaitMs) ? Math.max(0, configuredWaitMs) : 3000;
@@ -195,7 +221,7 @@ export function ensureSandboxPortForwardForPort(
         backoffFactor: 1.5,
       },
     );
-    if (stopState.health === true && !forceRestart) return true;
+    if (stopState.health === true && !forceRestart) return acceptSuccessfulForward();
     if (stopState.health === "occupied" || !stopSettled || !stopState.portReleased) return false;
   }
 
@@ -204,6 +230,11 @@ export function ensureSandboxPortForwardForPort(
     ["forward", "start", "--background", forwardTarget, sandboxName],
     {
       ignoreError: true,
+      // OpenShell 0.0.72 leaves the background SSH forward attached to the
+      // caller's inherited descriptors. Detach them so a scripted `recover`
+      // can finish after the foreground OpenShell command exits. Keep this
+      // until every supported OpenShell release redirects those descriptors.
+      stdio: "ignore",
     },
   );
   if (startResult.status !== 0) return false;
@@ -213,7 +244,7 @@ export function ensureSandboxPortForwardForPort(
   // of accepting an arbitrary reachable listener or failing on the first
   // metadata refresh.
   let health = isSandboxPortForwardHealthy(sandboxName, port, expectedBind);
-  if (health === true) return true;
+  if (health === true) return acceptSuccessfulForward();
   if (health === "occupied") return false;
   if (waitMs === 0) return false;
 
@@ -234,7 +265,7 @@ export function ensureSandboxPortForwardForPort(
       backoffFactor: 1.5,
     },
   );
-  return settled && !occupied;
+  return settled && !occupied && acceptSuccessfulForward();
 }
 
 export function ensureHermesDashboardPortForwardIfEnabled(sandboxName: string): boolean | null {

src/lib/actions/sandbox/gateway-restart.test.ts

@@ -18,6 +18,8 @@ describe("gateway restart failure markers", () => {
       [string, ReturnType<typeof classifyGatewayRestartFailure>["layer"]]
     > = [
       ["PRIVILEGED_CONTROL_UNAVAILABLE", "privileged control unavailable"],
+      ["SUPERVISOR_UNAVAILABLE", "privileged control unavailable"],
+      ["SUPERVISOR_NOT_RUNNING", "supervisor not running"],
       ["SUPERVISOR_REBUILD_REQUIRED", "privileged control unavailable"],
       ["SUPERVISOR_BUSY", "privileged control unavailable"],
       [MARKERS.SECRET_BOUNDARY_REFUSED, "secret-boundary refusal"],

src/lib/actions/sandbox/gateway-restart.ts

@@ -17,6 +17,7 @@ export type GatewayRestartCommandResult = {
 export type GatewayRestartFailureLayer =
   | "unsupported agent"
   | "privileged control unavailable"
+  | "supervisor not running"
   | "secret-boundary refusal"
   | "unsafe config path"
   | "config hash mismatch"
@@ -127,6 +128,12 @@ export function classifyGatewayRestartFailure(result: GatewayRestartCommandResul
 
   const output = gatewayRestartOutput(result);
   const detail = sanitizeGatewayRestartFailureDetail(output.trim());
+  if (output.includes("SUPERVISOR_NOT_RUNNING")) {
+    return {
+      layer: "supervisor not running",
+      detail: detail || "the in-sandbox gateway supervisor is not running",
+    };
+  }
   if (
     output.includes(MARKERS.ROOT_EXEC_UNAVAILABLE) ||
     output.includes("PRIVILEGED_CONTROL_UNAVAILABLE") ||