fix(gateway): respawn openclaw gateway after a managed restart (#6868)

scripts/lib/gateway-supervisor.sh

@@ -240,6 +240,122 @@ EOF
   return 1
 }
 
+NEMOCLAW_MANAGED_EXPECTED_EXIT_DIR="/run/nemoclaw"
+NEMOCLAW_MANAGED_EXPECTED_EXIT_MARKER="managed-gateway-expected-exit"
+NEMOCLAW_MANAGED_CONTROLLER_PATH="/usr/local/lib/nemoclaw/managed-gateway-control.py"
+
+gateway_control_managed_controller_argv_is_expected() {
+  [ "$#" -eq 5 ] || return 1
+  case "${1##*/}" in
+    python3) ;;
+    *) return 1 ;;
+  esac
+  [ "$2" = "-I" ] && [ "$3" = "$NEMOCLAW_MANAGED_CONTROLLER_PATH" ] || return 1
+  case "$4" in
+    restart | recover) ;;
+    *) return 1 ;;
+  esac
+  case "$5" in
+    '' | *[!0-9a-f]*) return 1 ;;
+  esac
+  [ "${#5}" -eq 64 ]
+}
+
+# A numeric PID in the marker is never authority on its own. Prove the named
+# controller is still the live root helper that published the lease, so an
+# orphaned marker cannot authorize a later unrelated exit.
+gateway_control_managed_controller_is_live() {
+  local pid="$1"
+  local expected_start_identity="$2"
+  local proc_root
+  local first_start second_start first_state second_state first_uids second_uids
+  local -a first_argv=()
+  local -a second_argv=()
+
+  case "$pid" in
+    '' | 0 | 1 | *[!0-9]*) return 1 ;;
+  esac
+  case "$expected_start_identity" in
+    '' | *[!0-9]*) return 1 ;;
+  esac
+  proc_root="$(gateway_control_proc_root)" || return 1
+  [ -r "${proc_root}/${pid}/status" ] \
+    && [ -r "${proc_root}/${pid}/cmdline" ] || return 1
+
+  first_start="$(gateway_control_pid_start_identity "$pid")" || return 1
+  first_state="$(gateway_control_pid_state "$pid")" || return 1
+  first_uids="$(awk '/^Uid:/ { print $2 ":" $3 ":" $4 ":" $5; exit }' "${proc_root}/${pid}/status")" || return 1
+  while IFS= read -r -d "" elem; do first_argv+=("$elem"); done <"${proc_root}/${pid}/cmdline" || return 1
+  gateway_control_managed_controller_argv_is_expected "${first_argv[@]}" || return 1
+
+  second_start="$(gateway_control_pid_start_identity "$pid")" || return 1
+  second_state="$(gateway_control_pid_state "$pid")" || return 1
+  second_uids="$(awk '/^Uid:/ { print $2 ":" $3 ":" $4 ":" $5; exit }' "${proc_root}/${pid}/status")" || return 1
+  while IFS= read -r -d "" elem; do second_argv+=("$elem"); done <"${proc_root}/${pid}/cmdline" || return 1
+  gateway_control_managed_controller_argv_is_expected "${second_argv[@]}" || return 1
+
+  [ "$first_start" = "$expected_start_identity" ] \
+    && [ "$second_start" = "$expected_start_identity" ] \
+    && [ "$first_uids" = "0:0:0:0" ] \
+    && [ "$second_uids" = "0:0:0:0" ] \
+    && [ "$first_state" != "Z" ] \
+    && [ "$second_state" != "Z" ] \
+    && [ "${first_argv[*]}" = "${second_argv[*]}" ]
+}
+
+# True when the root control helper published a lease authorizing this exact
+# gateway exit. The managed restart path terminates the gateway with SIGTERM
+# and then only waits for a replacement, so the entrypoint must relaunch a
+# leased exit instead of reading its clean status as an intentional shutdown.
+gateway_control_exit_was_host_authorized() {
+  local pid="$1"
+  local start_identity="$2"
+  local marker dir_metadata marker_metadata
+  local version marker_pid marker_start_identity controller_pid controller_start_identity extra
+  local trailing=""
+
+  case "$pid" in
+    '' | 0 | 1 | *[!0-9]*) return 1 ;;
+  esac
+  case "$start_identity" in
+    '' | *[!0-9]*) return 1 ;;
+  esac
+
+  [ -d "$NEMOCLAW_MANAGED_EXPECTED_EXIT_DIR" ] \
+    && [ ! -L "$NEMOCLAW_MANAGED_EXPECTED_EXIT_DIR" ] || return 1
+  dir_metadata="$(stat -c '%u:%g %a' "$NEMOCLAW_MANAGED_EXPECTED_EXIT_DIR" 2>/dev/null || true)"
+  [ "$dir_metadata" = "0:0 711" ] || return 1
+
+  marker="${NEMOCLAW_MANAGED_EXPECTED_EXIT_DIR}/${NEMOCLAW_MANAGED_EXPECTED_EXIT_MARKER}"
+  [ -f "$marker" ] && [ ! -L "$marker" ] || return 1
+  marker_metadata="$(stat -c '%u:%g %a %h' "$marker" 2>/dev/null || true)"
+  [ "$marker_metadata" = "0:0 444 1" ] || return 1
+
+  # bash 4.1+ named FDs ({var}<file) are not available on bash 3.2 (macOS).
+  # Use a grouped redirect instead — variables assigned inside {} remain in scope.
+  {
+    if ! IFS=' ' read -r \
+      version marker_pid marker_start_identity controller_pid controller_start_identity extra; then
+      return 1
+    fi
+    if IFS= read -r trailing || [ -n "$trailing" ]; then
+      return 1
+    fi
+  } <"$marker" || return 1
+
+  [ "$version" = "v1" ] \
+    && [ "$marker_pid" = "$pid" ] \
+    && [ "$marker_start_identity" = "$start_identity" ] \
+    && [ -z "${extra:-}" ] || return 1
+  case "$controller_pid" in
+    '' | 0 | 1 | *[!0-9]*) return 1 ;;
+  esac
+  case "$controller_start_identity" in
+    '' | *[!0-9]*) return 1 ;;
+  esac
+  gateway_control_managed_controller_is_live "$controller_pid" "$controller_start_identity"
+}
+
 gateway_control_stop_tracked_pid() {
   local pid="$1"
   local expected_start_identity="${2:-}"

scripts/managed-gateway-control.py

@@ -1533,12 +1533,17 @@ def _control(action: str, nonce: str) -> tuple[str, int, int]:
 
         expected_exit_lease = None
         try:
-            if old_identity is not None and spec.name == "hermes":
-                # The nonroot entrypoint owns the child and its crash budget.
-                # Publish an exact root-owned authorization before the pidfd
-                # signal. The marker names this live root controller so a
-                # delayed reap remains authorized without trusting wall time,
-                # while an orphaned marker fails closed as an ordinary crash.
+            if old_identity is not None:
+                # The nonroot entrypoint owns the child and its crash budget,
+                # and SIGTERM is indistinguishable from a self-requested
+                # shutdown once the child has exited: openclaw exits 0 on
+                # SIGTERM, which nemoclaw-start reads as an intentional stop
+                # and therefore does not respawn. Publish an exact root-owned
+                # authorization before the pidfd signal so the entrypoint
+                # relaunches the gateway this controller is about to wait for.
+                # The marker names this live root controller so a delayed reap
+                # remains authorized without trusting wall time, while an
+                # orphaned marker fails closed as an ordinary crash.
                 with _control_stage("publish-expected-exit"):
                     controller_identity = _controller_process_identity(reader)
                     expected_exit_lease = _publish_expected_exit_lease(

scripts/nemoclaw-start.sh

@@ -5495,7 +5495,9 @@ if [ "$(id -u)" -ne 0 ]; then
     wait "$EXITED_GATEWAY_PID" || RC=$?
     mark_openclaw_gateway_stopped
     if [ "$RC" -eq 0 ] \
-      && ! consume_gateway_watchdog_kill "${EXITED_GATEWAY_PID}:${EXITED_GATEWAY_START_IDENTITY}"; then
+      && ! consume_gateway_watchdog_kill "${EXITED_GATEWAY_PID}:${EXITED_GATEWAY_START_IDENTITY}" \
+      && ! gateway_control_exit_was_host_authorized \
+        "$EXITED_GATEWAY_PID" "$EXITED_GATEWAY_START_IDENTITY"; then
       exit 0
     fi
     NOW=$(date +%s)
@@ -5782,7 +5784,9 @@ while :; do
     continue
   fi
   if [ "$RC" -eq 0 ] \
-    && ! consume_gateway_watchdog_kill "${EXITED_GATEWAY_PID}:${EXITED_GATEWAY_START_IDENTITY}"; then
+    && ! consume_gateway_watchdog_kill "${EXITED_GATEWAY_PID}:${EXITED_GATEWAY_START_IDENTITY}" \
+    && ! gateway_control_exit_was_host_authorized \
+      "$EXITED_GATEWAY_PID" "$EXITED_GATEWAY_START_IDENTITY"; then
     exit 0
   fi
   NOW=$(date +%s)

src/lib/sandbox/config.ts

@@ -119,16 +119,23 @@ function configFail(lines: string | readonly string[], exitCode = 1): never {
 
 function restartSandboxAgentAfterConfigSet(
   sandboxName: string,
+  agentName: string,
   restartImpl?: ManagedGatewayRestart,
 ): void {
   const restart =
     restartImpl ??
     (require("../actions/sandbox/process-recovery").restartSandboxGateway as ManagedGatewayRestart);
   const result = restart(sandboxName);
   if (!result.ok) {
-    configFail(
-      `  Config was updated, but the managed gateway restart failed for '${sandboxName}'.`,
-    );
+    // The config was already written to disk (the CAS write above succeeded),
+    // but the running agent was not reloaded. Say so plainly and point at the
+    // idempotent retry rather than leaving disk and the live gateway silently
+    // diverged. The restart layer has already printed its own failure detail.
+    configFail([
+      `  Config was written to disk but NOT applied to the running agent.`,
+      `  The ${agentName} gateway restart did not complete for '${sandboxName}' (see the failure above).`,
+      `  Retry the restart with: nemoclaw ${shellQuote(sandboxName)} gateway restart`,
+    ]);
   }
 }
 
@@ -1252,7 +1259,7 @@ async function configSet(sandboxName: string, opts: ConfigSetOpts = {}): Promise
 
   // Restart if requested
   if (opts.restart) {
-    restartSandboxAgentAfterConfigSet(sandboxName);
+    restartSandboxAgentAfterConfigSet(sandboxName, target.agentName);
   } else {
     console.log("");
     for (const line of buildConfigSetRestartGuidance(sandboxName, target.agentName)) {

test/config-set.test.ts

@@ -125,18 +125,26 @@ describe("config set helpers", () => {
     it("routes --restart through the managed gateway supervisor flow", () => {
       const calls: string[] = [];
 
-      restartSandboxAgentAfterConfigSet("alpha", (sandboxName: string) => {
+      restartSandboxAgentAfterConfigSet("alpha", "openclaw", (sandboxName: string) => {
         calls.push(sandboxName);
         return { ok: true };
       });
 
       expect(calls).toEqual(["alpha"]);
     });
 
-    it("fails the config command when the managed restart fails", () => {
-      expect(() => restartSandboxAgentAfterConfigSet("alpha", () => ({ ok: false }))).toThrow(
-        "Config was updated, but the managed gateway restart failed for 'alpha'.",
-      );
+    it("fails with a written-but-not-applied message and a retry hint when the restart fails", () => {
+      let thrown: unknown;
+      try {
+        restartSandboxAgentAfterConfigSet("alpha", "openclaw", () => ({ ok: false }));
+      } catch (error) {
+        thrown = error;
+      }
+
+      const message = thrown instanceof Error ? thrown.message : String(thrown);
+      expect(message).toContain("written to disk but NOT applied to the running agent");
+      expect(message).toContain("openclaw gateway restart did not complete for 'alpha'");
+      expect(message).toContain("nemoclaw 'alpha' gateway restart");
     });
   });
 

test/managed-gateway-control.test.ts

@@ -689,23 +689,21 @@ with tempfile.TemporaryDirectory() as root:
         real_agent_spec = control._agent_spec
         real_gateway_candidates = control._gateway_candidates
         real_wait_for_healthy = control._wait_for_healthy_gateway
-        real_publish_lease = control._publish_expected_exit_lease
         control._detect_agent = lambda: "openclaw"
         control._agent_spec = lambda *_args: control.AgentSpec("openclaw", 18642)
         control._gateway_candidates = lambda reader, *_args: [reader.capture(43)]
         control._wait_for_healthy_gateway = lambda reader, *_args: reader.capture(43)
-        control._terminate_gateway = lambda *_args: None
-        control._publish_expected_exit_lease = lambda *_args: (_ for _ in ()).throw(
-            AssertionError("OpenClaw must not publish a Hermes crash-budget lease")
+        control._terminate_gateway = lambda _reader, identity: observe_expected_exit_lease(
+            identity, "openclaw-restart"
         )
         try:
             openclaw_restart = control._control("restart", "f" * 64)
+            openclaw_lease_cleared = not os.path.exists(lease_path)
         finally:
             control._detect_agent = real_detect_agent
             control._agent_spec = real_agent_spec
             control._gateway_candidates = real_gateway_candidates
             control._wait_for_healthy_gateway = real_wait_for_healthy
-            control._publish_expected_exit_lease = real_publish_lease
             control._terminate_gateway = replace_gateway
 
         real_wait_for_healthy = control._wait_for_healthy_gateway
@@ -936,6 +934,7 @@ with tempfile.TemporaryDirectory() as root:
             lease_observations,
             restart_lease_cleared,
             timeout_lease_cleared,
+            openclaw_lease_cleared,
         ],
         "timeout_refresh": [
             timeout_refresh,
@@ -1015,6 +1014,11 @@ describe("managed gateway root control", () => {
             identity: ["v1", 41, "333", expect.any(Number), "777"],
             secure: true,
           },
+          {
+            label: "openclaw-restart",
+            identity: ["v1", 43, "555", expect.any(Number), "777"],
+            secure: true,
+          },
           {
             label: "unhealthy-recover",
             identity: ["v1", 44, "666", expect.any(Number), "777"],
@@ -1023,6 +1027,7 @@ describe("managed gateway root control", () => {
         ],
         true,
         true,
+        true,
       ],
       timeout_refresh: [
         ["ok", 44, 45],