fix(release): resolve recovery, Hermes, and inference E2E blockers

agents/hermes/Dockerfile

@@ -32,6 +32,7 @@ FROM scratch AS hermes-agent-payload
 COPY agents/hermes/plugin/ /opt/nemoclaw-hermes-plugin/
 COPY agents/hermes/generate-config.ts /opt/nemoclaw-hermes-config/generate-config.ts
 COPY agents/hermes/config/ /opt/nemoclaw-hermes-config/config/
+COPY agents/hermes/patch-gateway-runtime-metadata.py /opt/nemoclaw-hermes-config/patch-gateway-runtime-metadata.py
 COPY agents/hermes/host/managed-tool-gateway-matrix.json /opt/nemoclaw-hermes-config/managed-tool-gateway-matrix.json
 COPY src/lib/tool-disclosure.ts /src/lib/tool-disclosure.ts
 COPY src/lib/messaging/ /src/lib/messaging/
@@ -245,6 +246,37 @@ RUN chmod 755 /usr/local/bin/nemoclaw-start /usr/local/lib/nemoclaw/sandbox-init
 RUN test -x /usr/local/lib/nemoclaw/validate-hermes-env-secret-boundary.py \
     || { echo "ERROR: validate-hermes-env-secret-boundary.py missing or not executable" >&2; exit 1; }
 
+# Hermes v0.18.0 writes gateway lifecycle metadata directly below HERMES_HOME.
+# Shields-up deliberately makes that config root root-owned and non-writable,
+# so a replacement cannot remove the old PID file and exits with a false PID
+# race. Keep config/hash locks intact and relocate only Hermes's central
+# PID/lock/status path helpers to the existing writable runtime directory.
+RUN /usr/bin/python3 -I \
+        /opt/nemoclaw-hermes-config/patch-gateway-runtime-metadata.py \
+        /opt/hermes/gateway/status.py \
+    && HERMES_HOME="$(mktemp -d)" /opt/hermes/.venv/bin/python - <<'PY'
+from pathlib import Path
+
+from gateway.status import (
+    _get_gateway_lock_path,
+    _get_pid_path,
+    _get_runtime_status_path,
+)
+from hermes_constants import get_hermes_home
+
+home = get_hermes_home()
+runtime = home / "runtime"
+assert _get_pid_path() == runtime / "gateway.pid"
+assert _get_gateway_lock_path() == runtime / "gateway.lock"
+assert _get_runtime_status_path() == runtime / "gateway_state.json"
+assert all(path.parent == runtime for path in (
+    _get_pid_path(),
+    _get_gateway_lock_path(),
+    _get_runtime_status_path(),
+))
+assert isinstance(home, Path)
+PY
+
 # Hermes v0.18.0 computes `sessions list` preview from the first user message,
 # while #5254's user-facing expectation is that the existing row reflects the
 # latest resumed/continued one-shot turn. Patch only the pinned query shape and

agents/hermes/mcp-config-transaction.py

@@ -48,7 +48,7 @@
 
 CONFIG_PATH = "/sandbox/.hermes/config.yaml"
 HERMES_DIR = "/sandbox/.hermes"
-GATEWAY_PID_PATH = f"{HERMES_DIR}/gateway.pid"
+GATEWAY_PID_PATH = f"{HERMES_DIR}/runtime/gateway.pid"
 STRICT_HASH_PATH = "/etc/nemoclaw/hermes.config-hash"
 GUARD_PATH = "/usr/local/lib/nemoclaw/hermes-runtime-config-guard.py"
 ROOT_LIFECYCLE_MARKER = "/run/nemoclaw/hermes-root-lifecycle"

agents/hermes/patch-gateway-runtime-metadata.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+"""Relocate pinned Hermes gateway metadata below its writable runtime directory.
+
+Hermes v0.18.0 stores ``gateway.pid``, ``gateway.lock``, and
+``gateway_state.json`` directly below ``HERMES_HOME``. NemoClaw shields-up
+correctly makes that config root root-owned and non-writable, so a managed
+gateway replacement cannot remove the old PID file or atomically refresh
+runtime status. The resulting replacement exits with "PID file race lost".
+
+NemoClaw already provisions ``HERMES_HOME/runtime`` as the writable lifecycle
+boundary. Patch only Hermes's central path helpers so all of its PID, lock,
+status, health, and CLI readers agree on that directory. The exact source-shape
+checks fail closed when the pinned Hermes implementation changes.
+
+Remove this patch when the minimum supported Hermes release natively separates
+writable gateway metadata from its configuration root.
+"""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+OLD_PID_HELPER = '''def _get_pid_path() -> Path:
+    """Return the path to the gateway PID file, respecting HERMES_HOME."""
+    home = get_hermes_home()
+    return home / "gateway.pid"
+'''
+NEW_PID_HELPER = '''def _get_pid_path() -> Path:
+    """Return the path to the gateway PID file, respecting HERMES_HOME."""
+    home = get_hermes_home()
+    return home / "runtime" / "gateway.pid"
+'''
+
+OLD_LOCK_HELPER = '''def _get_gateway_lock_path(pid_path: Optional[Path] = None) -> Path:
+    """Return the path to the runtime gateway lock file."""
+    if pid_path is not None:
+        return pid_path.with_name(_GATEWAY_LOCK_FILENAME)
+    home = get_hermes_home()
+    return home / _GATEWAY_LOCK_FILENAME
+'''
+NEW_LOCK_HELPER = '''def _get_gateway_lock_path(pid_path: Optional[Path] = None) -> Path:
+    """Return the path to the runtime gateway lock file."""
+    if pid_path is not None:
+        return pid_path.with_name(_GATEWAY_LOCK_FILENAME)
+    home = get_hermes_home()
+    return home / "runtime" / _GATEWAY_LOCK_FILENAME
+'''
+
+
+def patch_file(path: Path) -> None:
+    source = path.read_text(encoding="utf-8")
+    replacements = (
+        ("PID", OLD_PID_HELPER, NEW_PID_HELPER),
+        ("lock", OLD_LOCK_HELPER, NEW_LOCK_HELPER),
+    )
+
+    if all(source.count(old) == 0 and source.count(new) == 1 for _, old, new in replacements):
+        return
+
+    for label, old, new in replacements:
+        old_count = source.count(old)
+        new_count = source.count(new)
+        if old_count != 1 or new_count != 0:
+            raise SystemExit(
+                "ERROR: Hermes gateway runtime metadata source shape changed; "
+                f"expected one unpatched {label} helper, found {old_count} "
+                f"(already patched helpers: {new_count})"
+            )
+
+    for _, old, new in replacements:
+        source = source.replace(old, new)
+    path.write_text(source, encoding="utf-8")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "path",
+        nargs="?",
+        default="/opt/hermes/gateway/status.py",
+        help="Hermes gateway status module to patch",
+    )
+    args = parser.parse_args()
+    patch_file(Path(args.path))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

docs/changelog/2026-07-28.mdx

@@ -28,7 +28,10 @@ It also hardens compatible-provider switching, managed sandbox images, Jetson GP
   Host-side `config set` also accepts the exact `http://host.openshell.internal:<unprivileged-port>` shape only for supported provider `baseUrl` fields, while generic keys and other private URL shapes remain rejected.
   Retired NVIDIA Build MiniMax and Qwen model paths are no longer offered.
   For more information, refer to [Switch Inference Providers](/user-guide/openclaw/inference/manage-inference/switch-providers), [Meet Custom Endpoint Security Requirements](/user-guide/openclaw/inference/custom-endpoints/custom-endpoint-security), and [Set Up Ollama](/user-guide/openclaw/inference/local-inference/set-up-ollama).
-- Docker-driver recovery now unpauses a paused original container before reporting recovery, and resumed onboarding writes a secret-free same-name recreation journal before deletion so an interrupted run can continue or fail closed on ambiguous identity.
+- Docker-driver post-reboot recovery now waits for the labeled container to reach Docker readiness.
+  It restores the in-sandbox gateway and host forwards before reporting success, refreshes stale stopped-container evidence, and fails closed when delivery cannot be proven.
+  Recovery also unpauses a paused original container before reporting success.
+  Resumed onboarding writes a secret-free same-name recreation journal before deletion so an interrupted run can continue or fail closed on ambiguous identity.
   Uninstall checks for the `openshell` command after confirmation and before cleanup mutation, while source-checkout installation preserves an absolute `NEMOCLAW_OPENSHELL_BIN` selection during user-local OpenShell discovery.
   For more information, refer to [Recover and Rebuild Sandboxes](/user-guide/openclaw/manage-sandboxes/operate-sandboxes/recover-and-rebuild-sandboxes) and [Uninstall NemoClaw](/user-guide/openclaw/manage-sandboxes/operate-sandboxes/uninstall-nemoclaw).
 - Jetson setup now explains why host preparation was skipped when it encounters an unrecognized or unparseable release.
@@ -38,6 +41,8 @@ It also hardens compatible-provider switching, managed sandbox images, Jetson GP
 - Telegram channel status now returns `unreachable` with a nonzero exit when the Bot API startup probe receives a definitive HTTP error.
   Managed MCP tool discovery accepts compliant case-variant or parameterized SSE media types through the reviewed SDK runtime.
   Hermes image assembly normalizes executable modes before metadata validation, and locked restart integrity accepts the generated v1 managed MCP state record while retaining fail-closed hashing and failed-reload rollback.
+  Hermes now keeps gateway PID, lock, and status records in the writable runtime directory.
+  This lets a restart replace the tracked gateway while shields are up without weakening config locks or crash quarantine.
   For more information, refer to [Choose Messaging Channels](/user-guide/openclaw/manage-sandboxes/messaging-channels/choose-messaging-channels), [Manage MCP Servers with OpenClaw](/user-guide/openclaw/manage-sandboxes/mcp-servers/manage-mcp-servers), [Manage MCP Servers with Hermes](/user-guide/hermes/manage-sandboxes/mcp-servers/manage-mcp-servers), and the [NemoClaw CLI Commands Reference](/user-guide/openclaw/reference/commands).
 - Deep Agents now publishes the bounded approval, baseline, preset, custom-preset, and live-policy tasks that apply to its maintained runtime.
   The `claude-code` preset permits browser login only through `GET` and `POST` on `platform.claude.com/v1/oauth/**`, without widening unrelated Anthropic or telemetry access.

docs/reference/commands.mdx

@@ -1354,7 +1354,11 @@ For cloud-only providers, the output omits the NIM status line unless a NIM cont
 
 When the sandbox's recorded driver is `docker` and the host Docker daemon is not reachable, the command prints the `docker_unreachable` failure layer with the message `Docker daemon is not reachable.` as the first line of stdout, suppresses the host-side `Inference` probe (which otherwise hits the remote provider directly and is misleading when the local stack is down), and exits with a non-zero status.
 
-When the host Docker daemon is reachable but the per-sandbox container is stopped, the command prints the `sandbox_container_stopped` failure layer with the message `sandbox container exists but is not running.` as the first line of stdout, suppresses the host-side `Inference` probe, and exits with a non-zero status.
+When the host Docker daemon is reachable but the per-sandbox container is stopped, the initial preflight records the `sandbox_container_stopped` failure layer and suppresses the host-side `Inference` probe.
+If the owning OpenShell gateway is healthy but no longer lists the registered Docker-driver sandbox, status attempts post-reboot recovery from the labeled container.
+It waits for Docker readiness, restores the in-sandbox gateway and host forwards, and refreshes preflight before probing inference.
+A successful recovery clears the stale stopped-container failure.
+If Docker readiness or the agent delivery chain cannot be proven, status exits non-zero and reports the failed recovery layer.
 <AgentOnly variant="openclaw,hermes">
 If the sandbox's recorded dashboard port is also held by a foreign listener, the header escalates to the `sandbox_dashboard_port_conflict` failure layer with the message `sandbox container is stopped and the dashboard port is held by a foreign listener.` so the operator can recover the port before restarting the sandbox.
 </AgentOnly>

src/lib/actions/sandbox/process-recovery.ts

@@ -1152,6 +1152,17 @@ function checkAndRecoverSandboxProcessesWithoutHostLock(
           "the primary dashboard/API host forward is owned by another sandbox",
       };
     }
+    if (forwardHealthy === null) {
+      return {
+        checked: true,
+        wasRunning: true,
+        recovered: false,
+        forwardRecovered: false,
+        forwardRecoveryFailed: true,
+        forwardRecoveryFailureDetail:
+          "the primary dashboard/API host forward could not be verified because OpenShell forward state was unavailable",
+      };
+    }
     const dashboardForwardRecovered = ensureHermesDashboardPortForwardIfEnabled(sandboxName);
     const messagingForwardRecovered = recoverMessagingHostForward(sandboxName, { quiet });
     const declaredForwardsRecovered = recoverDeclaredAgentForwardPorts(sandboxName, recoveryPort, {

src/lib/actions/sandbox/status-flow.test.ts

@@ -490,8 +490,87 @@ describe("showSandboxStatus flow", () => {
     expect(output).toContain("Retry `openshell gateway start --name nemoclaw`");
     expect(output).toContain("If the gateway never becomes healthy");
     expect(harness.collectSandboxStatusSnapshotSpy).toHaveBeenCalledWith("alpha", {
+      preflight: {
+        failure: null,
+        failureLayer: "docker_unreachable",
+        suppressInferenceProbe: true,
+        exitCode: 1,
+      },
+    });
+  });
+
+  it("renders the refreshed preflight after Docker recovery", async () => {
+    const preflight = {
+      failure: {
+        layer: "sandbox_container_stopped" as const,
+        dockerUnreachable: false,
+      },
+      failureLayer: "sandbox_container_stopped" as const,
+      suppressInferenceProbe: true,
+      exitCode: 1 as const,
+    };
+    const harness = createStatusFlowHarness({
+      preflight,
+      postRecoveryPreflight: {
+        failure: null,
+        failureLayer: null,
+        suppressInferenceProbe: false,
+        exitCode: 0,
+      },
+    });
+
+    await expect(harness.showSandboxStatus("alpha")).resolves.toBeUndefined();
+
+    const output = harness.logSpy.mock.calls.flat().join("\n");
+    expect(output).not.toContain("Failure layer: sandbox_container_stopped");
+    expect(process.exitCode).toBeUndefined();
+    expect(harness.collectSandboxStatusSnapshotSpy).toHaveBeenCalledWith("alpha", {
+      preflight,
+    });
+  });
+
+  it("does not erase a dashboard-port conflict during Docker recovery", async () => {
+    const conflict = {
+      failure: {
+        layer: "sandbox_dashboard_port_conflict" as const,
+        dockerUnreachable: false,
+      },
+      failureLayer: "sandbox_dashboard_port_conflict" as const,
       suppressInferenceProbe: true,
+      exitCode: 1 as const,
+    };
+    const harness = createStatusFlowHarness({
+      preflight: conflict,
+      postRecoveryPreflight: conflict,
     });
+
+    await expect(harness.showSandboxStatus("alpha")).resolves.toBeUndefined();
+
+    const output = harness.logSpy.mock.calls.flat().join("\n");
+    expect(output).toContain("Failure layer: sandbox_dashboard_port_conflict");
+    expect(process.exitCode).toBe(1);
+  });
+
+  it("renders an agent delivery recovery failure as actionable and nonzero", async () => {
+    const harness = createStatusFlowHarness({
+      inferenceHealth: null,
+      lookup: {
+        state: "sandbox_recovery_failed",
+        output:
+          "  Docker restored sandbox 'alpha', but its agent delivery chain is not ready " +
+          "(forward-recovery: OpenShell forward state unavailable).",
+        recoveredSandbox: true,
+      },
+    });
+
+    await expect(harness.showSandboxStatus("alpha")).rejects.toThrow("process.exit(1)");
+
+    const output = harness.logSpy.mock.calls.flat().join("\n");
+    expect(output).toContain("restored from Docker");
+    expect(output).toContain("agent delivery chain could not be recovered safely");
+    expect(output).toContain("forward-recovery: OpenShell forward state unavailable");
+    expect(output).toContain("Retry `nemoclaw alpha recover`");
+    expect(output).not.toContain("Could not verify against live gateway");
   });
 
   it("renders missing gateway metadata after restart without claiming recovery", async () => {

src/lib/actions/sandbox/status-lookup-rendering.ts

@@ -47,11 +47,29 @@ export async function printSandboxGatewayLookupStatus(
     case "gateway_missing_after_restart":
       await printGatewayMissingAfterRestartLookupStatus(context);
       return;
+    case "sandbox_recovery_failed":
+      printSandboxRecoveryFailedLookupStatus(context);
+      return;
     default:
       await printUnknownGatewayLookupStatus(context);
   }
 }
 
+function printSandboxRecoveryFailedLookupStatus({
+  sandboxName,
+  lookup,
+}: SandboxGatewayLookupStatusContext): void {
+  console.log("");
+  console.log(
+    `  Sandbox '${sandboxName}' was restored from Docker, but its agent delivery chain could not be recovered safely.`,
+  );
+  if (lookup.output) console.log(lookup.output);
+  console.log(
+    `  Retry \`${CLI_NAME} ${sandboxName} recover\` after addressing the reported layer.`,
+  );
+  process.exit(1);
+}
+
 function printMissingLiveSandboxStatusGuidance(
   sandboxName: string,
   lookup: SandboxGatewayState,

src/lib/actions/sandbox/status-snapshot-inference-health.test.ts

@@ -1,7 +1,7 @@
 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-import { describe, expect, it } from "vitest";
+import { describe, expect, it, vi } from "vitest";
 
 import type { ProviderHealthStatus } from "../../inference/health";
 import type { SandboxEntry } from "../../state/registry";
@@ -43,6 +43,59 @@ function snapshotDeps(
 }
 
 describe("collectSandboxStatusSnapshot inference route health", () => {
+  it("restores the guarded agent and host-forward chain before probing a Docker-recovered sandbox", async () => {
+    const order: string[] = [];
+    const gateway: SandboxInferenceRouteHealth = {
+      ok: true,
+      endpoint: "https://inference.local/v1/models",
+      httpStatus: 200,
+      detail: "reachable",
+    };
+    const options = snapshotDeps(gateway);
+    options.deps.reconcile = async () => {
+      order.push("reconcile");
+      return {
+        state: "present",
+        output: "Phase: Ready",
+        recoveredSandbox: true,
+        recoverySandboxVia: "started-stopped-original",
+      };
+    };
+    const recoverSandboxProcesses = vi.fn(() => {
+      order.push("recover-agent-and-forward");
+      return {
+        checked: true,
+        wasRunning: false,
+        recovered: true,
+        forwardRecovered: true,
+      };
+    });
+    options.deps.probeSandboxInferenceGatewayHealthImpl = async () => {
+      order.push("probe-inference");
+      return gateway;
+    };
+
+    await collectSandboxStatusSnapshot("alpha", {
+      ...options,
+      deps: { ...options.deps, recoverSandboxProcesses },
+    });
+
+    expect(recoverSandboxProcesses).toHaveBeenCalledWith("alpha", { quiet: true });
+    expect(order).toEqual(["reconcile", "recover-agent-and-forward", "probe-inference"]);
+  });
+
+  it("does not mutate the agent or host forward during an ordinary present status lookup", async () => {
+    const options = snapshotDeps(null);
+    const recoverSandboxProcesses = vi.fn();
+
+    await collectSandboxStatusSnapshot("alpha", {
+      ...options,
+      deps: { ...options.deps, recoverSandboxProcesses },
+    });
+
+    expect(recoverSandboxProcesses).not.toHaveBeenCalled();
+  });
+
   it("labels a reachable route okLabel: reachable, not a bare healthy claim (#6846)", async () => {
     const gateway: SandboxInferenceRouteHealth = {
       ok: true,