diff --git a/agents/hermes/Dockerfile b/agents/hermes/Dockerfile index 12edfe44507..de6075ec362 100644 --- a/agents/hermes/Dockerfile +++ b/agents/hermes/Dockerfile @@ -32,6 +32,7 @@ FROM scratch AS hermes-agent-payload COPY agents/hermes/plugin/ /opt/nemoclaw-hermes-plugin/ COPY agents/hermes/generate-config.ts /opt/nemoclaw-hermes-config/generate-config.ts COPY agents/hermes/config/ /opt/nemoclaw-hermes-config/config/ +COPY agents/hermes/patch-gateway-runtime-metadata.py /opt/nemoclaw-hermes-config/patch-gateway-runtime-metadata.py COPY agents/hermes/host/managed-tool-gateway-matrix.json /opt/nemoclaw-hermes-config/managed-tool-gateway-matrix.json COPY src/lib/tool-disclosure.ts /src/lib/tool-disclosure.ts COPY src/lib/messaging/ /src/lib/messaging/ @@ -245,6 +246,37 @@ RUN chmod 755 /usr/local/bin/nemoclaw-start /usr/local/lib/nemoclaw/sandbox-init RUN test -x /usr/local/lib/nemoclaw/validate-hermes-env-secret-boundary.py \ || { echo "ERROR: validate-hermes-env-secret-boundary.py missing or not executable" >&2; exit 1; } +# Hermes v0.18.0 writes gateway lifecycle metadata directly below HERMES_HOME. +# Shields-up deliberately makes that config root root-owned and non-writable, +# so a replacement cannot remove the old PID file and exits with a false PID +# race. Keep config/hash locks intact and relocate only Hermes's central +# PID/lock/status path helpers to the existing writable runtime directory. +RUN /usr/bin/python3 -I \ + /opt/nemoclaw-hermes-config/patch-gateway-runtime-metadata.py \ + /opt/hermes/gateway/status.py \ + && HERMES_HOME="$(mktemp -d)" /opt/hermes/.venv/bin/python - <<'PY' +from pathlib import Path + +from gateway.status import ( + _get_gateway_lock_path, + _get_pid_path, + _get_runtime_status_path, +) +from hermes_constants import get_hermes_home + +home = get_hermes_home() +runtime = home / "runtime" +assert _get_pid_path() == runtime / "gateway.pid" +assert _get_gateway_lock_path() == runtime / "gateway.lock" +assert _get_runtime_status_path() == runtime / "gateway_state.json" +assert all(path.parent == runtime for path in ( + _get_pid_path(), + _get_gateway_lock_path(), + _get_runtime_status_path(), +)) +assert isinstance(home, Path) +PY + # Hermes v0.18.0 computes `sessions list` preview from the first user message, # while #5254's user-facing expectation is that the existing row reflects the # latest resumed/continued one-shot turn. Patch only the pinned query shape and diff --git a/agents/hermes/mcp-config-transaction.py b/agents/hermes/mcp-config-transaction.py index 4cd644316b2..a3fddc885a1 100755 --- a/agents/hermes/mcp-config-transaction.py +++ b/agents/hermes/mcp-config-transaction.py @@ -48,7 +48,7 @@ CONFIG_PATH = "/sandbox/.hermes/config.yaml" HERMES_DIR = "/sandbox/.hermes" -GATEWAY_PID_PATH = f"{HERMES_DIR}/gateway.pid" +GATEWAY_PID_PATH = f"{HERMES_DIR}/runtime/gateway.pid" STRICT_HASH_PATH = "/etc/nemoclaw/hermes.config-hash" GUARD_PATH = "/usr/local/lib/nemoclaw/hermes-runtime-config-guard.py" ROOT_LIFECYCLE_MARKER = "/run/nemoclaw/hermes-root-lifecycle" diff --git a/agents/hermes/patch-gateway-runtime-metadata.py b/agents/hermes/patch-gateway-runtime-metadata.py new file mode 100755 index 00000000000..b1f955838cf --- /dev/null +++ b/agents/hermes/patch-gateway-runtime-metadata.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Relocate pinned Hermes gateway metadata below its writable runtime directory. + +Hermes v0.18.0 stores ``gateway.pid``, ``gateway.lock``, and +``gateway_state.json`` directly below ``HERMES_HOME``. NemoClaw shields-up +correctly makes that config root root-owned and non-writable, so a managed +gateway replacement cannot remove the old PID file or atomically refresh +runtime status. The resulting replacement exits with "PID file race lost". + +NemoClaw already provisions ``HERMES_HOME/runtime`` as the writable lifecycle +boundary. Patch only Hermes's central path helpers so all of its PID, lock, +status, health, and CLI readers agree on that directory. The exact source-shape +checks fail closed when the pinned Hermes implementation changes. + +Remove this patch when the minimum supported Hermes release natively separates +writable gateway metadata from its configuration root. +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +OLD_PID_HELPER = '''def _get_pid_path() -> Path: + """Return the path to the gateway PID file, respecting HERMES_HOME.""" + home = get_hermes_home() + return home / "gateway.pid" +''' +NEW_PID_HELPER = '''def _get_pid_path() -> Path: + """Return the path to the gateway PID file, respecting HERMES_HOME.""" + home = get_hermes_home() + return home / "runtime" / "gateway.pid" +''' + +OLD_LOCK_HELPER = '''def _get_gateway_lock_path(pid_path: Optional[Path] = None) -> Path: + """Return the path to the runtime gateway lock file.""" + if pid_path is not None: + return pid_path.with_name(_GATEWAY_LOCK_FILENAME) + home = get_hermes_home() + return home / _GATEWAY_LOCK_FILENAME +''' +NEW_LOCK_HELPER = '''def _get_gateway_lock_path(pid_path: Optional[Path] = None) -> Path: + """Return the path to the runtime gateway lock file.""" + if pid_path is not None: + return pid_path.with_name(_GATEWAY_LOCK_FILENAME) + home = get_hermes_home() + return home / "runtime" / _GATEWAY_LOCK_FILENAME +''' + + +def patch_file(path: Path) -> None: + source = path.read_text(encoding="utf-8") + replacements = ( + ("PID", OLD_PID_HELPER, NEW_PID_HELPER), + ("lock", OLD_LOCK_HELPER, NEW_LOCK_HELPER), + ) + + if all(source.count(old) == 0 and source.count(new) == 1 for _, old, new in replacements): + return + + for label, old, new in replacements: + old_count = source.count(old) + new_count = source.count(new) + if old_count != 1 or new_count != 0: + raise SystemExit( + "ERROR: Hermes gateway runtime metadata source shape changed; " + f"expected one unpatched {label} helper, found {old_count} " + f"(already patched helpers: {new_count})" + ) + + for _, old, new in replacements: + source = source.replace(old, new) + path.write_text(source, encoding="utf-8") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument( + "path", + nargs="?", + default="/opt/hermes/gateway/status.py", + help="Hermes gateway status module to patch", + ) + args = parser.parse_args() + patch_file(Path(args.path)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/changelog/2026-07-28.mdx b/docs/changelog/2026-07-28.mdx index 9538ab72f40..38929b8c099 100644 --- a/docs/changelog/2026-07-28.mdx +++ b/docs/changelog/2026-07-28.mdx @@ -28,7 +28,10 @@ It also hardens compatible-provider switching, managed sandbox images, Jetson GP Host-side `config set` also accepts the exact `http://host.openshell.internal:` shape only for supported provider `baseUrl` fields, while generic keys and other private URL shapes remain rejected. Retired NVIDIA Build MiniMax and Qwen model paths are no longer offered. For more information, refer to [Switch Inference Providers](/user-guide/openclaw/inference/manage-inference/switch-providers), [Meet Custom Endpoint Security Requirements](/user-guide/openclaw/inference/custom-endpoints/custom-endpoint-security), and [Set Up Ollama](/user-guide/openclaw/inference/local-inference/set-up-ollama). -- Docker-driver recovery now unpauses a paused original container before reporting recovery, and resumed onboarding writes a secret-free same-name recreation journal before deletion so an interrupted run can continue or fail closed on ambiguous identity. +- Docker-driver post-reboot recovery now waits for the labeled container to reach Docker readiness. + It restores the in-sandbox gateway and host forwards before reporting success, refreshes stale stopped-container evidence, and fails closed when delivery cannot be proven. + Recovery also unpauses a paused original container before reporting success. + Resumed onboarding writes a secret-free same-name recreation journal before deletion so an interrupted run can continue or fail closed on ambiguous identity. Uninstall checks for the `openshell` command after confirmation and before cleanup mutation, while source-checkout installation preserves an absolute `NEMOCLAW_OPENSHELL_BIN` selection during user-local OpenShell discovery. For more information, refer to [Recover and Rebuild Sandboxes](/user-guide/openclaw/manage-sandboxes/operate-sandboxes/recover-and-rebuild-sandboxes) and [Uninstall NemoClaw](/user-guide/openclaw/manage-sandboxes/operate-sandboxes/uninstall-nemoclaw). - Jetson setup now explains why host preparation was skipped when it encounters an unrecognized or unparseable release. @@ -38,6 +41,8 @@ It also hardens compatible-provider switching, managed sandbox images, Jetson GP - Telegram channel status now returns `unreachable` with a nonzero exit when the Bot API startup probe receives a definitive HTTP error. Managed MCP tool discovery accepts compliant case-variant or parameterized SSE media types through the reviewed SDK runtime. Hermes image assembly normalizes executable modes before metadata validation, and locked restart integrity accepts the generated v1 managed MCP state record while retaining fail-closed hashing and failed-reload rollback. + Hermes now keeps gateway PID, lock, and status records in the writable runtime directory. + This lets a restart replace the tracked gateway while shields are up without weakening config locks or crash quarantine. For more information, refer to [Choose Messaging Channels](/user-guide/openclaw/manage-sandboxes/messaging-channels/choose-messaging-channels), [Manage MCP Servers with OpenClaw](/user-guide/openclaw/manage-sandboxes/mcp-servers/manage-mcp-servers), [Manage MCP Servers with Hermes](/user-guide/hermes/manage-sandboxes/mcp-servers/manage-mcp-servers), and the [NemoClaw CLI Commands Reference](/user-guide/openclaw/reference/commands). - Deep Agents now publishes the bounded approval, baseline, preset, custom-preset, and live-policy tasks that apply to its maintained runtime. The `claude-code` preset permits browser login only through `GET` and `POST` on `platform.claude.com/v1/oauth/**`, without widening unrelated Anthropic or telemetry access. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 28d9c8f78fa..bb3285ac7a3 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -1354,7 +1354,11 @@ For cloud-only providers, the output omits the NIM status line unless a NIM cont When the sandbox's recorded driver is `docker` and the host Docker daemon is not reachable, the command prints the `docker_unreachable` failure layer with the message `Docker daemon is not reachable.` as the first line of stdout, suppresses the host-side `Inference` probe (which otherwise hits the remote provider directly and is misleading when the local stack is down), and exits with a non-zero status. -When the host Docker daemon is reachable but the per-sandbox container is stopped, the command prints the `sandbox_container_stopped` failure layer with the message `sandbox container exists but is not running.` as the first line of stdout, suppresses the host-side `Inference` probe, and exits with a non-zero status. +When the host Docker daemon is reachable but the per-sandbox container is stopped, the initial preflight records the `sandbox_container_stopped` failure layer and suppresses the host-side `Inference` probe. +If the owning OpenShell gateway is healthy but no longer lists the registered Docker-driver sandbox, status attempts post-reboot recovery from the labeled container. +It waits for Docker readiness, restores the in-sandbox gateway and host forwards, and refreshes preflight before probing inference. +A successful recovery clears the stale stopped-container failure. +If Docker readiness or the agent delivery chain cannot be proven, status exits non-zero and reports the failed recovery layer. If the sandbox's recorded dashboard port is also held by a foreign listener, the header escalates to the `sandbox_dashboard_port_conflict` failure layer with the message `sandbox container is stopped and the dashboard port is held by a foreign listener.` so the operator can recover the port before restarting the sandbox. diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index c3f0d58372c..b8171b4cea7 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -1152,6 +1152,17 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( "the primary dashboard/API host forward is owned by another sandbox", }; } + if (forwardHealthy === null) { + return { + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: false, + forwardRecoveryFailed: true, + forwardRecoveryFailureDetail: + "the primary dashboard/API host forward could not be verified because OpenShell forward state was unavailable", + }; + } const dashboardForwardRecovered = ensureHermesDashboardPortForwardIfEnabled(sandboxName); const messagingForwardRecovered = recoverMessagingHostForward(sandboxName, { quiet }); const declaredForwardsRecovered = recoverDeclaredAgentForwardPorts(sandboxName, recoveryPort, { diff --git a/src/lib/actions/sandbox/status-flow.test.ts b/src/lib/actions/sandbox/status-flow.test.ts index 0e5e301f0a3..57b9a9508a3 100644 --- a/src/lib/actions/sandbox/status-flow.test.ts +++ b/src/lib/actions/sandbox/status-flow.test.ts @@ -490,8 +490,87 @@ describe("showSandboxStatus flow", () => { expect(output).toContain("Retry `openshell gateway start --name nemoclaw`"); expect(output).toContain("If the gateway never becomes healthy"); expect(harness.collectSandboxStatusSnapshotSpy).toHaveBeenCalledWith("alpha", { + preflight: { + failure: null, + failureLayer: "docker_unreachable", + suppressInferenceProbe: true, + exitCode: 1, + }, + }); + }); + + it("renders the refreshed preflight after Docker recovery", async () => { + const preflight = { + failure: { + layer: "sandbox_container_stopped" as const, + dockerUnreachable: false, + }, + failureLayer: "sandbox_container_stopped" as const, + suppressInferenceProbe: true, + exitCode: 1 as const, + }; + const harness = createStatusFlowHarness({ + preflight, + postRecoveryPreflight: { + failure: null, + failureLayer: null, + suppressInferenceProbe: false, + exitCode: 0, + }, + }); + + await expect(harness.showSandboxStatus("alpha")).resolves.toBeUndefined(); + + const output = harness.logSpy.mock.calls.flat().join("\n"); + expect(output).not.toContain("Failure layer: sandbox_container_stopped"); + expect(process.exitCode).toBeUndefined(); + expect(harness.collectSandboxStatusSnapshotSpy).toHaveBeenCalledWith("alpha", { + preflight, + }); + }); + + it("does not erase a dashboard-port conflict during Docker recovery", async () => { + const conflict = { + failure: { + layer: "sandbox_dashboard_port_conflict" as const, + dockerUnreachable: false, + }, + failureLayer: "sandbox_dashboard_port_conflict" as const, suppressInferenceProbe: true, + exitCode: 1 as const, + }; + const harness = createStatusFlowHarness({ + preflight: conflict, + postRecoveryPreflight: conflict, }); + + await expect(harness.showSandboxStatus("alpha")).resolves.toBeUndefined(); + + const output = harness.logSpy.mock.calls.flat().join("\n"); + expect(output).toContain("Failure layer: sandbox_dashboard_port_conflict"); + expect(process.exitCode).toBe(1); + }); + + it("renders an agent delivery recovery failure as actionable and nonzero", async () => { + const harness = createStatusFlowHarness({ + inferenceHealth: null, + lookup: { + state: "sandbox_recovery_failed", + output: + " Docker restored sandbox 'alpha', but its agent delivery chain is not ready " + + "(forward-recovery: OpenShell forward state unavailable).", + recoveredSandbox: true, + }, + }); + + await expect(harness.showSandboxStatus("alpha")).rejects.toThrow("process.exit(1)"); + + const output = harness.logSpy.mock.calls.flat().join("\n"); + expect(output).toContain("restored from Docker"); + expect(output).toContain("agent delivery chain could not be recovered safely"); + expect(output).toContain("forward-recovery: OpenShell forward state unavailable"); + expect(output).toContain("Retry `nemoclaw alpha recover`"); + expect(output).not.toContain("Could not verify against live gateway"); }); it("renders missing gateway metadata after restart without claiming recovery", async () => { diff --git a/src/lib/actions/sandbox/status-lookup-rendering.ts b/src/lib/actions/sandbox/status-lookup-rendering.ts index 92cab9b18e0..7343d944307 100644 --- a/src/lib/actions/sandbox/status-lookup-rendering.ts +++ b/src/lib/actions/sandbox/status-lookup-rendering.ts @@ -47,11 +47,29 @@ export async function printSandboxGatewayLookupStatus( case "gateway_missing_after_restart": await printGatewayMissingAfterRestartLookupStatus(context); return; + case "sandbox_recovery_failed": + printSandboxRecoveryFailedLookupStatus(context); + return; default: await printUnknownGatewayLookupStatus(context); } } +function printSandboxRecoveryFailedLookupStatus({ + sandboxName, + lookup, +}: SandboxGatewayLookupStatusContext): void { + console.log(""); + console.log( + ` Sandbox '${sandboxName}' was restored from Docker, but its agent delivery chain could not be recovered safely.`, + ); + if (lookup.output) console.log(lookup.output); + console.log( + ` Retry \`${CLI_NAME} ${sandboxName} recover\` after addressing the reported layer.`, + ); + process.exit(1); +} + function printMissingLiveSandboxStatusGuidance( sandboxName: string, lookup: SandboxGatewayState, diff --git a/src/lib/actions/sandbox/status-snapshot-inference-health.test.ts b/src/lib/actions/sandbox/status-snapshot-inference-health.test.ts index 87088dfb1fe..b75ae797de3 100644 --- a/src/lib/actions/sandbox/status-snapshot-inference-health.test.ts +++ b/src/lib/actions/sandbox/status-snapshot-inference-health.test.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import type { ProviderHealthStatus } from "../../inference/health"; import type { SandboxEntry } from "../../state/registry"; @@ -43,6 +43,59 @@ function snapshotDeps( } describe("collectSandboxStatusSnapshot inference route health", () => { + it("restores the guarded agent and host-forward chain before probing a Docker-recovered sandbox", async () => { + const order: string[] = []; + const gateway: SandboxInferenceRouteHealth = { + ok: true, + endpoint: "https://inference.local/v1/models", + httpStatus: 200, + detail: "reachable", + }; + const options = snapshotDeps(gateway); + options.deps.reconcile = async () => { + order.push("reconcile"); + return { + state: "present", + output: "Phase: Ready", + recoveredSandbox: true, + recoverySandboxVia: "started-stopped-original", + }; + }; + const recoverSandboxProcesses = vi.fn(() => { + order.push("recover-agent-and-forward"); + return { + checked: true, + wasRunning: false, + recovered: true, + forwardRecovered: true, + }; + }); + options.deps.probeSandboxInferenceGatewayHealthImpl = async () => { + order.push("probe-inference"); + return gateway; + }; + + await collectSandboxStatusSnapshot("alpha", { + ...options, + deps: { ...options.deps, recoverSandboxProcesses }, + }); + + expect(recoverSandboxProcesses).toHaveBeenCalledWith("alpha", { quiet: true }); + expect(order).toEqual(["reconcile", "recover-agent-and-forward", "probe-inference"]); + }); + + it("does not mutate the agent or host forward during an ordinary present status lookup", async () => { + const options = snapshotDeps(null); + const recoverSandboxProcesses = vi.fn(); + + await collectSandboxStatusSnapshot("alpha", { + ...options, + deps: { ...options.deps, recoverSandboxProcesses }, + }); + + expect(recoverSandboxProcesses).not.toHaveBeenCalled(); + }); + it("labels a reachable route okLabel: reachable, not a bare healthy claim (#6846)", async () => { const gateway: SandboxInferenceRouteHealth = { ok: true, diff --git a/src/lib/actions/sandbox/status-snapshot-recovery.test.ts b/src/lib/actions/sandbox/status-snapshot-recovery.test.ts new file mode 100644 index 00000000000..629f3600c61 --- /dev/null +++ b/src/lib/actions/sandbox/status-snapshot-recovery.test.ts @@ -0,0 +1,228 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import type { SandboxEntry } from "../../state/registry"; +import type { SandboxInferenceRouteHealth } from "./inference-route-health"; +import type { SandboxStatusPreflightResult } from "./status-preflight"; +import { collectSandboxStatusSnapshot, getSandboxStatusReport } from "./status-snapshot"; + +const sandbox: SandboxEntry = { + name: "alpha", + agent: "openclaw", + policies: [], + provider: "nvidia", + model: "nvidia/nemotron", + openshellDriver: "docker", + dashboardPort: 18789, +}; + +const stoppedPreflight: SandboxStatusPreflightResult = { + failure: { + layer: "sandbox_container_stopped", + dockerUnreachable: false, + }, + failureLayer: "sandbox_container_stopped", + suppressInferenceProbe: true, + exitCode: 1, +}; + +const clearPreflight: SandboxStatusPreflightResult = { + failure: null, + failureLayer: null, + suppressInferenceProbe: false, + exitCode: 0, +}; + +const conflictPreflight: SandboxStatusPreflightResult = { + failure: { + layer: "sandbox_dashboard_port_conflict", + dockerUnreachable: false, + }, + failureLayer: "sandbox_dashboard_port_conflict", + suppressInferenceProbe: true, + exitCode: 1, +}; + +const healthyRoute: SandboxInferenceRouteHealth = { + ok: true, + endpoint: "https://inference.local/v1/models", + httpStatus: 200, + detail: "reachable", +}; + +function recoveredLookup() { + return Promise.resolve({ + state: "present", + output: "Phase: Ready", + recoveredSandbox: true, + recoverySandboxVia: "started-stopped-original", + }); +} + +function snapshotDeps(recoveryResult: unknown) { + const probeProviderHealthImpl = vi.fn(() => null); + const probeSandboxInferenceGatewayHealthImpl = vi.fn(async () => healthyRoute); + return { + getSandbox: () => sandbox, + listSandboxes: () => ({ sandboxes: [sandbox], defaultSandbox: sandbox.name }), + reconcile: recoveredLookup, + captureOpenshellForStatusImpl: async () => { + throw new Error("live route lookup not needed"); + }, + probeProviderHealthImpl, + probeSandboxInferenceGatewayHealthImpl, + recoverSandboxProcesses: vi.fn(() => recoveryResult) as never, + }; +} + +describe("collectSandboxStatusSnapshot Docker recovery", () => { + it.each([ + [ + "inspection", + { + checked: false, + wasRunning: null, + recovered: false, + forwardRecovered: false, + }, + ], + [ + "secret-boundary", + { + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: false, + secretBoundaryRefused: true, + secretBoundaryReason: "persisted secret boundary refused recovery", + }, + ], + [ + "mcp-reconciliation", + { + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: false, + mcpReconciliationRefused: true, + mcpReconciliationReason: "MCP intent mismatch", + }, + ], + [ + "gateway-recovery", + { + checked: true, + wasRunning: false, + recovered: false, + forwardRecovered: false, + }, + ], + [ + "forward-recovery", + { + checked: true, + wasRunning: false, + recovered: true, + forwardRecovered: false, + }, + ], + [ + "forward-recovery", + { + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: false, + forwardRecoveryFailed: true, + forwardRecoveryFailureDetail: "OpenShell forward state unavailable", + }, + ], + ])("fails closed at the %s layer", async (layer, recoveryResult) => { + const deps = snapshotDeps(recoveryResult); + + const snapshot = await collectSandboxStatusSnapshot("alpha", { deps }); + + expect(snapshot.lookup.state).toBe("sandbox_recovery_failed"); + expect(snapshot.lookup.output).toContain(`(${layer}:`); + expect(deps.probeProviderHealthImpl).not.toHaveBeenCalled(); + expect(deps.probeSandboxInferenceGatewayHealthImpl).not.toHaveBeenCalled(); + }); + + it("accepts a recovered gateway only when the primary forward is proven", async () => { + const deps = snapshotDeps({ + checked: true, + wasRunning: false, + recovered: true, + forwardRecovered: true, + }); + + const snapshot = await collectSandboxStatusSnapshot("alpha", { deps }); + + expect(snapshot.lookup.state).toBe("present"); + expect(deps.probeSandboxInferenceGatewayHealthImpl).toHaveBeenCalledWith("alpha"); + }); + + it("keeps a terminal runtime result neutral", async () => { + const deps = snapshotDeps({ + checked: true, + wasRunning: null, + recovered: false, + forwardRecovered: false, + runtime: "terminal", + }); + + const snapshot = await collectSandboxStatusSnapshot("alpha", { deps }); + + expect(snapshot.lookup.state).toBe("present"); + }); +}); + +describe("getSandboxStatusReport Docker recovery preflight refresh", () => { + it("clears a stale stopped-container preflight after successful recovery", async () => { + const getSandboxStatusPreflightImpl = vi + .fn() + .mockResolvedValueOnce(stoppedPreflight) + .mockResolvedValueOnce(clearPreflight); + const deps = { + ...snapshotDeps({ + checked: true, + wasRunning: false, + recovered: true, + forwardRecovered: true, + }), + getSandboxStatusPreflightImpl, + }; + + const report = await getSandboxStatusReport("alpha", deps); + + expect(report.gatewayState).toBe("present"); + expect(report.failureLayer).toBeNull(); + expect(report.inferenceHealth).toMatchObject({ ok: true, probed: true }); + expect(getSandboxStatusPreflightImpl).toHaveBeenCalledTimes(2); + }); + + it("preserves a dashboard-port conflict observed before Docker recovery", async () => { + const getSandboxStatusPreflightImpl = vi + .fn() + .mockResolvedValueOnce(conflictPreflight) + .mockResolvedValueOnce(clearPreflight); + const deps = { + ...snapshotDeps({ + checked: true, + wasRunning: false, + recovered: true, + forwardRecovered: true, + }), + getSandboxStatusPreflightImpl, + }; + + const report = await getSandboxStatusReport("alpha", deps); + + expect(report.failureLayer).toBe("sandbox_dashboard_port_conflict"); + expect(report.inferenceHealth).toBeNull(); + expect(getSandboxStatusPreflightImpl).toHaveBeenCalledOnce(); + expect(deps.probeSandboxInferenceGatewayHealthImpl).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/actions/sandbox/status-snapshot.ts b/src/lib/actions/sandbox/status-snapshot.ts index 6363e0dd99d..7cf8e5db7bb 100644 --- a/src/lib/actions/sandbox/status-snapshot.ts +++ b/src/lib/actions/sandbox/status-snapshot.ts @@ -41,6 +41,7 @@ import { probeSandboxInferenceGatewayHealth } from "./inference-route-health"; import { getSandboxStatusPreflight, type SandboxStatusFailureLayer, + type SandboxStatusPreflightResult, withoutTerminalPhasePreflight, } from "./status-preflight"; import { @@ -211,6 +212,8 @@ export interface SandboxStatusSnapshot { inferenceHealth: ProviderHealthStatus | null; terminalRuntimeHealth: TerminalRuntimeOomProbeResult | null; servingProcessHealth: ServingProcessHealth | null; + /** Refreshed after Docker recovery so callers do not render stale stopped-container state. */ + postRecoveryPreflight?: SandboxStatusPreflightResult; } export interface SandboxStatusAgentInfo { @@ -255,6 +258,28 @@ export function resolveSandboxStatusAgent(agentName = "openclaw"): SandboxStatus type ReconcileSandboxGatewayState = (sandboxName: string) => Promise; type ProbeTerminalRuntimeHealth = (sandboxName: string) => TerminalRuntimeOomProbeResult; +type RecoverSandboxProcesses = + typeof import("./status/process-recovery")["checkAndRecoverSandboxProcesses"]; +type SandboxProcessRecoveryResult = ReturnType; + +type SandboxProcessRecoveryFailure = { + layer: + | "inspection" + | "secret-boundary" + | "mcp-reconciliation" + | "gateway-recovery" + | "forward-recovery" + | "recovery-error"; + detail: string; +}; + +function loadRecoverSandboxProcesses(): RecoverSandboxProcesses { + return ( + require("./status/process-recovery") as { + checkAndRecoverSandboxProcesses: RecoverSandboxProcesses; + } + ).checkAndRecoverSandboxProcesses; +} interface CollectSandboxStatusSnapshotDeps { getSandbox?: typeof registry.getSandbox; @@ -264,18 +289,101 @@ interface CollectSandboxStatusSnapshotDeps { probeSandboxInferenceGatewayHealthImpl?: ProbeSandboxInferenceGatewayHealth; reportInferenceProbeError?: (message: string) => void; probeTerminalRuntimeHealth?: ProbeTerminalRuntimeHealth; + recoverSandboxProcesses?: RecoverSandboxProcesses; reconcile?: ReconcileSandboxGatewayState; + getSandboxStatusPreflightImpl?: typeof getSandboxStatusPreflight; getBaselineExclusionRuntimeStatus?: typeof getBaselineExclusionRuntimeStatus; } -function reportInferenceProbeError(error: unknown, writer: (message: string) => void): void { +function sanitizedStatusDetail(error: unknown): string { const raw = error instanceof Error && error.message ? error.message : String(error); - const detail = redact(raw) + return redact(raw) .replace(/[\u0000-\u001f\u007f-\u009f]/g, " ") .replace(/\s+/g, " ") - .trim(); + .trim() + .slice(0, 240); +} + +function processRecoveryFailure( + result: SandboxProcessRecoveryResult, +): SandboxProcessRecoveryFailure | null { + if (!result.checked) { + return { + layer: "inspection", + detail: "the managed agent gateway could not be inspected", + }; + } + if ("secretBoundaryRefused" in result && result.secretBoundaryRefused) { + return { + layer: "secret-boundary", + detail: sanitizedStatusDetail( + "secretBoundaryReason" in result + ? result.secretBoundaryReason + : "the agent secret boundary refused recovery", + ), + }; + } + if ("mcpReconciliationRefused" in result && result.mcpReconciliationRefused) { + return { + layer: "mcp-reconciliation", + detail: sanitizedStatusDetail( + "mcpReconciliationReason" in result + ? result.mcpReconciliationReason + : "MCP reconciliation refused recovery", + ), + }; + } + if ("forwardRecoveryFailed" in result && result.forwardRecoveryFailed) { + return { + layer: "forward-recovery", + detail: sanitizedStatusDetail( + "forwardRecoveryFailureDetail" in result + ? result.forwardRecoveryFailureDetail + : "the host forward could not be restored", + ), + }; + } + if (result.wasRunning === false && result.recovered && !result.forwardRecovered) { + return { + layer: "forward-recovery", + detail: "the primary dashboard/API host forward was not proven after gateway recovery", + }; + } + if (result.wasRunning === false && !result.recovered) { + return { + layer: "gateway-recovery", + detail: "the managed agent gateway could not be restarted", + }; + } + if (result.wasRunning === null && (!("runtime" in result) || result.runtime !== "terminal")) { + return { + layer: "inspection", + detail: "the managed agent gateway recovery result was inconclusive", + }; + } + return null; +} + +async function refreshPreflightAfterDockerRecovery( + sandbox: registry.SandboxEntry | null, + initial: SandboxStatusPreflightResult, + getPreflight: typeof getSandboxStatusPreflight, +): Promise { + // A listener observed while the sandbox was stopped may belong to another + // process. Starting the container makes the ordinary preflight stop checking + // that port, so keep this ownership conflict authoritative. + if (initial.failureLayer === "sandbox_dashboard_port_conflict") return initial; + try { + return await getPreflight(sandbox); + } catch { + return initial; + } +} + +function reportInferenceProbeError(error: unknown, writer: (message: string) => void): void { + const detail = sanitizedStatusDetail(error); writer( - ` Warning: the authoritative inference.local probe could not run: ${detail.slice(0, 240) || "unknown error"}`, + ` Warning: the authoritative inference.local probe could not run: ${detail || "unknown error"}`, ); } @@ -283,6 +391,7 @@ export async function collectSandboxStatusSnapshot( sandboxName: string, opts: { suppressInferenceProbe?: boolean; + preflight?: SandboxStatusPreflightResult; deps?: CollectSandboxStatusSnapshotDeps; } = {}, ): Promise { @@ -304,6 +413,48 @@ export async function collectSandboxStatusSnapshot( output: ` Could not probe live gateway state: ${message}`, }; } + const dockerRecovered = lookup.recoveredSandbox === true; + if (lookup.state === "present" && lookup.recoveredSandbox) { + let failure: SandboxProcessRecoveryFailure | null; + try { + // Docker recovery makes the sandbox visible to OpenShell again, but a + // host reboot also tears down the managed agent process and port-forward. + // Reuse the guarded connect recovery only for this explicit mutation + // path, before status probes the delivery chain. + const recovery = (opts.deps?.recoverSandboxProcesses ?? loadRecoverSandboxProcesses())( + sandboxName, + { + quiet: true, + }, + ); + failure = processRecoveryFailure(recovery); + } catch (error) { + failure = { + layer: "recovery-error", + detail: sanitizedStatusDetail(error) || "agent and host-forward recovery failed", + }; + } + if (failure) { + lookup = { + ...lookup, + state: "sandbox_recovery_failed", + output: + ` Docker restored sandbox '${sandboxName}', but its agent delivery chain is not ready ` + + `(${failure.layer}: ${failure.detail}).`, + }; + } + } + const postRecoveryPreflight = + dockerRecovered && opts.preflight + ? await refreshPreflightAfterDockerRecovery( + sb, + opts.preflight, + opts.deps?.getSandboxStatusPreflightImpl ?? getSandboxStatusPreflight, + ) + : undefined; + const suppressInferenceProbe = + (postRecoveryPreflight ?? opts.preflight)?.suppressInferenceProbe ?? + opts.suppressInferenceProbe === true; let liveResult: Awaited> | null = null; let gatewayName: string | null = null; if (lookup.state === "present") { @@ -332,6 +483,7 @@ export async function collectSandboxStatusSnapshot( inferenceHealth: null, terminalRuntimeHealth: null, servingProcessHealth: null, + ...(postRecoveryPreflight ? { postRecoveryPreflight } : {}), }; } const live = @@ -374,7 +526,7 @@ export async function collectSandboxStatusSnapshot( let providerHealth: ProviderHealthStatus | null = null; try { providerHealth = maybeGetSandboxStatusInferenceHealth( - opts.suppressInferenceProbe === true, + suppressInferenceProbe, lookup.state === "present", (live && live.provider) || currentProvider, (live && live.model) || currentModel, @@ -394,7 +546,7 @@ export async function collectSandboxStatusSnapshot( // `inference.local` is authoritative because it is the route the agent uses. // Probe it independently of direct/upstream provider diagnostics, including // providers without a registered host-side health probe (#6192). - if (opts.suppressInferenceProbe !== true && lookup.state === "present") { + if (!suppressInferenceProbe && lookup.state === "present") { let gatewayChain: Awaited> = null; try { gatewayChain = await ( @@ -432,6 +584,7 @@ export async function collectSandboxStatusSnapshot( inferenceHealth, terminalRuntimeHealth, servingProcessHealth, + ...(postRecoveryPreflight ? { postRecoveryPreflight } : {}), }; } @@ -452,9 +605,11 @@ async function buildSandboxStatusReport( deps: CollectSandboxStatusSnapshotDeps, ): Promise { const getSandbox = deps.getSandbox ?? registry.getSandbox; - const preflight = await getSandboxStatusPreflight(getSandbox(sandboxName)); + const preflight = await (deps.getSandboxStatusPreflightImpl ?? getSandboxStatusPreflight)( + getSandbox(sandboxName), + ); const snapshot = await collectSandboxStatusSnapshot(sandboxName, { - suppressInferenceProbe: preflight.suppressInferenceProbe, + preflight, deps, }); const { @@ -471,7 +626,10 @@ async function buildSandboxStatusReport( } = snapshot; const dockerRuntime = lookup.state === "present" ? getSandboxDockerRuntime(sandboxName) : null; const phase = lookup.state === "present" ? parseSandboxPhase(lookup.output || "") : null; - const effectivePreflight = withoutTerminalPhasePreflight(preflight, phase); + const effectivePreflight = withoutTerminalPhasePreflight( + snapshot.postRecoveryPreflight ?? preflight, + phase, + ); const sandboxGpuEnabled = sb ? (sb.sandboxGpuEnabled ?? sb.gpuEnabled === true) : false; const policies = sb && Array.isArray(sb.policies) diff --git a/src/lib/actions/sandbox/status-text.ts b/src/lib/actions/sandbox/status-text.ts index 7c8bca85ba8..550128c7f64 100644 --- a/src/lib/actions/sandbox/status-text.ts +++ b/src/lib/actions/sandbox/status-text.ts @@ -24,7 +24,7 @@ import { } from "../../state/sandbox-session"; import type { SandboxDockerRuntime } from "./docker-health"; import type { SandboxGatewayState } from "./gateway-state"; -import { isSandboxGatewayRunningForStatus } from "./process-recovery"; +import { isSandboxGatewayRunningForStatus } from "./status/process-recovery"; import { isInferenceHealthFailing, resolveSandboxStatusDcodeAutoApprovalMode, diff --git a/src/lib/actions/sandbox/status.ts b/src/lib/actions/sandbox/status.ts index 9f15c04c4c5..4c1c92ceb74 100644 --- a/src/lib/actions/sandbox/status.ts +++ b/src/lib/actions/sandbox/status.ts @@ -71,7 +71,7 @@ export async function showSandboxStatus(sandboxName: string): Promise { // handles `gateway_error` by printing an actionable block + exit(1), so a // synthesized fallback keeps the user-visible contract intact. const snapshot = await collectSandboxStatusSnapshot(sandboxName, { - suppressInferenceProbe: preflight.suppressInferenceProbe, + preflight, }); const { sb, @@ -88,7 +88,10 @@ export async function showSandboxStatus(sandboxName: string): Promise { // recovery hint (#4495) and the Docker health line below (#3975). const dockerRuntime = lookup.state === "present" ? getSandboxDockerRuntime(sandboxName) : null; const phase = lookup.state === "present" ? parseSandboxPhase(lookup.output || "") : null; - const effectivePreflight = withoutTerminalPhasePreflight(preflight, phase); + const effectivePreflight = withoutTerminalPhasePreflight( + snapshot.postRecoveryPreflight ?? preflight, + phase, + ); const statusAgent = resolveSandboxStatusAgent(sb?.agent || "openclaw"); printSandboxStatusPreflightHeader(effectivePreflight); if (effectivePreflight.exitCode !== 0) { diff --git a/src/lib/actions/sandbox/status/process-recovery.ts b/src/lib/actions/sandbox/status/process-recovery.ts new file mode 100644 index 00000000000..8d7ee749236 --- /dev/null +++ b/src/lib/actions/sandbox/status/process-recovery.ts @@ -0,0 +1,11 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Keep status's process inspection and recovery consumers behind one edge to +// the lifecycle module. This preserves the source-architecture fan-in budget +// while both text rendering and snapshot collection share the same guarded +// recovery implementation. +export { + checkAndRecoverSandboxProcesses, + isSandboxGatewayRunningForStatus, +} from "../process-recovery"; diff --git a/src/lib/onboard/docker-driver-sandbox-recovery.test.ts b/src/lib/onboard/docker-driver-sandbox-recovery.test.ts index 6662d8acd1c..e7c98d4ace4 100644 --- a/src/lib/onboard/docker-driver-sandbox-recovery.test.ts +++ b/src/lib/onboard/docker-driver-sandbox-recovery.test.ts @@ -22,8 +22,24 @@ function fakeRename( return () => ({ status }); } -function fakeCapture(output: string): (args: readonly string[]) => string { - return () => output; +function fakeCapture( + output: string, + inspectOutputs: string[] = ["running\thealthy"], + onInspect: (opts?: Record) => void = () => undefined, +): (args: readonly string[], opts?: Record) => string { + let inspectIndex = 0; + return (args, opts) => { + switch (args[0]) { + case "inspect": { + onInspect(opts); + const index = Math.min(inspectIndex, inspectOutputs.length - 1); + inspectIndex += 1; + return inspectOutputs[index] ?? ""; + } + default: + return output; + } + }; } describe("findLabeledSandboxContainers", () => { @@ -57,11 +73,16 @@ describe("findLabeledSandboxContainers", () => { }); describe("recoverDockerDriverSandbox — running original (no-op)", () => { - it("reports recovered with via=started-running-original", () => { + it("waits for an already-running container to become healthy without starting it", () => { const start = vi.fn(fakeStart(0)); + const sleep = vi.fn(); const result = recoverDockerDriverSandbox("e2e-x", { - dockerCapture: fakeCapture("openshell-e2e-x\tUp 5 minutes\n"), + dockerCapture: fakeCapture("openshell-e2e-x\tUp 5 seconds\n", [ + "running\tstarting", + "running\thealthy", + ]), dockerStart: start, + sleep, }); expect(result).toEqual({ recovered: true, @@ -69,6 +90,7 @@ describe("recoverDockerDriverSandbox — running original (no-op)", () => { containerName: "openshell-e2e-x", }); expect(start).not.toHaveBeenCalled(); + expect(sleep).toHaveBeenCalledOnce(); }); }); @@ -106,6 +128,21 @@ describe("recoverDockerDriverSandbox — paused original (unpause)", () => { expect(result.detail).toContain("docker unpause"); expect(unpause).toHaveBeenCalledTimes(1); }); + + it("does not report recovery when an unpaused container reaches a terminal state", () => { + const result = recoverDockerDriverSandbox("e2e-x", { + dockerCapture: fakeCapture("openshell-e2e-x\tUp 3 hours (Paused)\n", ["dead\tunhealthy"]), + dockerUnpause: fakeStart(0), + sleep: vi.fn(), + }); + + expect(result).toMatchObject({ + recovered: false, + via: null, + containerName: "openshell-e2e-x", + detail: expect.stringContaining("runtime=dead, health=unhealthy"), + }); + }); }); describe("recoverDockerDriverSandbox — stopped original (start)", () => { @@ -136,6 +173,123 @@ describe("recoverDockerDriverSandbox — stopped original (start)", () => { expect(result.via).toBeNull(); expect(result.detail).toMatch(/docker start openshell-e2e-x failed.*125/); }); + + it("waits for Docker health to become ready before reporting recovery", () => { + const sleep = vi.fn(); + const result = recoverDockerDriverSandbox("e2e-x", { + dockerCapture: fakeCapture("openshell-e2e-x\tExited (137) 30 seconds ago\n", [ + "running\tstarting", + "running\tstarting", + "running\thealthy", + ]), + dockerStart: fakeStart(0), + sleep, + }); + + expect(result).toEqual({ + recovered: true, + via: "started-stopped-original", + containerName: "openshell-e2e-x", + }); + expect(sleep).toHaveBeenCalledTimes(2); + expect(sleep).toHaveBeenCalledWith(1_000); + }); + + it("accepts a running container whose image has no Docker health check", () => { + const sleep = vi.fn(); + const result = recoverDockerDriverSandbox("e2e-x", { + dockerCapture: fakeCapture("openshell-e2e-x\tExited (137) 30 seconds ago\n", [ + "running\tnone", + ]), + dockerStart: fakeStart(0), + sleep, + }); + + expect(result).toEqual({ + recovered: true, + via: "started-stopped-original", + containerName: "openshell-e2e-x", + }); + expect(sleep).not.toHaveBeenCalled(); + }); + + it.each([ + "dead", + "exited", + "removing", + ])("does not report recovery when the restarted container reaches terminal state %s", (runtimeState) => { + const sleep = vi.fn(); + const result = recoverDockerDriverSandbox("e2e-x", { + dockerCapture: fakeCapture("openshell-e2e-x\tExited (137) 30 seconds ago\n", [ + `${runtimeState}\tnone`, + ]), + dockerStart: fakeStart(0), + sleep, + }); + + expect(result).toEqual({ + recovered: false, + via: null, + containerName: "openshell-e2e-x", + detail: + "docker container openshell-e2e-x did not become ready after recovery " + + `(runtime=${runtimeState}, health=none)`, + }); + expect(sleep).not.toHaveBeenCalled(); + }); + + it("enforces the readiness deadline with an advancing clock", () => { + let currentMs = 0; + const inspectStartTimes: number[] = []; + const inspectTimeouts: number[] = []; + const capture = fakeCapture( + "openshell-e2e-x\tExited (137) 30 seconds ago\n", + ["running\tstarting"], + (opts) => { + inspectStartTimes.push(currentMs); + const timeout = Number(opts?.timeout); + inspectTimeouts.push(timeout); + currentMs += Math.min(4_900, timeout); + }, + ); + const sleep = vi.fn((ms: number) => { + currentMs += ms; + }); + const result = recoverDockerDriverSandbox("e2e-x", { + dockerCapture: capture, + dockerStart: fakeStart(0), + now: () => currentMs, + sleep, + }); + + expect(result.recovered).toBe(false); + expect(result.via).toBeNull(); + expect(result.detail).toContain("runtime=running, health=starting"); + expect(currentMs).toBe(90_000); + expect(inspectStartTimes.every((startedAt) => startedAt < 90_000)).toBe(true); + expect(inspectTimeouts.at(-1)).toBe(1_500); + }); + + it("fails closed when Docker inspect returns malformed readiness output", () => { + let currentMs = 0; + const result = recoverDockerDriverSandbox("e2e-x", { + dockerCapture: fakeCapture("openshell-e2e-x\tExited (137) 30 seconds ago\n", [ + "not-a-docker-state", + ]), + dockerStart: fakeStart(0), + now: () => currentMs, + sleep: (ms) => { + currentMs += ms; + }, + }); + + expect(result).toMatchObject({ + recovered: false, + via: null, + detail: expect.stringContaining("runtime=not-a-docker-state"), + }); + expect(currentMs).toBe(90_000); + }); }); describe("recoverDockerDriverSandbox — backup-only (rename + start)", () => { @@ -202,6 +356,25 @@ describe("recoverDockerDriverSandbox — backup-only (rename + start)", () => { expect(result.recovered).toBe(false); expect(result.detail).toMatch(/after backup rename failed.*1/); }); + + it("does not report a renamed backup recovered before Docker readiness", () => { + const result = recoverDockerDriverSandbox("e2e-x", { + dockerCapture: fakeCapture("openshell-e2e-x-nemoclaw-gpu-backup-1717280000000\tExited\n", [ + "running\tstarting", + "removing\tnone", + ]), + dockerRename: fakeRename(0), + dockerStart: fakeStart(0), + sleep: vi.fn(), + }); + + expect(result).toMatchObject({ + recovered: false, + via: null, + containerName: "openshell-e2e-x", + detail: expect.stringContaining("runtime=removing, health=none"), + }); + }); }); describe("recoverDockerDriverSandbox — collision and missing cases", () => { diff --git a/src/lib/onboard/docker-driver-sandbox-recovery.ts b/src/lib/onboard/docker-driver-sandbox-recovery.ts index 429660d6eb4..50d47d3ec07 100644 --- a/src/lib/onboard/docker-driver-sandbox-recovery.ts +++ b/src/lib/onboard/docker-driver-sandbox-recovery.ts @@ -4,6 +4,7 @@ // OpenShell-managed container labels — duplicated locally to keep // the module unit-testable. `docker-gpu-patch.ts` re-exports them as // the source of truth; a parity test pins drift. + export const OPENSHELL_MANAGED_BY_LABEL = "openshell.ai/managed-by"; export const OPENSHELL_MANAGED_BY_VALUE = "openshell"; export const OPENSHELL_SANDBOX_NAME_LABEL = "openshell.ai/sandbox-name"; @@ -68,7 +69,17 @@ function loadDockerUnpause(): DockerStartFn { const DOCKER_PROBE_TIMEOUT_MS = 5_000; const DOCKER_OPERATION_TIMEOUT_MS = 30_000; +const DOCKER_RECOVERY_READY_TIMEOUT_MS = 90_000; +const DOCKER_RECOVERY_READY_POLL_INTERVAL_MS = 1_000; +const DOCKER_RECOVERY_READY_MAX_ATTEMPTS = + Math.floor(DOCKER_RECOVERY_READY_TIMEOUT_MS / DOCKER_RECOVERY_READY_POLL_INTERVAL_MS) + 1; const MAX_DOCKER_CONTAINER_NAME_LENGTH = 253; +const DOCKER_RECOVERY_SLEEP_BUFFER = new Int32Array(new SharedArrayBuffer(4)); + +function sleepForDockerRecovery(ms: number): void { + if (ms <= 0 || !Number.isFinite(ms)) return; + Atomics.wait(DOCKER_RECOVERY_SLEEP_BUFFER, 0, 0, ms); +} /** * Names recovery dispatched on. Stable identifiers callers can log, @@ -104,7 +115,9 @@ export interface DockerDriverRecoveryDeps { newName: string, opts?: Record, ) => { status?: number | null }; - /** Injectable clock for deterministic backup-name normalization in tests. */ + /** Injectable sleep used by the post-start readiness wait. */ + sleep?: (ms: number) => void; + /** Injectable clock for deterministic readiness deadlines in tests. */ now?: () => number; } @@ -122,6 +135,7 @@ function depsWithDefaults(deps: DockerDriverRecoveryDeps) { dockerUnpause: deps.dockerUnpause ?? ((name, opts) => loadDockerUnpause()(name, opts)), dockerRename: deps.dockerRename ?? ((oldName, newName, opts) => loadDockerRename()(oldName, newName, opts)), + sleep: deps.sleep ?? sleepForDockerRecovery, now: deps.now ?? (() => Date.now()), }; } @@ -214,6 +228,81 @@ function buildBackupRestoreName(originalName: string): string { return originalName.slice(0, MAX_DOCKER_CONTAINER_NAME_LENGTH); } +interface ContainerReadinessResult { + ready: boolean; + detail: string; +} + +/** + * A successful `docker start` only proves that Docker accepted the lifecycle + * request. The image can remain in `health: starting` while its agent gateway + * is still unavailable, so post-reboot recovery must not report success until + * Docker's own readiness contract settles. + */ +function waitForRecoveredContainerReady( + containerName: string, + deps: ReturnType, +): ContainerReadinessResult { + let lastState = "unknown"; + const deadlineMs = deps.now() + DOCKER_RECOVERY_READY_TIMEOUT_MS; + for (let attempt = 0; attempt < DOCKER_RECOVERY_READY_MAX_ATTEMPTS; attempt += 1) { + const remainingMs = deadlineMs - deps.now(); + if (remainingMs <= 0) break; + const raw = deps.dockerCapture( + [ + "inspect", + "--type", + "container", + "--format", + "{{.State.Status}}\t{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}", + containerName, + ], + { ignoreError: true, timeout: Math.min(DOCKER_PROBE_TIMEOUT_MS, remainingMs) }, + ); + const [runtimeState = "", healthState = ""] = raw.trim().toLowerCase().split(/\s+/); + lastState = healthState + ? `runtime=${runtimeState || "unknown"}, health=${healthState}` + : `runtime=${runtimeState || "unknown"}`; + + if (runtimeState === "running" && (healthState === "healthy" || healthState === "none")) { + return { ready: true, detail: lastState }; + } + // These states cannot become ready without another lifecycle action. + if (["dead", "exited", "removing"].includes(runtimeState)) { + return { ready: false, detail: lastState }; + } + if (attempt + 1 >= DOCKER_RECOVERY_READY_MAX_ATTEMPTS) break; + + const postInspectRemainingMs = deadlineMs - deps.now(); + if (postInspectRemainingMs <= 0) break; + deps.sleep(Math.min(DOCKER_RECOVERY_READY_POLL_INTERVAL_MS, postInspectRemainingMs)); + } + return { ready: false, detail: lastState }; +} + +function recoveredAfterReadiness( + containerName: string, + via: DockerDriverRecoveryVia, + deps: ReturnType, +): DockerDriverRecoveryResult { + const readiness = waitForRecoveredContainerReady(containerName, deps); + if (!readiness.ready) { + return { + recovered: false, + via: null, + containerName, + detail: + `docker container ${containerName} did not become ready after recovery ` + + `(${readiness.detail})`, + }; + } + return { + recovered: true, + via, + containerName, + }; +} + /** * Attempt to recover the labeled sandbox container so OpenShell sees * the sandbox again. Caller must retry `openshell sandbox get ` @@ -258,17 +347,9 @@ export function recoverDockerDriverSandbox( detail: `docker unpause ${runningOriginal.name} failed (exit ${unpause.status ?? "unknown"}).`, }; } - return { - recovered: true, - via: "unpaused-original", - containerName: runningOriginal.name, - }; + return recoveredAfterReadiness(runningOriginal.name, "unpaused-original", d); } - return { - recovered: true, - via: "started-running-original", - containerName: runningOriginal.name, - }; + return recoveredAfterReadiness(runningOriginal.name, "started-running-original", d); } if (stoppedOriginal) { @@ -277,11 +358,7 @@ export function recoverDockerDriverSandbox( timeout: DOCKER_OPERATION_TIMEOUT_MS, }); if (result.status === 0) { - return { - recovered: true, - via: "started-stopped-original", - containerName: stoppedOriginal.name, - }; + return recoveredAfterReadiness(stoppedOriginal.name, "started-stopped-original", d); } return { recovered: false, @@ -308,11 +385,7 @@ export function recoverDockerDriverSandbox( timeout: DOCKER_OPERATION_TIMEOUT_MS, }); if (startResult.status === 0) { - return { - recovered: true, - via: "renamed-and-started-backup", - containerName: restoreName, - }; + return recoveredAfterReadiness(restoreName, "renamed-and-started-backup", d); } return { recovered: false, diff --git a/test/e2e/fixtures/compatible-anthropic-switch.ts b/test/e2e/fixtures/compatible-anthropic-switch.ts new file mode 100644 index 00000000000..6b51307486b --- /dev/null +++ b/test/e2e/fixtures/compatible-anthropic-switch.ts @@ -0,0 +1,71 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { HostCliClient } from "./clients/host.ts"; +import { resultText } from "./clients/index.ts"; + +export const COMPATIBLE_ANTHROPIC_PROVIDER = "compatible-anthropic-endpoint"; +export const COMPATIBLE_ANTHROPIC_CREDENTIAL_ENV = "COMPATIBLE_ANTHROPIC_API_KEY"; +const DEFAULT_COMPATIBLE_ANTHROPIC_CREDENTIAL = "test-compatible-anthropic-key"; + +export interface CompatibleAnthropicSwitchBinding { + endpointUrl: string; + credentialValue: string; +} + +export function compatibleAnthropicSwitchBinding( + endpointUrl: string, + runtimeEnv: NodeJS.ProcessEnv = process.env, +): CompatibleAnthropicSwitchBinding { + const normalizedEndpointUrl = endpointUrl.trim(); + if (!normalizedEndpointUrl) { + throw new Error( + "NEMOCLAW_SWITCH_ENDPOINT_URL is required for compatible Anthropic inference switches", + ); + } + const credentialValue = + runtimeEnv[COMPATIBLE_ANTHROPIC_CREDENTIAL_ENV] ?? DEFAULT_COMPATIBLE_ANTHROPIC_CREDENTIAL; + if (!credentialValue.trim()) { + throw new Error( + "COMPATIBLE_ANTHROPIC_API_KEY is required for compatible Anthropic inference switches", + ); + } + return { endpointUrl: normalizedEndpointUrl, credentialValue }; +} + +export function compatibleAnthropicSwitchEnv( + binding: CompatibleAnthropicSwitchBinding | null, +): NodeJS.ProcessEnv { + return binding ? { [COMPATIBLE_ANTHROPIC_CREDENTIAL_ENV]: binding.credentialValue } : {}; +} + +export async function requireCompatibleAnthropicProviderAbsent( + host: HostCliClient, + options: { + artifactName: string; + env: NodeJS.ProcessEnv; + gatewayName?: string; + }, +): Promise { + const gatewayName = options.gatewayName ?? "nemoclaw"; + const result = await host.command( + "openshell", + ["provider", "get", "-g", gatewayName, COMPATIBLE_ANTHROPIC_PROVIDER], + { + artifactName: options.artifactName, + env: options.env, + timeoutMs: 30_000, + }, + ); + const output = resultText(result); + if (result.exitCode === 0) { + throw new Error( + `Provider '${COMPATIBLE_ANTHROPIC_PROVIDER}' must be absent before this inference switch so NemoClaw can create and verify its rollback-safe binding.`, + ); + } + if (!/provider not found|requested entity was not found/iu.test(output)) { + throw new Error( + `Could not prove provider '${COMPATIBLE_ANTHROPIC_PROVIDER}' is absent: ${output}`, + ); + } +} diff --git a/test/e2e/fixtures/phases/lifecycle.ts b/test/e2e/fixtures/phases/lifecycle.ts index 7110af8df6c..b63c30d0507 100644 --- a/test/e2e/fixtures/phases/lifecycle.ts +++ b/test/e2e/fixtures/phases/lifecycle.ts @@ -30,9 +30,8 @@ export { // probe. const OPENSHELL_SANDBOX_NAME_LABEL = "openshell.ai/sandbox-name"; const DOCKER_PROBE_TIMEOUT_MS = 15_000; -// Status invocation can take several minutes on unfixed code while -// the gateway recovery path retries. Keep the budget generous; the -// bug is independent of latency. +// Recovery can take several minutes while gateway and host-forward +// readiness converge, so keep the status budget generous. const STATUS_TIMEOUT_MS = 5 * 60_000; const REBUILD_TIMEOUT_MS = 20 * 60_000; const SANDBOX_READY_ATTEMPTS = 30; @@ -355,10 +354,9 @@ export class LifecyclePhaseFixture { * make `openshell status` report the named gateway connected * without running `nemoclaw onboard --resume`. * - * We deliberately do NOT assert on the status exit code here - * because the bug is precisely that status "succeeds" at - * destroying state. The state-validation phase that follows is - * what catches the regression via the + * Status must exit zero to prove that the restored sandbox delivery + * path is ready. The state-validation phase that follows additionally + * verifies preservation via the * `local-registry-entry-present` and `docker-sandbox-container-present` * probes. * @@ -444,10 +442,9 @@ export class LifecyclePhaseFixture { // We invoke status through the host CLI client so artifacts are // captured and the command goes through the same // shellProbe/redaction layer the rest of the fixture code uses. - // Status is allowed to fail (exit non-zero) because on unfixed - // code it intentionally fails after destroying state — the - // post-action invariants are checked by state-validation. - const statusResult = await this.host.nemoclaw([instance.sandboxName, "status"], { + // Status must exit zero to prove the restored delivery path is ready; + // state-validation still verifies registry and container preservation. + const statusResult = await this.host.expectStatus(instance.sandboxName, { artifactName: `lifecycle-post-reboot-nemoclaw-status-${instance.sandboxName}`, env: buildAvailabilityProbeEnv(), timeoutMs: STATUS_TIMEOUT_MS, diff --git a/test/e2e/live/hermes-inference-switch-helpers.ts b/test/e2e/live/hermes-inference-switch-helpers.ts index a16e32db6db..95bd9219ecb 100644 --- a/test/e2e/live/hermes-inference-switch-helpers.ts +++ b/test/e2e/live/hermes-inference-switch-helpers.ts @@ -16,6 +16,12 @@ import { trustedSandboxShellScript, validateSandboxName, } from "../fixtures/clients/sandbox.ts"; +import { + type CompatibleAnthropicSwitchBinding, + compatibleAnthropicSwitchBinding, + compatibleAnthropicSwitchEnv, + requireCompatibleAnthropicProviderAbsent, +} from "../fixtures/compatible-anthropic-switch.ts"; import { expect } from "../fixtures/e2e-test.ts"; import type { FakeOpenAiCompatibleServer } from "../fixtures/fake-openai-compatible.ts"; import { @@ -511,42 +517,22 @@ async function startMockAnthropicProvider(): Promise Promise | void): void }, -): Promise { +): Promise { if (SWITCH_PROVIDER !== "compatible-anthropic-endpoint" || SWITCH_API !== "anthropic-messages") return null; const mock = mockAnthropicSwitchEnabled() ? await startMockAnthropicProvider() : undefined; mock && cleanup.add("close compatible Anthropic switch mock", () => mock.close()); - const endpointUrl = process.env.NEMOCLAW_SWITCH_ENDPOINT_URL ?? mock?.endpointUrl ?? ""; - const compatibleKey = process.env.COMPATIBLE_ANTHROPIC_API_KEY ?? "test-compatible-anthropic-key"; - expect( - endpointUrl, - "NEMOCLAW_SWITCH_ENDPOINT_URL is required for compatible Anthropic inference switches", - ).not.toBe(""); - expect( - compatibleKey, - "COMPATIBLE_ANTHROPIC_API_KEY is required for compatible Anthropic inference switches", - ).not.toBe(""); - const providerScript = [ - "set -euo pipefail", - "if openshell provider get -g nemoclaw compatible-anthropic-endpoint >/dev/null 2>&1; then", - " openshell provider delete -g nemoclaw compatible-anthropic-endpoint", - "fi", - 'openshell provider create -g nemoclaw --name compatible-anthropic-endpoint --type openai --credential COMPATIBLE_ANTHROPIC_API_KEY --config "OPENAI_BASE_URL=${SWITCH_OPENAI_ENDPOINT_URL}"', - ].join("\n"); - const result = await host.command("bash", ["-lc", providerScript], { - artifactName: "register-compatible-anthropic-switch-provider", - env: env(undefined, { - COMPATIBLE_ANTHROPIC_API_KEY: compatibleKey, - SWITCH_OPENAI_ENDPOINT_URL: openAiSurfaceEndpointUrl(endpointUrl), - }), - redactionValues: [compatibleKey], - timeoutMs: 120_000, + const binding = compatibleAnthropicSwitchBinding( + process.env.NEMOCLAW_SWITCH_ENDPOINT_URL ?? mock?.endpointUrl ?? "", + ); + await requireCompatibleAnthropicProviderAbsent(host, { + artifactName: "compatible-anthropic-provider-absent-before-switch", + env: env(), }); - expect(result.exitCode).toBe(0); - return endpointUrl; + return { ...binding, endpointUrl: openAiSurfaceEndpointUrl(binding.endpointUrl) }; } export async function installHermes( @@ -583,7 +569,11 @@ export async function runHermesInferenceSetWithRetry( host: HostCliClient, redactionValues: string[], compatibleMetadataArgs: string[], - options: { attempts?: number; delay?: (milliseconds: number) => Promise } = {}, + options: { + attempts?: number; + compatibleBinding?: CompatibleAnthropicSwitchBinding | null; + delay?: (milliseconds: number) => Promise; + } = {}, ): Promise { const args = [ CLI, @@ -604,7 +594,7 @@ export async function runHermesInferenceSetWithRetry( artifactName: verify ? `hermes-inference-set-${attempt}` : "hermes-inference-set-no-verify-after-transient-failures", - env: env(), + env: env(undefined, compatibleAnthropicSwitchEnv(options.compatibleBinding ?? null)), redactionValues, timeoutMs: 180_000, }), diff --git a/test/e2e/live/hermes-inference-switch.test.ts b/test/e2e/live/hermes-inference-switch.test.ts index ed414d82839..37152b12720 100644 --- a/test/e2e/live/hermes-inference-switch.test.ts +++ b/test/e2e/live/hermes-inference-switch.test.ts @@ -13,7 +13,6 @@ import { chatContent, cleanupHermesSwitch, compatibleAnthropicMetadataArgs, - ensureCompatibleAnthropicSwitchProvider, env, envHash, expectAuthenticatedBaselineInventoryRequest, @@ -36,6 +35,7 @@ import { PROXY_RESOLUTION_PROVIDER, parseHermesModelBlock, parseInferenceRoute, + prepareCompatibleAnthropicSwitchBinding, prepareProxyResolutionRoute, RUNTIME_SWITCH_API, registryState, @@ -180,13 +180,9 @@ test("Hermes inference set updates route/config and preserves live runtime", { ? await registerPublicNvidiaSwitchProvider(host, publicApiKey, env()) : null; publicProvider && expect(publicProvider.exitCode, resultText(publicProvider)).toBe(0); - const switchEndpointUrl = await ensureCompatibleAnthropicSwitchProvider(host, cleanup); - switchEndpointUrl && - (await expectOpenAiProvider( - host, - "compatible-anthropic-endpoint", - "COMPATIBLE_ANTHROPIC_API_KEY", - )); + const switchBinding = await prepareCompatibleAnthropicSwitchBinding(host, cleanup); + const switchEndpointUrl = switchBinding?.endpointUrl ?? null; + switchBinding && redactionValues.push(switchBinding.credentialValue); const pidBefore = await hermesGatewayPid(sandbox, "pid-before"); const envHashBefore = await envHash(sandbox, "env-hash-before"); @@ -197,10 +193,17 @@ test("Hermes inference set updates route/config and preserves live runtime", { host, redactionValues, compatibleMetadataArgs, + { compatibleBinding: switchBinding }, ); expect(switched.exitCode, resultText(switched)).toBe(0); expect(resultText(switched)).not.toContain("writing the in-sandbox config failed"); expect(resultText(switched)).toContain(`Inference route synced for '${SANDBOX_NAME}'`); + switchBinding && + (await expectOpenAiProvider( + host, + "compatible-anthropic-endpoint", + "COMPATIBLE_ANTHROPIC_API_KEY", + )); progress.phase("validate switched route and locked config"); const pidAfter = await hermesGatewayPid(sandbox, "pid-after"); diff --git a/test/e2e/live/hermes-root-entrypoint-smoke.test.ts b/test/e2e/live/hermes-root-entrypoint-smoke.test.ts index 72842de61cd..1f5fd480637 100644 --- a/test/e2e/live/hermes-root-entrypoint-smoke.test.ts +++ b/test/e2e/live/hermes-root-entrypoint-smoke.test.ts @@ -221,8 +221,8 @@ async function assertRuntimeLayout(probe: DockerProbe, container: string): Promi await expectContainerSh( probe, container, - "gateway.pid is not a regular top-level file", - "test -f /sandbox/.hermes/gateway.pid && test ! -L /sandbox/.hermes/gateway.pid", + "gateway.pid is not a regular runtime file", + "test -f /sandbox/.hermes/runtime/gateway.pid && test ! -L /sandbox/.hermes/runtime/gateway.pid && test ! -e /sandbox/.hermes/gateway.pid && test ! -L /sandbox/.hermes/gateway.pid", ); await expectContainerShFails( probe, @@ -448,7 +448,7 @@ test("hermes root-entrypoint smoke preserves runtime layout and legacy pid migra "gateway log has no PID race or config load failure", "Hermes v0.14 writable runtime directories are present", "build-only root caches are absent from the runtime image", - "gateway.pid is migrated to a regular top-level file", + "gateway.pid is stored as a regular file below the writable runtime directory", "gateway user cannot remove config.yaml from sticky config root", "Hermes API denies missing/wrong bearer tokens and accepts API_SERVER_KEY", "dashboard-home is sandbox-owned 0700 with 0600 allowlisted config/env", diff --git a/test/e2e/live/openclaw-inference-switch.test.ts b/test/e2e/live/openclaw-inference-switch.test.ts index 58cadedebc7..66a604468fb 100644 --- a/test/e2e/live/openclaw-inference-switch.test.ts +++ b/test/e2e/live/openclaw-inference-switch.test.ts @@ -24,6 +24,12 @@ import { trustedSandboxShellScript, validateSandboxName, } from "../fixtures/clients/sandbox.ts"; +import { + type CompatibleAnthropicSwitchBinding, + compatibleAnthropicSwitchBinding, + compatibleAnthropicSwitchEnv, + requireCompatibleAnthropicProviderAbsent, +} from "../fixtures/compatible-anthropic-switch.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; import { type FakeOpenAiCompatibleServer, @@ -194,13 +200,18 @@ async function runNemoclaw( host: HostCliClient, home: string, args: string[], - options: { artifactName: string; timeoutMs?: number; redactionValues?: string[] } = { + options: { + artifactName: string; + env?: NodeJS.ProcessEnv; + timeoutMs?: number; + redactionValues?: string[]; + } = { artifactName: "nemoclaw", }, ): Promise { return host.command("node", [CLI_ENTRYPOINT, ...args], { artifactName: options.artifactName, - env: commandEnv(home), + env: commandEnv(home, options.env), timeoutMs: options.timeoutMs ?? COMMAND_TIMEOUT_MS, redactionValues: options.redactionValues, }); @@ -376,44 +387,21 @@ async function startMockAnthropicProvider(): Promise { }; } -async function ensureCompatibleAnthropicSwitchProvider( +async function prepareCompatibleAnthropicSwitchBinding( host: HostCliClient, home: string, mockProvider: MockAnthropicProvider | undefined, -): Promise { +): Promise { if (SWITCH_PROVIDER !== "compatible-anthropic-endpoint") return null; if (SWITCH_INFERENCE_API !== "anthropic-messages") return null; const endpointUrl = process.env.NEMOCLAW_SWITCH_ENDPOINT_URL ?? mockProvider?.endpointUrl ?? ""; - const apiKey = process.env.COMPATIBLE_ANTHROPIC_API_KEY ?? "test-compatible-anthropic-key"; - expect( - endpointUrl, - "NEMOCLAW_SWITCH_ENDPOINT_URL is required for compatible Anthropic inference switches", - ).not.toBe(""); - expect( - apiKey, - "COMPATIBLE_ANTHROPIC_API_KEY is required for compatible Anthropic inference switches", - ).not.toBe(""); - - const providerScript = [ - "set -euo pipefail", - "if openshell provider get -g nemoclaw compatible-anthropic-endpoint >/dev/null 2>&1; then", - ' openshell provider update -g nemoclaw compatible-anthropic-endpoint --credential COMPATIBLE_ANTHROPIC_API_KEY --config "ANTHROPIC_BASE_URL=${SWITCH_ENDPOINT_URL}"', - "else", - ' openshell provider create -g nemoclaw --name compatible-anthropic-endpoint --type anthropic --credential COMPATIBLE_ANTHROPIC_API_KEY --config "ANTHROPIC_BASE_URL=${SWITCH_ENDPOINT_URL}"', - "fi", - ].join("\n"); - const provider = await host.command("bash", ["-lc", providerScript], { - artifactName: "register-compatible-anthropic-switch-provider", - env: commandEnv(home, { - COMPATIBLE_ANTHROPIC_API_KEY: apiKey, - SWITCH_ENDPOINT_URL: endpointUrl, - }), - redactionValues: [apiKey], - timeoutMs: COMMAND_TIMEOUT_MS, + const binding = compatibleAnthropicSwitchBinding(endpointUrl); + await requireCompatibleAnthropicProviderAbsent(host, { + artifactName: "compatible-anthropic-provider-absent-before-switch", + env: commandEnv(home), }); - expect(provider.exitCode, resultText(provider)).toBe(0); - return endpointUrl; + return binding; } async function openclawGatewayPid(sandbox: SandboxClient, home: string): Promise { @@ -833,7 +821,7 @@ async function runOpenClawInferenceSetWithRetry( host: HostCliClient, home: string, redactionValues: string[], - switchEndpointUrl: string | null, + switchBinding: CompatibleAnthropicSwitchBinding | null, ): Promise { const attempts = inferenceSetAttemptCount(process.env.NEMOCLAW_SWITCH_SET_ATTEMPTS); const compatibleCredentialEnv = (() => { @@ -846,10 +834,10 @@ async function runOpenClawInferenceSetWithRetry( return null; } })(); - const compatibleMetadataArgs = switchEndpointUrl + const compatibleMetadataArgs = switchBinding ? [ "--endpoint-url", - switchEndpointUrl, + switchBinding.endpointUrl, "--credential-env", compatibleCredentialEnv ?? "", "--inference-api", @@ -875,6 +863,7 @@ async function runOpenClawInferenceSetWithRetry( artifactName: verify ? `nemoclaw-inference-set-${attempt}` : "nemoclaw-inference-set-no-verify-after-transient-failures", + env: compatibleAnthropicSwitchEnv(switchBinding), redactionValues, timeoutMs: COMMAND_TIMEOUT_MS, }), @@ -1036,10 +1025,11 @@ test("openclaw-inference-switch: switches route and preserves live OpenClaw beha // Only the explicit Anthropic bridge supplies endpoint metadata. The // compatible baseline reuses its registered OpenShell provider, while the // public NVIDIA provider has no caller-supplied endpoint identity. - const switchEndpointUrl = + const switchBinding = SWITCH_PROVIDER === "compatible-anthropic-endpoint" - ? await ensureCompatibleAnthropicSwitchProvider(host, home, mockProvider) + ? await prepareCompatibleAnthropicSwitchBinding(host, home, mockProvider) : null; + switchBinding && redactionValues.push(switchBinding.credentialValue); progress.phase("switch the route and verify restart semantics"); expect(baseline.env.NEMOCLAW_PREFERRED_API).toBe("openai-completions"); @@ -1052,7 +1042,7 @@ test("openclaw-inference-switch: switches route and preserves live OpenClaw beha host, home, redactionValues, - switchEndpointUrl, + switchBinding, ); expect(switchResult.exitCode, resultText(switchResult)).toBe(0); expect( diff --git a/test/e2e/support/compatible-anthropic-switch.test.ts b/test/e2e/support/compatible-anthropic-switch.test.ts new file mode 100644 index 00000000000..c1378b7d41a --- /dev/null +++ b/test/e2e/support/compatible-anthropic-switch.test.ts @@ -0,0 +1,95 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import type { HostCliClient } from "../fixtures/clients/host.ts"; +import { + COMPATIBLE_ANTHROPIC_CREDENTIAL_ENV, + COMPATIBLE_ANTHROPIC_PROVIDER, + compatibleAnthropicSwitchBinding, + compatibleAnthropicSwitchEnv, + requireCompatibleAnthropicProviderAbsent, +} from "../fixtures/compatible-anthropic-switch.ts"; + +describe("compatible Anthropic inference switch setup", () => { + it("passes the direct binding credential only to the inference-set command", () => { + const binding = compatibleAnthropicSwitchBinding("http://host.openshell.internal:18766", { + COMPATIBLE_ANTHROPIC_API_KEY: "fixture-key", + }); + + expect(binding).toEqual({ + endpointUrl: "http://host.openshell.internal:18766", + credentialValue: "fixture-key", + }); + expect(compatibleAnthropicSwitchEnv(binding)).toEqual({ + [COMPATIBLE_ANTHROPIC_CREDENTIAL_ENV]: "fixture-key", + }); + expect(compatibleAnthropicSwitchEnv(null)).toEqual({}); + }); + + it("rejects a blank compatible Anthropic endpoint URL", () => { + expect(() => + compatibleAnthropicSwitchBinding(" ", { + COMPATIBLE_ANTHROPIC_API_KEY: "fixture-key", + }), + ).toThrow("NEMOCLAW_SWITCH_ENDPOINT_URL is required"); + }); + + it("rejects a blank compatible Anthropic credential", () => { + expect(() => + compatibleAnthropicSwitchBinding("http://host.openshell.internal:18766", { + COMPATIBLE_ANTHROPIC_API_KEY: " ", + }), + ).toThrow("COMPATIBLE_ANTHROPIC_API_KEY is required"); + }); + + it("requires the direct provider to be absent before inference set owns its creation", async () => { + const command = vi.fn().mockResolvedValue({ + exitCode: 1, + stderr: "Error: code: 'Some requested entity was not found', message: \"provider not found\"", + stdout: "", + }); + const host = { command } as unknown as HostCliClient; + const commandEnv = { OPENSHELL_GATEWAY: "nemoclaw" }; + + await expect( + requireCompatibleAnthropicProviderAbsent(host, { + artifactName: "compatible-anthropic-provider-absent", + env: commandEnv, + }), + ).resolves.toBeUndefined(); + expect(command).toHaveBeenCalledWith( + "openshell", + ["provider", "get", "-g", "nemoclaw", COMPATIBLE_ANTHROPIC_PROVIDER], + expect.objectContaining({ + artifactName: "compatible-anthropic-provider-absent", + env: commandEnv, + }), + ); + }); + + it("rejects a pre-existing or uninspectable direct provider", async () => { + const command = vi + .fn() + .mockResolvedValueOnce({ + exitCode: 0, + stderr: "", + stdout: `Name: ${COMPATIBLE_ANTHROPIC_PROVIDER}`, + }) + .mockResolvedValueOnce({ + exitCode: 1, + stderr: "gateway unavailable", + stdout: "", + }); + const host = { command } as unknown as HostCliClient; + const options = { artifactName: "provider-absent", env: {} }; + + await expect(requireCompatibleAnthropicProviderAbsent(host, options)).rejects.toThrow( + "must be absent", + ); + await expect(requireCompatibleAnthropicProviderAbsent(host, options)).rejects.toThrow( + "Could not prove", + ); + }); +}); diff --git a/test/e2e/support/e2e-phase-lifecycle.test.ts b/test/e2e/support/e2e-phase-lifecycle.test.ts index 257708a567d..bbe39eb0efa 100644 --- a/test/e2e/support/e2e-phase-lifecycle.test.ts +++ b/test/e2e/support/e2e-phase-lifecycle.test.ts @@ -168,7 +168,7 @@ describe("LifecyclePhaseFixture.simulate post-reboot-recovery (stop-original)", runner.enqueue(shellResult(0)); // container stop runner.enqueue(shellResult(0)); // user service restart runner.enqueue(shellResult(0, "Connected to nemoclaw\n")); // openshell status - runner.enqueue(shellResult(1, "Removed stale local registry entry.\n")); // status (non-zero on unfixed) + runner.enqueue(shellResult(0)); // status proves recovered delivery readiness const result = await prepared.simulate("post-reboot-recovery", instance()); @@ -198,7 +198,7 @@ describe("LifecyclePhaseFixture.simulate post-reboot-recovery (stop-original)", ]); }); - it("tolerates a non-zero status exit (the bug succeeds at destroying state)", async () => { + it("fails when status cannot prove post-reboot recovery", async () => { const runner = new FakeRunner(); const cleanup = new FakeCleanup(); const prepared = await preparedPostRebootFixture(runner, cleanup); @@ -212,11 +212,9 @@ describe("LifecyclePhaseFixture.simulate post-reboot-recovery (stop-original)", runner.enqueue(shellResult(0, "Connected to nemoclaw\n")); // openshell status runner.enqueue(shellResult(1, "Removed stale local registry entry.\n")); // status non-zero - const result = await prepared.simulate("post-reboot-recovery", instance()); - - // simulate() does not throw; the post-status invariants belong - // to the state-validation phase that runs after. - expect(result.steps.find((step) => step.id.startsWith("nemoclaw-status:"))).toBeTruthy(); + await expect(prepared.simulate("post-reboot-recovery", instance())).rejects.toThrow( + /nemoclaw e2e-ubuntu-repo-cloud-openclaw status failed: Removed stale local registry entry/, + ); }); it("fails when no Docker container carries the OpenShell sandbox-name label", async () => { @@ -277,7 +275,7 @@ describe("LifecyclePhaseFixture.simulate post-reboot-recovery (rename-to-gpu-bac runner.enqueue(shellResult(0)); // container stop runner.enqueue(shellResult(0)); // user service restart runner.enqueue(shellResult(0, "Connected to nemoclaw\n")); // openshell status - runner.enqueue(shellResult(1, "Removed stale local registry entry.\n")); // status + runner.enqueue(shellResult(0)); // status proves recovered delivery readiness const result = await prepared.simulate( "post-reboot-recovery", diff --git a/test/e2e/support/hermes-inference-switch-command-shape.test.ts b/test/e2e/support/hermes-inference-switch-command-shape.test.ts index bf51effd6f3..536bad419d7 100644 --- a/test/e2e/support/hermes-inference-switch-command-shape.test.ts +++ b/test/e2e/support/hermes-inference-switch-command-shape.test.ts @@ -8,6 +8,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { resolveAgentInferenceApi } from "../../../src/lib/inference/config.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import type { SandboxClient } from "../fixtures/clients/sandbox.ts"; +import { compatibleAnthropicSwitchBinding } from "../fixtures/compatible-anthropic-switch.ts"; import { DEFAULT_HOSTED_INFERENCE_MODEL } from "../fixtures/hosted-inference.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; import { @@ -329,18 +330,26 @@ describe("Hermes inference switch command shape", () => { stdout: "", }) .mockResolvedValueOnce({ exitCode: 0, stderr: "", stdout: "" }); + const compatibleBinding = compatibleAnthropicSwitchBinding( + "http://host.openshell.internal:18766/v1", + { COMPATIBLE_ANTHROPIC_API_KEY: "switch-key" }, + ); await expect( runHermesInferenceSetWithRetry( { command } as unknown as HostCliClient, - ["hosted-key"], - ["--inference-api", "anthropic-messages"], - { attempts: 1, delay: async () => {} }, + ["hosted-key", compatibleBinding.credentialValue], + compatibleAnthropicMetadataArgs(compatibleBinding.endpointUrl), + { attempts: 1, compatibleBinding, delay: async () => {} }, ), ).resolves.toMatchObject({ exitCode: 0 }); expect(command.mock.calls[0]?.[1]).not.toContain("--no-verify"); expect(command.mock.calls[1]?.[1]).toContain("--no-verify"); + expect(command.mock.calls[0]?.[2]).toMatchObject({ + env: { COMPATIBLE_ANTHROPIC_API_KEY: "switch-key" }, + redactionValues: ["hosted-key", "switch-key"], + }); expect(command.mock.calls[0]?.[2]?.env).not.toHaveProperty("NVIDIA_INFERENCE_API_KEY"); }); }); diff --git a/test/hermes-final-image-layout.test.ts b/test/hermes-final-image-layout.test.ts index 5828bf9db4b..887d8ac8e81 100644 --- a/test/hermes-final-image-layout.test.ts +++ b/test/hermes-final-image-layout.test.ts @@ -181,6 +181,7 @@ describe("Hermes final image layout", () => { "COPY agents/hermes/plugin/ /opt/nemoclaw-hermes-plugin/", "COPY agents/hermes/generate-config.ts /opt/nemoclaw-hermes-config/generate-config.ts", "COPY agents/hermes/config/ /opt/nemoclaw-hermes-config/config/", + "COPY agents/hermes/patch-gateway-runtime-metadata.py /opt/nemoclaw-hermes-config/patch-gateway-runtime-metadata.py", "COPY agents/hermes/host/managed-tool-gateway-matrix.json /opt/nemoclaw-hermes-config/managed-tool-gateway-matrix.json", "COPY src/lib/tool-disclosure.ts /src/lib/tool-disclosure.ts", "COPY src/lib/messaging/ /src/lib/messaging/", diff --git a/test/hermes-gateway-runtime-metadata-patch.test.ts b/test/hermes-gateway-runtime-metadata-patch.test.ts new file mode 100644 index 00000000000..00ed4a3f712 --- /dev/null +++ b/test/hermes-gateway-runtime-metadata-patch.test.ts @@ -0,0 +1,108 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; + +const ROOT = path.resolve(import.meta.dirname, ".."); +const PATCHER = path.join(ROOT, "agents", "hermes", "patch-gateway-runtime-metadata.py"); +const MCP_TRANSACTION = path.join(ROOT, "agents", "hermes", "mcp-config-transaction.py"); + +const UPSTREAM_FIXTURE = `from pathlib import Path +from typing import Optional + +HOME = Path(__file__).parent / "hermes-home" +_GATEWAY_LOCK_FILENAME = "gateway.lock" +_RUNTIME_STATUS_FILE = "gateway_state.json" + +def get_hermes_home() -> Path: + return HOME + +def _get_pid_path() -> Path: + """Return the path to the gateway PID file, respecting HERMES_HOME.""" + home = get_hermes_home() + return home / "gateway.pid" + +def _get_gateway_lock_path(pid_path: Optional[Path] = None) -> Path: + """Return the path to the runtime gateway lock file.""" + if pid_path is not None: + return pid_path.with_name(_GATEWAY_LOCK_FILENAME) + home = get_hermes_home() + return home / _GATEWAY_LOCK_FILENAME + +def _get_runtime_status_path() -> Path: + """Return the persisted runtime health/status file path.""" + return _get_pid_path().with_name(_RUNTIME_STATUS_FILE) + +if __name__ == "__main__": + print(_get_pid_path()) + print(_get_gateway_lock_path()) + print(_get_runtime_status_path()) +`; + +function runPatcher(fixture: string) { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-runtime-metadata-")); + const statusPath = path.join(tmp, "status.py"); + fs.writeFileSync(statusPath, fixture); + const result = spawnSync("python3", ["-I", PATCHER, statusPath], { + encoding: "utf-8", + timeout: 5000, + }); + return { result, statusPath, tmp }; +} + +describe("Hermes writable gateway runtime metadata", () => { + it("relocates every central gateway metadata reader and remains idempotent", () => { + const { result, statusPath, tmp } = runPatcher(UPSTREAM_FIXTURE); + try { + expect(result.status, result.stderr).toBe(0); + const second = spawnSync("python3", ["-I", PATCHER, statusPath], { + encoding: "utf-8", + timeout: 5000, + }); + expect(second.status, second.stderr).toBe(0); + + const probe = spawnSync("python3", ["-I", statusPath], { + encoding: "utf-8", + timeout: 5000, + }); + expect(probe.status, probe.stderr).toBe(0); + expect( + probe.stdout + .trim() + .split("\n") + .map((entry) => path.relative(tmp, entry)), + ).toEqual([ + "hermes-home/runtime/gateway.pid", + "hermes-home/runtime/gateway.lock", + "hermes-home/runtime/gateway_state.json", + ]); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + + it("fails closed when the pinned Hermes helper shape changes", () => { + const drifted = UPSTREAM_FIXTURE.replace( + 'return home / "gateway.pid"', + 'return home / "changed-gateway.pid"', + ); + const { result, statusPath, tmp } = runPatcher(drifted); + try { + expect(result.status).toBe(1); + expect(result.stderr).toContain("gateway runtime metadata source shape changed"); + expect(fs.readFileSync(statusPath, "utf-8")).toBe(drifted); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + + it("keeps the managed MCP identity reader on the relocated PID record", () => { + const source = fs.readFileSync(MCP_TRANSACTION, "utf-8"); + expect(source).toContain('GATEWAY_PID_PATH = f"{HERMES_DIR}/runtime/gateway.pid"'); + expect(source).not.toContain('GATEWAY_PID_PATH = f"{HERMES_DIR}/gateway.pid"'); + }); +}); diff --git a/test/process-recovery-forward-failure.test.ts b/test/process-recovery-forward-failure.test.ts index 779f7dc5357..1ea8f506d6e 100644 --- a/test/process-recovery-forward-failure.test.ts +++ b/test/process-recovery-forward-failure.test.ts @@ -89,6 +89,43 @@ function compactTeamsMessagingPlan(port = "3978") { } describe("checkAndRecoverSandboxProcesses primary forward failure", () => { + it("fails closed when OpenShell forward state is unavailable", () => { + const openshellRuntime = requireSource("../src/lib/adapters/openshell/runtime.ts"); + const agentRuntime = requireSource("../src/lib/agent/runtime.ts"); + const registry = requireSource("../src/lib/state/registry.ts"); + const childProcess = requireSource("node:child_process"); + + vi.spyOn(childProcess, "spawnSync").mockReturnValue({ + status: 0, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nRUNNING\n", + stderr: "", + } as never); + vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue(null); + vi.spyOn(registry, "getSandbox").mockReturnValue({ + name: "beta", + agent: "openclaw", + dashboardPort: 18789, + }); + vi.spyOn(openshellRuntime, "captureOpenshell").mockReturnValue({ + status: 1, + output: "OpenShell forward state unavailable", + }); + const runOpenshell = vi.spyOn(openshellRuntime, "runOpenshell"); + + expect( + withFakeOpenshellBinary(() => checkAndRecoverSandboxProcesses("beta", { quiet: true })), + ).toEqual({ + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: false, + forwardRecoveryFailed: true, + forwardRecoveryFailureDetail: + "the primary dashboard/API host forward could not be verified because OpenShell forward state was unavailable", + }); + expect(runOpenshell).not.toHaveBeenCalled(); + }); + it("reports failure when a messaging forward cannot recover even if the primary is healthy", () => { const openshellRuntime = requireSource("../src/lib/adapters/openshell/runtime.ts"); const agentRuntime = requireSource("../src/lib/agent/runtime.ts"); diff --git a/test/support/status-flow-test-harness.ts b/test/support/status-flow-test-harness.ts index 5526ce62bb9..556f400718c 100644 --- a/test/support/status-flow-test-harness.ts +++ b/test/support/status-flow-test-harness.ts @@ -66,6 +66,7 @@ export type StatusFlowHarnessOptions = { lookup?: SandboxGatewayState; lookupState?: "present" | "missing"; preflight?: SandboxStatusPreflightResult; + postRecoveryPreflight?: SandboxStatusPreflightResult; sandboxEntry?: Partial> & { agent?: string | null; agentVersion?: string | null; @@ -102,7 +103,9 @@ export function createStatusFlowHarness(options: StatusFlowHarnessOptions = {}): const statusPreflight = requireDist("../../src/lib/actions/sandbox/status-preflight.js"); const statusSnapshot = requireDist("../../src/lib/actions/sandbox/status-snapshot.js"); const dockerHealth = requireDist("../../src/lib/actions/sandbox/docker-health.js"); - const processRecovery = requireDist("../../src/lib/actions/sandbox/process-recovery.js"); + const statusProcessRecovery = requireDist( + "../../src/lib/actions/sandbox/status/process-recovery.js", + ); const resolve = requireDist("../../src/lib/adapters/openshell/resolve.js"); const agentRuntime = requireDist("../../src/lib/agent/runtime.js"); const nim = requireDist("../../src/lib/inference/nim.js"); @@ -187,6 +190,9 @@ export function createStatusFlowHarness(options: StatusFlowHarnessOptions = {}): ? null : { checked: false } : options.servingProcessHealth, + ...(options.postRecoveryPreflight + ? { postRecoveryPreflight: options.postRecoveryPreflight } + : {}), }); const getSandboxDockerRuntimeSpy = vi .spyOn(dockerHealth, "getSandboxDockerRuntime") @@ -195,7 +201,7 @@ export function createStatusFlowHarness(options: StatusFlowHarnessOptions = {}): health: "unhealthy", paused: false, }); - vi.spyOn(processRecovery, "isSandboxGatewayRunningForStatus").mockResolvedValue(false); + vi.spyOn(statusProcessRecovery, "isSandboxGatewayRunningForStatus").mockResolvedValue(false); vi.spyOn(resolve, "resolveOpenshell").mockReturnValue("/usr/bin/openshell"); vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue({ name: "openclaw" }); vi.spyOn(agentRuntime, "getAgentDisplayName").mockReturnValue("OpenClaw");