From 16c7b9e372b2bc4fa9df8c19384a54e786d9ed41 Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Tue, 28 Jul 2026 19:42:13 -0700 Subject: [PATCH 01/11] fix(release): resolve v0.0.97 E2E blockers Signed-off-by: Charan Jagwani --- agents/hermes/Dockerfile | 32 +++ agents/hermes/mcp-config-transaction.py | 2 +- .../hermes/patch-gateway-runtime-metadata.py | 92 +++++++ docs/changelog/2026-07-28.mdx | 11 +- docs/reference/commands.mdx | 6 +- src/lib/actions/sandbox/process-recovery.ts | 11 + .../sandbox/restore-gateway-pairing.test.ts | 105 +++++++- .../sandbox/restore-gateway-pairing.ts | 33 ++- src/lib/actions/sandbox/status-flow.test.ts | 79 ++++++ .../sandbox/status-lookup-rendering.ts | 18 ++ .../status-snapshot-inference-health.test.ts | 55 ++++- .../sandbox/status-snapshot-recovery.test.ts | 228 ++++++++++++++++++ src/lib/actions/sandbox/status-snapshot.ts | 176 +++++++++++++- src/lib/actions/sandbox/status-text.ts | 2 +- src/lib/actions/sandbox/status.ts | 7 +- .../sandbox/status/process-recovery.ts | 11 + .../docker-driver-sandbox-recovery.test.ts | 175 +++++++++++++- .../onboard/docker-driver-sandbox-recovery.ts | 115 +++++++-- .../fixtures/compatible-anthropic-switch.ts | 71 ++++++ test/e2e/fixtures/phases/lifecycle.ts | 19 +- .../live/hermes-inference-switch-helpers.ts | 52 ++-- test/e2e/live/hermes-inference-switch.test.ts | 19 +- .../live/hermes-root-entrypoint-smoke.test.ts | 6 +- .../live/openclaw-inference-switch.test.ts | 66 +++-- .../compatible-anthropic-switch.test.ts | 79 ++++++ test/e2e/support/e2e-phase-lifecycle.test.ts | 14 +- ...mes-inference-switch-command-shape.test.ts | 15 +- test/hermes-final-image-layout.test.ts | 1 + ...mes-gateway-runtime-metadata-patch.test.ts | 108 +++++++++ test/process-recovery-forward-failure.test.ts | 37 +++ test/support/status-flow-test-harness.ts | 10 +- 31 files changed, 1498 insertions(+), 157 deletions(-) create mode 100755 agents/hermes/patch-gateway-runtime-metadata.py create mode 100644 src/lib/actions/sandbox/status-snapshot-recovery.test.ts create mode 100644 src/lib/actions/sandbox/status/process-recovery.ts create mode 100644 test/e2e/fixtures/compatible-anthropic-switch.ts create mode 100644 test/e2e/support/compatible-anthropic-switch.test.ts create mode 100644 test/hermes-gateway-runtime-metadata-patch.test.ts diff --git a/agents/hermes/Dockerfile b/agents/hermes/Dockerfile index 12edfe44507..de6075ec362 100644 --- a/agents/hermes/Dockerfile +++ b/agents/hermes/Dockerfile @@ -32,6 +32,7 @@ FROM scratch AS hermes-agent-payload COPY agents/hermes/plugin/ /opt/nemoclaw-hermes-plugin/ COPY agents/hermes/generate-config.ts /opt/nemoclaw-hermes-config/generate-config.ts COPY agents/hermes/config/ /opt/nemoclaw-hermes-config/config/ +COPY agents/hermes/patch-gateway-runtime-metadata.py /opt/nemoclaw-hermes-config/patch-gateway-runtime-metadata.py COPY agents/hermes/host/managed-tool-gateway-matrix.json /opt/nemoclaw-hermes-config/managed-tool-gateway-matrix.json COPY src/lib/tool-disclosure.ts /src/lib/tool-disclosure.ts COPY src/lib/messaging/ /src/lib/messaging/ @@ -245,6 +246,37 @@ RUN chmod 755 /usr/local/bin/nemoclaw-start /usr/local/lib/nemoclaw/sandbox-init RUN test -x /usr/local/lib/nemoclaw/validate-hermes-env-secret-boundary.py \ || { echo "ERROR: validate-hermes-env-secret-boundary.py missing or not executable" >&2; exit 1; } +# Hermes v0.18.0 writes gateway lifecycle metadata directly below HERMES_HOME. +# Shields-up deliberately makes that config root root-owned and non-writable, +# so a replacement cannot remove the old PID file and exits with a false PID +# race. Keep config/hash locks intact and relocate only Hermes's central +# PID/lock/status path helpers to the existing writable runtime directory. +RUN /usr/bin/python3 -I \ + /opt/nemoclaw-hermes-config/patch-gateway-runtime-metadata.py \ + /opt/hermes/gateway/status.py \ + && HERMES_HOME="$(mktemp -d)" /opt/hermes/.venv/bin/python - <<'PY' +from pathlib import Path + +from gateway.status import ( + _get_gateway_lock_path, + _get_pid_path, + _get_runtime_status_path, +) +from hermes_constants import get_hermes_home + +home = get_hermes_home() +runtime = home / "runtime" +assert _get_pid_path() == runtime / "gateway.pid" +assert _get_gateway_lock_path() == runtime / "gateway.lock" +assert _get_runtime_status_path() == runtime / "gateway_state.json" +assert all(path.parent == runtime for path in ( + _get_pid_path(), + _get_gateway_lock_path(), + _get_runtime_status_path(), +)) +assert isinstance(home, Path) +PY + # Hermes v0.18.0 computes `sessions list` preview from the first user message, # while #5254's user-facing expectation is that the existing row reflects the # latest resumed/continued one-shot turn. Patch only the pinned query shape and diff --git a/agents/hermes/mcp-config-transaction.py b/agents/hermes/mcp-config-transaction.py index 4cd644316b2..a3fddc885a1 100755 --- a/agents/hermes/mcp-config-transaction.py +++ b/agents/hermes/mcp-config-transaction.py @@ -48,7 +48,7 @@ CONFIG_PATH = "/sandbox/.hermes/config.yaml" HERMES_DIR = "/sandbox/.hermes" -GATEWAY_PID_PATH = f"{HERMES_DIR}/gateway.pid" +GATEWAY_PID_PATH = f"{HERMES_DIR}/runtime/gateway.pid" STRICT_HASH_PATH = "/etc/nemoclaw/hermes.config-hash" GUARD_PATH = "/usr/local/lib/nemoclaw/hermes-runtime-config-guard.py" ROOT_LIFECYCLE_MARKER = "/run/nemoclaw/hermes-root-lifecycle" diff --git a/agents/hermes/patch-gateway-runtime-metadata.py b/agents/hermes/patch-gateway-runtime-metadata.py new file mode 100755 index 00000000000..b1f955838cf --- /dev/null +++ b/agents/hermes/patch-gateway-runtime-metadata.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Relocate pinned Hermes gateway metadata below its writable runtime directory. + +Hermes v0.18.0 stores ``gateway.pid``, ``gateway.lock``, and +``gateway_state.json`` directly below ``HERMES_HOME``. NemoClaw shields-up +correctly makes that config root root-owned and non-writable, so a managed +gateway replacement cannot remove the old PID file or atomically refresh +runtime status. The resulting replacement exits with "PID file race lost". + +NemoClaw already provisions ``HERMES_HOME/runtime`` as the writable lifecycle +boundary. Patch only Hermes's central path helpers so all of its PID, lock, +status, health, and CLI readers agree on that directory. The exact source-shape +checks fail closed when the pinned Hermes implementation changes. + +Remove this patch when the minimum supported Hermes release natively separates +writable gateway metadata from its configuration root. +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +OLD_PID_HELPER = '''def _get_pid_path() -> Path: + """Return the path to the gateway PID file, respecting HERMES_HOME.""" + home = get_hermes_home() + return home / "gateway.pid" +''' +NEW_PID_HELPER = '''def _get_pid_path() -> Path: + """Return the path to the gateway PID file, respecting HERMES_HOME.""" + home = get_hermes_home() + return home / "runtime" / "gateway.pid" +''' + +OLD_LOCK_HELPER = '''def _get_gateway_lock_path(pid_path: Optional[Path] = None) -> Path: + """Return the path to the runtime gateway lock file.""" + if pid_path is not None: + return pid_path.with_name(_GATEWAY_LOCK_FILENAME) + home = get_hermes_home() + return home / _GATEWAY_LOCK_FILENAME +''' +NEW_LOCK_HELPER = '''def _get_gateway_lock_path(pid_path: Optional[Path] = None) -> Path: + """Return the path to the runtime gateway lock file.""" + if pid_path is not None: + return pid_path.with_name(_GATEWAY_LOCK_FILENAME) + home = get_hermes_home() + return home / "runtime" / _GATEWAY_LOCK_FILENAME +''' + + +def patch_file(path: Path) -> None: + source = path.read_text(encoding="utf-8") + replacements = ( + ("PID", OLD_PID_HELPER, NEW_PID_HELPER), + ("lock", OLD_LOCK_HELPER, NEW_LOCK_HELPER), + ) + + if all(source.count(old) == 0 and source.count(new) == 1 for _, old, new in replacements): + return + + for label, old, new in replacements: + old_count = source.count(old) + new_count = source.count(new) + if old_count != 1 or new_count != 0: + raise SystemExit( + "ERROR: Hermes gateway runtime metadata source shape changed; " + f"expected one unpatched {label} helper, found {old_count} " + f"(already patched helpers: {new_count})" + ) + + for _, old, new in replacements: + source = source.replace(old, new) + path.write_text(source, encoding="utf-8") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument( + "path", + nargs="?", + default="/opt/hermes/gateway/status.py", + help="Hermes gateway status module to patch", + ) + args = parser.parse_args() + patch_file(Path(args.path)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/changelog/2026-07-28.mdx b/docs/changelog/2026-07-28.mdx index 9538ab72f40..949b3563f82 100644 --- a/docs/changelog/2026-07-28.mdx +++ b/docs/changelog/2026-07-28.mdx @@ -28,9 +28,14 @@ It also hardens compatible-provider switching, managed sandbox images, Jetson GP Host-side `config set` also accepts the exact `http://host.openshell.internal:` shape only for supported provider `baseUrl` fields, while generic keys and other private URL shapes remain rejected. Retired NVIDIA Build MiniMax and Qwen model paths are no longer offered. For more information, refer to [Switch Inference Providers](/user-guide/openclaw/inference/manage-inference/switch-providers), [Meet Custom Endpoint Security Requirements](/user-guide/openclaw/inference/custom-endpoints/custom-endpoint-security), and [Set Up Ollama](/user-guide/openclaw/inference/local-inference/set-up-ollama). -- Docker-driver recovery now unpauses a paused original container before reporting recovery, and resumed onboarding writes a secret-free same-name recreation journal before deletion so an interrupted run can continue or fail closed on ambiguous identity. +- Docker-driver post-reboot recovery now waits for the labeled container to reach Docker readiness. + It restores the in-sandbox gateway and host forwards before reporting success, refreshes stale stopped-container evidence, and fails closed when delivery cannot be proven. + Recovery also unpauses a paused original container before reporting success. + Resumed onboarding writes a secret-free same-name recreation journal before deletion so an interrupted run can continue or fail closed on ambiguous identity. + Restored snapshot clones retry pairing once only when local approval fails while the authenticated verifier reports a pending scope upgrade. + The retry preserves the local-device and single-request bounds. Uninstall checks for the `openshell` command after confirmation and before cleanup mutation, while source-checkout installation preserves an absolute `NEMOCLAW_OPENSHELL_BIN` selection during user-local OpenShell discovery. - For more information, refer to [Recover and Rebuild Sandboxes](/user-guide/openclaw/manage-sandboxes/operate-sandboxes/recover-and-rebuild-sandboxes) and [Uninstall NemoClaw](/user-guide/openclaw/manage-sandboxes/operate-sandboxes/uninstall-nemoclaw). + For more information, refer to [Recover and Rebuild Sandboxes](/user-guide/openclaw/manage-sandboxes/operate-sandboxes/recover-and-rebuild-sandboxes), [Create and Restore Snapshots](/user-guide/openclaw/manage-sandboxes/state-and-backups/create-and-restore-snapshots), and [Uninstall NemoClaw](/user-guide/openclaw/manage-sandboxes/operate-sandboxes/uninstall-nemoclaw). - Jetson setup now explains why host preparation was skipped when it encounters an unrecognized or unparseable release. GPU onboarding carries eligible numeric group IDs for real, non-symlink DRI render character devices into the recreated container while preserving the established Tegra device allowlist. The sandbox CUDA proof remains fail-closed, and physical IGX Orin validation of the reported configuration remains pending. @@ -38,6 +43,8 @@ It also hardens compatible-provider switching, managed sandbox images, Jetson GP - Telegram channel status now returns `unreachable` with a nonzero exit when the Bot API startup probe receives a definitive HTTP error. Managed MCP tool discovery accepts compliant case-variant or parameterized SSE media types through the reviewed SDK runtime. Hermes image assembly normalizes executable modes before metadata validation, and locked restart integrity accepts the generated v1 managed MCP state record while retaining fail-closed hashing and failed-reload rollback. + Hermes now keeps gateway PID, lock, and status records in the writable runtime directory. + This lets a restart replace the tracked gateway while shields are up without weakening config locks or crash quarantine. For more information, refer to [Choose Messaging Channels](/user-guide/openclaw/manage-sandboxes/messaging-channels/choose-messaging-channels), [Manage MCP Servers with OpenClaw](/user-guide/openclaw/manage-sandboxes/mcp-servers/manage-mcp-servers), [Manage MCP Servers with Hermes](/user-guide/hermes/manage-sandboxes/mcp-servers/manage-mcp-servers), and the [NemoClaw CLI Commands Reference](/user-guide/openclaw/reference/commands). - Deep Agents now publishes the bounded approval, baseline, preset, custom-preset, and live-policy tasks that apply to its maintained runtime. The `claude-code` preset permits browser login only through `GET` and `POST` on `platform.claude.com/v1/oauth/**`, without widening unrelated Anthropic or telemetry access. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 28d9c8f78fa..bb3285ac7a3 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -1354,7 +1354,11 @@ For cloud-only providers, the output omits the NIM status line unless a NIM cont When the sandbox's recorded driver is `docker` and the host Docker daemon is not reachable, the command prints the `docker_unreachable` failure layer with the message `Docker daemon is not reachable.` as the first line of stdout, suppresses the host-side `Inference` probe (which otherwise hits the remote provider directly and is misleading when the local stack is down), and exits with a non-zero status. -When the host Docker daemon is reachable but the per-sandbox container is stopped, the command prints the `sandbox_container_stopped` failure layer with the message `sandbox container exists but is not running.` as the first line of stdout, suppresses the host-side `Inference` probe, and exits with a non-zero status. +When the host Docker daemon is reachable but the per-sandbox container is stopped, the initial preflight records the `sandbox_container_stopped` failure layer and suppresses the host-side `Inference` probe. +If the owning OpenShell gateway is healthy but no longer lists the registered Docker-driver sandbox, status attempts post-reboot recovery from the labeled container. +It waits for Docker readiness, restores the in-sandbox gateway and host forwards, and refreshes preflight before probing inference. +A successful recovery clears the stale stopped-container failure. +If Docker readiness or the agent delivery chain cannot be proven, status exits non-zero and reports the failed recovery layer. If the sandbox's recorded dashboard port is also held by a foreign listener, the header escalates to the `sandbox_dashboard_port_conflict` failure layer with the message `sandbox container is stopped and the dashboard port is held by a foreign listener.` so the operator can recover the port before restarting the sandbox. diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index c3f0d58372c..b8171b4cea7 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -1152,6 +1152,17 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( "the primary dashboard/API host forward is owned by another sandbox", }; } + if (forwardHealthy === null) { + return { + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: false, + forwardRecoveryFailed: true, + forwardRecoveryFailureDetail: + "the primary dashboard/API host forward could not be verified because OpenShell forward state was unavailable", + }; + } const dashboardForwardRecovered = ensureHermesDashboardPortForwardIfEnabled(sandboxName); const messagingForwardRecovered = recoverMessagingHostForward(sandboxName, { quiet }); const declaredForwardsRecovered = recoverDeclaredAgentForwardPorts(sandboxName, recoveryPort, { diff --git a/src/lib/actions/sandbox/restore-gateway-pairing.test.ts b/src/lib/actions/sandbox/restore-gateway-pairing.test.ts index 81d499af287..ff98e0749d3 100644 --- a/src/lib/actions/sandbox/restore-gateway-pairing.test.ts +++ b/src/lib/actions/sandbox/restore-gateway-pairing.test.ts @@ -171,7 +171,60 @@ describe("establishRestoredSandboxGatewayPairing", () => { expect(verifyGatewayPairing).not.toHaveBeenCalled(); }); - it("fails after one ordinary verifier without retrying the handshake (#7431)", async () => { + it("retries one failed clone approval when verification reports a pending scope upgrade (#7431)", async () => { + const order: string[] = []; + const restartRestoredSandboxGateway = vi.fn(() => order.push("restart")); + const warmupScopeUpgrade = vi.fn(() => order.push("warmup")); + const approveRestoredClonePairing = vi + .fn() + .mockImplementationOnce(() => { + order.push("approve:failed"); + return "approve-failed" as const; + }) + .mockImplementationOnce(() => { + order.push("approve:succeeded"); + return "approved-one" as const; + }); + const verifyGatewayPairing = vi + .fn() + .mockImplementationOnce(() => { + order.push("verify:pending"); + return { + ok: false as const, + failureLayer: "scope-upgrade-pending" as const, + }; + }) + .mockImplementationOnce(() => { + order.push("verify:authenticated"); + return { ok: true as const }; + }); + + await establishRestoredSandboxGatewayPairing("beta", { + restartRestoredSandboxGateway, + warmupScopeUpgrade, + approveRestoredClonePairing, + verifyGatewayPairing, + }); + + expect(order).toEqual([ + "restart", + "warmup", + "approve:failed", + "restart", + "verify:pending", + "restart", + "warmup", + "approve:succeeded", + "restart", + "verify:authenticated", + ]); + expect(restartRestoredSandboxGateway).toHaveBeenCalledTimes(4); + expect(warmupScopeUpgrade).toHaveBeenCalledTimes(2); + expect(approveRestoredClonePairing).toHaveBeenCalledTimes(2); + expect(verifyGatewayPairing).toHaveBeenCalledTimes(2); + }); + + it("fails after the bounded clone approval retry cannot authenticate pairing (#7431)", async () => { const restartRestoredSandboxGateway = vi.fn(); const warmupScopeUpgrade = vi.fn(); const approveRestoredClonePairing = vi.fn(() => "approve-failed" as const); @@ -190,6 +243,56 @@ describe("establishRestoredSandboxGatewayPairing", () => { ).rejects.toThrow( "authenticated gateway verification run failed (scope-upgrade-pending; approval=approve-failed)", ); + expect(restartRestoredSandboxGateway).toHaveBeenCalledTimes(4); + expect(warmupScopeUpgrade).toHaveBeenCalledTimes(2); + expect(approveRestoredClonePairing).toHaveBeenCalledTimes(2); + expect(verifyGatewayPairing).toHaveBeenCalledTimes(2); + }); + + it("does not retry a failed clone approval for an unrelated verifier failure (#7431)", async () => { + const restartRestoredSandboxGateway = vi.fn(); + const warmupScopeUpgrade = vi.fn(); + const approveRestoredClonePairing = vi.fn(() => "approve-failed" as const); + const verifyGatewayPairing = vi.fn(() => ({ + ok: false as const, + failureLayer: "gateway-connect-failure" as const, + })); + + await expect( + establishRestoredSandboxGatewayPairing("beta", { + restartRestoredSandboxGateway, + warmupScopeUpgrade, + approveRestoredClonePairing, + verifyGatewayPairing, + }), + ).rejects.toThrow( + "authenticated gateway verification run failed (gateway-connect-failure; approval=approve-failed)", + ); + expect(restartRestoredSandboxGateway).toHaveBeenCalledTimes(2); + expect(warmupScopeUpgrade).toHaveBeenCalledOnce(); + expect(approveRestoredClonePairing).toHaveBeenCalledOnce(); + expect(verifyGatewayPairing).toHaveBeenCalledOnce(); + }); + + it("does not retry a completed clone approval when verification remains pending (#7431)", async () => { + const restartRestoredSandboxGateway = vi.fn(); + const warmupScopeUpgrade = vi.fn(); + const approveRestoredClonePairing = vi.fn(() => "approved-one" as const); + const verifyGatewayPairing = vi.fn(() => ({ + ok: false as const, + failureLayer: "scope-upgrade-pending" as const, + })); + + await expect( + establishRestoredSandboxGatewayPairing("beta", { + restartRestoredSandboxGateway, + warmupScopeUpgrade, + approveRestoredClonePairing, + verifyGatewayPairing, + }), + ).rejects.toThrow( + "authenticated gateway verification run failed (scope-upgrade-pending; approval=approved-one)", + ); expect(restartRestoredSandboxGateway).toHaveBeenCalledTimes(2); expect(warmupScopeUpgrade).toHaveBeenCalledOnce(); expect(approveRestoredClonePairing).toHaveBeenCalledOnce(); diff --git a/src/lib/actions/sandbox/restore-gateway-pairing.ts b/src/lib/actions/sandbox/restore-gateway-pairing.ts index 50cff79caff..e70c1b1ca43 100644 --- a/src/lib/actions/sandbox/restore-gateway-pairing.ts +++ b/src/lib/actions/sandbox/restore-gateway-pairing.ts @@ -29,6 +29,8 @@ const RESTORED_CLONE_PAIRING_BUDGET = { timeoutMs: CONNECT_AUTO_PAIR_TIMEOUT_MS, } as const; +const RESTORED_CLONE_APPROVE_FAILURE_ATTEMPTS = 2; + class RestoreGatewayPairingClassifiedError extends Error {} type RestoredSandboxGatewayRestartDeps = { @@ -78,17 +80,28 @@ export async function establishRestoredSandboxGatewayPairing( targetSandbox: string, deps: RestoreGatewayPairingDeps = defaultRestoreGatewayPairingDeps(), ): Promise { - // Deliberately do not retry this authorization sequence internally. A fixed - // failure lets the caller retry the restore command as a new bounded attempt. try { - deps.restartRestoredSandboxGateway(targetSandbox); - deps.warmupScopeUpgrade(targetSandbox); - const approvalReceipt = deps.approveRestoredClonePairing(targetSandbox) ?? "exec-failed"; - // Publish the clone's approved pairing transition before the one ordinary - // authenticated verifier. The verifier alone decides success. - deps.restartRestoredSandboxGateway(targetSandbox); - const verification = deps.verifyGatewayPairing(targetSandbox); - if (!verification.ok) { + for (let attempt = 1; attempt <= RESTORED_CLONE_APPROVE_FAILURE_ATTEMPTS; attempt += 1) { + deps.restartRestoredSandboxGateway(targetSandbox); + deps.warmupScopeUpgrade(targetSandbox); + const approvalReceipt = deps.approveRestoredClonePairing(targetSandbox) ?? "exec-failed"; + // Publish the clone's approved pairing transition before the ordinary + // authenticated verifier. The verifier alone decides success. + deps.restartRestoredSandboxGateway(targetSandbox); + const verification = deps.verifyGatewayPairing(targetSandbox); + if (verification.ok) { + return; + } + // The canonical approval command can create its own local scope-upgrade + // request and report approve-failed. Retry that exact pending transition + // once with the same local-device and one-request bounds. + if ( + attempt < RESTORED_CLONE_APPROVE_FAILURE_ATTEMPTS && + approvalReceipt === "approve-failed" && + verification.failureLayer === "scope-upgrade-pending" + ) { + continue; + } throw new RestoreGatewayPairingClassifiedError( `the authenticated gateway verification run failed (${verification.failureLayer}; approval=${approvalReceipt})`, ); diff --git a/src/lib/actions/sandbox/status-flow.test.ts b/src/lib/actions/sandbox/status-flow.test.ts index 0e5e301f0a3..57b9a9508a3 100644 --- a/src/lib/actions/sandbox/status-flow.test.ts +++ b/src/lib/actions/sandbox/status-flow.test.ts @@ -490,8 +490,87 @@ describe("showSandboxStatus flow", () => { expect(output).toContain("Retry `openshell gateway start --name nemoclaw`"); expect(output).toContain("If the gateway never becomes healthy"); expect(harness.collectSandboxStatusSnapshotSpy).toHaveBeenCalledWith("alpha", { + preflight: { + failure: null, + failureLayer: "docker_unreachable", + suppressInferenceProbe: true, + exitCode: 1, + }, + }); + }); + + it("renders the refreshed preflight after Docker recovery", async () => { + const preflight = { + failure: { + layer: "sandbox_container_stopped" as const, + dockerUnreachable: false, + }, + failureLayer: "sandbox_container_stopped" as const, + suppressInferenceProbe: true, + exitCode: 1 as const, + }; + const harness = createStatusFlowHarness({ + preflight, + postRecoveryPreflight: { + failure: null, + failureLayer: null, + suppressInferenceProbe: false, + exitCode: 0, + }, + }); + + await expect(harness.showSandboxStatus("alpha")).resolves.toBeUndefined(); + + const output = harness.logSpy.mock.calls.flat().join("\n"); + expect(output).not.toContain("Failure layer: sandbox_container_stopped"); + expect(process.exitCode).toBeUndefined(); + expect(harness.collectSandboxStatusSnapshotSpy).toHaveBeenCalledWith("alpha", { + preflight, + }); + }); + + it("does not erase a dashboard-port conflict during Docker recovery", async () => { + const conflict = { + failure: { + layer: "sandbox_dashboard_port_conflict" as const, + dockerUnreachable: false, + }, + failureLayer: "sandbox_dashboard_port_conflict" as const, suppressInferenceProbe: true, + exitCode: 1 as const, + }; + const harness = createStatusFlowHarness({ + preflight: conflict, + postRecoveryPreflight: conflict, }); + + await expect(harness.showSandboxStatus("alpha")).resolves.toBeUndefined(); + + const output = harness.logSpy.mock.calls.flat().join("\n"); + expect(output).toContain("Failure layer: sandbox_dashboard_port_conflict"); + expect(process.exitCode).toBe(1); + }); + + it("renders an agent delivery recovery failure as actionable and nonzero", async () => { + const harness = createStatusFlowHarness({ + inferenceHealth: null, + lookup: { + state: "sandbox_recovery_failed", + output: + " Docker restored sandbox 'alpha', but its agent delivery chain is not ready " + + "(forward-recovery: OpenShell forward state unavailable).", + recoveredSandbox: true, + }, + }); + + await expect(harness.showSandboxStatus("alpha")).rejects.toThrow("process.exit(1)"); + + const output = harness.logSpy.mock.calls.flat().join("\n"); + expect(output).toContain("restored from Docker"); + expect(output).toContain("agent delivery chain could not be recovered safely"); + expect(output).toContain("forward-recovery: OpenShell forward state unavailable"); + expect(output).toContain("Retry `nemoclaw alpha recover`"); + expect(output).not.toContain("Could not verify against live gateway"); }); it("renders missing gateway metadata after restart without claiming recovery", async () => { diff --git a/src/lib/actions/sandbox/status-lookup-rendering.ts b/src/lib/actions/sandbox/status-lookup-rendering.ts index 92cab9b18e0..7343d944307 100644 --- a/src/lib/actions/sandbox/status-lookup-rendering.ts +++ b/src/lib/actions/sandbox/status-lookup-rendering.ts @@ -47,11 +47,29 @@ export async function printSandboxGatewayLookupStatus( case "gateway_missing_after_restart": await printGatewayMissingAfterRestartLookupStatus(context); return; + case "sandbox_recovery_failed": + printSandboxRecoveryFailedLookupStatus(context); + return; default: await printUnknownGatewayLookupStatus(context); } } +function printSandboxRecoveryFailedLookupStatus({ + sandboxName, + lookup, +}: SandboxGatewayLookupStatusContext): void { + console.log(""); + console.log( + ` Sandbox '${sandboxName}' was restored from Docker, but its agent delivery chain could not be recovered safely.`, + ); + if (lookup.output) console.log(lookup.output); + console.log( + ` Retry \`${CLI_NAME} ${sandboxName} recover\` after addressing the reported layer.`, + ); + process.exit(1); +} + function printMissingLiveSandboxStatusGuidance( sandboxName: string, lookup: SandboxGatewayState, diff --git a/src/lib/actions/sandbox/status-snapshot-inference-health.test.ts b/src/lib/actions/sandbox/status-snapshot-inference-health.test.ts index 87088dfb1fe..b75ae797de3 100644 --- a/src/lib/actions/sandbox/status-snapshot-inference-health.test.ts +++ b/src/lib/actions/sandbox/status-snapshot-inference-health.test.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import type { ProviderHealthStatus } from "../../inference/health"; import type { SandboxEntry } from "../../state/registry"; @@ -43,6 +43,59 @@ function snapshotDeps( } describe("collectSandboxStatusSnapshot inference route health", () => { + it("restores the guarded agent and host-forward chain before probing a Docker-recovered sandbox", async () => { + const order: string[] = []; + const gateway: SandboxInferenceRouteHealth = { + ok: true, + endpoint: "https://inference.local/v1/models", + httpStatus: 200, + detail: "reachable", + }; + const options = snapshotDeps(gateway); + options.deps.reconcile = async () => { + order.push("reconcile"); + return { + state: "present", + output: "Phase: Ready", + recoveredSandbox: true, + recoverySandboxVia: "started-stopped-original", + }; + }; + const recoverSandboxProcesses = vi.fn(() => { + order.push("recover-agent-and-forward"); + return { + checked: true, + wasRunning: false, + recovered: true, + forwardRecovered: true, + }; + }); + options.deps.probeSandboxInferenceGatewayHealthImpl = async () => { + order.push("probe-inference"); + return gateway; + }; + + await collectSandboxStatusSnapshot("alpha", { + ...options, + deps: { ...options.deps, recoverSandboxProcesses }, + }); + + expect(recoverSandboxProcesses).toHaveBeenCalledWith("alpha", { quiet: true }); + expect(order).toEqual(["reconcile", "recover-agent-and-forward", "probe-inference"]); + }); + + it("does not mutate the agent or host forward during an ordinary present status lookup", async () => { + const options = snapshotDeps(null); + const recoverSandboxProcesses = vi.fn(); + + await collectSandboxStatusSnapshot("alpha", { + ...options, + deps: { ...options.deps, recoverSandboxProcesses }, + }); + + expect(recoverSandboxProcesses).not.toHaveBeenCalled(); + }); + it("labels a reachable route okLabel: reachable, not a bare healthy claim (#6846)", async () => { const gateway: SandboxInferenceRouteHealth = { ok: true, diff --git a/src/lib/actions/sandbox/status-snapshot-recovery.test.ts b/src/lib/actions/sandbox/status-snapshot-recovery.test.ts new file mode 100644 index 00000000000..629f3600c61 --- /dev/null +++ b/src/lib/actions/sandbox/status-snapshot-recovery.test.ts @@ -0,0 +1,228 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import type { SandboxEntry } from "../../state/registry"; +import type { SandboxInferenceRouteHealth } from "./inference-route-health"; +import type { SandboxStatusPreflightResult } from "./status-preflight"; +import { collectSandboxStatusSnapshot, getSandboxStatusReport } from "./status-snapshot"; + +const sandbox: SandboxEntry = { + name: "alpha", + agent: "openclaw", + policies: [], + provider: "nvidia", + model: "nvidia/nemotron", + openshellDriver: "docker", + dashboardPort: 18789, +}; + +const stoppedPreflight: SandboxStatusPreflightResult = { + failure: { + layer: "sandbox_container_stopped", + dockerUnreachable: false, + }, + failureLayer: "sandbox_container_stopped", + suppressInferenceProbe: true, + exitCode: 1, +}; + +const clearPreflight: SandboxStatusPreflightResult = { + failure: null, + failureLayer: null, + suppressInferenceProbe: false, + exitCode: 0, +}; + +const conflictPreflight: SandboxStatusPreflightResult = { + failure: { + layer: "sandbox_dashboard_port_conflict", + dockerUnreachable: false, + }, + failureLayer: "sandbox_dashboard_port_conflict", + suppressInferenceProbe: true, + exitCode: 1, +}; + +const healthyRoute: SandboxInferenceRouteHealth = { + ok: true, + endpoint: "https://inference.local/v1/models", + httpStatus: 200, + detail: "reachable", +}; + +function recoveredLookup() { + return Promise.resolve({ + state: "present", + output: "Phase: Ready", + recoveredSandbox: true, + recoverySandboxVia: "started-stopped-original", + }); +} + +function snapshotDeps(recoveryResult: unknown) { + const probeProviderHealthImpl = vi.fn(() => null); + const probeSandboxInferenceGatewayHealthImpl = vi.fn(async () => healthyRoute); + return { + getSandbox: () => sandbox, + listSandboxes: () => ({ sandboxes: [sandbox], defaultSandbox: sandbox.name }), + reconcile: recoveredLookup, + captureOpenshellForStatusImpl: async () => { + throw new Error("live route lookup not needed"); + }, + probeProviderHealthImpl, + probeSandboxInferenceGatewayHealthImpl, + recoverSandboxProcesses: vi.fn(() => recoveryResult) as never, + }; +} + +describe("collectSandboxStatusSnapshot Docker recovery", () => { + it.each([ + [ + "inspection", + { + checked: false, + wasRunning: null, + recovered: false, + forwardRecovered: false, + }, + ], + [ + "secret-boundary", + { + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: false, + secretBoundaryRefused: true, + secretBoundaryReason: "persisted secret boundary refused recovery", + }, + ], + [ + "mcp-reconciliation", + { + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: false, + mcpReconciliationRefused: true, + mcpReconciliationReason: "MCP intent mismatch", + }, + ], + [ + "gateway-recovery", + { + checked: true, + wasRunning: false, + recovered: false, + forwardRecovered: false, + }, + ], + [ + "forward-recovery", + { + checked: true, + wasRunning: false, + recovered: true, + forwardRecovered: false, + }, + ], + [ + "forward-recovery", + { + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: false, + forwardRecoveryFailed: true, + forwardRecoveryFailureDetail: "OpenShell forward state unavailable", + }, + ], + ])("fails closed at the %s layer", async (layer, recoveryResult) => { + const deps = snapshotDeps(recoveryResult); + + const snapshot = await collectSandboxStatusSnapshot("alpha", { deps }); + + expect(snapshot.lookup.state).toBe("sandbox_recovery_failed"); + expect(snapshot.lookup.output).toContain(`(${layer}:`); + expect(deps.probeProviderHealthImpl).not.toHaveBeenCalled(); + expect(deps.probeSandboxInferenceGatewayHealthImpl).not.toHaveBeenCalled(); + }); + + it("accepts a recovered gateway only when the primary forward is proven", async () => { + const deps = snapshotDeps({ + checked: true, + wasRunning: false, + recovered: true, + forwardRecovered: true, + }); + + const snapshot = await collectSandboxStatusSnapshot("alpha", { deps }); + + expect(snapshot.lookup.state).toBe("present"); + expect(deps.probeSandboxInferenceGatewayHealthImpl).toHaveBeenCalledWith("alpha"); + }); + + it("keeps a terminal runtime result neutral", async () => { + const deps = snapshotDeps({ + checked: true, + wasRunning: null, + recovered: false, + forwardRecovered: false, + runtime: "terminal", + }); + + const snapshot = await collectSandboxStatusSnapshot("alpha", { deps }); + + expect(snapshot.lookup.state).toBe("present"); + }); +}); + +describe("getSandboxStatusReport Docker recovery preflight refresh", () => { + it("clears a stale stopped-container preflight after successful recovery", async () => { + const getSandboxStatusPreflightImpl = vi + .fn() + .mockResolvedValueOnce(stoppedPreflight) + .mockResolvedValueOnce(clearPreflight); + const deps = { + ...snapshotDeps({ + checked: true, + wasRunning: false, + recovered: true, + forwardRecovered: true, + }), + getSandboxStatusPreflightImpl, + }; + + const report = await getSandboxStatusReport("alpha", deps); + + expect(report.gatewayState).toBe("present"); + expect(report.failureLayer).toBeNull(); + expect(report.inferenceHealth).toMatchObject({ ok: true, probed: true }); + expect(getSandboxStatusPreflightImpl).toHaveBeenCalledTimes(2); + }); + + it("preserves a dashboard-port conflict observed before Docker recovery", async () => { + const getSandboxStatusPreflightImpl = vi + .fn() + .mockResolvedValueOnce(conflictPreflight) + .mockResolvedValueOnce(clearPreflight); + const deps = { + ...snapshotDeps({ + checked: true, + wasRunning: false, + recovered: true, + forwardRecovered: true, + }), + getSandboxStatusPreflightImpl, + }; + + const report = await getSandboxStatusReport("alpha", deps); + + expect(report.failureLayer).toBe("sandbox_dashboard_port_conflict"); + expect(report.inferenceHealth).toBeNull(); + expect(getSandboxStatusPreflightImpl).toHaveBeenCalledOnce(); + expect(deps.probeSandboxInferenceGatewayHealthImpl).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/actions/sandbox/status-snapshot.ts b/src/lib/actions/sandbox/status-snapshot.ts index 6363e0dd99d..7cf8e5db7bb 100644 --- a/src/lib/actions/sandbox/status-snapshot.ts +++ b/src/lib/actions/sandbox/status-snapshot.ts @@ -41,6 +41,7 @@ import { probeSandboxInferenceGatewayHealth } from "./inference-route-health"; import { getSandboxStatusPreflight, type SandboxStatusFailureLayer, + type SandboxStatusPreflightResult, withoutTerminalPhasePreflight, } from "./status-preflight"; import { @@ -211,6 +212,8 @@ export interface SandboxStatusSnapshot { inferenceHealth: ProviderHealthStatus | null; terminalRuntimeHealth: TerminalRuntimeOomProbeResult | null; servingProcessHealth: ServingProcessHealth | null; + /** Refreshed after Docker recovery so callers do not render stale stopped-container state. */ + postRecoveryPreflight?: SandboxStatusPreflightResult; } export interface SandboxStatusAgentInfo { @@ -255,6 +258,28 @@ export function resolveSandboxStatusAgent(agentName = "openclaw"): SandboxStatus type ReconcileSandboxGatewayState = (sandboxName: string) => Promise; type ProbeTerminalRuntimeHealth = (sandboxName: string) => TerminalRuntimeOomProbeResult; +type RecoverSandboxProcesses = + typeof import("./status/process-recovery")["checkAndRecoverSandboxProcesses"]; +type SandboxProcessRecoveryResult = ReturnType; + +type SandboxProcessRecoveryFailure = { + layer: + | "inspection" + | "secret-boundary" + | "mcp-reconciliation" + | "gateway-recovery" + | "forward-recovery" + | "recovery-error"; + detail: string; +}; + +function loadRecoverSandboxProcesses(): RecoverSandboxProcesses { + return ( + require("./status/process-recovery") as { + checkAndRecoverSandboxProcesses: RecoverSandboxProcesses; + } + ).checkAndRecoverSandboxProcesses; +} interface CollectSandboxStatusSnapshotDeps { getSandbox?: typeof registry.getSandbox; @@ -264,18 +289,101 @@ interface CollectSandboxStatusSnapshotDeps { probeSandboxInferenceGatewayHealthImpl?: ProbeSandboxInferenceGatewayHealth; reportInferenceProbeError?: (message: string) => void; probeTerminalRuntimeHealth?: ProbeTerminalRuntimeHealth; + recoverSandboxProcesses?: RecoverSandboxProcesses; reconcile?: ReconcileSandboxGatewayState; + getSandboxStatusPreflightImpl?: typeof getSandboxStatusPreflight; getBaselineExclusionRuntimeStatus?: typeof getBaselineExclusionRuntimeStatus; } -function reportInferenceProbeError(error: unknown, writer: (message: string) => void): void { +function sanitizedStatusDetail(error: unknown): string { const raw = error instanceof Error && error.message ? error.message : String(error); - const detail = redact(raw) + return redact(raw) .replace(/[\u0000-\u001f\u007f-\u009f]/g, " ") .replace(/\s+/g, " ") - .trim(); + .trim() + .slice(0, 240); +} + +function processRecoveryFailure( + result: SandboxProcessRecoveryResult, +): SandboxProcessRecoveryFailure | null { + if (!result.checked) { + return { + layer: "inspection", + detail: "the managed agent gateway could not be inspected", + }; + } + if ("secretBoundaryRefused" in result && result.secretBoundaryRefused) { + return { + layer: "secret-boundary", + detail: sanitizedStatusDetail( + "secretBoundaryReason" in result + ? result.secretBoundaryReason + : "the agent secret boundary refused recovery", + ), + }; + } + if ("mcpReconciliationRefused" in result && result.mcpReconciliationRefused) { + return { + layer: "mcp-reconciliation", + detail: sanitizedStatusDetail( + "mcpReconciliationReason" in result + ? result.mcpReconciliationReason + : "MCP reconciliation refused recovery", + ), + }; + } + if ("forwardRecoveryFailed" in result && result.forwardRecoveryFailed) { + return { + layer: "forward-recovery", + detail: sanitizedStatusDetail( + "forwardRecoveryFailureDetail" in result + ? result.forwardRecoveryFailureDetail + : "the host forward could not be restored", + ), + }; + } + if (result.wasRunning === false && result.recovered && !result.forwardRecovered) { + return { + layer: "forward-recovery", + detail: "the primary dashboard/API host forward was not proven after gateway recovery", + }; + } + if (result.wasRunning === false && !result.recovered) { + return { + layer: "gateway-recovery", + detail: "the managed agent gateway could not be restarted", + }; + } + if (result.wasRunning === null && (!("runtime" in result) || result.runtime !== "terminal")) { + return { + layer: "inspection", + detail: "the managed agent gateway recovery result was inconclusive", + }; + } + return null; +} + +async function refreshPreflightAfterDockerRecovery( + sandbox: registry.SandboxEntry | null, + initial: SandboxStatusPreflightResult, + getPreflight: typeof getSandboxStatusPreflight, +): Promise { + // A listener observed while the sandbox was stopped may belong to another + // process. Starting the container makes the ordinary preflight stop checking + // that port, so keep this ownership conflict authoritative. + if (initial.failureLayer === "sandbox_dashboard_port_conflict") return initial; + try { + return await getPreflight(sandbox); + } catch { + return initial; + } +} + +function reportInferenceProbeError(error: unknown, writer: (message: string) => void): void { + const detail = sanitizedStatusDetail(error); writer( - ` Warning: the authoritative inference.local probe could not run: ${detail.slice(0, 240) || "unknown error"}`, + ` Warning: the authoritative inference.local probe could not run: ${detail || "unknown error"}`, ); } @@ -283,6 +391,7 @@ export async function collectSandboxStatusSnapshot( sandboxName: string, opts: { suppressInferenceProbe?: boolean; + preflight?: SandboxStatusPreflightResult; deps?: CollectSandboxStatusSnapshotDeps; } = {}, ): Promise { @@ -304,6 +413,48 @@ export async function collectSandboxStatusSnapshot( output: ` Could not probe live gateway state: ${message}`, }; } + const dockerRecovered = lookup.recoveredSandbox === true; + if (lookup.state === "present" && lookup.recoveredSandbox) { + let failure: SandboxProcessRecoveryFailure | null; + try { + // Docker recovery makes the sandbox visible to OpenShell again, but a + // host reboot also tears down the managed agent process and port-forward. + // Reuse the guarded connect recovery only for this explicit mutation + // path, before status probes the delivery chain. + const recovery = (opts.deps?.recoverSandboxProcesses ?? loadRecoverSandboxProcesses())( + sandboxName, + { + quiet: true, + }, + ); + failure = processRecoveryFailure(recovery); + } catch (error) { + failure = { + layer: "recovery-error", + detail: sanitizedStatusDetail(error) || "agent and host-forward recovery failed", + }; + } + if (failure) { + lookup = { + ...lookup, + state: "sandbox_recovery_failed", + output: + ` Docker restored sandbox '${sandboxName}', but its agent delivery chain is not ready ` + + `(${failure.layer}: ${failure.detail}).`, + }; + } + } + const postRecoveryPreflight = + dockerRecovered && opts.preflight + ? await refreshPreflightAfterDockerRecovery( + sb, + opts.preflight, + opts.deps?.getSandboxStatusPreflightImpl ?? getSandboxStatusPreflight, + ) + : undefined; + const suppressInferenceProbe = + (postRecoveryPreflight ?? opts.preflight)?.suppressInferenceProbe ?? + opts.suppressInferenceProbe === true; let liveResult: Awaited> | null = null; let gatewayName: string | null = null; if (lookup.state === "present") { @@ -332,6 +483,7 @@ export async function collectSandboxStatusSnapshot( inferenceHealth: null, terminalRuntimeHealth: null, servingProcessHealth: null, + ...(postRecoveryPreflight ? { postRecoveryPreflight } : {}), }; } const live = @@ -374,7 +526,7 @@ export async function collectSandboxStatusSnapshot( let providerHealth: ProviderHealthStatus | null = null; try { providerHealth = maybeGetSandboxStatusInferenceHealth( - opts.suppressInferenceProbe === true, + suppressInferenceProbe, lookup.state === "present", (live && live.provider) || currentProvider, (live && live.model) || currentModel, @@ -394,7 +546,7 @@ export async function collectSandboxStatusSnapshot( // `inference.local` is authoritative because it is the route the agent uses. // Probe it independently of direct/upstream provider diagnostics, including // providers without a registered host-side health probe (#6192). - if (opts.suppressInferenceProbe !== true && lookup.state === "present") { + if (!suppressInferenceProbe && lookup.state === "present") { let gatewayChain: Awaited> = null; try { gatewayChain = await ( @@ -432,6 +584,7 @@ export async function collectSandboxStatusSnapshot( inferenceHealth, terminalRuntimeHealth, servingProcessHealth, + ...(postRecoveryPreflight ? { postRecoveryPreflight } : {}), }; } @@ -452,9 +605,11 @@ async function buildSandboxStatusReport( deps: CollectSandboxStatusSnapshotDeps, ): Promise { const getSandbox = deps.getSandbox ?? registry.getSandbox; - const preflight = await getSandboxStatusPreflight(getSandbox(sandboxName)); + const preflight = await (deps.getSandboxStatusPreflightImpl ?? getSandboxStatusPreflight)( + getSandbox(sandboxName), + ); const snapshot = await collectSandboxStatusSnapshot(sandboxName, { - suppressInferenceProbe: preflight.suppressInferenceProbe, + preflight, deps, }); const { @@ -471,7 +626,10 @@ async function buildSandboxStatusReport( } = snapshot; const dockerRuntime = lookup.state === "present" ? getSandboxDockerRuntime(sandboxName) : null; const phase = lookup.state === "present" ? parseSandboxPhase(lookup.output || "") : null; - const effectivePreflight = withoutTerminalPhasePreflight(preflight, phase); + const effectivePreflight = withoutTerminalPhasePreflight( + snapshot.postRecoveryPreflight ?? preflight, + phase, + ); const sandboxGpuEnabled = sb ? (sb.sandboxGpuEnabled ?? sb.gpuEnabled === true) : false; const policies = sb && Array.isArray(sb.policies) diff --git a/src/lib/actions/sandbox/status-text.ts b/src/lib/actions/sandbox/status-text.ts index 7c8bca85ba8..550128c7f64 100644 --- a/src/lib/actions/sandbox/status-text.ts +++ b/src/lib/actions/sandbox/status-text.ts @@ -24,7 +24,7 @@ import { } from "../../state/sandbox-session"; import type { SandboxDockerRuntime } from "./docker-health"; import type { SandboxGatewayState } from "./gateway-state"; -import { isSandboxGatewayRunningForStatus } from "./process-recovery"; +import { isSandboxGatewayRunningForStatus } from "./status/process-recovery"; import { isInferenceHealthFailing, resolveSandboxStatusDcodeAutoApprovalMode, diff --git a/src/lib/actions/sandbox/status.ts b/src/lib/actions/sandbox/status.ts index 9f15c04c4c5..4c1c92ceb74 100644 --- a/src/lib/actions/sandbox/status.ts +++ b/src/lib/actions/sandbox/status.ts @@ -71,7 +71,7 @@ export async function showSandboxStatus(sandboxName: string): Promise { // handles `gateway_error` by printing an actionable block + exit(1), so a // synthesized fallback keeps the user-visible contract intact. const snapshot = await collectSandboxStatusSnapshot(sandboxName, { - suppressInferenceProbe: preflight.suppressInferenceProbe, + preflight, }); const { sb, @@ -88,7 +88,10 @@ export async function showSandboxStatus(sandboxName: string): Promise { // recovery hint (#4495) and the Docker health line below (#3975). const dockerRuntime = lookup.state === "present" ? getSandboxDockerRuntime(sandboxName) : null; const phase = lookup.state === "present" ? parseSandboxPhase(lookup.output || "") : null; - const effectivePreflight = withoutTerminalPhasePreflight(preflight, phase); + const effectivePreflight = withoutTerminalPhasePreflight( + snapshot.postRecoveryPreflight ?? preflight, + phase, + ); const statusAgent = resolveSandboxStatusAgent(sb?.agent || "openclaw"); printSandboxStatusPreflightHeader(effectivePreflight); if (effectivePreflight.exitCode !== 0) { diff --git a/src/lib/actions/sandbox/status/process-recovery.ts b/src/lib/actions/sandbox/status/process-recovery.ts new file mode 100644 index 00000000000..8d7ee749236 --- /dev/null +++ b/src/lib/actions/sandbox/status/process-recovery.ts @@ -0,0 +1,11 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Keep status's process inspection and recovery consumers behind one edge to +// the lifecycle module. This preserves the source-architecture fan-in budget +// while both text rendering and snapshot collection share the same guarded +// recovery implementation. +export { + checkAndRecoverSandboxProcesses, + isSandboxGatewayRunningForStatus, +} from "../process-recovery"; diff --git a/src/lib/onboard/docker-driver-sandbox-recovery.test.ts b/src/lib/onboard/docker-driver-sandbox-recovery.test.ts index 6662d8acd1c..580f62ea185 100644 --- a/src/lib/onboard/docker-driver-sandbox-recovery.test.ts +++ b/src/lib/onboard/docker-driver-sandbox-recovery.test.ts @@ -22,8 +22,17 @@ function fakeRename( return () => ({ status }); } -function fakeCapture(output: string): (args: readonly string[]) => string { - return () => output; +function fakeCapture( + output: string, + inspectOutputs: string[] = ["running\thealthy"], +): (args: readonly string[]) => string { + let inspectIndex = 0; + return (args) => { + if (args[0] !== "inspect") return output; + const index = Math.min(inspectIndex, inspectOutputs.length - 1); + inspectIndex += 1; + return inspectOutputs[index] ?? ""; + }; } describe("findLabeledSandboxContainers", () => { @@ -57,11 +66,16 @@ describe("findLabeledSandboxContainers", () => { }); describe("recoverDockerDriverSandbox — running original (no-op)", () => { - it("reports recovered with via=started-running-original", () => { + it("waits for an already-running container to become healthy without starting it", () => { const start = vi.fn(fakeStart(0)); + const sleep = vi.fn(); const result = recoverDockerDriverSandbox("e2e-x", { - dockerCapture: fakeCapture("openshell-e2e-x\tUp 5 minutes\n"), + dockerCapture: fakeCapture("openshell-e2e-x\tUp 5 seconds\n", [ + "running\tstarting", + "running\thealthy", + ]), dockerStart: start, + sleep, }); expect(result).toEqual({ recovered: true, @@ -69,6 +83,7 @@ describe("recoverDockerDriverSandbox — running original (no-op)", () => { containerName: "openshell-e2e-x", }); expect(start).not.toHaveBeenCalled(); + expect(sleep).toHaveBeenCalledOnce(); }); }); @@ -106,6 +121,21 @@ describe("recoverDockerDriverSandbox — paused original (unpause)", () => { expect(result.detail).toContain("docker unpause"); expect(unpause).toHaveBeenCalledTimes(1); }); + + it("does not report recovery when an unpaused container reaches a terminal state", () => { + const result = recoverDockerDriverSandbox("e2e-x", { + dockerCapture: fakeCapture("openshell-e2e-x\tUp 3 hours (Paused)\n", ["dead\tunhealthy"]), + dockerUnpause: fakeStart(0), + sleep: vi.fn(), + }); + + expect(result).toMatchObject({ + recovered: false, + via: null, + containerName: "openshell-e2e-x", + detail: expect.stringContaining("runtime=dead, health=unhealthy"), + }); + }); }); describe("recoverDockerDriverSandbox — stopped original (start)", () => { @@ -136,6 +166,124 @@ describe("recoverDockerDriverSandbox — stopped original (start)", () => { expect(result.via).toBeNull(); expect(result.detail).toMatch(/docker start openshell-e2e-x failed.*125/); }); + + it("waits for Docker health to become ready before reporting recovery", () => { + const sleep = vi.fn(); + const result = recoverDockerDriverSandbox("e2e-x", { + dockerCapture: fakeCapture("openshell-e2e-x\tExited (137) 30 seconds ago\n", [ + "running\tstarting", + "running\tstarting", + "running\thealthy", + ]), + dockerStart: fakeStart(0), + sleep, + }); + + expect(result).toEqual({ + recovered: true, + via: "started-stopped-original", + containerName: "openshell-e2e-x", + }); + expect(sleep).toHaveBeenCalledTimes(2); + expect(sleep).toHaveBeenCalledWith(1_000); + }); + + it("accepts a running container whose image has no Docker health check", () => { + const sleep = vi.fn(); + const result = recoverDockerDriverSandbox("e2e-x", { + dockerCapture: fakeCapture("openshell-e2e-x\tExited (137) 30 seconds ago\n", [ + "running\tnone", + ]), + dockerStart: fakeStart(0), + sleep, + }); + + expect(result).toEqual({ + recovered: true, + via: "started-stopped-original", + containerName: "openshell-e2e-x", + }); + expect(sleep).not.toHaveBeenCalled(); + }); + + it.each([ + "dead", + "exited", + "removing", + ])("does not report recovery when the restarted container reaches terminal state %s", (runtimeState) => { + const sleep = vi.fn(); + const result = recoverDockerDriverSandbox("e2e-x", { + dockerCapture: fakeCapture("openshell-e2e-x\tExited (137) 30 seconds ago\n", [ + `${runtimeState}\tnone`, + ]), + dockerStart: fakeStart(0), + sleep, + }); + + expect(result).toEqual({ + recovered: false, + via: null, + containerName: "openshell-e2e-x", + detail: + "docker container openshell-e2e-x did not become ready after recovery " + + `(runtime=${runtimeState}, health=none)`, + }); + expect(sleep).not.toHaveBeenCalled(); + }); + + it("enforces the readiness deadline with an advancing clock", () => { + let currentMs = 0; + const inspectStartTimes: number[] = []; + const inspectTimeouts: number[] = []; + const capture = fakeCapture("openshell-e2e-x\tExited (137) 30 seconds ago\n", [ + "running\tstarting", + ]); + const sleep = vi.fn((ms: number) => { + currentMs += ms; + }); + const result = recoverDockerDriverSandbox("e2e-x", { + dockerCapture: (args, opts) => { + if (args[0] === "inspect") { + inspectStartTimes.push(currentMs); + const timeout = Number(opts?.timeout); + inspectTimeouts.push(timeout); + currentMs += Math.min(4_900, timeout); + } + return capture(args); + }, + dockerStart: fakeStart(0), + now: () => currentMs, + sleep, + }); + + expect(result.recovered).toBe(false); + expect(result.via).toBeNull(); + expect(result.detail).toContain("runtime=running, health=starting"); + expect(currentMs).toBe(90_000); + expect(inspectStartTimes.every((startedAt) => startedAt < 90_000)).toBe(true); + expect(inspectTimeouts.at(-1)).toBe(1_500); + }); + + it("fails closed when Docker inspect returns malformed readiness output", () => { + let currentMs = 0; + const result = recoverDockerDriverSandbox("e2e-x", { + dockerCapture: fakeCapture("openshell-e2e-x\tExited (137) 30 seconds ago\n", [ + "not-a-docker-state", + ]), + dockerStart: fakeStart(0), + now: () => currentMs, + sleep: (ms) => { + currentMs += ms; + }, + }); + + expect(result).toMatchObject({ + recovered: false, + via: null, + detail: expect.stringContaining("runtime=not-a-docker-state"), + }); + expect(currentMs).toBe(90_000); + }); }); describe("recoverDockerDriverSandbox — backup-only (rename + start)", () => { @@ -202,6 +350,25 @@ describe("recoverDockerDriverSandbox — backup-only (rename + start)", () => { expect(result.recovered).toBe(false); expect(result.detail).toMatch(/after backup rename failed.*1/); }); + + it("does not report a renamed backup recovered before Docker readiness", () => { + const result = recoverDockerDriverSandbox("e2e-x", { + dockerCapture: fakeCapture("openshell-e2e-x-nemoclaw-gpu-backup-1717280000000\tExited\n", [ + "running\tstarting", + "removing\tnone", + ]), + dockerRename: fakeRename(0), + dockerStart: fakeStart(0), + sleep: vi.fn(), + }); + + expect(result).toMatchObject({ + recovered: false, + via: null, + containerName: "openshell-e2e-x", + detail: expect.stringContaining("runtime=removing, health=none"), + }); + }); }); describe("recoverDockerDriverSandbox — collision and missing cases", () => { diff --git a/src/lib/onboard/docker-driver-sandbox-recovery.ts b/src/lib/onboard/docker-driver-sandbox-recovery.ts index 429660d6eb4..50d47d3ec07 100644 --- a/src/lib/onboard/docker-driver-sandbox-recovery.ts +++ b/src/lib/onboard/docker-driver-sandbox-recovery.ts @@ -4,6 +4,7 @@ // OpenShell-managed container labels — duplicated locally to keep // the module unit-testable. `docker-gpu-patch.ts` re-exports them as // the source of truth; a parity test pins drift. + export const OPENSHELL_MANAGED_BY_LABEL = "openshell.ai/managed-by"; export const OPENSHELL_MANAGED_BY_VALUE = "openshell"; export const OPENSHELL_SANDBOX_NAME_LABEL = "openshell.ai/sandbox-name"; @@ -68,7 +69,17 @@ function loadDockerUnpause(): DockerStartFn { const DOCKER_PROBE_TIMEOUT_MS = 5_000; const DOCKER_OPERATION_TIMEOUT_MS = 30_000; +const DOCKER_RECOVERY_READY_TIMEOUT_MS = 90_000; +const DOCKER_RECOVERY_READY_POLL_INTERVAL_MS = 1_000; +const DOCKER_RECOVERY_READY_MAX_ATTEMPTS = + Math.floor(DOCKER_RECOVERY_READY_TIMEOUT_MS / DOCKER_RECOVERY_READY_POLL_INTERVAL_MS) + 1; const MAX_DOCKER_CONTAINER_NAME_LENGTH = 253; +const DOCKER_RECOVERY_SLEEP_BUFFER = new Int32Array(new SharedArrayBuffer(4)); + +function sleepForDockerRecovery(ms: number): void { + if (ms <= 0 || !Number.isFinite(ms)) return; + Atomics.wait(DOCKER_RECOVERY_SLEEP_BUFFER, 0, 0, ms); +} /** * Names recovery dispatched on. Stable identifiers callers can log, @@ -104,7 +115,9 @@ export interface DockerDriverRecoveryDeps { newName: string, opts?: Record, ) => { status?: number | null }; - /** Injectable clock for deterministic backup-name normalization in tests. */ + /** Injectable sleep used by the post-start readiness wait. */ + sleep?: (ms: number) => void; + /** Injectable clock for deterministic readiness deadlines in tests. */ now?: () => number; } @@ -122,6 +135,7 @@ function depsWithDefaults(deps: DockerDriverRecoveryDeps) { dockerUnpause: deps.dockerUnpause ?? ((name, opts) => loadDockerUnpause()(name, opts)), dockerRename: deps.dockerRename ?? ((oldName, newName, opts) => loadDockerRename()(oldName, newName, opts)), + sleep: deps.sleep ?? sleepForDockerRecovery, now: deps.now ?? (() => Date.now()), }; } @@ -214,6 +228,81 @@ function buildBackupRestoreName(originalName: string): string { return originalName.slice(0, MAX_DOCKER_CONTAINER_NAME_LENGTH); } +interface ContainerReadinessResult { + ready: boolean; + detail: string; +} + +/** + * A successful `docker start` only proves that Docker accepted the lifecycle + * request. The image can remain in `health: starting` while its agent gateway + * is still unavailable, so post-reboot recovery must not report success until + * Docker's own readiness contract settles. + */ +function waitForRecoveredContainerReady( + containerName: string, + deps: ReturnType, +): ContainerReadinessResult { + let lastState = "unknown"; + const deadlineMs = deps.now() + DOCKER_RECOVERY_READY_TIMEOUT_MS; + for (let attempt = 0; attempt < DOCKER_RECOVERY_READY_MAX_ATTEMPTS; attempt += 1) { + const remainingMs = deadlineMs - deps.now(); + if (remainingMs <= 0) break; + const raw = deps.dockerCapture( + [ + "inspect", + "--type", + "container", + "--format", + "{{.State.Status}}\t{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}", + containerName, + ], + { ignoreError: true, timeout: Math.min(DOCKER_PROBE_TIMEOUT_MS, remainingMs) }, + ); + const [runtimeState = "", healthState = ""] = raw.trim().toLowerCase().split(/\s+/); + lastState = healthState + ? `runtime=${runtimeState || "unknown"}, health=${healthState}` + : `runtime=${runtimeState || "unknown"}`; + + if (runtimeState === "running" && (healthState === "healthy" || healthState === "none")) { + return { ready: true, detail: lastState }; + } + // These states cannot become ready without another lifecycle action. + if (["dead", "exited", "removing"].includes(runtimeState)) { + return { ready: false, detail: lastState }; + } + if (attempt + 1 >= DOCKER_RECOVERY_READY_MAX_ATTEMPTS) break; + + const postInspectRemainingMs = deadlineMs - deps.now(); + if (postInspectRemainingMs <= 0) break; + deps.sleep(Math.min(DOCKER_RECOVERY_READY_POLL_INTERVAL_MS, postInspectRemainingMs)); + } + return { ready: false, detail: lastState }; +} + +function recoveredAfterReadiness( + containerName: string, + via: DockerDriverRecoveryVia, + deps: ReturnType, +): DockerDriverRecoveryResult { + const readiness = waitForRecoveredContainerReady(containerName, deps); + if (!readiness.ready) { + return { + recovered: false, + via: null, + containerName, + detail: + `docker container ${containerName} did not become ready after recovery ` + + `(${readiness.detail})`, + }; + } + return { + recovered: true, + via, + containerName, + }; +} + /** * Attempt to recover the labeled sandbox container so OpenShell sees * the sandbox again. Caller must retry `openshell sandbox get ` @@ -258,17 +347,9 @@ export function recoverDockerDriverSandbox( detail: `docker unpause ${runningOriginal.name} failed (exit ${unpause.status ?? "unknown"}).`, }; } - return { - recovered: true, - via: "unpaused-original", - containerName: runningOriginal.name, - }; + return recoveredAfterReadiness(runningOriginal.name, "unpaused-original", d); } - return { - recovered: true, - via: "started-running-original", - containerName: runningOriginal.name, - }; + return recoveredAfterReadiness(runningOriginal.name, "started-running-original", d); } if (stoppedOriginal) { @@ -277,11 +358,7 @@ export function recoverDockerDriverSandbox( timeout: DOCKER_OPERATION_TIMEOUT_MS, }); if (result.status === 0) { - return { - recovered: true, - via: "started-stopped-original", - containerName: stoppedOriginal.name, - }; + return recoveredAfterReadiness(stoppedOriginal.name, "started-stopped-original", d); } return { recovered: false, @@ -308,11 +385,7 @@ export function recoverDockerDriverSandbox( timeout: DOCKER_OPERATION_TIMEOUT_MS, }); if (startResult.status === 0) { - return { - recovered: true, - via: "renamed-and-started-backup", - containerName: restoreName, - }; + return recoveredAfterReadiness(restoreName, "renamed-and-started-backup", d); } return { recovered: false, diff --git a/test/e2e/fixtures/compatible-anthropic-switch.ts b/test/e2e/fixtures/compatible-anthropic-switch.ts new file mode 100644 index 00000000000..6b51307486b --- /dev/null +++ b/test/e2e/fixtures/compatible-anthropic-switch.ts @@ -0,0 +1,71 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { HostCliClient } from "./clients/host.ts"; +import { resultText } from "./clients/index.ts"; + +export const COMPATIBLE_ANTHROPIC_PROVIDER = "compatible-anthropic-endpoint"; +export const COMPATIBLE_ANTHROPIC_CREDENTIAL_ENV = "COMPATIBLE_ANTHROPIC_API_KEY"; +const DEFAULT_COMPATIBLE_ANTHROPIC_CREDENTIAL = "test-compatible-anthropic-key"; + +export interface CompatibleAnthropicSwitchBinding { + endpointUrl: string; + credentialValue: string; +} + +export function compatibleAnthropicSwitchBinding( + endpointUrl: string, + runtimeEnv: NodeJS.ProcessEnv = process.env, +): CompatibleAnthropicSwitchBinding { + const normalizedEndpointUrl = endpointUrl.trim(); + if (!normalizedEndpointUrl) { + throw new Error( + "NEMOCLAW_SWITCH_ENDPOINT_URL is required for compatible Anthropic inference switches", + ); + } + const credentialValue = + runtimeEnv[COMPATIBLE_ANTHROPIC_CREDENTIAL_ENV] ?? DEFAULT_COMPATIBLE_ANTHROPIC_CREDENTIAL; + if (!credentialValue.trim()) { + throw new Error( + "COMPATIBLE_ANTHROPIC_API_KEY is required for compatible Anthropic inference switches", + ); + } + return { endpointUrl: normalizedEndpointUrl, credentialValue }; +} + +export function compatibleAnthropicSwitchEnv( + binding: CompatibleAnthropicSwitchBinding | null, +): NodeJS.ProcessEnv { + return binding ? { [COMPATIBLE_ANTHROPIC_CREDENTIAL_ENV]: binding.credentialValue } : {}; +} + +export async function requireCompatibleAnthropicProviderAbsent( + host: HostCliClient, + options: { + artifactName: string; + env: NodeJS.ProcessEnv; + gatewayName?: string; + }, +): Promise { + const gatewayName = options.gatewayName ?? "nemoclaw"; + const result = await host.command( + "openshell", + ["provider", "get", "-g", gatewayName, COMPATIBLE_ANTHROPIC_PROVIDER], + { + artifactName: options.artifactName, + env: options.env, + timeoutMs: 30_000, + }, + ); + const output = resultText(result); + if (result.exitCode === 0) { + throw new Error( + `Provider '${COMPATIBLE_ANTHROPIC_PROVIDER}' must be absent before this inference switch so NemoClaw can create and verify its rollback-safe binding.`, + ); + } + if (!/provider not found|requested entity was not found/iu.test(output)) { + throw new Error( + `Could not prove provider '${COMPATIBLE_ANTHROPIC_PROVIDER}' is absent: ${output}`, + ); + } +} diff --git a/test/e2e/fixtures/phases/lifecycle.ts b/test/e2e/fixtures/phases/lifecycle.ts index 7110af8df6c..b63c30d0507 100644 --- a/test/e2e/fixtures/phases/lifecycle.ts +++ b/test/e2e/fixtures/phases/lifecycle.ts @@ -30,9 +30,8 @@ export { // probe. const OPENSHELL_SANDBOX_NAME_LABEL = "openshell.ai/sandbox-name"; const DOCKER_PROBE_TIMEOUT_MS = 15_000; -// Status invocation can take several minutes on unfixed code while -// the gateway recovery path retries. Keep the budget generous; the -// bug is independent of latency. +// Recovery can take several minutes while gateway and host-forward +// readiness converge, so keep the status budget generous. const STATUS_TIMEOUT_MS = 5 * 60_000; const REBUILD_TIMEOUT_MS = 20 * 60_000; const SANDBOX_READY_ATTEMPTS = 30; @@ -355,10 +354,9 @@ export class LifecyclePhaseFixture { * make `openshell status` report the named gateway connected * without running `nemoclaw onboard --resume`. * - * We deliberately do NOT assert on the status exit code here - * because the bug is precisely that status "succeeds" at - * destroying state. The state-validation phase that follows is - * what catches the regression via the + * Status must exit zero to prove that the restored sandbox delivery + * path is ready. The state-validation phase that follows additionally + * verifies preservation via the * `local-registry-entry-present` and `docker-sandbox-container-present` * probes. * @@ -444,10 +442,9 @@ export class LifecyclePhaseFixture { // We invoke status through the host CLI client so artifacts are // captured and the command goes through the same // shellProbe/redaction layer the rest of the fixture code uses. - // Status is allowed to fail (exit non-zero) because on unfixed - // code it intentionally fails after destroying state — the - // post-action invariants are checked by state-validation. - const statusResult = await this.host.nemoclaw([instance.sandboxName, "status"], { + // Status must exit zero to prove the restored delivery path is ready; + // state-validation still verifies registry and container preservation. + const statusResult = await this.host.expectStatus(instance.sandboxName, { artifactName: `lifecycle-post-reboot-nemoclaw-status-${instance.sandboxName}`, env: buildAvailabilityProbeEnv(), timeoutMs: STATUS_TIMEOUT_MS, diff --git a/test/e2e/live/hermes-inference-switch-helpers.ts b/test/e2e/live/hermes-inference-switch-helpers.ts index a16e32db6db..95bd9219ecb 100644 --- a/test/e2e/live/hermes-inference-switch-helpers.ts +++ b/test/e2e/live/hermes-inference-switch-helpers.ts @@ -16,6 +16,12 @@ import { trustedSandboxShellScript, validateSandboxName, } from "../fixtures/clients/sandbox.ts"; +import { + type CompatibleAnthropicSwitchBinding, + compatibleAnthropicSwitchBinding, + compatibleAnthropicSwitchEnv, + requireCompatibleAnthropicProviderAbsent, +} from "../fixtures/compatible-anthropic-switch.ts"; import { expect } from "../fixtures/e2e-test.ts"; import type { FakeOpenAiCompatibleServer } from "../fixtures/fake-openai-compatible.ts"; import { @@ -511,42 +517,22 @@ async function startMockAnthropicProvider(): Promise Promise | void): void }, -): Promise { +): Promise { if (SWITCH_PROVIDER !== "compatible-anthropic-endpoint" || SWITCH_API !== "anthropic-messages") return null; const mock = mockAnthropicSwitchEnabled() ? await startMockAnthropicProvider() : undefined; mock && cleanup.add("close compatible Anthropic switch mock", () => mock.close()); - const endpointUrl = process.env.NEMOCLAW_SWITCH_ENDPOINT_URL ?? mock?.endpointUrl ?? ""; - const compatibleKey = process.env.COMPATIBLE_ANTHROPIC_API_KEY ?? "test-compatible-anthropic-key"; - expect( - endpointUrl, - "NEMOCLAW_SWITCH_ENDPOINT_URL is required for compatible Anthropic inference switches", - ).not.toBe(""); - expect( - compatibleKey, - "COMPATIBLE_ANTHROPIC_API_KEY is required for compatible Anthropic inference switches", - ).not.toBe(""); - const providerScript = [ - "set -euo pipefail", - "if openshell provider get -g nemoclaw compatible-anthropic-endpoint >/dev/null 2>&1; then", - " openshell provider delete -g nemoclaw compatible-anthropic-endpoint", - "fi", - 'openshell provider create -g nemoclaw --name compatible-anthropic-endpoint --type openai --credential COMPATIBLE_ANTHROPIC_API_KEY --config "OPENAI_BASE_URL=${SWITCH_OPENAI_ENDPOINT_URL}"', - ].join("\n"); - const result = await host.command("bash", ["-lc", providerScript], { - artifactName: "register-compatible-anthropic-switch-provider", - env: env(undefined, { - COMPATIBLE_ANTHROPIC_API_KEY: compatibleKey, - SWITCH_OPENAI_ENDPOINT_URL: openAiSurfaceEndpointUrl(endpointUrl), - }), - redactionValues: [compatibleKey], - timeoutMs: 120_000, + const binding = compatibleAnthropicSwitchBinding( + process.env.NEMOCLAW_SWITCH_ENDPOINT_URL ?? mock?.endpointUrl ?? "", + ); + await requireCompatibleAnthropicProviderAbsent(host, { + artifactName: "compatible-anthropic-provider-absent-before-switch", + env: env(), }); - expect(result.exitCode).toBe(0); - return endpointUrl; + return { ...binding, endpointUrl: openAiSurfaceEndpointUrl(binding.endpointUrl) }; } export async function installHermes( @@ -583,7 +569,11 @@ export async function runHermesInferenceSetWithRetry( host: HostCliClient, redactionValues: string[], compatibleMetadataArgs: string[], - options: { attempts?: number; delay?: (milliseconds: number) => Promise } = {}, + options: { + attempts?: number; + compatibleBinding?: CompatibleAnthropicSwitchBinding | null; + delay?: (milliseconds: number) => Promise; + } = {}, ): Promise { const args = [ CLI, @@ -604,7 +594,7 @@ export async function runHermesInferenceSetWithRetry( artifactName: verify ? `hermes-inference-set-${attempt}` : "hermes-inference-set-no-verify-after-transient-failures", - env: env(), + env: env(undefined, compatibleAnthropicSwitchEnv(options.compatibleBinding ?? null)), redactionValues, timeoutMs: 180_000, }), diff --git a/test/e2e/live/hermes-inference-switch.test.ts b/test/e2e/live/hermes-inference-switch.test.ts index ed414d82839..37152b12720 100644 --- a/test/e2e/live/hermes-inference-switch.test.ts +++ b/test/e2e/live/hermes-inference-switch.test.ts @@ -13,7 +13,6 @@ import { chatContent, cleanupHermesSwitch, compatibleAnthropicMetadataArgs, - ensureCompatibleAnthropicSwitchProvider, env, envHash, expectAuthenticatedBaselineInventoryRequest, @@ -36,6 +35,7 @@ import { PROXY_RESOLUTION_PROVIDER, parseHermesModelBlock, parseInferenceRoute, + prepareCompatibleAnthropicSwitchBinding, prepareProxyResolutionRoute, RUNTIME_SWITCH_API, registryState, @@ -180,13 +180,9 @@ test("Hermes inference set updates route/config and preserves live runtime", { ? await registerPublicNvidiaSwitchProvider(host, publicApiKey, env()) : null; publicProvider && expect(publicProvider.exitCode, resultText(publicProvider)).toBe(0); - const switchEndpointUrl = await ensureCompatibleAnthropicSwitchProvider(host, cleanup); - switchEndpointUrl && - (await expectOpenAiProvider( - host, - "compatible-anthropic-endpoint", - "COMPATIBLE_ANTHROPIC_API_KEY", - )); + const switchBinding = await prepareCompatibleAnthropicSwitchBinding(host, cleanup); + const switchEndpointUrl = switchBinding?.endpointUrl ?? null; + switchBinding && redactionValues.push(switchBinding.credentialValue); const pidBefore = await hermesGatewayPid(sandbox, "pid-before"); const envHashBefore = await envHash(sandbox, "env-hash-before"); @@ -197,10 +193,17 @@ test("Hermes inference set updates route/config and preserves live runtime", { host, redactionValues, compatibleMetadataArgs, + { compatibleBinding: switchBinding }, ); expect(switched.exitCode, resultText(switched)).toBe(0); expect(resultText(switched)).not.toContain("writing the in-sandbox config failed"); expect(resultText(switched)).toContain(`Inference route synced for '${SANDBOX_NAME}'`); + switchBinding && + (await expectOpenAiProvider( + host, + "compatible-anthropic-endpoint", + "COMPATIBLE_ANTHROPIC_API_KEY", + )); progress.phase("validate switched route and locked config"); const pidAfter = await hermesGatewayPid(sandbox, "pid-after"); diff --git a/test/e2e/live/hermes-root-entrypoint-smoke.test.ts b/test/e2e/live/hermes-root-entrypoint-smoke.test.ts index 72842de61cd..1f5fd480637 100644 --- a/test/e2e/live/hermes-root-entrypoint-smoke.test.ts +++ b/test/e2e/live/hermes-root-entrypoint-smoke.test.ts @@ -221,8 +221,8 @@ async function assertRuntimeLayout(probe: DockerProbe, container: string): Promi await expectContainerSh( probe, container, - "gateway.pid is not a regular top-level file", - "test -f /sandbox/.hermes/gateway.pid && test ! -L /sandbox/.hermes/gateway.pid", + "gateway.pid is not a regular runtime file", + "test -f /sandbox/.hermes/runtime/gateway.pid && test ! -L /sandbox/.hermes/runtime/gateway.pid && test ! -e /sandbox/.hermes/gateway.pid && test ! -L /sandbox/.hermes/gateway.pid", ); await expectContainerShFails( probe, @@ -448,7 +448,7 @@ test("hermes root-entrypoint smoke preserves runtime layout and legacy pid migra "gateway log has no PID race or config load failure", "Hermes v0.14 writable runtime directories are present", "build-only root caches are absent from the runtime image", - "gateway.pid is migrated to a regular top-level file", + "gateway.pid is stored as a regular file below the writable runtime directory", "gateway user cannot remove config.yaml from sticky config root", "Hermes API denies missing/wrong bearer tokens and accepts API_SERVER_KEY", "dashboard-home is sandbox-owned 0700 with 0600 allowlisted config/env", diff --git a/test/e2e/live/openclaw-inference-switch.test.ts b/test/e2e/live/openclaw-inference-switch.test.ts index 58cadedebc7..66a604468fb 100644 --- a/test/e2e/live/openclaw-inference-switch.test.ts +++ b/test/e2e/live/openclaw-inference-switch.test.ts @@ -24,6 +24,12 @@ import { trustedSandboxShellScript, validateSandboxName, } from "../fixtures/clients/sandbox.ts"; +import { + type CompatibleAnthropicSwitchBinding, + compatibleAnthropicSwitchBinding, + compatibleAnthropicSwitchEnv, + requireCompatibleAnthropicProviderAbsent, +} from "../fixtures/compatible-anthropic-switch.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; import { type FakeOpenAiCompatibleServer, @@ -194,13 +200,18 @@ async function runNemoclaw( host: HostCliClient, home: string, args: string[], - options: { artifactName: string; timeoutMs?: number; redactionValues?: string[] } = { + options: { + artifactName: string; + env?: NodeJS.ProcessEnv; + timeoutMs?: number; + redactionValues?: string[]; + } = { artifactName: "nemoclaw", }, ): Promise { return host.command("node", [CLI_ENTRYPOINT, ...args], { artifactName: options.artifactName, - env: commandEnv(home), + env: commandEnv(home, options.env), timeoutMs: options.timeoutMs ?? COMMAND_TIMEOUT_MS, redactionValues: options.redactionValues, }); @@ -376,44 +387,21 @@ async function startMockAnthropicProvider(): Promise { }; } -async function ensureCompatibleAnthropicSwitchProvider( +async function prepareCompatibleAnthropicSwitchBinding( host: HostCliClient, home: string, mockProvider: MockAnthropicProvider | undefined, -): Promise { +): Promise { if (SWITCH_PROVIDER !== "compatible-anthropic-endpoint") return null; if (SWITCH_INFERENCE_API !== "anthropic-messages") return null; const endpointUrl = process.env.NEMOCLAW_SWITCH_ENDPOINT_URL ?? mockProvider?.endpointUrl ?? ""; - const apiKey = process.env.COMPATIBLE_ANTHROPIC_API_KEY ?? "test-compatible-anthropic-key"; - expect( - endpointUrl, - "NEMOCLAW_SWITCH_ENDPOINT_URL is required for compatible Anthropic inference switches", - ).not.toBe(""); - expect( - apiKey, - "COMPATIBLE_ANTHROPIC_API_KEY is required for compatible Anthropic inference switches", - ).not.toBe(""); - - const providerScript = [ - "set -euo pipefail", - "if openshell provider get -g nemoclaw compatible-anthropic-endpoint >/dev/null 2>&1; then", - ' openshell provider update -g nemoclaw compatible-anthropic-endpoint --credential COMPATIBLE_ANTHROPIC_API_KEY --config "ANTHROPIC_BASE_URL=${SWITCH_ENDPOINT_URL}"', - "else", - ' openshell provider create -g nemoclaw --name compatible-anthropic-endpoint --type anthropic --credential COMPATIBLE_ANTHROPIC_API_KEY --config "ANTHROPIC_BASE_URL=${SWITCH_ENDPOINT_URL}"', - "fi", - ].join("\n"); - const provider = await host.command("bash", ["-lc", providerScript], { - artifactName: "register-compatible-anthropic-switch-provider", - env: commandEnv(home, { - COMPATIBLE_ANTHROPIC_API_KEY: apiKey, - SWITCH_ENDPOINT_URL: endpointUrl, - }), - redactionValues: [apiKey], - timeoutMs: COMMAND_TIMEOUT_MS, + const binding = compatibleAnthropicSwitchBinding(endpointUrl); + await requireCompatibleAnthropicProviderAbsent(host, { + artifactName: "compatible-anthropic-provider-absent-before-switch", + env: commandEnv(home), }); - expect(provider.exitCode, resultText(provider)).toBe(0); - return endpointUrl; + return binding; } async function openclawGatewayPid(sandbox: SandboxClient, home: string): Promise { @@ -833,7 +821,7 @@ async function runOpenClawInferenceSetWithRetry( host: HostCliClient, home: string, redactionValues: string[], - switchEndpointUrl: string | null, + switchBinding: CompatibleAnthropicSwitchBinding | null, ): Promise { const attempts = inferenceSetAttemptCount(process.env.NEMOCLAW_SWITCH_SET_ATTEMPTS); const compatibleCredentialEnv = (() => { @@ -846,10 +834,10 @@ async function runOpenClawInferenceSetWithRetry( return null; } })(); - const compatibleMetadataArgs = switchEndpointUrl + const compatibleMetadataArgs = switchBinding ? [ "--endpoint-url", - switchEndpointUrl, + switchBinding.endpointUrl, "--credential-env", compatibleCredentialEnv ?? "", "--inference-api", @@ -875,6 +863,7 @@ async function runOpenClawInferenceSetWithRetry( artifactName: verify ? `nemoclaw-inference-set-${attempt}` : "nemoclaw-inference-set-no-verify-after-transient-failures", + env: compatibleAnthropicSwitchEnv(switchBinding), redactionValues, timeoutMs: COMMAND_TIMEOUT_MS, }), @@ -1036,10 +1025,11 @@ test("openclaw-inference-switch: switches route and preserves live OpenClaw beha // Only the explicit Anthropic bridge supplies endpoint metadata. The // compatible baseline reuses its registered OpenShell provider, while the // public NVIDIA provider has no caller-supplied endpoint identity. - const switchEndpointUrl = + const switchBinding = SWITCH_PROVIDER === "compatible-anthropic-endpoint" - ? await ensureCompatibleAnthropicSwitchProvider(host, home, mockProvider) + ? await prepareCompatibleAnthropicSwitchBinding(host, home, mockProvider) : null; + switchBinding && redactionValues.push(switchBinding.credentialValue); progress.phase("switch the route and verify restart semantics"); expect(baseline.env.NEMOCLAW_PREFERRED_API).toBe("openai-completions"); @@ -1052,7 +1042,7 @@ test("openclaw-inference-switch: switches route and preserves live OpenClaw beha host, home, redactionValues, - switchEndpointUrl, + switchBinding, ); expect(switchResult.exitCode, resultText(switchResult)).toBe(0); expect( diff --git a/test/e2e/support/compatible-anthropic-switch.test.ts b/test/e2e/support/compatible-anthropic-switch.test.ts new file mode 100644 index 00000000000..6466880b6ea --- /dev/null +++ b/test/e2e/support/compatible-anthropic-switch.test.ts @@ -0,0 +1,79 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import type { HostCliClient } from "../fixtures/clients/host.ts"; +import { + COMPATIBLE_ANTHROPIC_CREDENTIAL_ENV, + COMPATIBLE_ANTHROPIC_PROVIDER, + compatibleAnthropicSwitchBinding, + compatibleAnthropicSwitchEnv, + requireCompatibleAnthropicProviderAbsent, +} from "../fixtures/compatible-anthropic-switch.ts"; + +describe("compatible Anthropic inference switch setup", () => { + it("passes the direct binding credential only to the inference-set command", () => { + const binding = compatibleAnthropicSwitchBinding("http://host.openshell.internal:18766", { + COMPATIBLE_ANTHROPIC_API_KEY: "fixture-key", + }); + + expect(binding).toEqual({ + endpointUrl: "http://host.openshell.internal:18766", + credentialValue: "fixture-key", + }); + expect(compatibleAnthropicSwitchEnv(binding)).toEqual({ + [COMPATIBLE_ANTHROPIC_CREDENTIAL_ENV]: "fixture-key", + }); + expect(compatibleAnthropicSwitchEnv(null)).toEqual({}); + }); + + it("requires the direct provider to be absent before inference set owns its creation", async () => { + const command = vi.fn().mockResolvedValue({ + exitCode: 1, + stderr: "Error: code: 'Some requested entity was not found', message: \"provider not found\"", + stdout: "", + }); + const host = { command } as unknown as HostCliClient; + const commandEnv = { OPENSHELL_GATEWAY: "nemoclaw" }; + + await expect( + requireCompatibleAnthropicProviderAbsent(host, { + artifactName: "compatible-anthropic-provider-absent", + env: commandEnv, + }), + ).resolves.toBeUndefined(); + expect(command).toHaveBeenCalledWith( + "openshell", + ["provider", "get", "-g", "nemoclaw", COMPATIBLE_ANTHROPIC_PROVIDER], + expect.objectContaining({ + artifactName: "compatible-anthropic-provider-absent", + env: commandEnv, + }), + ); + }); + + it("rejects a pre-existing or uninspectable direct provider", async () => { + const command = vi + .fn() + .mockResolvedValueOnce({ + exitCode: 0, + stderr: "", + stdout: `Name: ${COMPATIBLE_ANTHROPIC_PROVIDER}`, + }) + .mockResolvedValueOnce({ + exitCode: 1, + stderr: "gateway unavailable", + stdout: "", + }); + const host = { command } as unknown as HostCliClient; + const options = { artifactName: "provider-absent", env: {} }; + + await expect(requireCompatibleAnthropicProviderAbsent(host, options)).rejects.toThrow( + "must be absent", + ); + await expect(requireCompatibleAnthropicProviderAbsent(host, options)).rejects.toThrow( + "Could not prove", + ); + }); +}); diff --git a/test/e2e/support/e2e-phase-lifecycle.test.ts b/test/e2e/support/e2e-phase-lifecycle.test.ts index 257708a567d..bbe39eb0efa 100644 --- a/test/e2e/support/e2e-phase-lifecycle.test.ts +++ b/test/e2e/support/e2e-phase-lifecycle.test.ts @@ -168,7 +168,7 @@ describe("LifecyclePhaseFixture.simulate post-reboot-recovery (stop-original)", runner.enqueue(shellResult(0)); // container stop runner.enqueue(shellResult(0)); // user service restart runner.enqueue(shellResult(0, "Connected to nemoclaw\n")); // openshell status - runner.enqueue(shellResult(1, "Removed stale local registry entry.\n")); // status (non-zero on unfixed) + runner.enqueue(shellResult(0)); // status proves recovered delivery readiness const result = await prepared.simulate("post-reboot-recovery", instance()); @@ -198,7 +198,7 @@ describe("LifecyclePhaseFixture.simulate post-reboot-recovery (stop-original)", ]); }); - it("tolerates a non-zero status exit (the bug succeeds at destroying state)", async () => { + it("fails when status cannot prove post-reboot recovery", async () => { const runner = new FakeRunner(); const cleanup = new FakeCleanup(); const prepared = await preparedPostRebootFixture(runner, cleanup); @@ -212,11 +212,9 @@ describe("LifecyclePhaseFixture.simulate post-reboot-recovery (stop-original)", runner.enqueue(shellResult(0, "Connected to nemoclaw\n")); // openshell status runner.enqueue(shellResult(1, "Removed stale local registry entry.\n")); // status non-zero - const result = await prepared.simulate("post-reboot-recovery", instance()); - - // simulate() does not throw; the post-status invariants belong - // to the state-validation phase that runs after. - expect(result.steps.find((step) => step.id.startsWith("nemoclaw-status:"))).toBeTruthy(); + await expect(prepared.simulate("post-reboot-recovery", instance())).rejects.toThrow( + /nemoclaw e2e-ubuntu-repo-cloud-openclaw status failed: Removed stale local registry entry/, + ); }); it("fails when no Docker container carries the OpenShell sandbox-name label", async () => { @@ -277,7 +275,7 @@ describe("LifecyclePhaseFixture.simulate post-reboot-recovery (rename-to-gpu-bac runner.enqueue(shellResult(0)); // container stop runner.enqueue(shellResult(0)); // user service restart runner.enqueue(shellResult(0, "Connected to nemoclaw\n")); // openshell status - runner.enqueue(shellResult(1, "Removed stale local registry entry.\n")); // status + runner.enqueue(shellResult(0)); // status proves recovered delivery readiness const result = await prepared.simulate( "post-reboot-recovery", diff --git a/test/e2e/support/hermes-inference-switch-command-shape.test.ts b/test/e2e/support/hermes-inference-switch-command-shape.test.ts index bf51effd6f3..536bad419d7 100644 --- a/test/e2e/support/hermes-inference-switch-command-shape.test.ts +++ b/test/e2e/support/hermes-inference-switch-command-shape.test.ts @@ -8,6 +8,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { resolveAgentInferenceApi } from "../../../src/lib/inference/config.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import type { SandboxClient } from "../fixtures/clients/sandbox.ts"; +import { compatibleAnthropicSwitchBinding } from "../fixtures/compatible-anthropic-switch.ts"; import { DEFAULT_HOSTED_INFERENCE_MODEL } from "../fixtures/hosted-inference.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; import { @@ -329,18 +330,26 @@ describe("Hermes inference switch command shape", () => { stdout: "", }) .mockResolvedValueOnce({ exitCode: 0, stderr: "", stdout: "" }); + const compatibleBinding = compatibleAnthropicSwitchBinding( + "http://host.openshell.internal:18766/v1", + { COMPATIBLE_ANTHROPIC_API_KEY: "switch-key" }, + ); await expect( runHermesInferenceSetWithRetry( { command } as unknown as HostCliClient, - ["hosted-key"], - ["--inference-api", "anthropic-messages"], - { attempts: 1, delay: async () => {} }, + ["hosted-key", compatibleBinding.credentialValue], + compatibleAnthropicMetadataArgs(compatibleBinding.endpointUrl), + { attempts: 1, compatibleBinding, delay: async () => {} }, ), ).resolves.toMatchObject({ exitCode: 0 }); expect(command.mock.calls[0]?.[1]).not.toContain("--no-verify"); expect(command.mock.calls[1]?.[1]).toContain("--no-verify"); + expect(command.mock.calls[0]?.[2]).toMatchObject({ + env: { COMPATIBLE_ANTHROPIC_API_KEY: "switch-key" }, + redactionValues: ["hosted-key", "switch-key"], + }); expect(command.mock.calls[0]?.[2]?.env).not.toHaveProperty("NVIDIA_INFERENCE_API_KEY"); }); }); diff --git a/test/hermes-final-image-layout.test.ts b/test/hermes-final-image-layout.test.ts index 5828bf9db4b..887d8ac8e81 100644 --- a/test/hermes-final-image-layout.test.ts +++ b/test/hermes-final-image-layout.test.ts @@ -181,6 +181,7 @@ describe("Hermes final image layout", () => { "COPY agents/hermes/plugin/ /opt/nemoclaw-hermes-plugin/", "COPY agents/hermes/generate-config.ts /opt/nemoclaw-hermes-config/generate-config.ts", "COPY agents/hermes/config/ /opt/nemoclaw-hermes-config/config/", + "COPY agents/hermes/patch-gateway-runtime-metadata.py /opt/nemoclaw-hermes-config/patch-gateway-runtime-metadata.py", "COPY agents/hermes/host/managed-tool-gateway-matrix.json /opt/nemoclaw-hermes-config/managed-tool-gateway-matrix.json", "COPY src/lib/tool-disclosure.ts /src/lib/tool-disclosure.ts", "COPY src/lib/messaging/ /src/lib/messaging/", diff --git a/test/hermes-gateway-runtime-metadata-patch.test.ts b/test/hermes-gateway-runtime-metadata-patch.test.ts new file mode 100644 index 00000000000..00ed4a3f712 --- /dev/null +++ b/test/hermes-gateway-runtime-metadata-patch.test.ts @@ -0,0 +1,108 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; + +const ROOT = path.resolve(import.meta.dirname, ".."); +const PATCHER = path.join(ROOT, "agents", "hermes", "patch-gateway-runtime-metadata.py"); +const MCP_TRANSACTION = path.join(ROOT, "agents", "hermes", "mcp-config-transaction.py"); + +const UPSTREAM_FIXTURE = `from pathlib import Path +from typing import Optional + +HOME = Path(__file__).parent / "hermes-home" +_GATEWAY_LOCK_FILENAME = "gateway.lock" +_RUNTIME_STATUS_FILE = "gateway_state.json" + +def get_hermes_home() -> Path: + return HOME + +def _get_pid_path() -> Path: + """Return the path to the gateway PID file, respecting HERMES_HOME.""" + home = get_hermes_home() + return home / "gateway.pid" + +def _get_gateway_lock_path(pid_path: Optional[Path] = None) -> Path: + """Return the path to the runtime gateway lock file.""" + if pid_path is not None: + return pid_path.with_name(_GATEWAY_LOCK_FILENAME) + home = get_hermes_home() + return home / _GATEWAY_LOCK_FILENAME + +def _get_runtime_status_path() -> Path: + """Return the persisted runtime health/status file path.""" + return _get_pid_path().with_name(_RUNTIME_STATUS_FILE) + +if __name__ == "__main__": + print(_get_pid_path()) + print(_get_gateway_lock_path()) + print(_get_runtime_status_path()) +`; + +function runPatcher(fixture: string) { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-runtime-metadata-")); + const statusPath = path.join(tmp, "status.py"); + fs.writeFileSync(statusPath, fixture); + const result = spawnSync("python3", ["-I", PATCHER, statusPath], { + encoding: "utf-8", + timeout: 5000, + }); + return { result, statusPath, tmp }; +} + +describe("Hermes writable gateway runtime metadata", () => { + it("relocates every central gateway metadata reader and remains idempotent", () => { + const { result, statusPath, tmp } = runPatcher(UPSTREAM_FIXTURE); + try { + expect(result.status, result.stderr).toBe(0); + const second = spawnSync("python3", ["-I", PATCHER, statusPath], { + encoding: "utf-8", + timeout: 5000, + }); + expect(second.status, second.stderr).toBe(0); + + const probe = spawnSync("python3", ["-I", statusPath], { + encoding: "utf-8", + timeout: 5000, + }); + expect(probe.status, probe.stderr).toBe(0); + expect( + probe.stdout + .trim() + .split("\n") + .map((entry) => path.relative(tmp, entry)), + ).toEqual([ + "hermes-home/runtime/gateway.pid", + "hermes-home/runtime/gateway.lock", + "hermes-home/runtime/gateway_state.json", + ]); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + + it("fails closed when the pinned Hermes helper shape changes", () => { + const drifted = UPSTREAM_FIXTURE.replace( + 'return home / "gateway.pid"', + 'return home / "changed-gateway.pid"', + ); + const { result, statusPath, tmp } = runPatcher(drifted); + try { + expect(result.status).toBe(1); + expect(result.stderr).toContain("gateway runtime metadata source shape changed"); + expect(fs.readFileSync(statusPath, "utf-8")).toBe(drifted); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + + it("keeps the managed MCP identity reader on the relocated PID record", () => { + const source = fs.readFileSync(MCP_TRANSACTION, "utf-8"); + expect(source).toContain('GATEWAY_PID_PATH = f"{HERMES_DIR}/runtime/gateway.pid"'); + expect(source).not.toContain('GATEWAY_PID_PATH = f"{HERMES_DIR}/gateway.pid"'); + }); +}); diff --git a/test/process-recovery-forward-failure.test.ts b/test/process-recovery-forward-failure.test.ts index 779f7dc5357..1ea8f506d6e 100644 --- a/test/process-recovery-forward-failure.test.ts +++ b/test/process-recovery-forward-failure.test.ts @@ -89,6 +89,43 @@ function compactTeamsMessagingPlan(port = "3978") { } describe("checkAndRecoverSandboxProcesses primary forward failure", () => { + it("fails closed when OpenShell forward state is unavailable", () => { + const openshellRuntime = requireSource("../src/lib/adapters/openshell/runtime.ts"); + const agentRuntime = requireSource("../src/lib/agent/runtime.ts"); + const registry = requireSource("../src/lib/state/registry.ts"); + const childProcess = requireSource("node:child_process"); + + vi.spyOn(childProcess, "spawnSync").mockReturnValue({ + status: 0, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nRUNNING\n", + stderr: "", + } as never); + vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue(null); + vi.spyOn(registry, "getSandbox").mockReturnValue({ + name: "beta", + agent: "openclaw", + dashboardPort: 18789, + }); + vi.spyOn(openshellRuntime, "captureOpenshell").mockReturnValue({ + status: 1, + output: "OpenShell forward state unavailable", + }); + const runOpenshell = vi.spyOn(openshellRuntime, "runOpenshell"); + + expect( + withFakeOpenshellBinary(() => checkAndRecoverSandboxProcesses("beta", { quiet: true })), + ).toEqual({ + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: false, + forwardRecoveryFailed: true, + forwardRecoveryFailureDetail: + "the primary dashboard/API host forward could not be verified because OpenShell forward state was unavailable", + }); + expect(runOpenshell).not.toHaveBeenCalled(); + }); + it("reports failure when a messaging forward cannot recover even if the primary is healthy", () => { const openshellRuntime = requireSource("../src/lib/adapters/openshell/runtime.ts"); const agentRuntime = requireSource("../src/lib/agent/runtime.ts"); diff --git a/test/support/status-flow-test-harness.ts b/test/support/status-flow-test-harness.ts index 5526ce62bb9..556f400718c 100644 --- a/test/support/status-flow-test-harness.ts +++ b/test/support/status-flow-test-harness.ts @@ -66,6 +66,7 @@ export type StatusFlowHarnessOptions = { lookup?: SandboxGatewayState; lookupState?: "present" | "missing"; preflight?: SandboxStatusPreflightResult; + postRecoveryPreflight?: SandboxStatusPreflightResult; sandboxEntry?: Partial> & { agent?: string | null; agentVersion?: string | null; @@ -102,7 +103,9 @@ export function createStatusFlowHarness(options: StatusFlowHarnessOptions = {}): const statusPreflight = requireDist("../../src/lib/actions/sandbox/status-preflight.js"); const statusSnapshot = requireDist("../../src/lib/actions/sandbox/status-snapshot.js"); const dockerHealth = requireDist("../../src/lib/actions/sandbox/docker-health.js"); - const processRecovery = requireDist("../../src/lib/actions/sandbox/process-recovery.js"); + const statusProcessRecovery = requireDist( + "../../src/lib/actions/sandbox/status/process-recovery.js", + ); const resolve = requireDist("../../src/lib/adapters/openshell/resolve.js"); const agentRuntime = requireDist("../../src/lib/agent/runtime.js"); const nim = requireDist("../../src/lib/inference/nim.js"); @@ -187,6 +190,9 @@ export function createStatusFlowHarness(options: StatusFlowHarnessOptions = {}): ? null : { checked: false } : options.servingProcessHealth, + ...(options.postRecoveryPreflight + ? { postRecoveryPreflight: options.postRecoveryPreflight } + : {}), }); const getSandboxDockerRuntimeSpy = vi .spyOn(dockerHealth, "getSandboxDockerRuntime") @@ -195,7 +201,7 @@ export function createStatusFlowHarness(options: StatusFlowHarnessOptions = {}): health: "unhealthy", paused: false, }); - vi.spyOn(processRecovery, "isSandboxGatewayRunningForStatus").mockResolvedValue(false); + vi.spyOn(statusProcessRecovery, "isSandboxGatewayRunningForStatus").mockResolvedValue(false); vi.spyOn(resolve, "resolveOpenshell").mockReturnValue("/usr/bin/openshell"); vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue({ name: "openclaw" }); vi.spyOn(agentRuntime, "getAgentDisplayName").mockReturnValue("OpenClaw"); From 24eabea350015996b3799490287038d64ba1c1b9 Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Tue, 28 Jul 2026 19:48:40 -0700 Subject: [PATCH 02/11] test(release): keep Docker readiness setup linear Signed-off-by: Charan Jagwani --- .../docker-driver-sandbox-recovery.test.ts | 42 +++++++++++-------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/src/lib/onboard/docker-driver-sandbox-recovery.test.ts b/src/lib/onboard/docker-driver-sandbox-recovery.test.ts index 580f62ea185..e7c98d4ace4 100644 --- a/src/lib/onboard/docker-driver-sandbox-recovery.test.ts +++ b/src/lib/onboard/docker-driver-sandbox-recovery.test.ts @@ -25,13 +25,20 @@ function fakeRename( function fakeCapture( output: string, inspectOutputs: string[] = ["running\thealthy"], -): (args: readonly string[]) => string { + onInspect: (opts?: Record) => void = () => undefined, +): (args: readonly string[], opts?: Record) => string { let inspectIndex = 0; - return (args) => { - if (args[0] !== "inspect") return output; - const index = Math.min(inspectIndex, inspectOutputs.length - 1); - inspectIndex += 1; - return inspectOutputs[index] ?? ""; + return (args, opts) => { + switch (args[0]) { + case "inspect": { + onInspect(opts); + const index = Math.min(inspectIndex, inspectOutputs.length - 1); + inspectIndex += 1; + return inspectOutputs[index] ?? ""; + } + default: + return output; + } }; } @@ -235,22 +242,21 @@ describe("recoverDockerDriverSandbox — stopped original (start)", () => { let currentMs = 0; const inspectStartTimes: number[] = []; const inspectTimeouts: number[] = []; - const capture = fakeCapture("openshell-e2e-x\tExited (137) 30 seconds ago\n", [ - "running\tstarting", - ]); + const capture = fakeCapture( + "openshell-e2e-x\tExited (137) 30 seconds ago\n", + ["running\tstarting"], + (opts) => { + inspectStartTimes.push(currentMs); + const timeout = Number(opts?.timeout); + inspectTimeouts.push(timeout); + currentMs += Math.min(4_900, timeout); + }, + ); const sleep = vi.fn((ms: number) => { currentMs += ms; }); const result = recoverDockerDriverSandbox("e2e-x", { - dockerCapture: (args, opts) => { - if (args[0] === "inspect") { - inspectStartTimes.push(currentMs); - const timeout = Number(opts?.timeout); - inspectTimeouts.push(timeout); - currentMs += Math.min(4_900, timeout); - } - return capture(args); - }, + dockerCapture: capture, dockerStart: fakeStart(0), now: () => currentMs, sleep, From da7fd520dd72e47998e73384e3606daeff6e945d Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Tue, 28 Jul 2026 20:05:01 -0700 Subject: [PATCH 03/11] test(e2e): cover Anthropic binding guards Signed-off-by: Charan Jagwani --- .../support/compatible-anthropic-switch.test.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/e2e/support/compatible-anthropic-switch.test.ts b/test/e2e/support/compatible-anthropic-switch.test.ts index 6466880b6ea..c1378b7d41a 100644 --- a/test/e2e/support/compatible-anthropic-switch.test.ts +++ b/test/e2e/support/compatible-anthropic-switch.test.ts @@ -28,6 +28,22 @@ describe("compatible Anthropic inference switch setup", () => { expect(compatibleAnthropicSwitchEnv(null)).toEqual({}); }); + it("rejects a blank compatible Anthropic endpoint URL", () => { + expect(() => + compatibleAnthropicSwitchBinding(" ", { + COMPATIBLE_ANTHROPIC_API_KEY: "fixture-key", + }), + ).toThrow("NEMOCLAW_SWITCH_ENDPOINT_URL is required"); + }); + + it("rejects a blank compatible Anthropic credential", () => { + expect(() => + compatibleAnthropicSwitchBinding("http://host.openshell.internal:18766", { + COMPATIBLE_ANTHROPIC_API_KEY: " ", + }), + ).toThrow("COMPATIBLE_ANTHROPIC_API_KEY is required"); + }); + it("requires the direct provider to be absent before inference set owns its creation", async () => { const command = vi.fn().mockResolvedValue({ exitCode: 1, From c435538e921be1fbb1b6b21cdfa05adf5cb2e583 Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Tue, 28 Jul 2026 20:45:50 -0700 Subject: [PATCH 04/11] fix(snapshot): retry bounded clone pairing list failure Signed-off-by: Charan Jagwani --- docs/changelog/2026-07-28.mdx | 2 +- .../sandbox/restore-gateway-pairing.test.ts | 122 ++++++++++++++++++ .../sandbox/restore-gateway-pairing.ts | 14 +- 3 files changed, 130 insertions(+), 8 deletions(-) diff --git a/docs/changelog/2026-07-28.mdx b/docs/changelog/2026-07-28.mdx index 949b3563f82..5f1f5d75f51 100644 --- a/docs/changelog/2026-07-28.mdx +++ b/docs/changelog/2026-07-28.mdx @@ -32,7 +32,7 @@ It also hardens compatible-provider switching, managed sandbox images, Jetson GP It restores the in-sandbox gateway and host forwards before reporting success, refreshes stale stopped-container evidence, and fails closed when delivery cannot be proven. Recovery also unpauses a paused original container before reporting success. Resumed onboarding writes a secret-free same-name recreation journal before deletion so an interrupted run can continue or fail closed on ambiguous identity. - Restored snapshot clones retry pairing once only when local approval fails while the authenticated verifier reports a pending scope upgrade. + When the local pairing pass cannot list or approve the clone request, restored snapshot clones retry pairing once only if the authenticated verifier reports a pending scope upgrade. The retry preserves the local-device and single-request bounds. Uninstall checks for the `openshell` command after confirmation and before cleanup mutation, while source-checkout installation preserves an absolute `NEMOCLAW_OPENSHELL_BIN` selection during user-local OpenShell discovery. For more information, refer to [Recover and Rebuild Sandboxes](/user-guide/openclaw/manage-sandboxes/operate-sandboxes/recover-and-rebuild-sandboxes), [Create and Restore Snapshots](/user-guide/openclaw/manage-sandboxes/state-and-backups/create-and-restore-snapshots), and [Uninstall NemoClaw](/user-guide/openclaw/manage-sandboxes/operate-sandboxes/uninstall-nemoclaw). diff --git a/src/lib/actions/sandbox/restore-gateway-pairing.test.ts b/src/lib/actions/sandbox/restore-gateway-pairing.test.ts index ff98e0749d3..7cce7170409 100644 --- a/src/lib/actions/sandbox/restore-gateway-pairing.test.ts +++ b/src/lib/actions/sandbox/restore-gateway-pairing.test.ts @@ -249,6 +249,128 @@ describe("establishRestoredSandboxGatewayPairing", () => { expect(verifyGatewayPairing).toHaveBeenCalledTimes(2); }); + it("retries one failed clone list when verification reports a pending scope upgrade (#7431)", async () => { + const order: string[] = []; + const restartRestoredSandboxGateway = vi.fn(() => order.push("restart")); + const warmupScopeUpgrade = vi.fn(() => order.push("warmup")); + const approveRestoredClonePairing = vi + .fn() + .mockImplementationOnce(() => { + order.push("approve:list-failed"); + return "list-failed" as const; + }) + .mockImplementationOnce(() => { + order.push("approve:succeeded"); + return "approved-one" as const; + }); + const verifyGatewayPairing = vi + .fn() + .mockImplementationOnce(() => { + order.push("verify:pending"); + return { + ok: false as const, + failureLayer: "scope-upgrade-pending" as const, + }; + }) + .mockImplementationOnce(() => { + order.push("verify:authenticated"); + return { ok: true as const }; + }); + + await establishRestoredSandboxGatewayPairing("beta", { + restartRestoredSandboxGateway, + warmupScopeUpgrade, + approveRestoredClonePairing, + verifyGatewayPairing, + }); + + expect(order).toEqual([ + "restart", + "warmup", + "approve:list-failed", + "restart", + "verify:pending", + "restart", + "warmup", + "approve:succeeded", + "restart", + "verify:authenticated", + ]); + expect(restartRestoredSandboxGateway).toHaveBeenCalledTimes(4); + expect(warmupScopeUpgrade).toHaveBeenCalledTimes(2); + expect(approveRestoredClonePairing).toHaveBeenCalledTimes(2); + expect(verifyGatewayPairing).toHaveBeenCalledTimes(2); + }); + + it("retries one failed clone list for a pending scope upgrade, then fails closed (#7431)", async () => { + const order: string[] = []; + const restartRestoredSandboxGateway = vi.fn(() => order.push("restart")); + const warmupScopeUpgrade = vi.fn(() => order.push("warmup")); + const approveRestoredClonePairing = vi.fn(() => { + order.push("approve:list-failed"); + return "list-failed" as const; + }); + const verifyGatewayPairing = vi.fn(() => { + order.push("verify:pending"); + return { + ok: false as const, + failureLayer: "scope-upgrade-pending" as const, + }; + }); + + await expect( + establishRestoredSandboxGatewayPairing("beta", { + restartRestoredSandboxGateway, + warmupScopeUpgrade, + approveRestoredClonePairing, + verifyGatewayPairing, + }), + ).rejects.toThrow( + "authenticated gateway verification run failed (scope-upgrade-pending; approval=list-failed)", + ); + expect(order).toEqual([ + "restart", + "warmup", + "approve:list-failed", + "restart", + "verify:pending", + "restart", + "warmup", + "approve:list-failed", + "restart", + "verify:pending", + ]); + expect(restartRestoredSandboxGateway).toHaveBeenCalledTimes(4); + expect(warmupScopeUpgrade).toHaveBeenCalledTimes(2); + expect(approveRestoredClonePairing).toHaveBeenCalledTimes(2); + expect(verifyGatewayPairing).toHaveBeenCalledTimes(2); + }); + + it("does not retry a failed clone list for an unrelated verifier failure (#7431)", async () => { + const restartRestoredSandboxGateway = vi.fn(); + const warmupScopeUpgrade = vi.fn(); + const approveRestoredClonePairing = vi.fn(() => "list-failed" as const); + const verifyGatewayPairing = vi.fn(() => ({ + ok: false as const, + failureLayer: "gateway-connect-failure" as const, + })); + + await expect( + establishRestoredSandboxGatewayPairing("beta", { + restartRestoredSandboxGateway, + warmupScopeUpgrade, + approveRestoredClonePairing, + verifyGatewayPairing, + }), + ).rejects.toThrow( + "authenticated gateway verification run failed (gateway-connect-failure; approval=list-failed)", + ); + expect(restartRestoredSandboxGateway).toHaveBeenCalledTimes(2); + expect(warmupScopeUpgrade).toHaveBeenCalledOnce(); + expect(approveRestoredClonePairing).toHaveBeenCalledOnce(); + expect(verifyGatewayPairing).toHaveBeenCalledOnce(); + }); + it("does not retry a failed clone approval for an unrelated verifier failure (#7431)", async () => { const restartRestoredSandboxGateway = vi.fn(); const warmupScopeUpgrade = vi.fn(); diff --git a/src/lib/actions/sandbox/restore-gateway-pairing.ts b/src/lib/actions/sandbox/restore-gateway-pairing.ts index e70c1b1ca43..8d0995125e9 100644 --- a/src/lib/actions/sandbox/restore-gateway-pairing.ts +++ b/src/lib/actions/sandbox/restore-gateway-pairing.ts @@ -29,7 +29,7 @@ const RESTORED_CLONE_PAIRING_BUDGET = { timeoutMs: CONNECT_AUTO_PAIR_TIMEOUT_MS, } as const; -const RESTORED_CLONE_APPROVE_FAILURE_ATTEMPTS = 2; +const RESTORED_CLONE_PAIRING_ATTEMPTS = 2; class RestoreGatewayPairingClassifiedError extends Error {} @@ -81,7 +81,7 @@ export async function establishRestoredSandboxGatewayPairing( deps: RestoreGatewayPairingDeps = defaultRestoreGatewayPairingDeps(), ): Promise { try { - for (let attempt = 1; attempt <= RESTORED_CLONE_APPROVE_FAILURE_ATTEMPTS; attempt += 1) { + for (let attempt = 1; attempt <= RESTORED_CLONE_PAIRING_ATTEMPTS; attempt += 1) { deps.restartRestoredSandboxGateway(targetSandbox); deps.warmupScopeUpgrade(targetSandbox); const approvalReceipt = deps.approveRestoredClonePairing(targetSandbox) ?? "exec-failed"; @@ -92,12 +92,12 @@ export async function establishRestoredSandboxGatewayPairing( if (verification.ok) { return; } - // The canonical approval command can create its own local scope-upgrade - // request and report approve-failed. Retry that exact pending transition - // once with the same local-device and one-request bounds. + // The bounded approval pass can fail while listing or approving the + // clone's local request. Retry once only when the authenticated verifier + // independently proves that exact scope-upgrade transition is pending. if ( - attempt < RESTORED_CLONE_APPROVE_FAILURE_ATTEMPTS && - approvalReceipt === "approve-failed" && + attempt < RESTORED_CLONE_PAIRING_ATTEMPTS && + (approvalReceipt === "list-failed" || approvalReceipt === "approve-failed") && verification.failureLayer === "scope-upgrade-pending" ) { continue; From c1634b051934faabb8669ead2cbbf487b1ccb99f Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Tue, 28 Jul 2026 22:21:44 -0700 Subject: [PATCH 05/11] fix(snapshot): use bounded stored auth for clone list Signed-off-by: Charan Jagwani --- docs/changelog/2026-07-28.mdx | 6 +- .../openclaw-2026.7.1-dependency-review.md | 16 +- .../patch-openclaw-device-self-approval.mts | 56 +++++++ .../sandbox/auto-pair-approval.test.ts | 57 ++++++- src/lib/actions/sandbox/auto-pair-approval.ts | 35 +++-- ...claw-device-self-approval-patch-harness.ts | 29 +++- ...penclaw-real-device-self-approval-proof.ts | 79 +++++++++- .../openclaw-device-stored-auth-patch.test.ts | 145 ++++++++++++++++++ 8 files changed, 399 insertions(+), 24 deletions(-) diff --git a/docs/changelog/2026-07-28.mdx b/docs/changelog/2026-07-28.mdx index 5f1f5d75f51..f9da711b1e2 100644 --- a/docs/changelog/2026-07-28.mdx +++ b/docs/changelog/2026-07-28.mdx @@ -32,8 +32,10 @@ It also hardens compatible-provider switching, managed sandbox images, Jetson GP It restores the in-sandbox gateway and host forwards before reporting success, refreshes stale stopped-container evidence, and fails closed when delivery cannot be proven. Recovery also unpauses a paused original container before reporting success. Resumed onboarding writes a secret-free same-name recreation journal before deletion so an interrupted run can continue or fail closed on ambiguous identity. - When the local pairing pass cannot list or approve the clone request, restored snapshot clones retry pairing once only if the authenticated verifier reports a pending scope upgrade. - The retry preserves the local-device and single-request bounds. + Restored snapshot clones now remove the shared gateway URL, port, and token before listing the clone request, then select pairing-scoped stored-device authentication only after an exact local repair preflight. + The selected list path cannot fall back to the shared gateway credential or local pairing-state output. + When that local pairing pass cannot list or approve the request, the clone retries pairing once only if the authenticated verifier reports a pending scope upgrade. + The credential transition and retry preserve the local-device and single-request bounds. Uninstall checks for the `openshell` command after confirmation and before cleanup mutation, while source-checkout installation preserves an absolute `NEMOCLAW_OPENSHELL_BIN` selection during user-local OpenShell discovery. For more information, refer to [Recover and Rebuild Sandboxes](/user-guide/openclaw/manage-sandboxes/operate-sandboxes/recover-and-rebuild-sandboxes), [Create and Restore Snapshots](/user-guide/openclaw/manage-sandboxes/state-and-backups/create-and-restore-snapshots), and [Uninstall NemoClaw](/user-guide/openclaw/manage-sandboxes/operate-sandboxes/uninstall-nemoclaw). - Jetson setup now explains why host preparation was skipped when it encounters an unrecognized or unparseable release. diff --git a/docs/security/openclaw-2026.7.1-dependency-review.md b/docs/security/openclaw-2026.7.1-dependency-review.md index 62b1c17ec8f..ac5fed57364 100644 --- a/docs/security/openclaw-2026.7.1-dependency-review.md +++ b/docs/security/openclaw-2026.7.1-dependency-review.md @@ -250,8 +250,20 @@ stored device token. Once that credential exists, the patch automatically retains CLI identity on ordinary loopback shared-token calls; the upstream local-backend omission remains unchanged. This restores device-scope enforcement without moving the gateway credential into OpenClaw state. After -bootstrap, list calls and every `devices approve` remove the gateway URL, port, -and shared token so the bounded approval flow uses that device credential. +bootstrap, every `devices approve` removes the gateway URL, port, and shared +token so the bounded approval flow uses that device credential. + +Restored-clone approval adds a separate child-only +`NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH=1` marker to its JSON list +subprocess after removing the gateway URL, port, and shared token. The compiled +CLI honors that marker only when OpenClaw's own pairing state contains exactly +one unambiguous same-device CLI repair requesting `operator.write` against a +paired `operator.pairing` baseline. It then performs one live +`device.pair.list` call with pairing-scoped stored-device authentication, +bypassing any `gateway.auth.token` in config. The selected path does not retry +with shared credentials or return local pairing-state output. The host helper +never reads or writes OpenClaw's pending or paired state; OpenClaw uses that +state only to select the narrower authentication mode. ## Gateway Startup Migration Compatibility diff --git a/scripts/patch-openclaw-device-self-approval.mts b/scripts/patch-openclaw-device-self-approval.mts index 13f9f48a352..1d26c318b3d 100644 --- a/scripts/patch-openclaw-device-self-approval.mts +++ b/scripts/patch-openclaw-device-self-approval.mts @@ -46,6 +46,8 @@ const CLI_APPROVE_MARKER = const CLI_SCOPE_MARKER = "nemoclaw: reach gateway for bounded same-device scope approval"; const CLI_RETRY_MARKER = "nemoclaw: keep bounded stored device auth fail closed"; const CLI_LIST_MARKER = "nemoclaw: preflight bounded stored device auth before live pairing list"; +const CLI_STANDALONE_LIST_MARKER = + "nemoclaw: select stored device auth for bounded standalone pairing list"; const CALL_FORCE_IDENTITY_MARKER = "nemoclaw: force device identity for loopback pairing bootstrap"; const CALL_STORED_IDENTITY_MARKER = "nemoclaw: retain stored CLI device identity for loopback shared-token scope enforcement"; @@ -55,6 +57,7 @@ const CLI_APPLIED_MARKERS = [ CLI_SCOPE_MARKER, CLI_RETRY_MARKER, CLI_LIST_MARKER, + CLI_STANDALONE_LIST_MARKER, ] as const; const AUTH_SCOPE_UPGRADE_MARKER = "nemoclaw: route bounded CLI device-token scope upgrade into pairing"; @@ -217,6 +220,35 @@ const CLI_HELPER = [ "\t};", "}", "", + "async function resolveNemoClawStoredDeviceListCallOpts(opts) {", + '\tif (process.env.NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH !== "1" || opts.json !== true || normalizeOptionalString(opts.url) || normalizeOptionalString(opts.token) || normalizeOptionalString(opts.password)) return;', + "\ttry {", + "\t\tconst nemoclawLocalList = await listDevicePairing();", + "\t\tconst nemoclawLocalPending = Array.isArray(nemoclawLocalList.pending) ? nemoclawLocalList.pending : [];", + "\t\tconst nemoclawLocalPaired = Array.isArray(nemoclawLocalList.paired) ? nemoclawLocalList.paired : [];", + "\t\tconst nemoclawCandidates = nemoclawLocalPending.filter((request) => {", + '\t\t\tif (!request || typeof request !== "object" || Array.isArray(request)) return false;', + "\t\t\tconst nemoclawRequestId = normalizeOptionalString(request.requestId);", + "\t\t\tconst nemoclawDeviceId = normalizeOptionalString(request.deviceId);", + "\t\t\tif (!nemoclawRequestId || !nemoclawDeviceId || request.isRepair !== true) return false;", + "\t\t\tif (nemoclawLocalPending.filter((candidate) => normalizeOptionalString(candidate?.requestId) === nemoclawRequestId).length !== 1) return false;", + "\t\t\tconst nemoclawRawScopes = request.scopes;", + '\t\t\tif (!Array.isArray(nemoclawRawScopes) || !nemoclawRawScopes.some((scope) => normalizeOptionalString(scope) === "operator.write")) return false;', + "\t\t\tconst nemoclawPairedMatches = nemoclawLocalPaired.filter((device) => normalizeOptionalString(device?.deviceId) === nemoclawDeviceId);", + "\t\t\tif (nemoclawPairedMatches.length !== 1) return false;", + "\t\t\treturn resolveNemoClawSelfRepairPairingContext(request, nemoclawPairedMatches[0]).useStoredDeviceAuth;", + "\t\t});", + "\t\tif (nemoclawCandidates.length !== 1) return;", + "\t\treturn {", + "\t\t\tscopes: [PAIRING_SCOPE],", + "\t\t\tuseStoredDeviceAuth: true,", + "\t\t\trequiredStoredDeviceAuthScopes: [PAIRING_SCOPE]", + "\t\t};", + "\t} catch {", + "\t\treturn;", + "\t}", + "}", + "", ].join("\n"); const CLI_REPLACEMENT = [ @@ -252,6 +284,21 @@ const CLI_LIST_CALL_TARGET = '\t\treturn parseDevicePairingList(await callGatewayCli("device.pair.list", opts, {}));'; const CLI_LIST_CALL_REPLACEMENT = '\t\treturn parseDevicePairingList(await callGatewayCli("device.pair.list", opts, {}, callOpts));'; +const CLI_STANDALONE_LIST_TARGET = [ + "async function runDevicesListCommand(opts) {", + "\tlet list;", + "\ttry {", + "\t\tlist = await listPairingWithFallback(opts);", +].join("\n"); +const CLI_STANDALONE_LIST_REPLACEMENT = [ + "async function runDevicesListCommand(opts) {", + "\tconst nemoclawListCallOpts = await resolveNemoClawStoredDeviceListCallOpts(opts);", + "\tlet list;", + "\ttry {", + "\t\tlist = nemoclawListCallOpts", + '\t\t\t? parseDevicePairingList(await callGatewayCli("device.pair.list", opts, {}, nemoclawListCallOpts)) // nemoclaw: select stored device auth for bounded standalone pairing list (#4462)', + "\t\t\t: await listPairingWithFallback(opts);", +].join("\n"); const CLI_CONTEXT_TARGET = [ "async function resolveApprovePairingGatewayContext(opts, requestId) {", @@ -891,6 +938,7 @@ const FILE_SPECS: FileSpec[] = [ selector(source) { return ( source.includes("async function approvePairingWithFallback(opts, requestId)") && + source.includes("async function runDevicesListCommand(opts)") && source.includes("function resolveApprovePairingScopesForRequest(request, paired)") && source.includes('callGatewayCli("device.pair.approve"') && CLI_SELECTOR_DEPENDENCIES.every((dependency) => source.includes(dependency)) @@ -950,6 +998,14 @@ const FILE_SPECS: FileSpec[] = [ file, ); if (result.error) return { source, status: "no-match", error: result.error }; + result = replaceExactlyOnce( + result.source, + CLI_STANDALONE_LIST_TARGET, + CLI_STANDALONE_LIST_REPLACEMENT, + "devices CLI bounded standalone list target", + file, + ); + if (result.error) return { source, status: "no-match", error: result.error }; result = replaceExactlyOnce( result.source, CLI_CONTEXT_TARGET, diff --git a/src/lib/actions/sandbox/auto-pair-approval.test.ts b/src/lib/actions/sandbox/auto-pair-approval.test.ts index 5578efbe953..2a33aed40c8 100644 --- a/src/lib/actions/sandbox/auto-pair-approval.test.ts +++ b/src/lib/actions/sandbox/auto-pair-approval.test.ts @@ -41,7 +41,11 @@ describe("buildAutoPairApprovalScript (#4263/#4616)", () => { }); expect(ordinary).not.toContain("local_identity_public_key"); + expect(ordinary).toContain("env=None"); + expect(ordinary).not.toContain("NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH"); expect(restoredClone).toContain("local_identity_public_key"); + expect(restoredClone).toContain("env=local_device_list_env(os.environ)"); + expect(restoredClone).toContain("env['NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH'] = '1'"); expect(restoredClone).toContain("if not related_pending:"); expect(restoredClone).toContain("len(related_pending) > 1"); expect(restoredClone).toContain("pending = related_pending"); @@ -98,6 +102,7 @@ describe("auto-pair approval pass behaviour (#4616)", () => { try { const approvalsFile = path.join(tmpDir, "approvals.log"); const approveEnvFile = path.join(tmpDir, "approve-env.log"); + const listEnvFile = path.join(tmpDir, "list-env.log"); const pending = [ { requestId: "ok-webchat", @@ -149,6 +154,15 @@ describe("auto-pair approval pass behaviour (#4616)", () => { const fs = require("fs"); const args = process.argv.slice(2); if (args[0] === "devices" && args[1] === "list") { + fs.appendFileSync( + ${JSON.stringify(listEnvFile)}, + [ + process.env.OPENCLAW_GATEWAY_URL || "unset", + process.env.OPENCLAW_GATEWAY_PORT || "unset", + process.env.OPENCLAW_GATEWAY_TOKEN || "unset", + process.env.NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH || "unset", + ].join(":") + "\\n", + ); process.stdout.write(${JSON.stringify(`${listResponse}\n`)}); process.exit(0); } @@ -160,6 +174,7 @@ if (args[0] === "devices" && args[1] === "approve") { process.env.OPENCLAW_GATEWAY_URL || "unset", process.env.OPENCLAW_GATEWAY_PORT || "unset", process.env.OPENCLAW_GATEWAY_TOKEN || "unset", + process.env.NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH || "unset", ].join(":") + "\\n", ); process.stdout.write("{}\\n"); @@ -188,10 +203,18 @@ process.exit(2); const approveEnv = fs.existsSync(approveEnvFile) ? fs.readFileSync(approveEnvFile, "utf-8").trim().split("\n").filter(Boolean) : []; + const listEnv = fs.existsSync(listEnvFile) + ? fs.readFileSync(listEnvFile, "utf-8").trim().split("\n").filter(Boolean) + : []; expect(approvals).toEqual(["ok-webchat", "ok-cli", "ok-agent-cli"]); + expect(listEnv).toEqual(["ws://127.0.0.1:18789:18789:secret-token:unset"]); // Gateway env stripped on the approve subprocess (#4462 workaround). - expect(approveEnv).toEqual(["unset:unset:unset", "unset:unset:unset", "unset:unset:unset"]); + expect(approveEnv).toEqual([ + "unset:unset:unset:unset", + "unset:unset:unset:unset", + "unset:unset:unset:unset", + ]); expect(result.stdout).toContain(`${SUMMARY_MARKER}=3`); } finally { fs.rmSync(tmpDir, { recursive: true, force: true }); @@ -201,7 +224,7 @@ process.exit(2); const pyIt = spawnSync("sh", ["-c", "command -v python3"], { stdio: "ignore" }).status === 0 ? it : it.skip; - pyIt("approves only one exact local clone pairing transition on a shared gateway", () => { + const approveOnlyOneLocalClonePairing = () => { const policy = readAutoPairApprovalPolicyModule(); expect(policy).toBeTruthy(); const script = buildAutoPairApprovalScript( @@ -219,6 +242,7 @@ process.exit(2); const identityDir = path.join(stateDir, "identity"); const approvalsFile = path.join(tmpDir, "approvals.log"); const approveEnvFile = path.join(tmpDir, "approve-env.log"); + const listEnvFile = path.join(tmpDir, "list-env.log"); fs.mkdirSync(identityDir, { recursive: true }); const publicKey = "y3vjb9p8tAecivI1l5f1Hdc9QdZJSt3BmLkJMM7wZD8"; const deviceId = "04a4c561c730435e9f6a2e38d2e7b929bcbec2ea1c37d3dd053f3341ecce4e47"; @@ -236,6 +260,23 @@ process.exit(2); const fs = require("fs"); const args = process.argv.slice(2); if (args[0] === "devices" && args[1] === "list") { + fs.appendFileSync( + ${JSON.stringify(listEnvFile)}, + [ + process.env.OPENCLAW_GATEWAY_URL || "unset", + process.env.OPENCLAW_GATEWAY_PORT || "unset", + process.env.OPENCLAW_GATEWAY_TOKEN || "unset", + process.env.NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH || "unset", + ].join(":") + "\\n", + ); + if ( + process.env.OPENCLAW_GATEWAY_URL || + process.env.OPENCLAW_GATEWAY_PORT || + process.env.OPENCLAW_GATEWAY_TOKEN + ) { + process.stderr.write("restored clone list retained shared gateway credentials\\n"); + process.exit(3); + } process.stdout.write(process.env.NEMOCLAW_LIST_RESPONSE + "\\n"); process.exit(0); } @@ -251,6 +292,7 @@ if (args[0] === "devices" && args[1] === "approve") { process.env.OPENCLAW_GATEWAY_URL || "unset", process.env.OPENCLAW_GATEWAY_PORT || "unset", process.env.OPENCLAW_GATEWAY_TOKEN || "unset", + process.env.NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH || "unset", ].join(":") + "\\n", ); process.stdout.write("{}\\n"); @@ -286,6 +328,7 @@ process.exit(2); const resetLogs = () => { fs.rmSync(approvalsFile, { force: true }); fs.rmSync(approveEnvFile, { force: true }); + fs.rmSync(listEnvFile, { force: true }); }; const localRequest = { requestId: "clone-pairing", @@ -309,7 +352,8 @@ process.exit(2); expect(initial.stdout).toContain(`${SUMMARY_MARKER}=1`); expect(initial.stdout).toContain(`${RECEIPT_MARKER}=approved-one`); expect(readApprovals()).toEqual(["clone-pairing"]); - expect(fs.readFileSync(approveEnvFile, "utf-8").trim()).toBe("unset:unset:unset"); + expect(fs.readFileSync(listEnvFile, "utf-8").trim()).toBe("unset:unset:unset:1"); + expect(fs.readFileSync(approveEnvFile, "utf-8").trim()).toBe("unset:unset:unset:unset"); resetLogs(); const repairRequest = { @@ -403,7 +447,12 @@ process.exit(2); } finally { fs.rmSync(tmpDir, { recursive: true, force: true }); } - }); + }; + pyIt( + "approves only one exact local clone pairing transition on a shared gateway", + approveOnlyOneLocalClonePairing, + 30_000, + ); it("leaves a failed compatibility-shaped approval retryable without editing device state", () => { if (spawnSync("sh", ["-c", "command -v python3"], { stdio: "ignore" }).status !== 0) { diff --git a/src/lib/actions/sandbox/auto-pair-approval.ts b/src/lib/actions/sandbox/auto-pair-approval.ts index 357b990aaf5..41943cb12a8 100644 --- a/src/lib/actions/sandbox/auto-pair-approval.ts +++ b/src/lib/actions/sandbox/auto-pair-approval.ts @@ -26,14 +26,16 @@ * semantics. In the reviewed OpenClaw 2026.6.10, a gateway-pinned * `devices approve` for a scope-upgrade can request the upgraded scopes for * its own connection and return the pending-scope failure it is trying to - * resolve. The sourced runtime environment makes the list call inspect the - * same live gateway through local loopback, while the approval call also - * strips OPENCLAW_GATEWAY_URL/PORT/TOKEN from the child env. The reviewed dist - * patch then forces OpenClaw's existing local-only stored-device-auth path for - * the exact bounded self-repair shape so a shared token reloaded from config - * cannot take precedence. Remove this compatibility path when OpenClaw can - * complete scope upgrades natively through device-token auth using - * operator.pairing. + * resolve. The sourced runtime environment makes an ordinary recovery list + * call inspect the same live gateway through local loopback. A restored clone + * is already post-bootstrap, so its list call and every approval call strip + * OPENCLAW_GATEWAY_URL/PORT/TOKEN from the child env. A clone-only marker lets + * the reviewed dist patch select stored-device auth for the list only after an + * exact local repair preflight. The approval command independently forces that + * same local-only stored-device-auth path for the exact bounded self-repair + * shape, so a shared token reloaded from config cannot take precedence. Remove + * this compatibility path when OpenClaw can complete scope upgrades natively + * through device-token auth using operator.pairing. */ import { spawnSync } from "node:child_process"; @@ -165,6 +167,20 @@ def exit_with_receipt(receipt): const maxApprovals = options.budget?.maxApprovals ?? AUTO_PAIR_MAX_APPROVALS; const listTimeoutS = options.budget?.listTimeoutS ?? AUTO_PAIR_LIST_TIMEOUT_S; const approveTimeoutS = options.budget?.approveTimeoutS ?? AUTO_PAIR_APPROVE_TIMEOUT_S; + // A restored clone already has post-bootstrap device identity. Match the + // startup watcher's post-bootstrap list path by dropping the explicit shared + // gateway credential triplet, then privately request the patched bounded + // stored-device list path. Ordinary connect/doctor recovery keeps its + // existing bootstrap-capable list environment. + const listEnvPrelude = options.localDeviceOnly + ? ` +def local_device_list_env(source_env): + env = gateway_approval_env(source_env) + env['NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH'] = '1' + return env +` + : ""; + const listEnv = options.localDeviceOnly ? "local_device_list_env(os.environ)" : "None"; const localDeviceFilter = options.localDeviceOnly ? ` # Snapshot restore shares one gateway across the source sandbox and its clone. @@ -325,6 +341,7 @@ try: gateway_approval_env = policy_globals['gateway_approval_env'] except Exception: ${exitWithReceipt("policy-missing")} +${listEnvPrelude} OPENCLAW = os.environ.get('OPENCLAW_BIN', 'openclaw') MAX_APPROVALS = ${maxApprovals} @@ -332,7 +349,7 @@ MAX_APPROVALS = ${maxApprovals} try: proc = subprocess.run( [OPENCLAW, 'devices', 'list', '--json'], - capture_output=True, text=True, timeout=${listTimeoutS}, + capture_output=True, text=True, timeout=${listTimeoutS}, env=${listEnv}, ) except (subprocess.TimeoutExpired, FileNotFoundError, OSError): ${exitWithReceipt("list-failed")} diff --git a/test/helpers/openclaw-device-self-approval-patch-harness.ts b/test/helpers/openclaw-device-self-approval-patch-harness.ts index 9b75450522f..5528bfe991f 100644 --- a/test/helpers/openclaw-device-self-approval-patch-harness.ts +++ b/test/helpers/openclaw-device-self-approval-patch-harness.ts @@ -49,19 +49,32 @@ const OPERATOR_ROLE = "operator"; const GATEWAY_CLIENT_NAMES = { CLI: "cli" }; const GATEWAY_CLIENT_MODES = { CLI: "cli" }; const KNOWN_NON_ADMIN_OPERATOR_SCOPES = new Set(["operator.pairing", "operator.read", "operator.write"]); +const process = { env: {} }; const gatewayCalls = []; +const runtimeJson = []; let pairingList = { pending: [], paired: [] }; let localPairingList = { pending: [], paired: [] }; +let localPairingFailure; +let listFailures = []; let approvalFailures = []; function setPairingLists(localList, liveList = localList) { localPairingList = localList; pairingList = liveList; } +function setStoredDeviceListMarker(value) { + if (value) process.env.NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH = "1"; + else delete process.env.NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH; +} +function setLocalPairingFailure(value) { localPairingFailure = value; } +function setListFailures(errors) { listFailures = errors; } function withProgress(_options, callback) { return callback(); } function parseTimeoutMsWithFallback(value, fallback) { return value ?? fallback; } async function callGateway(options) { gatewayCalls.push(options); - if (options.method === "device.pair.list") return pairingList; + if (options.method === "device.pair.list") { + if (listFailures.length > 0) throw listFailures.shift(); + return pairingList; + } if (options.method === "device.pair.approve" && approvalFailures.length > 0) { throw approvalFailures.shift(); } @@ -132,6 +145,7 @@ function lookupPairedDevice(pairedByDeviceId, request) { return pairedByDeviceId.get(normalizeOptionalString(request.deviceId)); } async function listDevicePairing() { + if (localPairingFailure) throw localPairingFailure; return localPairingList; } async function listPairingWithFallback(opts) { @@ -141,6 +155,19 @@ async function listPairingWithFallback(opts) { throw error; } } +const defaultRuntime = { writeJson(value) { runtimeJson.push(value); } }; +async function runDevicesListCommand(opts) { + let list; + try { + list = await listPairingWithFallback(opts); + } catch (error) { + throw error; + } + if (opts.json) { + defaultRuntime.writeJson(list); + return; + } +} function resolveApprovePairingScopesForRequest(request, paired) { const operatorScopes = resolvePendingOperatorApprovalScopes(request, paired); if (operatorScopes.length === 0) return; diff --git a/test/helpers/openclaw-real-device-self-approval-proof.ts b/test/helpers/openclaw-real-device-self-approval-proof.ts index ed874deb97f..5f63280e1ec 100644 --- a/test/helpers/openclaw-real-device-self-approval-proof.ts +++ b/test/helpers/openclaw-real-device-self-approval-proof.ts @@ -224,6 +224,20 @@ function requireRealStoredDeviceAuthLinkage(sources: DistSource[], cliSource: Di ], "devices CLI bounded pairing-list preflight", ); + requireOrderedMarkers( + cliSource.source, + [ + "async function resolveNemoClawStoredDeviceListCallOpts(opts)", + "NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH", + "nemoclawCandidates.length !== 1", + "useStoredDeviceAuth: true", + "async function runDevicesListCommand(opts)", + "await resolveNemoClawStoredDeviceListCallOpts(opts)", + 'callGatewayCli("device.pair.list", opts, {}, nemoclawListCallOpts)', + "nemoclaw: select stored device auth for bounded standalone pairing list", + ], + "devices CLI bounded standalone stored-auth list", + ); requireOrderedMarkers( cliSource.source, [ @@ -978,12 +992,14 @@ async function runLiveStoredDeviceAuthSelfApprovalProof(options: ProofOptions): OPENCLAW_SKIP_CHANNELS: "1", OPENCLAW_SKIP_PROVIDERS: "1", OPENCLAW_STATE_DIR: stateDir, + // The child inherits VITEST=true, which otherwise suppresses real CLI JSON. + OPENCLAW_TEST_RUNTIME_LOG: "1", }; - const runCli = (args: string[]) => + const runCli = (args: string[], cliEnv: NodeJS.ProcessEnv = env) => spawnSync(options.nodeExecutable, [openclawEntry, ...args], { cwd: packageDir, encoding: "utf8", - env, + env: cliEnv, timeout: Math.min(options.timeoutMs, 60_000), }); @@ -1043,7 +1059,7 @@ async function runLiveStoredDeviceAuthSelfApprovalProof(options: ProofOptions): ); await stopChild(gateway); - writeGatewayConfig({ mode: "token" }); + writeGatewayConfig({ mode: "token", token: gatewayToken }); gateway = startGateway({ ...env, OPENCLAW_GATEWAY_TOKEN: gatewayToken }, true); await waitForGatewayReady(gateway, port, options.timeoutMs); @@ -1101,8 +1117,59 @@ async function runLiveStoredDeviceAuthSelfApprovalProof(options: ProofOptions): const configuredGateway = asRecord(configuredBeforeApproval.gateway); const configuredAuth = asRecord(configuredGateway?.auth); requireLiveProof( - configuredAuth?.mode === "token" && configuredAuth.token === undefined, - "gateway token auth was not isolated from the stored-device-auth client", + configuredAuth?.mode === "token" && configuredAuth.token === gatewayToken, + "production-shaped gateway token auth configuration missing", + ); + + const markedListEnv = { + ...env, + NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH: "1", + }; + const hiddenDeviceAuthPath = `${deviceAuthPath}.hidden`; + fs.renameSync(deviceAuthPath, hiddenDeviceAuthPath); + try { + const listWithoutStoredAuth = runCli(["devices", "list", "--json"], markedListEnv); + requireLiveProof( + listWithoutStoredAuth.status !== 0, + "marked device list fell back to the configured shared token without stored device auth", + ); + } finally { + fs.renameSync(hiddenDeviceAuthPath, deviceAuthPath); + } + + const strippedList = runCli(["devices", "list", "--json"], markedListEnv); + requireSuccess( + strippedList, + "list the exact real same-device repair with the stripped post-bootstrap client", + ); + const strippedListValue: unknown = JSON.parse(String(strippedList.stdout)); + const strippedListObject = asRecord(strippedListValue); + requireLiveProof(strippedListObject, "stripped post-bootstrap device list was not an object"); + requireLiveProof( + Array.isArray(strippedListObject.pending) && Array.isArray(strippedListObject.paired), + "stripped post-bootstrap device list did not contain pending and paired arrays", + ); + const strippedPending = strippedListObject.pending + .map(asRecord) + .filter((request) => request !== null); + requireLiveProof( + strippedPending.length === 1, + `stripped post-bootstrap device list returned ${strippedPending.length} pending requests`, + ); + const listedRepair = strippedPending[0] as Record; + requireLiveProof( + listedRepair.requestId === repair.requestId && + listedRepair.deviceId === repair.deviceId && + listedRepair.publicKey === repair.publicKey && + listedRepair.clientId === repair.clientId && + listedRepair.clientMode === repair.clientMode && + listedRepair.isRepair === repair.isRepair, + "stripped post-bootstrap device list did not return the exact same-device repair", + ); + requireExactScopes( + listedRepair.scopes, + ["operator.write"], + "stripped post-bootstrap same-device repair scopes", ); const approval = runCli(["devices", "approve", String(repair.requestId), "--json"]); @@ -1152,7 +1219,7 @@ async function runLiveStoredDeviceAuthSelfApprovalProof(options: ProofOptions): const configuredGatewayAfter = asRecord(configuredAfterApproval.gateway); const configuredAuthAfter = asRecord(configuredGatewayAfter?.auth); requireLiveProof( - configuredAuthAfter?.mode === "token" && configuredAuthAfter.token === undefined, + configuredAuthAfter?.mode === "token" && configuredAuthAfter.token === gatewayToken, "gateway token auth configuration changed during stored-device-auth approval", ); } catch (error) { diff --git a/test/openclaw-device-stored-auth-patch.test.ts b/test/openclaw-device-stored-auth-patch.test.ts index 7ffbf7fd39d..39354a32061 100644 --- a/test/openclaw-device-stored-auth-patch.test.ts +++ b/test/openclaw-device-stored-auth-patch.test.ts @@ -16,6 +16,151 @@ import { } from "./helpers/openclaw-device-self-approval-patch-harness"; describe("OpenClaw bounded stored-device-auth selection (#4462)", () => { + it("uses pairing-scoped stored auth only for a marked exact repair list", async () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-device-cli-marked-list-")); + const dist = path.join(tmp, "dist"); + fs.mkdirSync(dist); + writeFixtureDist(dist); + try { + expect(runPatch(dist).status).toBe(0); + const source = fs.readFileSync(path.join(dist, "devices-cli.runtime-fixture.js"), "utf8"); + const runtime = runFixture<{ + calls: Array>; + output: Array>; + list: (opts: Record) => Promise; + setListFailures: (errors: Error[]) => void; + setMarker: (value: boolean) => void; + setPairingLists: (local: Record, live?: Record) => void; + }>( + source, + `({ + calls: gatewayCalls, + output: runtimeJson, + list: runDevicesListCommand, + setListFailures, + setMarker: setStoredDeviceListMarker, + setPairingLists, + })`, + ); + const exactList = { pending: [validPending()], paired: [validPaired()] }; + runtime.setPairingLists(exactList); + + await runtime.list({ json: true }); + expect(runtime.calls).toHaveLength(1); + expect(runtime.calls[0]).not.toHaveProperty("useStoredDeviceAuth"); + expect(runtime.output).toEqual([exactList]); + + runtime.calls.length = 0; + runtime.output.length = 0; + runtime.setMarker(true); + await runtime.list({ json: true }); + expect(runtime.calls).toHaveLength(1); + expect(runtime.calls[0]).toMatchObject({ + method: "device.pair.list", + scopes: ["operator.pairing"], + useStoredDeviceAuth: true, + requiredStoredDeviceAuthScopes: ["operator.pairing"], + }); + expect(runtime.output).toEqual([exactList]); + + runtime.calls.length = 0; + runtime.output.length = 0; + runtime.setListFailures([new Error("stored device list denied")]); + await expect(runtime.list({ json: true })).rejects.toThrow("stored device list denied"); + expect(runtime.calls).toHaveLength(1); + expect(runtime.calls[0]).toMatchObject({ useStoredDeviceAuth: true }); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + + it("fails closed to the ordinary list path when marked local repair evidence is unsafe", async () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-device-cli-list-bounds-")); + const dist = path.join(tmp, "dist"); + fs.mkdirSync(dist); + writeFixtureDist(dist); + try { + expect(runPatch(dist).status).toBe(0); + const source = fs.readFileSync(path.join(dist, "devices-cli.runtime-fixture.js"), "utf8"); + const runtime = runFixture<{ + resolve: (opts: Record) => Promise | undefined>; + setFailure: (value: Error | undefined) => void; + setMarker: (value: boolean) => void; + setPairingLists: (local: Record, live?: Record) => void; + }>( + source, + `({ + resolve: resolveNemoClawStoredDeviceListCallOpts, + setFailure: setLocalPairingFailure, + setMarker: setStoredDeviceListMarker, + setPairingLists, + })`, + ); + const exactList = { pending: [validPending()], paired: [validPaired()] }; + + runtime.setPairingLists(exactList); + expect(await runtime.resolve({ json: true })).toBeUndefined(); + runtime.setMarker(true); + expect(await runtime.resolve({ json: false })).toBeUndefined(); + for (const explicit of [ + { url: "ws://127.0.0.1:18789" }, + { token: "explicit-token" }, + { password: "explicit-password" }, + ]) { + expect(await runtime.resolve({ json: true, ...explicit })).toBeUndefined(); + } + expect(await runtime.resolve({ json: true })).toEqual({ + scopes: ["operator.pairing"], + useStoredDeviceAuth: true, + requiredStoredDeviceAuthScopes: ["operator.pairing"], + }); + + const unsafeLists = [ + { pending: null, paired: null }, + { pending: [], paired: [validPaired()] }, + { pending: [validPending({ requestId: " " })], paired: [validPaired()] }, + { pending: [validPending()], paired: [] }, + { pending: [validPending({ publicKey: "other-key" })], paired: [validPaired()] }, + { pending: [validPending({ clientId: "openclaw-control-ui" })], paired: [validPaired()] }, + { pending: [validPending({ clientMode: "webchat" })], paired: [validPaired()] }, + { pending: [validPending({ role: "node", roles: ["node"] })], paired: [validPaired()] }, + { + pending: [validPending({ roles: ["operator", "node"] })], + paired: [validPaired()], + }, + { pending: [validPending({ scopes: [] })], paired: [validPaired()] }, + { pending: [validPending({ scopes: ["operator.pairing"] })], paired: [validPaired()] }, + { pending: [validPending({ scopes: ["operator.admin"] })], paired: [validPaired()] }, + { pending: [validPending({ scopes: ["operator.unknown"] })], paired: [validPaired()] }, + { + pending: [validPending({ scopes: ["operator.write", "operator.write"] })], + paired: [validPaired()], + }, + { pending: [validPending({ isRepair: false })], paired: [validPaired()] }, + { + pending: [validPending(), validPending({ requestId: "request-2" })], + paired: [validPaired()], + }, + { + pending: [validPending(), validPending({ deviceId: "device-2" })], + paired: [validPaired()], + }, + { pending: [validPending()], paired: [validPaired(), validPaired()] }, + ]; + for (const unsafeList of unsafeLists) { + runtime.setPairingLists(unsafeList); + expect(await runtime.resolve({ json: true })).toBeUndefined(); + } + + runtime.setPairingLists(exactList); + runtime.setFailure(new Error("local pairing state unreadable")); + expect(await runtime.resolve({ json: true })).toBeUndefined(); + runtime.setFailure(undefined); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + it("forwards stored device auth only for exact same-device transitions", async () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-device-cli-stored-auth-")); const dist = path.join(tmp, "dist"); From 8f48594f8748af83cb6f394272b1b7fba1aac017 Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Tue, 28 Jul 2026 23:16:32 -0700 Subject: [PATCH 06/11] fix(snapshot): accept paired pre-convergence list Signed-off-by: Charan Jagwani --- docs/changelog/2026-07-28.mdx | 2 +- .../openclaw-2026.7.1-dependency-review.md | 6 ++- .../patch-openclaw-device-self-approval.mts | 2 +- ...penclaw-real-device-self-approval-proof.ts | 6 +++ .../openclaw-device-stored-auth-patch.test.ts | 41 +++++++++++-------- 5 files changed, 37 insertions(+), 20 deletions(-) diff --git a/docs/changelog/2026-07-28.mdx b/docs/changelog/2026-07-28.mdx index f9da711b1e2..a33a648a9e6 100644 --- a/docs/changelog/2026-07-28.mdx +++ b/docs/changelog/2026-07-28.mdx @@ -32,7 +32,7 @@ It also hardens compatible-provider switching, managed sandbox images, Jetson GP It restores the in-sandbox gateway and host forwards before reporting success, refreshes stale stopped-container evidence, and fails closed when delivery cannot be proven. Recovery also unpauses a paused original container before reporting success. Resumed onboarding writes a secret-free same-name recreation journal before deletion so an interrupted run can continue or fail closed on ambiguous identity. - Restored snapshot clones now remove the shared gateway URL, port, and token before listing the clone request, then select pairing-scoped stored-device authentication only after an exact local repair preflight. + Restored snapshot clones now remove the shared gateway URL, port, and token before listing the clone request, then select pairing-scoped stored-device authentication only after an exact local paired scope-transition preflight. The selected list path cannot fall back to the shared gateway credential or local pairing-state output. When that local pairing pass cannot list or approve the request, the clone retries pairing once only if the authenticated verifier reports a pending scope upgrade. The credential transition and retry preserve the local-device and single-request bounds. diff --git a/docs/security/openclaw-2026.7.1-dependency-review.md b/docs/security/openclaw-2026.7.1-dependency-review.md index ac5fed57364..004e50211be 100644 --- a/docs/security/openclaw-2026.7.1-dependency-review.md +++ b/docs/security/openclaw-2026.7.1-dependency-review.md @@ -257,8 +257,10 @@ Restored-clone approval adds a separate child-only `NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH=1` marker to its JSON list subprocess after removing the gateway URL, port, and shared token. The compiled CLI honors that marker only when OpenClaw's own pairing state contains exactly -one unambiguous same-device CLI repair requesting `operator.write` against a -paired `operator.pairing` baseline. It then performs one live +one unambiguous same-device CLI transition requesting `operator.write` against +a paired `operator.pairing` baseline. The accepted transition can be an +explicit repair or the paired pre-convergence form with `isRepair: false`. +It then performs one live `device.pair.list` call with pairing-scoped stored-device authentication, bypassing any `gateway.auth.token` in config. The selected path does not retry with shared credentials or return local pairing-state output. The host helper diff --git a/scripts/patch-openclaw-device-self-approval.mts b/scripts/patch-openclaw-device-self-approval.mts index 1d26c318b3d..cbb278554a7 100644 --- a/scripts/patch-openclaw-device-self-approval.mts +++ b/scripts/patch-openclaw-device-self-approval.mts @@ -230,7 +230,7 @@ const CLI_HELPER = [ '\t\t\tif (!request || typeof request !== "object" || Array.isArray(request)) return false;', "\t\t\tconst nemoclawRequestId = normalizeOptionalString(request.requestId);", "\t\t\tconst nemoclawDeviceId = normalizeOptionalString(request.deviceId);", - "\t\t\tif (!nemoclawRequestId || !nemoclawDeviceId || request.isRepair !== true) return false;", + "\t\t\tif (!nemoclawRequestId || !nemoclawDeviceId) return false;", "\t\t\tif (nemoclawLocalPending.filter((candidate) => normalizeOptionalString(candidate?.requestId) === nemoclawRequestId).length !== 1) return false;", "\t\t\tconst nemoclawRawScopes = request.scopes;", '\t\t\tif (!Array.isArray(nemoclawRawScopes) || !nemoclawRawScopes.some((scope) => normalizeOptionalString(scope) === "operator.write")) return false;', diff --git a/test/helpers/openclaw-real-device-self-approval-proof.ts b/test/helpers/openclaw-real-device-self-approval-proof.ts index 5f63280e1ec..8900ae3b83f 100644 --- a/test/helpers/openclaw-real-device-self-approval-proof.ts +++ b/test/helpers/openclaw-real-device-self-approval-proof.ts @@ -1113,6 +1113,12 @@ async function runLiveStoredDeviceAuthSelfApprovalProof(options: ProofOptions): typeof repair.requestId === "string" && repair.requestId.length > 0, "real same-device repair request id missing", ); + // Snapshot clone startup can retain the initial isRepair=false request + // after the pairing-only approval creates the matching paired baseline. + // Exercise that production ordering through the real CLI/gateway path, + // not only the standalone classifier proof below. + repair.isRepair = false; + fs.writeFileSync(pendingPath, JSON.stringify(pending)); const configuredBeforeApproval = readJsonObject(configPath, "real gateway config"); const configuredGateway = asRecord(configuredBeforeApproval.gateway); const configuredAuth = asRecord(configuredGateway?.auth); diff --git a/test/openclaw-device-stored-auth-patch.test.ts b/test/openclaw-device-stored-auth-patch.test.ts index 39354a32061..21e04cc1784 100644 --- a/test/openclaw-device-stored-auth-patch.test.ts +++ b/test/openclaw-device-stored-auth-patch.test.ts @@ -16,7 +16,7 @@ import { } from "./helpers/openclaw-device-self-approval-patch-harness"; describe("OpenClaw bounded stored-device-auth selection (#4462)", () => { - it("uses pairing-scoped stored auth only for a marked exact repair list", async () => { + it("uses pairing-scoped stored auth only for a marked exact same-device write transition", async () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-device-cli-marked-list-")); const dist = path.join(tmp, "dist"); fs.mkdirSync(dist); @@ -42,26 +42,35 @@ describe("OpenClaw bounded stored-device-auth selection (#4462)", () => { setPairingLists, })`, ); - const exactList = { pending: [validPending()], paired: [validPaired()] }; - runtime.setPairingLists(exactList); + const exactRepairList = { pending: [validPending()], paired: [validPaired()] }; + runtime.setPairingLists(exactRepairList); await runtime.list({ json: true }); expect(runtime.calls).toHaveLength(1); expect(runtime.calls[0]).not.toHaveProperty("useStoredDeviceAuth"); - expect(runtime.output).toEqual([exactList]); + expect(runtime.output).toEqual([exactRepairList]); - runtime.calls.length = 0; - runtime.output.length = 0; runtime.setMarker(true); - await runtime.list({ json: true }); - expect(runtime.calls).toHaveLength(1); - expect(runtime.calls[0]).toMatchObject({ - method: "device.pair.list", - scopes: ["operator.pairing"], - useStoredDeviceAuth: true, - requiredStoredDeviceAuthScopes: ["operator.pairing"], - }); - expect(runtime.output).toEqual([exactList]); + for (const exactList of [ + exactRepairList, + { + pending: [validPending({ isRepair: false })], + paired: [validPaired()], + }, + ]) { + runtime.calls.length = 0; + runtime.output.length = 0; + runtime.setPairingLists(exactList); + await runtime.list({ json: true }); + expect(runtime.calls).toHaveLength(1); + expect(runtime.calls[0]).toMatchObject({ + method: "device.pair.list", + scopes: ["operator.pairing"], + useStoredDeviceAuth: true, + requiredStoredDeviceAuthScopes: ["operator.pairing"], + }); + expect(runtime.output).toEqual([exactList]); + } runtime.calls.length = 0; runtime.output.length = 0; @@ -136,7 +145,7 @@ describe("OpenClaw bounded stored-device-auth selection (#4462)", () => { pending: [validPending({ scopes: ["operator.write", "operator.write"] })], paired: [validPaired()], }, - { pending: [validPending({ isRepair: false })], paired: [validPaired()] }, + { pending: [validPending({ isRepair: false })], paired: [] }, { pending: [validPending(), validPending({ requestId: "request-2" })], paired: [validPaired()], From 07b721137db55425546f6e028ba7691c5a89ed51 Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Tue, 28 Jul 2026 23:40:51 -0700 Subject: [PATCH 07/11] chore(ci): retry credentialed E2E gate Signed-off-by: Charan Jagwani From 6046ffd21a93418e21079ccac2737bf3159d8d0e Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Tue, 28 Jul 2026 23:55:04 -0700 Subject: [PATCH 08/11] chore(ci): renew credentialed E2E gate Signed-off-by: Charan Jagwani From ea7d6130e77d91286b8ef29700345de946cc0503 Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Wed, 29 Jul 2026 00:11:57 -0700 Subject: [PATCH 09/11] chore(ci): queue maintainer E2E fallback Signed-off-by: Charan Jagwani From 6fd2332818546bb242f0d38b1764ed2a95b42cfc Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Wed, 29 Jul 2026 01:35:24 -0700 Subject: [PATCH 10/11] fix(snapshot): converge restored clone pairing auth Signed-off-by: Charan Jagwani --- docs/changelog/2026-07-28.mdx | 9 +- .../openclaw-2026.7.1-dependency-review.md | 60 +++++++--- .../patch-openclaw-device-self-approval.mts | 16 ++- .../sandbox/auto-pair-approval.test.ts | 94 +++++++++++++-- src/lib/actions/sandbox/auto-pair-approval.ts | 108 ++++++++++++++---- .../sandbox/connect-autopair-budget.test.ts | 4 + .../sandbox/connect-autopair-budget.ts | 12 +- .../sandbox/restore-gateway-pairing.test.ts | 35 ++++-- .../sandbox/restore-gateway-pairing.ts | 21 +++- ...claw-device-self-approval-patch-harness.ts | 4 + ...penclaw-real-device-self-approval-proof.ts | 83 +++++++++++--- .../openclaw-device-stored-auth-patch.test.ts | 59 ++++++++++ .../auto-pair-approval.test.ts | 6 +- 13 files changed, 427 insertions(+), 84 deletions(-) diff --git a/docs/changelog/2026-07-28.mdx b/docs/changelog/2026-07-28.mdx index a33a648a9e6..341f9e96170 100644 --- a/docs/changelog/2026-07-28.mdx +++ b/docs/changelog/2026-07-28.mdx @@ -32,8 +32,13 @@ It also hardens compatible-provider switching, managed sandbox images, Jetson GP It restores the in-sandbox gateway and host forwards before reporting success, refreshes stale stopped-container evidence, and fails closed when delivery cannot be proven. Recovery also unpauses a paused original container before reporting success. Resumed onboarding writes a secret-free same-name recreation journal before deletion so an interrupted run can continue or fail closed on ambiguous identity. - Restored snapshot clones now remove the shared gateway URL, port, and token before listing the clone request, then select pairing-scoped stored-device authentication only after an exact local paired scope-transition preflight. - The selected list path cannot fall back to the shared gateway credential or local pairing-state output. + Restored snapshot clones now run one list-only, forced-identity handshake with the shared gateway credential to request and store the clone's `operator.pairing` device credential. + That handshake cannot approve a request. + NemoClaw then removes the shared gateway URL, port, and token before an exact stored-device-auth list and the canonical approval of one matching local scope transition. + The stored-auth list cannot fall back to the shared gateway credential or local pairing-state output, and the approval command independently uses the same stripped credential boundary. + If the exact stored-auth evidence changes between listing and approval, a child-only guard stops the approval before OpenClaw can use its ordinary gateway or `operator.admin` fallback. + Both list subprocesses have 15-second deadlines, and each clone approval pass has a 45-second outer bound. + Fixed output-free receipts classify failures without returning command output or request identifiers. When that local pairing pass cannot list or approve the request, the clone retries pairing once only if the authenticated verifier reports a pending scope upgrade. The credential transition and retry preserve the local-device and single-request bounds. Uninstall checks for the `openshell` command after confirmation and before cleanup mutation, while source-checkout installation preserves an absolute `NEMOCLAW_OPENSHELL_BIN` selection during user-local OpenShell discovery. diff --git a/docs/security/openclaw-2026.7.1-dependency-review.md b/docs/security/openclaw-2026.7.1-dependency-review.md index 004e50211be..ee7367bfab8 100644 --- a/docs/security/openclaw-2026.7.1-dependency-review.md +++ b/docs/security/openclaw-2026.7.1-dependency-review.md @@ -253,19 +253,53 @@ enforcement without moving the gateway credential into OpenClaw state. After bootstrap, every `devices approve` removes the gateway URL, port, and shared token so the bounded approval flow uses that device credential. -Restored-clone approval adds a separate child-only -`NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH=1` marker to its JSON list -subprocess after removing the gateway URL, port, and shared token. The compiled -CLI honors that marker only when OpenClaw's own pairing state contains exactly -one unambiguous same-device CLI transition requesting `operator.write` against -a paired `operator.pairing` baseline. The accepted transition can be an -explicit repair or the paired pre-convergence form with `isRepair: false`. -It then performs one live -`device.pair.list` call with pairing-scoped stored-device authentication, -bypassing any `gateway.auth.token` in config. The selected path does not retry -with shared credentials or return local pairing-state output. The host helper -never reads or writes OpenClaw's pending or paired state; OpenClaw uses that -state only to select the narrower authentication mode. +Restored-clone approval first runs a list-only CLI subprocess with the shared +gateway URL, port, and token plus the child-only +`NEMOCLAW_OPENCLAW_FORCE_DEVICE_PAIRING=1` marker. The marker preserves the +clone's device identity for this loopback shared-token handshake. OpenClaw +requests `operator.pairing`, receives the canonical server-issued device token, +and stores that token in its private device-auth store. The subprocess only +invokes `devices list`; it cannot approve or select a pending request. NemoClaw +uses the response only to confirm that it is valid JSON with a pending-request +array. No request from this shared-auth response is selected or approved. + +NemoClaw then removes the shared gateway URL, port, and token and clears the +forced-identity marker before adding the separate child-only +`NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH=1` marker to the next JSON list +subprocess. The compiled CLI honors that marker only when OpenClaw's own pairing +state contains exactly one unambiguous same-device CLI transition requesting +`operator.write` against a paired `operator.pairing` baseline. The accepted +transition can be an explicit repair or the paired pre-convergence form with +`isRepair: false`. It then performs one live `device.pair.list` call with +pairing-scoped stored-device authentication, bypassing any +`gateway.auth.token` in config. The host helper independently validates the +listed request against the clone's device identity, public key, client, role, +and bounded scopes before it invokes OpenClaw's canonical approval command with +the shared credential triplet removed. The selected list never retries with +shared credentials or returns local pairing-state output. + +For the separate approval subprocess, NemoClaw keeps the shared credential +triplet removed and adds the child-only +`NEMOCLAW_OPENCLAW_REQUIRE_STORED_DEVICE_APPROVAL=1` marker. The patched CLI +rechecks the exact local request, paired baseline, and stored-device +authentication context before it performs the live list and approval. If that +evidence is missing, changes, or cannot authenticate between the host's list +and the approval, OpenClaw aborts before `device.pair.approve` and before its +ordinary gateway or `operator.admin` fallback. The marker does not affect +ordinary approval callers. The host helper never reads or writes OpenClaw's +pending or paired state; OpenClaw uses that state only to select the narrower +authentication mode and remains the only pairing-state writer. + +Each list subprocess has a 15-second external deadline, which exceeds +OpenClaw's internal 10-second device-list deadline. The single approval attempt +has a 10-second deadline, and the restored-clone pass has a 45-second outer +bound that preserves five seconds for shell and Python startup. Fixed, +output-free receipts distinguish credential-list timeout or failure, +stored-list timeout, execution failure, nonzero exit, empty output, invalid +output, selector rejection or ambiguity, approval failure, and success. They +never include child command output or request identifiers. NemoClaw retries the +pass once only when its authenticated verifier independently reports that the +scope upgrade is still pending. ## Gateway Startup Migration Compatibility diff --git a/scripts/patch-openclaw-device-self-approval.mts b/scripts/patch-openclaw-device-self-approval.mts index cbb278554a7..1da09ad32f7 100644 --- a/scripts/patch-openclaw-device-self-approval.mts +++ b/scripts/patch-openclaw-device-self-approval.mts @@ -48,6 +48,8 @@ const CLI_RETRY_MARKER = "nemoclaw: keep bounded stored device auth fail closed" const CLI_LIST_MARKER = "nemoclaw: preflight bounded stored device auth before live pairing list"; const CLI_STANDALONE_LIST_MARKER = "nemoclaw: select stored device auth for bounded standalone pairing list"; +const CLI_REQUIRED_APPROVAL_MARKER = + "nemoclaw: require stored device auth for restored-clone approval"; const CALL_FORCE_IDENTITY_MARKER = "nemoclaw: force device identity for loopback pairing bootstrap"; const CALL_STORED_IDENTITY_MARKER = "nemoclaw: retain stored CLI device identity for loopback shared-token scope enforcement"; @@ -58,6 +60,7 @@ const CLI_APPLIED_MARKERS = [ CLI_RETRY_MARKER, CLI_LIST_MARKER, CLI_STANDALONE_LIST_MARKER, + CLI_REQUIRED_APPROVAL_MARKER, ] as const; const AUTH_SCOPE_UPGRADE_MARKER = "nemoclaw: route bounded CLI device-token scope upgrade into pairing"; @@ -323,6 +326,7 @@ const CLI_CONTEXT_TARGET = [ ].join("\n"); const CLI_CONTEXT_REPLACEMENT = [ "async function resolveApprovePairingGatewayContext(opts, requestId) {", + `\tconst nemoclawRequireStoredDeviceAuth = process.env.NEMOCLAW_OPENCLAW_REQUIRE_STORED_DEVICE_APPROVAL === "1"; // ${CLI_REQUIRED_APPROVAL_MARKER} (#4462)`, "\tlet nemoclawLocalStoredAuthCandidate = false;", "\ttry {", "\t\tconst nemoclawLocalList = await listDevicePairing();", @@ -332,6 +336,12 @@ const CLI_CONTEXT_REPLACEMENT = [ "\t\t\tnemoclawLocalStoredAuthCandidate = resolveNemoClawSelfRepairPairingContext(nemoclawLocalRequest, nemoclawLocalPaired).useStoredDeviceAuth;", "\t\t}", "\t} catch {}", + "\tif (nemoclawRequireStoredDeviceAuth && !nemoclawLocalStoredAuthCandidate) return {", + "\t\toriginalRequest: null,", + "\t\tscopes: void 0,", + "\t\tnemoclawUseStoredDeviceAuth: false,", + "\t\tnemoclawRefuseUnsafeApproval: true", + "\t};", "\ttry {", "\t\tconst nemoclawListCallOpts = nemoclawLocalStoredAuthCandidate ? {", "\t\t\tscopes: [PAIRING_SCOPE],", @@ -344,7 +354,7 @@ const CLI_CONTEXT_REPLACEMENT = [ "\t\t\toriginalRequest: null,", "\t\t\tscopes: void 0,", "\t\t\tnemoclawUseStoredDeviceAuth: false,", - "\t\t\tnemoclawRefuseUnsafeApproval: nemoclawLocalStoredAuthCandidate", + "\t\t\tnemoclawRefuseUnsafeApproval: nemoclawRequireStoredDeviceAuth || nemoclawLocalStoredAuthCandidate", "\t\t};", "\t\tconst paired = lookupPairedDevice(indexPairedDevices(list.paired), request);", "\t\tconst nemoclawSelfRepairContext = resolveNemoClawSelfRepairPairingContext(request, paired);", @@ -353,14 +363,14 @@ const CLI_CONTEXT_REPLACEMENT = [ "\t\t\toriginalRequest: request,", "\t\t\tscopes: resolveApprovePairingScopesForRequest(request, paired),", "\t\t\tnemoclawUseStoredDeviceAuth,", - "\t\t\tnemoclawRefuseUnsafeApproval: nemoclawLocalStoredAuthCandidate && !nemoclawUseStoredDeviceAuth", + "\t\t\tnemoclawRefuseUnsafeApproval: (nemoclawRequireStoredDeviceAuth || nemoclawLocalStoredAuthCandidate) && !nemoclawUseStoredDeviceAuth", "\t\t};", "\t} catch {", "\t\treturn {", "\t\t\toriginalRequest: null,", "\t\t\tscopes: void 0,", "\t\t\tnemoclawUseStoredDeviceAuth: false,", - "\t\t\tnemoclawRefuseUnsafeApproval: nemoclawLocalStoredAuthCandidate", + "\t\t\tnemoclawRefuseUnsafeApproval: nemoclawRequireStoredDeviceAuth || nemoclawLocalStoredAuthCandidate", "\t\t};", "\t}", "}", diff --git a/src/lib/actions/sandbox/auto-pair-approval.test.ts b/src/lib/actions/sandbox/auto-pair-approval.test.ts index 2a33aed40c8..5fe1d97ca68 100644 --- a/src/lib/actions/sandbox/auto-pair-approval.test.ts +++ b/src/lib/actions/sandbox/auto-pair-approval.test.ts @@ -43,7 +43,14 @@ describe("buildAutoPairApprovalScript (#4263/#4616)", () => { expect(ordinary).not.toContain("local_identity_public_key"); expect(ordinary).toContain("env=None"); expect(ordinary).not.toContain("NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH"); + expect(ordinary).not.toContain("NEMOCLAW_OPENCLAW_FORCE_DEVICE_PAIRING"); + expect(ordinary).not.toContain("NEMOCLAW_OPENCLAW_REQUIRE_STORED_DEVICE_APPROVAL"); expect(restoredClone).toContain("local_identity_public_key"); + expect(restoredClone.match(/\[OPENCLAW, 'devices', 'list', '--json'\]/g)).toHaveLength(2); + expect(restoredClone).toContain("env['NEMOCLAW_OPENCLAW_FORCE_DEVICE_PAIRING'] = '1'"); + expect(restoredClone).toContain( + "env['NEMOCLAW_OPENCLAW_REQUIRE_STORED_DEVICE_APPROVAL'] = '1'", + ); expect(restoredClone).toContain("env=local_device_list_env(os.environ)"); expect(restoredClone).toContain("env['NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH'] = '1'"); expect(restoredClone).toContain("if not related_pending:"); @@ -74,9 +81,20 @@ describe("buildAutoPairApprovalScript (#4263/#4616)", () => { }); it("accepts exactly one terminal fixed receipt", () => { - expect( - parseAutoPairApprovalReceipt(`ignored setup output\n${RECEIPT_MARKER}=approved-one\n`), - ).toBe("approved-one"); + for (const receipt of [ + "credential-list-timeout", + "credential-list-failed", + "list-timeout", + "list-exec-failed", + "list-command-failed", + "list-empty-output", + "list-invalid-output", + "approved-one", + ] as const) { + expect( + parseAutoPairApprovalReceipt(`ignored setup output\n${RECEIPT_MARKER}=${receipt}\n`), + ).toBe(receipt); + } for (const output of [ `${RECEIPT_MARKER}=approved-one\nlater output\n`, `${RECEIPT_MARKER}=approve-failed\n${RECEIPT_MARKER}=approved-one\n`, @@ -243,6 +261,7 @@ process.exit(2); const approvalsFile = path.join(tmpDir, "approvals.log"); const approveEnvFile = path.join(tmpDir, "approve-env.log"); const listEnvFile = path.join(tmpDir, "list-env.log"); + const deviceAuthReadyFile = path.join(tmpDir, "device-auth-ready"); fs.mkdirSync(identityDir, { recursive: true }); const publicKey = "y3vjb9p8tAecivI1l5f1Hdc9QdZJSt3BmLkJMM7wZD8"; const deviceId = "04a4c561c730435e9f6a2e38d2e7b929bcbec2ea1c37d3dd053f3341ecce4e47"; @@ -267,8 +286,28 @@ if (args[0] === "devices" && args[1] === "list") { process.env.OPENCLAW_GATEWAY_PORT || "unset", process.env.OPENCLAW_GATEWAY_TOKEN || "unset", process.env.NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH || "unset", + process.env.NEMOCLAW_OPENCLAW_FORCE_DEVICE_PAIRING || "unset", + process.env.NEMOCLAW_OPENCLAW_REQUIRE_STORED_DEVICE_APPROVAL || "unset", ].join(":") + "\\n", ); + if (process.env.NEMOCLAW_OPENCLAW_FORCE_DEVICE_PAIRING === "1") { + if ( + !process.env.OPENCLAW_GATEWAY_URL || + !process.env.OPENCLAW_GATEWAY_PORT || + !process.env.OPENCLAW_GATEWAY_TOKEN || + process.env.NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH + ) { + process.stderr.write("credential convergence environment was not bounded\\n"); + process.exit(4); + } + if (process.env.NEMOCLAW_CREDENTIAL_LIST_FAIL === "1") { + process.stderr.write("raw credential list output must stay private\\n"); + process.exit(5); + } + fs.writeFileSync(${JSON.stringify(deviceAuthReadyFile)}, "ready"); + process.stdout.write(process.env.NEMOCLAW_CREDENTIAL_LIST_RESPONSE + "\\n"); + process.exit(0); + } if ( process.env.OPENCLAW_GATEWAY_URL || process.env.OPENCLAW_GATEWAY_PORT || @@ -277,10 +316,25 @@ if (args[0] === "devices" && args[1] === "list") { process.stderr.write("restored clone list retained shared gateway credentials\\n"); process.exit(3); } + if (!fs.existsSync(${JSON.stringify(deviceAuthReadyFile)})) { + process.stderr.write("stored device auth did not converge\\n"); + process.exit(6); + } process.stdout.write(process.env.NEMOCLAW_LIST_RESPONSE + "\\n"); process.exit(0); } if (args[0] === "devices" && args[1] === "approve") { + if ( + process.env.OPENCLAW_GATEWAY_URL || + process.env.OPENCLAW_GATEWAY_PORT || + process.env.OPENCLAW_GATEWAY_TOKEN || + process.env.NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH || + process.env.NEMOCLAW_OPENCLAW_FORCE_DEVICE_PAIRING || + process.env.NEMOCLAW_OPENCLAW_REQUIRE_STORED_DEVICE_APPROVAL !== "1" + ) { + process.stderr.write("restored clone approval environment was not fail closed\\n"); + process.exit(7); + } if (process.env.NEMOCLAW_APPROVE_FAIL === "1") { process.stderr.write("raw approval output must stay private\\n"); process.exit(1); @@ -293,6 +347,8 @@ if (args[0] === "devices" && args[1] === "approve") { process.env.OPENCLAW_GATEWAY_PORT || "unset", process.env.OPENCLAW_GATEWAY_TOKEN || "unset", process.env.NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH || "unset", + process.env.NEMOCLAW_OPENCLAW_FORCE_DEVICE_PAIRING || "unset", + process.env.NEMOCLAW_OPENCLAW_REQUIRE_STORED_DEVICE_APPROVAL || "unset", ].join(":") + "\\n", ); process.stdout.write("{}\\n"); @@ -304,13 +360,20 @@ process.exit(2); ); const run = ( pending: unknown[], - options: { rawListResponse?: string; failApproval?: boolean } = {}, - ) => - spawnSync("sh", ["-c", script], { + options: { + rawListResponse?: string; + failApproval?: boolean; + failCredentialList?: boolean; + } = {}, + ) => { + fs.rmSync(deviceAuthReadyFile, { force: true }); + return spawnSync("sh", ["-c", script], { encoding: "utf-8", env: { ...process.env, PATH: `${tmpDir}:/usr/bin:/bin`, + NEMOCLAW_CREDENTIAL_LIST_RESPONSE: JSON.stringify({ pending: [], paired: [] }), + NEMOCLAW_CREDENTIAL_LIST_FAIL: options.failCredentialList ? "1" : "0", NEMOCLAW_LIST_RESPONSE: options.rawListResponse ?? JSON.stringify({ pending, paired: [] }), NEMOCLAW_APPROVE_FAIL: options.failApproval ? "1" : "0", @@ -321,6 +384,7 @@ process.exit(2); }, timeout: 10_000, }); + }; const readApprovals = () => fs.existsSync(approvalsFile) ? fs.readFileSync(approvalsFile, "utf-8").trim().split("\n").filter(Boolean) @@ -352,8 +416,13 @@ process.exit(2); expect(initial.stdout).toContain(`${SUMMARY_MARKER}=1`); expect(initial.stdout).toContain(`${RECEIPT_MARKER}=approved-one`); expect(readApprovals()).toEqual(["clone-pairing"]); - expect(fs.readFileSync(listEnvFile, "utf-8").trim()).toBe("unset:unset:unset:1"); - expect(fs.readFileSync(approveEnvFile, "utf-8").trim()).toBe("unset:unset:unset:unset"); + expect(fs.readFileSync(listEnvFile, "utf-8").trim().split("\n")).toEqual([ + "ws://127.0.0.1:18789:18789:secret-token:unset:1:unset", + "unset:unset:unset:1:unset:unset", + ]); + expect(fs.readFileSync(approveEnvFile, "utf-8").trim()).toBe( + "unset:unset:unset:unset:unset:1", + ); resetLogs(); const repairRequest = { @@ -391,9 +460,16 @@ process.exit(2); resetLogs(); const listFailed = run([], { rawListResponse: "raw list output must stay private" }); - expect(listFailed.stdout).toContain(`${RECEIPT_MARKER}=list-failed`); + expect(listFailed.stdout).toContain(`${RECEIPT_MARKER}=list-invalid-output`); expect(`${listFailed.stdout}${listFailed.stderr}`).not.toContain("raw list output"); + resetLogs(); + const credentialListFailed = run([], { failCredentialList: true }); + expect(credentialListFailed.stdout).toContain(`${RECEIPT_MARKER}=credential-list-failed`); + expect(`${credentialListFailed.stdout}${credentialListFailed.stderr}`).not.toContain( + "raw credential list output", + ); + resetLogs(); const noMatch = run([foreignRequest]); expect(noMatch.stdout).toContain(`${RECEIPT_MARKER}=clone-no-match`); diff --git a/src/lib/actions/sandbox/auto-pair-approval.ts b/src/lib/actions/sandbox/auto-pair-approval.ts index 41943cb12a8..e72ee2af78e 100644 --- a/src/lib/actions/sandbox/auto-pair-approval.ts +++ b/src/lib/actions/sandbox/auto-pair-approval.ts @@ -23,19 +23,22 @@ * unknown clients are ignored, never approved. * * Workaround boundary (NemoClaw#4462): OpenClaw owns device-pairing approval - * semantics. In the reviewed OpenClaw 2026.6.10, a gateway-pinned + * semantics. In the reviewed OpenClaw 2026.7.1, a gateway-pinned * `devices approve` for a scope-upgrade can request the upgraded scopes for * its own connection and return the pending-scope failure it is trying to * resolve. The sourced runtime environment makes an ordinary recovery list * call inspect the same live gateway through local loopback. A restored clone - * is already post-bootstrap, so its list call and every approval call strip - * OPENCLAW_GATEWAY_URL/PORT/TOKEN from the child env. A clone-only marker lets - * the reviewed dist patch select stored-device auth for the list only after an - * exact local repair preflight. The approval command independently forces that - * same local-only stored-device-auth path for the exact bounded self-repair - * shape, so a shared token reloaded from config cannot take precedence. Remove - * this compatibility path when OpenClaw can complete scope upgrades natively - * through device-token auth using operator.pairing. + * can have the server-side pairing baseline before its CLI has stored the + * matching pairing-only token, so it first performs one forced-identity + * shared-auth list to converge that local credential. Its request-enumeration + * list and every approval call then strip OPENCLAW_GATEWAY_URL/PORT/TOKEN from + * the child env. A clone-only marker lets the reviewed dist patch select + * stored-device auth for the list only after an exact local repair preflight. + * The approval command independently forces that same local-only + * stored-device-auth path for the exact bounded self-repair shape, so a shared + * token reloaded from config cannot take precedence. Remove this compatibility + * path when OpenClaw can complete scope upgrades natively through device-token + * auth using operator.pairing. */ import { spawnSync } from "node:child_process"; @@ -57,7 +60,7 @@ const AUTO_PAIR_APPROVE_TIMEOUT_S = 1; // Per-surface budget overrides. The connect/probe/finalization surfaces (#4504) // supply a tighter budget — a single realistic pending CLI/webchat scope -// upgrade (maxApprovals = 1) on the watcher's 10s approve budget with a 15s +// upgrade (maxApprovals = 1) on the watcher's 10s approve budget with a 30s // outer cap — via ./connect-autopair-budget. The doctor surface (#4616) uses // the defaults above to drain a backlog. Callers that omit a field inherit the // default, so the historical doctor payload stays byte-stable. @@ -89,6 +92,13 @@ export type AutoPairApprovalResult = { export type AutoPairApprovalReceipt = | "policy-missing" | "exec-failed" + | "credential-list-timeout" + | "credential-list-failed" + | "list-timeout" + | "list-exec-failed" + | "list-command-failed" + | "list-empty-output" + | "list-invalid-output" | "list-failed" | "clone-no-match" | "clone-ambiguous" @@ -97,7 +107,7 @@ export type AutoPairApprovalReceipt = | "approved-one"; const AUTO_PAIR_RECEIPT_LINE_RE = - /^__NEMOCLAW_AUTO_PAIR_RECEIPT__=(policy-missing|exec-failed|list-failed|clone-no-match|clone-ambiguous|request-rejected|approve-failed|approved-one)$/; + /^__NEMOCLAW_AUTO_PAIR_RECEIPT__=(policy-missing|exec-failed|credential-list-timeout|credential-list-failed|list-timeout|list-exec-failed|list-command-failed|list-empty-output|list-invalid-output|list-failed|clone-no-match|clone-ambiguous|request-rejected|approve-failed|approved-one)$/; /** * Parse one fixed receipt only when it is the sole receipt and terminal output @@ -167,20 +177,68 @@ def exit_with_receipt(receipt): const maxApprovals = options.budget?.maxApprovals ?? AUTO_PAIR_MAX_APPROVALS; const listTimeoutS = options.budget?.listTimeoutS ?? AUTO_PAIR_LIST_TIMEOUT_S; const approveTimeoutS = options.budget?.approveTimeoutS ?? AUTO_PAIR_APPROVE_TIMEOUT_S; - // A restored clone already has post-bootstrap device identity. Match the - // startup watcher's post-bootstrap list path by dropping the explicit shared - // gateway credential triplet, then privately request the patched bounded + // A restored clone already has post-bootstrap device identity but can still + // be missing its locally stored pairing token. Converge that credential + // through one forced-identity shared-auth handshake, then match the startup + // watcher's post-bootstrap list path by dropping the explicit shared gateway + // credential triplet and privately requesting the patched bounded // stored-device list path. Ordinary connect/doctor recovery keeps its // existing bootstrap-capable list environment. const listEnvPrelude = options.localDeviceOnly ? ` +def local_device_credential_env(source_env): + env = dict(source_env) + env.pop('NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH', None) + env.pop('NEMOCLAW_OPENCLAW_REQUIRE_STORED_DEVICE_APPROVAL', None) + env['NEMOCLAW_OPENCLAW_FORCE_DEVICE_PAIRING'] = '1' + return env + def local_device_list_env(source_env): env = gateway_approval_env(source_env) + env.pop('NEMOCLAW_OPENCLAW_FORCE_DEVICE_PAIRING', None) + env.pop('NEMOCLAW_OPENCLAW_REQUIRE_STORED_DEVICE_APPROVAL', None) env['NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH'] = '1' return env + +def local_device_approval_env(source_env): + env = gateway_approval_env(source_env) + env.pop('NEMOCLAW_OPENCLAW_FORCE_DEVICE_PAIRING', None) + env.pop('NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH', None) + env['NEMOCLAW_OPENCLAW_REQUIRE_STORED_DEVICE_APPROVAL'] = '1' + return env ` : ""; const listEnv = options.localDeviceOnly ? "local_device_list_env(os.environ)" : "None"; + const approveEnv = options.localDeviceOnly + ? "local_device_approval_env(os.environ)" + : "gateway_approval_env(os.environ)"; + const localDeviceCredentialConvergence = options.localDeviceOnly + ? ` +# A fresh clone can have a canonical pairing record before the requesting CLI +# has received and stored its pairing-scoped device token. Complete one +# forced-identity, shared-auth handshake first; it cannot approve a request and +# requests only the device-list pairing scope. The marked list below then +# strips shared credentials and requires the converged stored device token. +try: + credential_proc = subprocess.run( + [OPENCLAW, 'devices', 'list', '--json'], + capture_output=True, text=True, timeout=${listTimeoutS}, + env=local_device_credential_env(os.environ), + ) +except subprocess.TimeoutExpired: + ${exitWithReceipt("credential-list-timeout")} +except (FileNotFoundError, OSError): + ${exitWithReceipt("credential-list-failed")} +if credential_proc.returncode != 0 or not credential_proc.stdout.strip(): + ${exitWithReceipt("credential-list-failed")} +try: + credential_data = json.loads(credential_proc.stdout) +except ValueError: + ${exitWithReceipt("credential-list-failed")} +if not isinstance(credential_data, dict) or not isinstance(credential_data.get('pending'), list): + ${exitWithReceipt("credential-list-failed")} +` + : ""; const localDeviceFilter = options.localDeviceOnly ? ` # Snapshot restore shares one gateway across the source sandbox and its clone. @@ -345,25 +403,29 @@ ${listEnvPrelude} OPENCLAW = os.environ.get('OPENCLAW_BIN', 'openclaw') MAX_APPROVALS = ${maxApprovals} - +${localDeviceCredentialConvergence} try: proc = subprocess.run( [OPENCLAW, 'devices', 'list', '--json'], capture_output=True, text=True, timeout=${listTimeoutS}, env=${listEnv}, ) -except (subprocess.TimeoutExpired, FileNotFoundError, OSError): - ${exitWithReceipt("list-failed")} -if proc.returncode != 0 or not proc.stdout.strip(): - ${exitWithReceipt("list-failed")} +except subprocess.TimeoutExpired: + ${exitWithReceipt("list-timeout")} +except (FileNotFoundError, OSError): + ${exitWithReceipt("list-exec-failed")} +if proc.returncode != 0: + ${exitWithReceipt("list-command-failed")} +if not proc.stdout.strip(): + ${exitWithReceipt("list-empty-output")} try: data = json.loads(proc.stdout) except ValueError: - ${exitWithReceipt("list-failed")} + ${exitWithReceipt("list-invalid-output")} if not isinstance(data, dict): - ${exitWithReceipt("list-failed")} + ${exitWithReceipt("list-invalid-output")} pending = data.get('pending') if not isinstance(pending, list): - ${exitWithReceipt("list-failed")}${localDeviceFilter} + ${exitWithReceipt("list-invalid-output")}${localDeviceFilter} approved_count = 0 attempted_count = 0 seen_request_ids = set() @@ -379,7 +441,7 @@ for device in pending: if not decision['allowed']: ${rejectedRequestAction} seen_request_ids.add(request_id) - approve_env = gateway_approval_env(os.environ) + approve_env = ${approveEnv} attempted_count += 1 try: approve_proc = subprocess.run( diff --git a/src/lib/actions/sandbox/connect-autopair-budget.test.ts b/src/lib/actions/sandbox/connect-autopair-budget.test.ts index 8c3fc446da3..f314e94e0d7 100644 --- a/src/lib/actions/sandbox/connect-autopair-budget.test.ts +++ b/src/lib/actions/sandbox/connect-autopair-budget.test.ts @@ -30,6 +30,10 @@ describe("connect auto-pair budget", () => { expect(CONNECT_AUTO_PAIR_TIMEOUT_MS - innerWorstCaseMs).toBeGreaterThanOrEqual(5000); }); + it("does not preempt the reviewed OpenClaw device-list deadline", () => { + expect(CONNECT_AUTO_PAIR_LIST_TIMEOUT_S).toBeGreaterThan(10); + }); + it("uses positive, whole-number budgets", () => { for (const value of [ CONNECT_AUTO_PAIR_MAX_APPROVALS, diff --git a/src/lib/actions/sandbox/connect-autopair-budget.ts b/src/lib/actions/sandbox/connect-autopair-budget.ts index bf3396f6b90..a79a75e5da9 100644 --- a/src/lib/actions/sandbox/connect-autopair-budget.ts +++ b/src/lib/actions/sandbox/connect-autopair-budget.ts @@ -9,11 +9,11 @@ export const CONNECT_AUTO_PAIR_MAX_APPROVALS = 1; // `openclaw devices list` budget (seconds), interpolated into the in-sandbox // script so the invariant below is asserted on real values, not source text. -// A cold OpenClaw 2026.6.10 CLI can take just over 2s to load its runtime -// preloads on supported but resource-constrained hosts, so 5s prevents the -// finalization recovery from timing out before it can observe the pending -// request (#4504). -export const CONNECT_AUTO_PAIR_LIST_TIMEOUT_S = 5; +// OpenClaw 2026.7.1 gives its device-list gateway call 10s after CLI startup. +// Keep the enclosing subprocess above that internal deadline so a supported +// but resource-constrained host cannot terminate the command before OpenClaw +// can classify its own result (#4504). +export const CONNECT_AUTO_PAIR_LIST_TIMEOUT_S = 15; // `openclaw devices approve` budget (seconds); matches the in-sandbox watcher's // RUN_TIMEOUT_SECS = 10 (nemoclaw-start.sh). export const CONNECT_AUTO_PAIR_APPROVE_TIMEOUT_S = 10; @@ -23,4 +23,4 @@ export const CONNECT_AUTO_PAIR_APPROVE_TIMEOUT_S = 10; // timer starts at `sh` spawn before the proxy env is sourced and python3 // launches; the 5s slack prevents the outer timeout from terminating a // legitimate slow approve mid-loop, which would strand the allowlisted request. -export const CONNECT_AUTO_PAIR_TIMEOUT_MS = 20_000; +export const CONNECT_AUTO_PAIR_TIMEOUT_MS = 30_000; diff --git a/src/lib/actions/sandbox/restore-gateway-pairing.test.ts b/src/lib/actions/sandbox/restore-gateway-pairing.test.ts index 7cce7170409..169a3f8f7b7 100644 --- a/src/lib/actions/sandbox/restore-gateway-pairing.test.ts +++ b/src/lib/actions/sandbox/restore-gateway-pairing.test.ts @@ -3,8 +3,13 @@ import { afterEach, describe, expect, it, vi } from "vitest"; +import { + CONNECT_AUTO_PAIR_APPROVE_TIMEOUT_S, + CONNECT_AUTO_PAIR_LIST_TIMEOUT_S, +} from "./connect-autopair-budget"; import { establishRestoredSandboxGatewayPairing, + RESTORED_CLONE_PAIRING_TIMEOUT_MS, restartRestoredSandboxGateway, } from "./restore-gateway-pairing"; @@ -12,6 +17,16 @@ afterEach(() => { vi.restoreAllMocks(); }); +describe("restored clone pairing budget", () => { + it("covers credential convergence, stored-auth list, and one approval with startup slack", () => { + const innerWorstCaseMs = + (CONNECT_AUTO_PAIR_LIST_TIMEOUT_S * 2 + CONNECT_AUTO_PAIR_APPROVE_TIMEOUT_S) * 1000; + + expect(RESTORED_CLONE_PAIRING_TIMEOUT_MS).toBeGreaterThan(innerWorstCaseMs); + expect(RESTORED_CLONE_PAIRING_TIMEOUT_MS - innerWorstCaseMs).toBeGreaterThanOrEqual(5000); + }); +}); + describe("establishRestoredSandboxGatewayPairing", () => { it("restarts the restored gateway before warm-up and after approval (#7431)", async () => { const order: string[] = []; @@ -249,15 +264,15 @@ describe("establishRestoredSandboxGatewayPairing", () => { expect(verifyGatewayPairing).toHaveBeenCalledTimes(2); }); - it("retries one failed clone list when verification reports a pending scope upgrade (#7431)", async () => { + it("retries one failed credential-convergence list when verification reports a pending scope upgrade (#7431)", async () => { const order: string[] = []; const restartRestoredSandboxGateway = vi.fn(() => order.push("restart")); const warmupScopeUpgrade = vi.fn(() => order.push("warmup")); const approveRestoredClonePairing = vi .fn() .mockImplementationOnce(() => { - order.push("approve:list-failed"); - return "list-failed" as const; + order.push("approve:credential-list-failed"); + return "credential-list-failed" as const; }) .mockImplementationOnce(() => { order.push("approve:succeeded"); @@ -287,7 +302,7 @@ describe("establishRestoredSandboxGatewayPairing", () => { expect(order).toEqual([ "restart", "warmup", - "approve:list-failed", + "approve:credential-list-failed", "restart", "verify:pending", "restart", @@ -302,13 +317,13 @@ describe("establishRestoredSandboxGatewayPairing", () => { expect(verifyGatewayPairing).toHaveBeenCalledTimes(2); }); - it("retries one failed clone list for a pending scope upgrade, then fails closed (#7431)", async () => { + it("retries one timed-out stored-auth list for a pending scope upgrade, then fails closed (#7431)", async () => { const order: string[] = []; const restartRestoredSandboxGateway = vi.fn(() => order.push("restart")); const warmupScopeUpgrade = vi.fn(() => order.push("warmup")); const approveRestoredClonePairing = vi.fn(() => { - order.push("approve:list-failed"); - return "list-failed" as const; + order.push("approve:list-timeout"); + return "list-timeout" as const; }); const verifyGatewayPairing = vi.fn(() => { order.push("verify:pending"); @@ -326,17 +341,17 @@ describe("establishRestoredSandboxGatewayPairing", () => { verifyGatewayPairing, }), ).rejects.toThrow( - "authenticated gateway verification run failed (scope-upgrade-pending; approval=list-failed)", + "authenticated gateway verification run failed (scope-upgrade-pending; approval=list-timeout)", ); expect(order).toEqual([ "restart", "warmup", - "approve:list-failed", + "approve:list-timeout", "restart", "verify:pending", "restart", "warmup", - "approve:list-failed", + "approve:list-timeout", "restart", "verify:pending", ]); diff --git a/src/lib/actions/sandbox/restore-gateway-pairing.ts b/src/lib/actions/sandbox/restore-gateway-pairing.ts index 8d0995125e9..7810fc42d16 100644 --- a/src/lib/actions/sandbox/restore-gateway-pairing.ts +++ b/src/lib/actions/sandbox/restore-gateway-pairing.ts @@ -22,14 +22,31 @@ export type RestoreGatewayPairingDeps = { verifyGatewayPairing: (sandboxName: string) => RestoreGatewayPairingVerificationResult; }; +// Restored-clone approval performs one pairing-scoped credential-convergence +// list before the ordinary stored-auth list. Add that one bounded list to the +// connect-time outer cap while preserving its five seconds of startup slack. +export const RESTORED_CLONE_PAIRING_TIMEOUT_MS = + CONNECT_AUTO_PAIR_TIMEOUT_MS + CONNECT_AUTO_PAIR_LIST_TIMEOUT_S * 1000; + const RESTORED_CLONE_PAIRING_BUDGET = { maxApprovals: CONNECT_AUTO_PAIR_MAX_APPROVALS, listTimeoutS: CONNECT_AUTO_PAIR_LIST_TIMEOUT_S, approveTimeoutS: CONNECT_AUTO_PAIR_APPROVE_TIMEOUT_S, - timeoutMs: CONNECT_AUTO_PAIR_TIMEOUT_MS, + timeoutMs: RESTORED_CLONE_PAIRING_TIMEOUT_MS, } as const; const RESTORED_CLONE_PAIRING_ATTEMPTS = 2; +const RETRYABLE_RESTORED_CLONE_APPROVAL_RECEIPTS = new Set([ + "credential-list-timeout", + "credential-list-failed", + "list-timeout", + "list-exec-failed", + "list-command-failed", + "list-empty-output", + "list-invalid-output", + "list-failed", + "approve-failed", +]); class RestoreGatewayPairingClassifiedError extends Error {} @@ -97,7 +114,7 @@ export async function establishRestoredSandboxGatewayPairing( // independently proves that exact scope-upgrade transition is pending. if ( attempt < RESTORED_CLONE_PAIRING_ATTEMPTS && - (approvalReceipt === "list-failed" || approvalReceipt === "approve-failed") && + RETRYABLE_RESTORED_CLONE_APPROVAL_RECEIPTS.has(approvalReceipt) && verification.failureLayer === "scope-upgrade-pending" ) { continue; diff --git a/test/helpers/openclaw-device-self-approval-patch-harness.ts b/test/helpers/openclaw-device-self-approval-patch-harness.ts index 5528bfe991f..393c5c5049a 100644 --- a/test/helpers/openclaw-device-self-approval-patch-harness.ts +++ b/test/helpers/openclaw-device-self-approval-patch-harness.ts @@ -65,6 +65,10 @@ function setStoredDeviceListMarker(value) { if (value) process.env.NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH = "1"; else delete process.env.NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH; } +function setRequireStoredDeviceApprovalMarker(value) { + if (value) process.env.NEMOCLAW_OPENCLAW_REQUIRE_STORED_DEVICE_APPROVAL = "1"; + else delete process.env.NEMOCLAW_OPENCLAW_REQUIRE_STORED_DEVICE_APPROVAL; +} function setLocalPairingFailure(value) { localPairingFailure = value; } function setListFailures(errors) { listFailures = errors; } function withProgress(_options, callback) { return callback(); } diff --git a/test/helpers/openclaw-real-device-self-approval-proof.ts b/test/helpers/openclaw-real-device-self-approval-proof.ts index 8900ae3b83f..77131a1bded 100644 --- a/test/helpers/openclaw-real-device-self-approval-proof.ts +++ b/test/helpers/openclaw-real-device-self-approval-proof.ts @@ -8,6 +8,8 @@ import net from "node:net"; import path from "node:path"; import { pathToFileURL } from "node:url"; +import { CONNECT_AUTO_PAIR_LIST_TIMEOUT_S } from "../../src/lib/actions/sandbox/connect-autopair-budget"; + interface ProofOptions { dist: string; nodeExecutable: string; @@ -210,6 +212,18 @@ function requireRealStoredDeviceAuthLinkage(sources: DistSource[], cliSource: Di ], "devices CLI stored-auth bridge", ); + requireOrderedMarkers( + cliSource.source, + [ + "async function resolveApprovePairingGatewayContext(opts, requestId)", + "NEMOCLAW_OPENCLAW_REQUIRE_STORED_DEVICE_APPROVAL", + "nemoclaw: require stored device auth for restored-clone approval", + "let nemoclawLocalStoredAuthCandidate = false;", + "if (nemoclawRequireStoredDeviceAuth && !nemoclawLocalStoredAuthCandidate)", + "nemoclawRefuseUnsafeApproval: true", + ], + "devices CLI restored-clone required stored auth", + ); requireOrderedMarkers( cliSource.source, [ @@ -976,6 +990,9 @@ async function runLiveStoredDeviceAuthSelfApprovalProof(options: ProofOptions): ); writeGatewayConfig({ mode: "none" }); const { + NEMOCLAW_OPENCLAW_FORCE_DEVICE_PAIRING: _forceDevicePairing, + NEMOCLAW_OPENCLAW_REQUIRE_STORED_DEVICE_APPROVAL: _requireStoredDeviceApproval, + NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH: _useStoredDeviceListAuth, OPENCLAW_GATEWAY_PASSWORD: _gatewayPassword, OPENCLAW_GATEWAY_PORT: _gatewayPort, OPENCLAW_GATEWAY_TOKEN: _gatewayToken, @@ -995,12 +1012,16 @@ async function runLiveStoredDeviceAuthSelfApprovalProof(options: ProofOptions): // The child inherits VITEST=true, which otherwise suppresses real CLI JSON. OPENCLAW_TEST_RUNTIME_LOG: "1", }; - const runCli = (args: string[], cliEnv: NodeJS.ProcessEnv = env) => + const runCli = ( + args: string[], + cliEnv: NodeJS.ProcessEnv = env, + timeoutMs = Math.min(options.timeoutMs, 60_000), + ) => spawnSync(options.nodeExecutable, [openclawEntry, ...args], { cwd: packageDir, encoding: "utf8", env: cliEnv, - timeout: Math.min(options.timeoutMs, 60_000), + timeout: Math.min(options.timeoutMs, timeoutMs), }); const startGateway = (gatewayEnv: NodeJS.ProcessEnv, append: boolean) => { @@ -1133,17 +1154,50 @@ async function runLiveStoredDeviceAuthSelfApprovalProof(options: ProofOptions): }; const hiddenDeviceAuthPath = `${deviceAuthPath}.hidden`; fs.renameSync(deviceAuthPath, hiddenDeviceAuthPath); - try { - const listWithoutStoredAuth = runCli(["devices", "list", "--json"], markedListEnv); - requireLiveProof( - listWithoutStoredAuth.status !== 0, - "marked device list fell back to the configured shared token without stored device auth", - ); - } finally { - fs.renameSync(hiddenDeviceAuthPath, deviceAuthPath); - } + const listWithoutStoredAuth = runCli( + ["devices", "list", "--json"], + markedListEnv, + CONNECT_AUTO_PAIR_LIST_TIMEOUT_S * 1000, + ); + requireLiveProof( + listWithoutStoredAuth.status !== 0, + "marked device list fell back to the configured shared token without stored device auth", + ); + + const credentialList = runCli( + ["devices", "list", "--json"], + { + ...env, + NEMOCLAW_OPENCLAW_FORCE_DEVICE_PAIRING: "1", + OPENCLAW_GATEWAY_TOKEN: gatewayToken, + }, + CONNECT_AUTO_PAIR_LIST_TIMEOUT_S * 1000, + ); + requireSuccess( + credentialList, + "converge the real pairing-scoped local device credential through shared auth", + ); + const convergedAuthStore = readJsonObject(deviceAuthPath, "converged real stored device auth"); + const convergedOperator = requireOperatorToken( + convergedAuthStore, + "converged real stored device auth", + ); + requireLiveProof( + convergedOperator.token === serverTokenBefore, + "credential convergence stored a token outside the paired device baseline", + ); + requireExactScopes( + convergedOperator.scopes, + ["operator.pairing"], + "converged real stored operator scopes", + ); + fs.rmSync(hiddenDeviceAuthPath); - const strippedList = runCli(["devices", "list", "--json"], markedListEnv); + const strippedList = runCli( + ["devices", "list", "--json"], + markedListEnv, + CONNECT_AUTO_PAIR_LIST_TIMEOUT_S * 1000, + ); requireSuccess( strippedList, "list the exact real same-device repair with the stripped post-bootstrap client", @@ -1178,7 +1232,10 @@ async function runLiveStoredDeviceAuthSelfApprovalProof(options: ProofOptions): "stripped post-bootstrap same-device repair scopes", ); - const approval = runCli(["devices", "approve", String(repair.requestId), "--json"]); + const approval = runCli(["devices", "approve", String(repair.requestId), "--json"], { + ...env, + NEMOCLAW_OPENCLAW_REQUIRE_STORED_DEVICE_APPROVAL: "1", + }); requireSuccess(approval, "approve real same-device repair with stored device auth"); const pendingAfter = readJsonObject(pendingPath, "real pending state after approval"); diff --git a/test/openclaw-device-stored-auth-patch.test.ts b/test/openclaw-device-stored-auth-patch.test.ts index 21e04cc1784..8ff7597af54 100644 --- a/test/openclaw-device-stored-auth-patch.test.ts +++ b/test/openclaw-device-stored-auth-patch.test.ts @@ -181,15 +181,18 @@ describe("OpenClaw bounded stored-device-auth selection (#4462)", () => { const runtime = runFixture<{ approve: (opts: Record, requestId: string) => Promise; calls: Array>; + setMarker: (value: boolean) => void; setList: (value: Record) => void; }>( source, `({ approve: approvePairingWithFallback, calls: gatewayCalls, + setMarker: setRequireStoredDeviceApprovalMarker, setList: setPairingLists, })`, ); + runtime.setMarker(true); for (const pending of [validPending(), validPending({ isRepair: false })]) { runtime.calls.length = 0; runtime.setList({ pending: [pending], paired: [validPaired()] }); @@ -214,6 +217,62 @@ describe("OpenClaw bounded stored-device-auth selection (#4462)", () => { } }); + it("does not downgrade a marked restored-clone approval when local evidence disappears", async () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-device-cli-required-auth-")); + const dist = path.join(tmp, "dist"); + fs.mkdirSync(dist); + writeFixtureDist(dist); + try { + expect(runPatch(dist).status).toBe(0); + const source = fs.readFileSync(path.join(dist, "devices-cli.runtime-fixture.js"), "utf8"); + const runtime = runFixture<{ + approve: (opts: Record, requestId: string) => Promise; + calls: Array>; + setMarker: (value: boolean) => void; + setLists: (local: Record, live?: Record) => void; + }>( + source, + `({ + approve: approvePairingWithFallback, + calls: gatewayCalls, + setMarker: setRequireStoredDeviceApprovalMarker, + setLists: setPairingLists, + })`, + ); + runtime.setLists( + { pending: [], paired: [] }, + { pending: [validPending()], paired: [validPaired()] }, + ); + runtime.setMarker(true); + + await expect( + runtime.approve({ json: true, token: "configured-shared-token" }, "request-1"), + ).rejects.toThrow("bounded same-device approval context changed before gateway approval"); + expect(runtime.calls).toEqual([]); + + runtime.setMarker(false); + await expect( + runtime.approve({ json: true, token: "configured-shared-token" }, "request-1"), + ).resolves.toEqual({ requestId: "request-1", approved: true }); + expect(runtime.calls).toHaveLength(2); + expect(runtime.calls).toEqual([ + expect.objectContaining({ + method: "device.pair.list", + token: "configured-shared-token", + }), + expect.objectContaining({ + method: "device.pair.approve", + token: "configured-shared-token", + }), + ]); + expect(runtime.calls).not.toContainEqual( + expect.objectContaining({ useStoredDeviceAuth: true }), + ); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + it.each([ ["missing paired view", validPending(), undefined, true], ["mismatched device", validPending(), validPaired({ deviceId: "device-2" }), true], diff --git a/test/sandbox-connect-inference/auto-pair-approval.test.ts b/test/sandbox-connect-inference/auto-pair-approval.test.ts index 7ba528e9286..c4d5cda27ad 100644 --- a/test/sandbox-connect-inference/auto-pair-approval.test.ts +++ b/test/sandbox-connect-inference/auto-pair-approval.test.ts @@ -410,7 +410,7 @@ describe("sandbox connect scope-upgrade approval on recover/probe (#4504)", () = ); it( - "approve timeout matches the watcher, cold list gets 5s, and both stay within the outer cap", + "approve timeout matches the watcher, list exceeds OpenClaw's deadline, and both stay within the outer cap", testTimeoutOptions(20_000), () => { const { tmpDir, stateFile, sandboxName } = setupFixture( @@ -439,9 +439,9 @@ describe("sandbox connect scope-upgrade approval on recover/probe (#4504)", () = expect(script).toContain(`MAX_APPROVALS = ${CONNECT_AUTO_PAIR_MAX_APPROVALS}`); // Approve budget matches the in-sandbox watcher RUN_TIMEOUT_SECS = 10; - // list budget covers a cold OpenClaw 2026.6.10 CLI load. + // list budget exceeds OpenClaw 2026.7.1's own 10s gateway-call deadline. expect(CONNECT_AUTO_PAIR_APPROVE_TIMEOUT_S).toBe(10); - expect(CONNECT_AUTO_PAIR_LIST_TIMEOUT_S).toBe(5); + expect(CONNECT_AUTO_PAIR_LIST_TIMEOUT_S).toBe(15); // Budget invariant: the inner worst case (list + approve × MAX_APPROVALS) // must stay STRICTLY below the outer spawnSync cap. The outer timer starts From 45d83ed7610d57e1d4695450f840e75ff9bc9254 Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Wed, 29 Jul 2026 04:35:32 -0700 Subject: [PATCH 11/11] chore(release): separate snapshot E2E changes Signed-off-by: Charan Jagwani --- docs/changelog/2026-07-28.mdx | 11 +- .../openclaw-2026.7.1-dependency-review.md | 52 +--- .../patch-openclaw-device-self-approval.mts | 72 +----- .../sandbox/auto-pair-approval.test.ts | 147 +---------- src/lib/actions/sandbox/auto-pair-approval.ts | 121 ++------- .../sandbox/connect-autopair-budget.test.ts | 4 - .../sandbox/connect-autopair-budget.ts | 12 +- .../sandbox/restore-gateway-pairing.test.ts | 242 +----------------- .../sandbox/restore-gateway-pairing.ts | 52 +--- ...claw-device-self-approval-patch-harness.ts | 33 +-- ...penclaw-real-device-self-approval-proof.ts | 146 +---------- .../openclaw-device-stored-auth-patch.test.ts | 213 --------------- .../auto-pair-approval.test.ts | 6 +- 13 files changed, 68 insertions(+), 1043 deletions(-) diff --git a/docs/changelog/2026-07-28.mdx b/docs/changelog/2026-07-28.mdx index 341f9e96170..38929b8c099 100644 --- a/docs/changelog/2026-07-28.mdx +++ b/docs/changelog/2026-07-28.mdx @@ -32,17 +32,8 @@ It also hardens compatible-provider switching, managed sandbox images, Jetson GP It restores the in-sandbox gateway and host forwards before reporting success, refreshes stale stopped-container evidence, and fails closed when delivery cannot be proven. Recovery also unpauses a paused original container before reporting success. Resumed onboarding writes a secret-free same-name recreation journal before deletion so an interrupted run can continue or fail closed on ambiguous identity. - Restored snapshot clones now run one list-only, forced-identity handshake with the shared gateway credential to request and store the clone's `operator.pairing` device credential. - That handshake cannot approve a request. - NemoClaw then removes the shared gateway URL, port, and token before an exact stored-device-auth list and the canonical approval of one matching local scope transition. - The stored-auth list cannot fall back to the shared gateway credential or local pairing-state output, and the approval command independently uses the same stripped credential boundary. - If the exact stored-auth evidence changes between listing and approval, a child-only guard stops the approval before OpenClaw can use its ordinary gateway or `operator.admin` fallback. - Both list subprocesses have 15-second deadlines, and each clone approval pass has a 45-second outer bound. - Fixed output-free receipts classify failures without returning command output or request identifiers. - When that local pairing pass cannot list or approve the request, the clone retries pairing once only if the authenticated verifier reports a pending scope upgrade. - The credential transition and retry preserve the local-device and single-request bounds. Uninstall checks for the `openshell` command after confirmation and before cleanup mutation, while source-checkout installation preserves an absolute `NEMOCLAW_OPENSHELL_BIN` selection during user-local OpenShell discovery. - For more information, refer to [Recover and Rebuild Sandboxes](/user-guide/openclaw/manage-sandboxes/operate-sandboxes/recover-and-rebuild-sandboxes), [Create and Restore Snapshots](/user-guide/openclaw/manage-sandboxes/state-and-backups/create-and-restore-snapshots), and [Uninstall NemoClaw](/user-guide/openclaw/manage-sandboxes/operate-sandboxes/uninstall-nemoclaw). + For more information, refer to [Recover and Rebuild Sandboxes](/user-guide/openclaw/manage-sandboxes/operate-sandboxes/recover-and-rebuild-sandboxes) and [Uninstall NemoClaw](/user-guide/openclaw/manage-sandboxes/operate-sandboxes/uninstall-nemoclaw). - Jetson setup now explains why host preparation was skipped when it encounters an unrecognized or unparseable release. GPU onboarding carries eligible numeric group IDs for real, non-symlink DRI render character devices into the recreated container while preserving the established Tegra device allowlist. The sandbox CUDA proof remains fail-closed, and physical IGX Orin validation of the reported configuration remains pending. diff --git a/docs/security/openclaw-2026.7.1-dependency-review.md b/docs/security/openclaw-2026.7.1-dependency-review.md index ee7367bfab8..62b1c17ec8f 100644 --- a/docs/security/openclaw-2026.7.1-dependency-review.md +++ b/docs/security/openclaw-2026.7.1-dependency-review.md @@ -250,56 +250,8 @@ stored device token. Once that credential exists, the patch automatically retains CLI identity on ordinary loopback shared-token calls; the upstream local-backend omission remains unchanged. This restores device-scope enforcement without moving the gateway credential into OpenClaw state. After -bootstrap, every `devices approve` removes the gateway URL, port, and shared -token so the bounded approval flow uses that device credential. - -Restored-clone approval first runs a list-only CLI subprocess with the shared -gateway URL, port, and token plus the child-only -`NEMOCLAW_OPENCLAW_FORCE_DEVICE_PAIRING=1` marker. The marker preserves the -clone's device identity for this loopback shared-token handshake. OpenClaw -requests `operator.pairing`, receives the canonical server-issued device token, -and stores that token in its private device-auth store. The subprocess only -invokes `devices list`; it cannot approve or select a pending request. NemoClaw -uses the response only to confirm that it is valid JSON with a pending-request -array. No request from this shared-auth response is selected or approved. - -NemoClaw then removes the shared gateway URL, port, and token and clears the -forced-identity marker before adding the separate child-only -`NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH=1` marker to the next JSON list -subprocess. The compiled CLI honors that marker only when OpenClaw's own pairing -state contains exactly one unambiguous same-device CLI transition requesting -`operator.write` against a paired `operator.pairing` baseline. The accepted -transition can be an explicit repair or the paired pre-convergence form with -`isRepair: false`. It then performs one live `device.pair.list` call with -pairing-scoped stored-device authentication, bypassing any -`gateway.auth.token` in config. The host helper independently validates the -listed request against the clone's device identity, public key, client, role, -and bounded scopes before it invokes OpenClaw's canonical approval command with -the shared credential triplet removed. The selected list never retries with -shared credentials or returns local pairing-state output. - -For the separate approval subprocess, NemoClaw keeps the shared credential -triplet removed and adds the child-only -`NEMOCLAW_OPENCLAW_REQUIRE_STORED_DEVICE_APPROVAL=1` marker. The patched CLI -rechecks the exact local request, paired baseline, and stored-device -authentication context before it performs the live list and approval. If that -evidence is missing, changes, or cannot authenticate between the host's list -and the approval, OpenClaw aborts before `device.pair.approve` and before its -ordinary gateway or `operator.admin` fallback. The marker does not affect -ordinary approval callers. The host helper never reads or writes OpenClaw's -pending or paired state; OpenClaw uses that state only to select the narrower -authentication mode and remains the only pairing-state writer. - -Each list subprocess has a 15-second external deadline, which exceeds -OpenClaw's internal 10-second device-list deadline. The single approval attempt -has a 10-second deadline, and the restored-clone pass has a 45-second outer -bound that preserves five seconds for shell and Python startup. Fixed, -output-free receipts distinguish credential-list timeout or failure, -stored-list timeout, execution failure, nonzero exit, empty output, invalid -output, selector rejection or ambiguity, approval failure, and success. They -never include child command output or request identifiers. NemoClaw retries the -pass once only when its authenticated verifier independently reports that the -scope upgrade is still pending. +bootstrap, list calls and every `devices approve` remove the gateway URL, port, +and shared token so the bounded approval flow uses that device credential. ## Gateway Startup Migration Compatibility diff --git a/scripts/patch-openclaw-device-self-approval.mts b/scripts/patch-openclaw-device-self-approval.mts index 1da09ad32f7..13f9f48a352 100644 --- a/scripts/patch-openclaw-device-self-approval.mts +++ b/scripts/patch-openclaw-device-self-approval.mts @@ -46,10 +46,6 @@ const CLI_APPROVE_MARKER = const CLI_SCOPE_MARKER = "nemoclaw: reach gateway for bounded same-device scope approval"; const CLI_RETRY_MARKER = "nemoclaw: keep bounded stored device auth fail closed"; const CLI_LIST_MARKER = "nemoclaw: preflight bounded stored device auth before live pairing list"; -const CLI_STANDALONE_LIST_MARKER = - "nemoclaw: select stored device auth for bounded standalone pairing list"; -const CLI_REQUIRED_APPROVAL_MARKER = - "nemoclaw: require stored device auth for restored-clone approval"; const CALL_FORCE_IDENTITY_MARKER = "nemoclaw: force device identity for loopback pairing bootstrap"; const CALL_STORED_IDENTITY_MARKER = "nemoclaw: retain stored CLI device identity for loopback shared-token scope enforcement"; @@ -59,8 +55,6 @@ const CLI_APPLIED_MARKERS = [ CLI_SCOPE_MARKER, CLI_RETRY_MARKER, CLI_LIST_MARKER, - CLI_STANDALONE_LIST_MARKER, - CLI_REQUIRED_APPROVAL_MARKER, ] as const; const AUTH_SCOPE_UPGRADE_MARKER = "nemoclaw: route bounded CLI device-token scope upgrade into pairing"; @@ -223,35 +217,6 @@ const CLI_HELPER = [ "\t};", "}", "", - "async function resolveNemoClawStoredDeviceListCallOpts(opts) {", - '\tif (process.env.NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH !== "1" || opts.json !== true || normalizeOptionalString(opts.url) || normalizeOptionalString(opts.token) || normalizeOptionalString(opts.password)) return;', - "\ttry {", - "\t\tconst nemoclawLocalList = await listDevicePairing();", - "\t\tconst nemoclawLocalPending = Array.isArray(nemoclawLocalList.pending) ? nemoclawLocalList.pending : [];", - "\t\tconst nemoclawLocalPaired = Array.isArray(nemoclawLocalList.paired) ? nemoclawLocalList.paired : [];", - "\t\tconst nemoclawCandidates = nemoclawLocalPending.filter((request) => {", - '\t\t\tif (!request || typeof request !== "object" || Array.isArray(request)) return false;', - "\t\t\tconst nemoclawRequestId = normalizeOptionalString(request.requestId);", - "\t\t\tconst nemoclawDeviceId = normalizeOptionalString(request.deviceId);", - "\t\t\tif (!nemoclawRequestId || !nemoclawDeviceId) return false;", - "\t\t\tif (nemoclawLocalPending.filter((candidate) => normalizeOptionalString(candidate?.requestId) === nemoclawRequestId).length !== 1) return false;", - "\t\t\tconst nemoclawRawScopes = request.scopes;", - '\t\t\tif (!Array.isArray(nemoclawRawScopes) || !nemoclawRawScopes.some((scope) => normalizeOptionalString(scope) === "operator.write")) return false;', - "\t\t\tconst nemoclawPairedMatches = nemoclawLocalPaired.filter((device) => normalizeOptionalString(device?.deviceId) === nemoclawDeviceId);", - "\t\t\tif (nemoclawPairedMatches.length !== 1) return false;", - "\t\t\treturn resolveNemoClawSelfRepairPairingContext(request, nemoclawPairedMatches[0]).useStoredDeviceAuth;", - "\t\t});", - "\t\tif (nemoclawCandidates.length !== 1) return;", - "\t\treturn {", - "\t\t\tscopes: [PAIRING_SCOPE],", - "\t\t\tuseStoredDeviceAuth: true,", - "\t\t\trequiredStoredDeviceAuthScopes: [PAIRING_SCOPE]", - "\t\t};", - "\t} catch {", - "\t\treturn;", - "\t}", - "}", - "", ].join("\n"); const CLI_REPLACEMENT = [ @@ -287,21 +252,6 @@ const CLI_LIST_CALL_TARGET = '\t\treturn parseDevicePairingList(await callGatewayCli("device.pair.list", opts, {}));'; const CLI_LIST_CALL_REPLACEMENT = '\t\treturn parseDevicePairingList(await callGatewayCli("device.pair.list", opts, {}, callOpts));'; -const CLI_STANDALONE_LIST_TARGET = [ - "async function runDevicesListCommand(opts) {", - "\tlet list;", - "\ttry {", - "\t\tlist = await listPairingWithFallback(opts);", -].join("\n"); -const CLI_STANDALONE_LIST_REPLACEMENT = [ - "async function runDevicesListCommand(opts) {", - "\tconst nemoclawListCallOpts = await resolveNemoClawStoredDeviceListCallOpts(opts);", - "\tlet list;", - "\ttry {", - "\t\tlist = nemoclawListCallOpts", - '\t\t\t? parseDevicePairingList(await callGatewayCli("device.pair.list", opts, {}, nemoclawListCallOpts)) // nemoclaw: select stored device auth for bounded standalone pairing list (#4462)', - "\t\t\t: await listPairingWithFallback(opts);", -].join("\n"); const CLI_CONTEXT_TARGET = [ "async function resolveApprovePairingGatewayContext(opts, requestId) {", @@ -326,7 +276,6 @@ const CLI_CONTEXT_TARGET = [ ].join("\n"); const CLI_CONTEXT_REPLACEMENT = [ "async function resolveApprovePairingGatewayContext(opts, requestId) {", - `\tconst nemoclawRequireStoredDeviceAuth = process.env.NEMOCLAW_OPENCLAW_REQUIRE_STORED_DEVICE_APPROVAL === "1"; // ${CLI_REQUIRED_APPROVAL_MARKER} (#4462)`, "\tlet nemoclawLocalStoredAuthCandidate = false;", "\ttry {", "\t\tconst nemoclawLocalList = await listDevicePairing();", @@ -336,12 +285,6 @@ const CLI_CONTEXT_REPLACEMENT = [ "\t\t\tnemoclawLocalStoredAuthCandidate = resolveNemoClawSelfRepairPairingContext(nemoclawLocalRequest, nemoclawLocalPaired).useStoredDeviceAuth;", "\t\t}", "\t} catch {}", - "\tif (nemoclawRequireStoredDeviceAuth && !nemoclawLocalStoredAuthCandidate) return {", - "\t\toriginalRequest: null,", - "\t\tscopes: void 0,", - "\t\tnemoclawUseStoredDeviceAuth: false,", - "\t\tnemoclawRefuseUnsafeApproval: true", - "\t};", "\ttry {", "\t\tconst nemoclawListCallOpts = nemoclawLocalStoredAuthCandidate ? {", "\t\t\tscopes: [PAIRING_SCOPE],", @@ -354,7 +297,7 @@ const CLI_CONTEXT_REPLACEMENT = [ "\t\t\toriginalRequest: null,", "\t\t\tscopes: void 0,", "\t\t\tnemoclawUseStoredDeviceAuth: false,", - "\t\t\tnemoclawRefuseUnsafeApproval: nemoclawRequireStoredDeviceAuth || nemoclawLocalStoredAuthCandidate", + "\t\t\tnemoclawRefuseUnsafeApproval: nemoclawLocalStoredAuthCandidate", "\t\t};", "\t\tconst paired = lookupPairedDevice(indexPairedDevices(list.paired), request);", "\t\tconst nemoclawSelfRepairContext = resolveNemoClawSelfRepairPairingContext(request, paired);", @@ -363,14 +306,14 @@ const CLI_CONTEXT_REPLACEMENT = [ "\t\t\toriginalRequest: request,", "\t\t\tscopes: resolveApprovePairingScopesForRequest(request, paired),", "\t\t\tnemoclawUseStoredDeviceAuth,", - "\t\t\tnemoclawRefuseUnsafeApproval: (nemoclawRequireStoredDeviceAuth || nemoclawLocalStoredAuthCandidate) && !nemoclawUseStoredDeviceAuth", + "\t\t\tnemoclawRefuseUnsafeApproval: nemoclawLocalStoredAuthCandidate && !nemoclawUseStoredDeviceAuth", "\t\t};", "\t} catch {", "\t\treturn {", "\t\t\toriginalRequest: null,", "\t\t\tscopes: void 0,", "\t\t\tnemoclawUseStoredDeviceAuth: false,", - "\t\t\tnemoclawRefuseUnsafeApproval: nemoclawRequireStoredDeviceAuth || nemoclawLocalStoredAuthCandidate", + "\t\t\tnemoclawRefuseUnsafeApproval: nemoclawLocalStoredAuthCandidate", "\t\t};", "\t}", "}", @@ -948,7 +891,6 @@ const FILE_SPECS: FileSpec[] = [ selector(source) { return ( source.includes("async function approvePairingWithFallback(opts, requestId)") && - source.includes("async function runDevicesListCommand(opts)") && source.includes("function resolveApprovePairingScopesForRequest(request, paired)") && source.includes('callGatewayCli("device.pair.approve"') && CLI_SELECTOR_DEPENDENCIES.every((dependency) => source.includes(dependency)) @@ -1008,14 +950,6 @@ const FILE_SPECS: FileSpec[] = [ file, ); if (result.error) return { source, status: "no-match", error: result.error }; - result = replaceExactlyOnce( - result.source, - CLI_STANDALONE_LIST_TARGET, - CLI_STANDALONE_LIST_REPLACEMENT, - "devices CLI bounded standalone list target", - file, - ); - if (result.error) return { source, status: "no-match", error: result.error }; result = replaceExactlyOnce( result.source, CLI_CONTEXT_TARGET, diff --git a/src/lib/actions/sandbox/auto-pair-approval.test.ts b/src/lib/actions/sandbox/auto-pair-approval.test.ts index 5fe1d97ca68..5578efbe953 100644 --- a/src/lib/actions/sandbox/auto-pair-approval.test.ts +++ b/src/lib/actions/sandbox/auto-pair-approval.test.ts @@ -41,18 +41,7 @@ describe("buildAutoPairApprovalScript (#4263/#4616)", () => { }); expect(ordinary).not.toContain("local_identity_public_key"); - expect(ordinary).toContain("env=None"); - expect(ordinary).not.toContain("NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH"); - expect(ordinary).not.toContain("NEMOCLAW_OPENCLAW_FORCE_DEVICE_PAIRING"); - expect(ordinary).not.toContain("NEMOCLAW_OPENCLAW_REQUIRE_STORED_DEVICE_APPROVAL"); expect(restoredClone).toContain("local_identity_public_key"); - expect(restoredClone.match(/\[OPENCLAW, 'devices', 'list', '--json'\]/g)).toHaveLength(2); - expect(restoredClone).toContain("env['NEMOCLAW_OPENCLAW_FORCE_DEVICE_PAIRING'] = '1'"); - expect(restoredClone).toContain( - "env['NEMOCLAW_OPENCLAW_REQUIRE_STORED_DEVICE_APPROVAL'] = '1'", - ); - expect(restoredClone).toContain("env=local_device_list_env(os.environ)"); - expect(restoredClone).toContain("env['NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH'] = '1'"); expect(restoredClone).toContain("if not related_pending:"); expect(restoredClone).toContain("len(related_pending) > 1"); expect(restoredClone).toContain("pending = related_pending"); @@ -81,20 +70,9 @@ describe("buildAutoPairApprovalScript (#4263/#4616)", () => { }); it("accepts exactly one terminal fixed receipt", () => { - for (const receipt of [ - "credential-list-timeout", - "credential-list-failed", - "list-timeout", - "list-exec-failed", - "list-command-failed", - "list-empty-output", - "list-invalid-output", - "approved-one", - ] as const) { - expect( - parseAutoPairApprovalReceipt(`ignored setup output\n${RECEIPT_MARKER}=${receipt}\n`), - ).toBe(receipt); - } + expect( + parseAutoPairApprovalReceipt(`ignored setup output\n${RECEIPT_MARKER}=approved-one\n`), + ).toBe("approved-one"); for (const output of [ `${RECEIPT_MARKER}=approved-one\nlater output\n`, `${RECEIPT_MARKER}=approve-failed\n${RECEIPT_MARKER}=approved-one\n`, @@ -120,7 +98,6 @@ describe("auto-pair approval pass behaviour (#4616)", () => { try { const approvalsFile = path.join(tmpDir, "approvals.log"); const approveEnvFile = path.join(tmpDir, "approve-env.log"); - const listEnvFile = path.join(tmpDir, "list-env.log"); const pending = [ { requestId: "ok-webchat", @@ -172,15 +149,6 @@ describe("auto-pair approval pass behaviour (#4616)", () => { const fs = require("fs"); const args = process.argv.slice(2); if (args[0] === "devices" && args[1] === "list") { - fs.appendFileSync( - ${JSON.stringify(listEnvFile)}, - [ - process.env.OPENCLAW_GATEWAY_URL || "unset", - process.env.OPENCLAW_GATEWAY_PORT || "unset", - process.env.OPENCLAW_GATEWAY_TOKEN || "unset", - process.env.NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH || "unset", - ].join(":") + "\\n", - ); process.stdout.write(${JSON.stringify(`${listResponse}\n`)}); process.exit(0); } @@ -192,7 +160,6 @@ if (args[0] === "devices" && args[1] === "approve") { process.env.OPENCLAW_GATEWAY_URL || "unset", process.env.OPENCLAW_GATEWAY_PORT || "unset", process.env.OPENCLAW_GATEWAY_TOKEN || "unset", - process.env.NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH || "unset", ].join(":") + "\\n", ); process.stdout.write("{}\\n"); @@ -221,18 +188,10 @@ process.exit(2); const approveEnv = fs.existsSync(approveEnvFile) ? fs.readFileSync(approveEnvFile, "utf-8").trim().split("\n").filter(Boolean) : []; - const listEnv = fs.existsSync(listEnvFile) - ? fs.readFileSync(listEnvFile, "utf-8").trim().split("\n").filter(Boolean) - : []; expect(approvals).toEqual(["ok-webchat", "ok-cli", "ok-agent-cli"]); - expect(listEnv).toEqual(["ws://127.0.0.1:18789:18789:secret-token:unset"]); // Gateway env stripped on the approve subprocess (#4462 workaround). - expect(approveEnv).toEqual([ - "unset:unset:unset:unset", - "unset:unset:unset:unset", - "unset:unset:unset:unset", - ]); + expect(approveEnv).toEqual(["unset:unset:unset", "unset:unset:unset", "unset:unset:unset"]); expect(result.stdout).toContain(`${SUMMARY_MARKER}=3`); } finally { fs.rmSync(tmpDir, { recursive: true, force: true }); @@ -242,7 +201,7 @@ process.exit(2); const pyIt = spawnSync("sh", ["-c", "command -v python3"], { stdio: "ignore" }).status === 0 ? it : it.skip; - const approveOnlyOneLocalClonePairing = () => { + pyIt("approves only one exact local clone pairing transition on a shared gateway", () => { const policy = readAutoPairApprovalPolicyModule(); expect(policy).toBeTruthy(); const script = buildAutoPairApprovalScript( @@ -260,8 +219,6 @@ process.exit(2); const identityDir = path.join(stateDir, "identity"); const approvalsFile = path.join(tmpDir, "approvals.log"); const approveEnvFile = path.join(tmpDir, "approve-env.log"); - const listEnvFile = path.join(tmpDir, "list-env.log"); - const deviceAuthReadyFile = path.join(tmpDir, "device-auth-ready"); fs.mkdirSync(identityDir, { recursive: true }); const publicKey = "y3vjb9p8tAecivI1l5f1Hdc9QdZJSt3BmLkJMM7wZD8"; const deviceId = "04a4c561c730435e9f6a2e38d2e7b929bcbec2ea1c37d3dd053f3341ecce4e47"; @@ -279,62 +236,10 @@ process.exit(2); const fs = require("fs"); const args = process.argv.slice(2); if (args[0] === "devices" && args[1] === "list") { - fs.appendFileSync( - ${JSON.stringify(listEnvFile)}, - [ - process.env.OPENCLAW_GATEWAY_URL || "unset", - process.env.OPENCLAW_GATEWAY_PORT || "unset", - process.env.OPENCLAW_GATEWAY_TOKEN || "unset", - process.env.NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH || "unset", - process.env.NEMOCLAW_OPENCLAW_FORCE_DEVICE_PAIRING || "unset", - process.env.NEMOCLAW_OPENCLAW_REQUIRE_STORED_DEVICE_APPROVAL || "unset", - ].join(":") + "\\n", - ); - if (process.env.NEMOCLAW_OPENCLAW_FORCE_DEVICE_PAIRING === "1") { - if ( - !process.env.OPENCLAW_GATEWAY_URL || - !process.env.OPENCLAW_GATEWAY_PORT || - !process.env.OPENCLAW_GATEWAY_TOKEN || - process.env.NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH - ) { - process.stderr.write("credential convergence environment was not bounded\\n"); - process.exit(4); - } - if (process.env.NEMOCLAW_CREDENTIAL_LIST_FAIL === "1") { - process.stderr.write("raw credential list output must stay private\\n"); - process.exit(5); - } - fs.writeFileSync(${JSON.stringify(deviceAuthReadyFile)}, "ready"); - process.stdout.write(process.env.NEMOCLAW_CREDENTIAL_LIST_RESPONSE + "\\n"); - process.exit(0); - } - if ( - process.env.OPENCLAW_GATEWAY_URL || - process.env.OPENCLAW_GATEWAY_PORT || - process.env.OPENCLAW_GATEWAY_TOKEN - ) { - process.stderr.write("restored clone list retained shared gateway credentials\\n"); - process.exit(3); - } - if (!fs.existsSync(${JSON.stringify(deviceAuthReadyFile)})) { - process.stderr.write("stored device auth did not converge\\n"); - process.exit(6); - } process.stdout.write(process.env.NEMOCLAW_LIST_RESPONSE + "\\n"); process.exit(0); } if (args[0] === "devices" && args[1] === "approve") { - if ( - process.env.OPENCLAW_GATEWAY_URL || - process.env.OPENCLAW_GATEWAY_PORT || - process.env.OPENCLAW_GATEWAY_TOKEN || - process.env.NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH || - process.env.NEMOCLAW_OPENCLAW_FORCE_DEVICE_PAIRING || - process.env.NEMOCLAW_OPENCLAW_REQUIRE_STORED_DEVICE_APPROVAL !== "1" - ) { - process.stderr.write("restored clone approval environment was not fail closed\\n"); - process.exit(7); - } if (process.env.NEMOCLAW_APPROVE_FAIL === "1") { process.stderr.write("raw approval output must stay private\\n"); process.exit(1); @@ -346,9 +251,6 @@ if (args[0] === "devices" && args[1] === "approve") { process.env.OPENCLAW_GATEWAY_URL || "unset", process.env.OPENCLAW_GATEWAY_PORT || "unset", process.env.OPENCLAW_GATEWAY_TOKEN || "unset", - process.env.NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH || "unset", - process.env.NEMOCLAW_OPENCLAW_FORCE_DEVICE_PAIRING || "unset", - process.env.NEMOCLAW_OPENCLAW_REQUIRE_STORED_DEVICE_APPROVAL || "unset", ].join(":") + "\\n", ); process.stdout.write("{}\\n"); @@ -360,20 +262,13 @@ process.exit(2); ); const run = ( pending: unknown[], - options: { - rawListResponse?: string; - failApproval?: boolean; - failCredentialList?: boolean; - } = {}, - ) => { - fs.rmSync(deviceAuthReadyFile, { force: true }); - return spawnSync("sh", ["-c", script], { + options: { rawListResponse?: string; failApproval?: boolean } = {}, + ) => + spawnSync("sh", ["-c", script], { encoding: "utf-8", env: { ...process.env, PATH: `${tmpDir}:/usr/bin:/bin`, - NEMOCLAW_CREDENTIAL_LIST_RESPONSE: JSON.stringify({ pending: [], paired: [] }), - NEMOCLAW_CREDENTIAL_LIST_FAIL: options.failCredentialList ? "1" : "0", NEMOCLAW_LIST_RESPONSE: options.rawListResponse ?? JSON.stringify({ pending, paired: [] }), NEMOCLAW_APPROVE_FAIL: options.failApproval ? "1" : "0", @@ -384,7 +279,6 @@ process.exit(2); }, timeout: 10_000, }); - }; const readApprovals = () => fs.existsSync(approvalsFile) ? fs.readFileSync(approvalsFile, "utf-8").trim().split("\n").filter(Boolean) @@ -392,7 +286,6 @@ process.exit(2); const resetLogs = () => { fs.rmSync(approvalsFile, { force: true }); fs.rmSync(approveEnvFile, { force: true }); - fs.rmSync(listEnvFile, { force: true }); }; const localRequest = { requestId: "clone-pairing", @@ -416,13 +309,7 @@ process.exit(2); expect(initial.stdout).toContain(`${SUMMARY_MARKER}=1`); expect(initial.stdout).toContain(`${RECEIPT_MARKER}=approved-one`); expect(readApprovals()).toEqual(["clone-pairing"]); - expect(fs.readFileSync(listEnvFile, "utf-8").trim().split("\n")).toEqual([ - "ws://127.0.0.1:18789:18789:secret-token:unset:1:unset", - "unset:unset:unset:1:unset:unset", - ]); - expect(fs.readFileSync(approveEnvFile, "utf-8").trim()).toBe( - "unset:unset:unset:unset:unset:1", - ); + expect(fs.readFileSync(approveEnvFile, "utf-8").trim()).toBe("unset:unset:unset"); resetLogs(); const repairRequest = { @@ -460,16 +347,9 @@ process.exit(2); resetLogs(); const listFailed = run([], { rawListResponse: "raw list output must stay private" }); - expect(listFailed.stdout).toContain(`${RECEIPT_MARKER}=list-invalid-output`); + expect(listFailed.stdout).toContain(`${RECEIPT_MARKER}=list-failed`); expect(`${listFailed.stdout}${listFailed.stderr}`).not.toContain("raw list output"); - resetLogs(); - const credentialListFailed = run([], { failCredentialList: true }); - expect(credentialListFailed.stdout).toContain(`${RECEIPT_MARKER}=credential-list-failed`); - expect(`${credentialListFailed.stdout}${credentialListFailed.stderr}`).not.toContain( - "raw credential list output", - ); - resetLogs(); const noMatch = run([foreignRequest]); expect(noMatch.stdout).toContain(`${RECEIPT_MARKER}=clone-no-match`); @@ -523,12 +403,7 @@ process.exit(2); } finally { fs.rmSync(tmpDir, { recursive: true, force: true }); } - }; - pyIt( - "approves only one exact local clone pairing transition on a shared gateway", - approveOnlyOneLocalClonePairing, - 30_000, - ); + }); it("leaves a failed compatibility-shaped approval retryable without editing device state", () => { if (spawnSync("sh", ["-c", "command -v python3"], { stdio: "ignore" }).status !== 0) { diff --git a/src/lib/actions/sandbox/auto-pair-approval.ts b/src/lib/actions/sandbox/auto-pair-approval.ts index e72ee2af78e..357b990aaf5 100644 --- a/src/lib/actions/sandbox/auto-pair-approval.ts +++ b/src/lib/actions/sandbox/auto-pair-approval.ts @@ -23,22 +23,17 @@ * unknown clients are ignored, never approved. * * Workaround boundary (NemoClaw#4462): OpenClaw owns device-pairing approval - * semantics. In the reviewed OpenClaw 2026.7.1, a gateway-pinned + * semantics. In the reviewed OpenClaw 2026.6.10, a gateway-pinned * `devices approve` for a scope-upgrade can request the upgraded scopes for * its own connection and return the pending-scope failure it is trying to - * resolve. The sourced runtime environment makes an ordinary recovery list - * call inspect the same live gateway through local loopback. A restored clone - * can have the server-side pairing baseline before its CLI has stored the - * matching pairing-only token, so it first performs one forced-identity - * shared-auth list to converge that local credential. Its request-enumeration - * list and every approval call then strip OPENCLAW_GATEWAY_URL/PORT/TOKEN from - * the child env. A clone-only marker lets the reviewed dist patch select - * stored-device auth for the list only after an exact local repair preflight. - * The approval command independently forces that same local-only - * stored-device-auth path for the exact bounded self-repair shape, so a shared - * token reloaded from config cannot take precedence. Remove this compatibility - * path when OpenClaw can complete scope upgrades natively through device-token - * auth using operator.pairing. + * resolve. The sourced runtime environment makes the list call inspect the + * same live gateway through local loopback, while the approval call also + * strips OPENCLAW_GATEWAY_URL/PORT/TOKEN from the child env. The reviewed dist + * patch then forces OpenClaw's existing local-only stored-device-auth path for + * the exact bounded self-repair shape so a shared token reloaded from config + * cannot take precedence. Remove this compatibility path when OpenClaw can + * complete scope upgrades natively through device-token auth using + * operator.pairing. */ import { spawnSync } from "node:child_process"; @@ -60,7 +55,7 @@ const AUTO_PAIR_APPROVE_TIMEOUT_S = 1; // Per-surface budget overrides. The connect/probe/finalization surfaces (#4504) // supply a tighter budget — a single realistic pending CLI/webchat scope -// upgrade (maxApprovals = 1) on the watcher's 10s approve budget with a 30s +// upgrade (maxApprovals = 1) on the watcher's 10s approve budget with a 15s // outer cap — via ./connect-autopair-budget. The doctor surface (#4616) uses // the defaults above to drain a backlog. Callers that omit a field inherit the // default, so the historical doctor payload stays byte-stable. @@ -92,13 +87,6 @@ export type AutoPairApprovalResult = { export type AutoPairApprovalReceipt = | "policy-missing" | "exec-failed" - | "credential-list-timeout" - | "credential-list-failed" - | "list-timeout" - | "list-exec-failed" - | "list-command-failed" - | "list-empty-output" - | "list-invalid-output" | "list-failed" | "clone-no-match" | "clone-ambiguous" @@ -107,7 +95,7 @@ export type AutoPairApprovalReceipt = | "approved-one"; const AUTO_PAIR_RECEIPT_LINE_RE = - /^__NEMOCLAW_AUTO_PAIR_RECEIPT__=(policy-missing|exec-failed|credential-list-timeout|credential-list-failed|list-timeout|list-exec-failed|list-command-failed|list-empty-output|list-invalid-output|list-failed|clone-no-match|clone-ambiguous|request-rejected|approve-failed|approved-one)$/; + /^__NEMOCLAW_AUTO_PAIR_RECEIPT__=(policy-missing|exec-failed|list-failed|clone-no-match|clone-ambiguous|request-rejected|approve-failed|approved-one)$/; /** * Parse one fixed receipt only when it is the sole receipt and terminal output @@ -177,68 +165,6 @@ def exit_with_receipt(receipt): const maxApprovals = options.budget?.maxApprovals ?? AUTO_PAIR_MAX_APPROVALS; const listTimeoutS = options.budget?.listTimeoutS ?? AUTO_PAIR_LIST_TIMEOUT_S; const approveTimeoutS = options.budget?.approveTimeoutS ?? AUTO_PAIR_APPROVE_TIMEOUT_S; - // A restored clone already has post-bootstrap device identity but can still - // be missing its locally stored pairing token. Converge that credential - // through one forced-identity shared-auth handshake, then match the startup - // watcher's post-bootstrap list path by dropping the explicit shared gateway - // credential triplet and privately requesting the patched bounded - // stored-device list path. Ordinary connect/doctor recovery keeps its - // existing bootstrap-capable list environment. - const listEnvPrelude = options.localDeviceOnly - ? ` -def local_device_credential_env(source_env): - env = dict(source_env) - env.pop('NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH', None) - env.pop('NEMOCLAW_OPENCLAW_REQUIRE_STORED_DEVICE_APPROVAL', None) - env['NEMOCLAW_OPENCLAW_FORCE_DEVICE_PAIRING'] = '1' - return env - -def local_device_list_env(source_env): - env = gateway_approval_env(source_env) - env.pop('NEMOCLAW_OPENCLAW_FORCE_DEVICE_PAIRING', None) - env.pop('NEMOCLAW_OPENCLAW_REQUIRE_STORED_DEVICE_APPROVAL', None) - env['NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH'] = '1' - return env - -def local_device_approval_env(source_env): - env = gateway_approval_env(source_env) - env.pop('NEMOCLAW_OPENCLAW_FORCE_DEVICE_PAIRING', None) - env.pop('NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH', None) - env['NEMOCLAW_OPENCLAW_REQUIRE_STORED_DEVICE_APPROVAL'] = '1' - return env -` - : ""; - const listEnv = options.localDeviceOnly ? "local_device_list_env(os.environ)" : "None"; - const approveEnv = options.localDeviceOnly - ? "local_device_approval_env(os.environ)" - : "gateway_approval_env(os.environ)"; - const localDeviceCredentialConvergence = options.localDeviceOnly - ? ` -# A fresh clone can have a canonical pairing record before the requesting CLI -# has received and stored its pairing-scoped device token. Complete one -# forced-identity, shared-auth handshake first; it cannot approve a request and -# requests only the device-list pairing scope. The marked list below then -# strips shared credentials and requires the converged stored device token. -try: - credential_proc = subprocess.run( - [OPENCLAW, 'devices', 'list', '--json'], - capture_output=True, text=True, timeout=${listTimeoutS}, - env=local_device_credential_env(os.environ), - ) -except subprocess.TimeoutExpired: - ${exitWithReceipt("credential-list-timeout")} -except (FileNotFoundError, OSError): - ${exitWithReceipt("credential-list-failed")} -if credential_proc.returncode != 0 or not credential_proc.stdout.strip(): - ${exitWithReceipt("credential-list-failed")} -try: - credential_data = json.loads(credential_proc.stdout) -except ValueError: - ${exitWithReceipt("credential-list-failed")} -if not isinstance(credential_data, dict) or not isinstance(credential_data.get('pending'), list): - ${exitWithReceipt("credential-list-failed")} -` - : ""; const localDeviceFilter = options.localDeviceOnly ? ` # Snapshot restore shares one gateway across the source sandbox and its clone. @@ -399,33 +325,28 @@ try: gateway_approval_env = policy_globals['gateway_approval_env'] except Exception: ${exitWithReceipt("policy-missing")} -${listEnvPrelude} OPENCLAW = os.environ.get('OPENCLAW_BIN', 'openclaw') MAX_APPROVALS = ${maxApprovals} -${localDeviceCredentialConvergence} + try: proc = subprocess.run( [OPENCLAW, 'devices', 'list', '--json'], - capture_output=True, text=True, timeout=${listTimeoutS}, env=${listEnv}, + capture_output=True, text=True, timeout=${listTimeoutS}, ) -except subprocess.TimeoutExpired: - ${exitWithReceipt("list-timeout")} -except (FileNotFoundError, OSError): - ${exitWithReceipt("list-exec-failed")} -if proc.returncode != 0: - ${exitWithReceipt("list-command-failed")} -if not proc.stdout.strip(): - ${exitWithReceipt("list-empty-output")} +except (subprocess.TimeoutExpired, FileNotFoundError, OSError): + ${exitWithReceipt("list-failed")} +if proc.returncode != 0 or not proc.stdout.strip(): + ${exitWithReceipt("list-failed")} try: data = json.loads(proc.stdout) except ValueError: - ${exitWithReceipt("list-invalid-output")} + ${exitWithReceipt("list-failed")} if not isinstance(data, dict): - ${exitWithReceipt("list-invalid-output")} + ${exitWithReceipt("list-failed")} pending = data.get('pending') if not isinstance(pending, list): - ${exitWithReceipt("list-invalid-output")}${localDeviceFilter} + ${exitWithReceipt("list-failed")}${localDeviceFilter} approved_count = 0 attempted_count = 0 seen_request_ids = set() @@ -441,7 +362,7 @@ for device in pending: if not decision['allowed']: ${rejectedRequestAction} seen_request_ids.add(request_id) - approve_env = ${approveEnv} + approve_env = gateway_approval_env(os.environ) attempted_count += 1 try: approve_proc = subprocess.run( diff --git a/src/lib/actions/sandbox/connect-autopair-budget.test.ts b/src/lib/actions/sandbox/connect-autopair-budget.test.ts index f314e94e0d7..8c3fc446da3 100644 --- a/src/lib/actions/sandbox/connect-autopair-budget.test.ts +++ b/src/lib/actions/sandbox/connect-autopair-budget.test.ts @@ -30,10 +30,6 @@ describe("connect auto-pair budget", () => { expect(CONNECT_AUTO_PAIR_TIMEOUT_MS - innerWorstCaseMs).toBeGreaterThanOrEqual(5000); }); - it("does not preempt the reviewed OpenClaw device-list deadline", () => { - expect(CONNECT_AUTO_PAIR_LIST_TIMEOUT_S).toBeGreaterThan(10); - }); - it("uses positive, whole-number budgets", () => { for (const value of [ CONNECT_AUTO_PAIR_MAX_APPROVALS, diff --git a/src/lib/actions/sandbox/connect-autopair-budget.ts b/src/lib/actions/sandbox/connect-autopair-budget.ts index a79a75e5da9..bf3396f6b90 100644 --- a/src/lib/actions/sandbox/connect-autopair-budget.ts +++ b/src/lib/actions/sandbox/connect-autopair-budget.ts @@ -9,11 +9,11 @@ export const CONNECT_AUTO_PAIR_MAX_APPROVALS = 1; // `openclaw devices list` budget (seconds), interpolated into the in-sandbox // script so the invariant below is asserted on real values, not source text. -// OpenClaw 2026.7.1 gives its device-list gateway call 10s after CLI startup. -// Keep the enclosing subprocess above that internal deadline so a supported -// but resource-constrained host cannot terminate the command before OpenClaw -// can classify its own result (#4504). -export const CONNECT_AUTO_PAIR_LIST_TIMEOUT_S = 15; +// A cold OpenClaw 2026.6.10 CLI can take just over 2s to load its runtime +// preloads on supported but resource-constrained hosts, so 5s prevents the +// finalization recovery from timing out before it can observe the pending +// request (#4504). +export const CONNECT_AUTO_PAIR_LIST_TIMEOUT_S = 5; // `openclaw devices approve` budget (seconds); matches the in-sandbox watcher's // RUN_TIMEOUT_SECS = 10 (nemoclaw-start.sh). export const CONNECT_AUTO_PAIR_APPROVE_TIMEOUT_S = 10; @@ -23,4 +23,4 @@ export const CONNECT_AUTO_PAIR_APPROVE_TIMEOUT_S = 10; // timer starts at `sh` spawn before the proxy env is sourced and python3 // launches; the 5s slack prevents the outer timeout from terminating a // legitimate slow approve mid-loop, which would strand the allowlisted request. -export const CONNECT_AUTO_PAIR_TIMEOUT_MS = 30_000; +export const CONNECT_AUTO_PAIR_TIMEOUT_MS = 20_000; diff --git a/src/lib/actions/sandbox/restore-gateway-pairing.test.ts b/src/lib/actions/sandbox/restore-gateway-pairing.test.ts index 169a3f8f7b7..81d499af287 100644 --- a/src/lib/actions/sandbox/restore-gateway-pairing.test.ts +++ b/src/lib/actions/sandbox/restore-gateway-pairing.test.ts @@ -3,13 +3,8 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import { - CONNECT_AUTO_PAIR_APPROVE_TIMEOUT_S, - CONNECT_AUTO_PAIR_LIST_TIMEOUT_S, -} from "./connect-autopair-budget"; import { establishRestoredSandboxGatewayPairing, - RESTORED_CLONE_PAIRING_TIMEOUT_MS, restartRestoredSandboxGateway, } from "./restore-gateway-pairing"; @@ -17,16 +12,6 @@ afterEach(() => { vi.restoreAllMocks(); }); -describe("restored clone pairing budget", () => { - it("covers credential convergence, stored-auth list, and one approval with startup slack", () => { - const innerWorstCaseMs = - (CONNECT_AUTO_PAIR_LIST_TIMEOUT_S * 2 + CONNECT_AUTO_PAIR_APPROVE_TIMEOUT_S) * 1000; - - expect(RESTORED_CLONE_PAIRING_TIMEOUT_MS).toBeGreaterThan(innerWorstCaseMs); - expect(RESTORED_CLONE_PAIRING_TIMEOUT_MS - innerWorstCaseMs).toBeGreaterThanOrEqual(5000); - }); -}); - describe("establishRestoredSandboxGatewayPairing", () => { it("restarts the restored gateway before warm-up and after approval (#7431)", async () => { const order: string[] = []; @@ -186,60 +171,7 @@ describe("establishRestoredSandboxGatewayPairing", () => { expect(verifyGatewayPairing).not.toHaveBeenCalled(); }); - it("retries one failed clone approval when verification reports a pending scope upgrade (#7431)", async () => { - const order: string[] = []; - const restartRestoredSandboxGateway = vi.fn(() => order.push("restart")); - const warmupScopeUpgrade = vi.fn(() => order.push("warmup")); - const approveRestoredClonePairing = vi - .fn() - .mockImplementationOnce(() => { - order.push("approve:failed"); - return "approve-failed" as const; - }) - .mockImplementationOnce(() => { - order.push("approve:succeeded"); - return "approved-one" as const; - }); - const verifyGatewayPairing = vi - .fn() - .mockImplementationOnce(() => { - order.push("verify:pending"); - return { - ok: false as const, - failureLayer: "scope-upgrade-pending" as const, - }; - }) - .mockImplementationOnce(() => { - order.push("verify:authenticated"); - return { ok: true as const }; - }); - - await establishRestoredSandboxGatewayPairing("beta", { - restartRestoredSandboxGateway, - warmupScopeUpgrade, - approveRestoredClonePairing, - verifyGatewayPairing, - }); - - expect(order).toEqual([ - "restart", - "warmup", - "approve:failed", - "restart", - "verify:pending", - "restart", - "warmup", - "approve:succeeded", - "restart", - "verify:authenticated", - ]); - expect(restartRestoredSandboxGateway).toHaveBeenCalledTimes(4); - expect(warmupScopeUpgrade).toHaveBeenCalledTimes(2); - expect(approveRestoredClonePairing).toHaveBeenCalledTimes(2); - expect(verifyGatewayPairing).toHaveBeenCalledTimes(2); - }); - - it("fails after the bounded clone approval retry cannot authenticate pairing (#7431)", async () => { + it("fails after one ordinary verifier without retrying the handshake (#7431)", async () => { const restartRestoredSandboxGateway = vi.fn(); const warmupScopeUpgrade = vi.fn(); const approveRestoredClonePairing = vi.fn(() => "approve-failed" as const); @@ -258,178 +190,6 @@ describe("establishRestoredSandboxGatewayPairing", () => { ).rejects.toThrow( "authenticated gateway verification run failed (scope-upgrade-pending; approval=approve-failed)", ); - expect(restartRestoredSandboxGateway).toHaveBeenCalledTimes(4); - expect(warmupScopeUpgrade).toHaveBeenCalledTimes(2); - expect(approveRestoredClonePairing).toHaveBeenCalledTimes(2); - expect(verifyGatewayPairing).toHaveBeenCalledTimes(2); - }); - - it("retries one failed credential-convergence list when verification reports a pending scope upgrade (#7431)", async () => { - const order: string[] = []; - const restartRestoredSandboxGateway = vi.fn(() => order.push("restart")); - const warmupScopeUpgrade = vi.fn(() => order.push("warmup")); - const approveRestoredClonePairing = vi - .fn() - .mockImplementationOnce(() => { - order.push("approve:credential-list-failed"); - return "credential-list-failed" as const; - }) - .mockImplementationOnce(() => { - order.push("approve:succeeded"); - return "approved-one" as const; - }); - const verifyGatewayPairing = vi - .fn() - .mockImplementationOnce(() => { - order.push("verify:pending"); - return { - ok: false as const, - failureLayer: "scope-upgrade-pending" as const, - }; - }) - .mockImplementationOnce(() => { - order.push("verify:authenticated"); - return { ok: true as const }; - }); - - await establishRestoredSandboxGatewayPairing("beta", { - restartRestoredSandboxGateway, - warmupScopeUpgrade, - approveRestoredClonePairing, - verifyGatewayPairing, - }); - - expect(order).toEqual([ - "restart", - "warmup", - "approve:credential-list-failed", - "restart", - "verify:pending", - "restart", - "warmup", - "approve:succeeded", - "restart", - "verify:authenticated", - ]); - expect(restartRestoredSandboxGateway).toHaveBeenCalledTimes(4); - expect(warmupScopeUpgrade).toHaveBeenCalledTimes(2); - expect(approveRestoredClonePairing).toHaveBeenCalledTimes(2); - expect(verifyGatewayPairing).toHaveBeenCalledTimes(2); - }); - - it("retries one timed-out stored-auth list for a pending scope upgrade, then fails closed (#7431)", async () => { - const order: string[] = []; - const restartRestoredSandboxGateway = vi.fn(() => order.push("restart")); - const warmupScopeUpgrade = vi.fn(() => order.push("warmup")); - const approveRestoredClonePairing = vi.fn(() => { - order.push("approve:list-timeout"); - return "list-timeout" as const; - }); - const verifyGatewayPairing = vi.fn(() => { - order.push("verify:pending"); - return { - ok: false as const, - failureLayer: "scope-upgrade-pending" as const, - }; - }); - - await expect( - establishRestoredSandboxGatewayPairing("beta", { - restartRestoredSandboxGateway, - warmupScopeUpgrade, - approveRestoredClonePairing, - verifyGatewayPairing, - }), - ).rejects.toThrow( - "authenticated gateway verification run failed (scope-upgrade-pending; approval=list-timeout)", - ); - expect(order).toEqual([ - "restart", - "warmup", - "approve:list-timeout", - "restart", - "verify:pending", - "restart", - "warmup", - "approve:list-timeout", - "restart", - "verify:pending", - ]); - expect(restartRestoredSandboxGateway).toHaveBeenCalledTimes(4); - expect(warmupScopeUpgrade).toHaveBeenCalledTimes(2); - expect(approveRestoredClonePairing).toHaveBeenCalledTimes(2); - expect(verifyGatewayPairing).toHaveBeenCalledTimes(2); - }); - - it("does not retry a failed clone list for an unrelated verifier failure (#7431)", async () => { - const restartRestoredSandboxGateway = vi.fn(); - const warmupScopeUpgrade = vi.fn(); - const approveRestoredClonePairing = vi.fn(() => "list-failed" as const); - const verifyGatewayPairing = vi.fn(() => ({ - ok: false as const, - failureLayer: "gateway-connect-failure" as const, - })); - - await expect( - establishRestoredSandboxGatewayPairing("beta", { - restartRestoredSandboxGateway, - warmupScopeUpgrade, - approveRestoredClonePairing, - verifyGatewayPairing, - }), - ).rejects.toThrow( - "authenticated gateway verification run failed (gateway-connect-failure; approval=list-failed)", - ); - expect(restartRestoredSandboxGateway).toHaveBeenCalledTimes(2); - expect(warmupScopeUpgrade).toHaveBeenCalledOnce(); - expect(approveRestoredClonePairing).toHaveBeenCalledOnce(); - expect(verifyGatewayPairing).toHaveBeenCalledOnce(); - }); - - it("does not retry a failed clone approval for an unrelated verifier failure (#7431)", async () => { - const restartRestoredSandboxGateway = vi.fn(); - const warmupScopeUpgrade = vi.fn(); - const approveRestoredClonePairing = vi.fn(() => "approve-failed" as const); - const verifyGatewayPairing = vi.fn(() => ({ - ok: false as const, - failureLayer: "gateway-connect-failure" as const, - })); - - await expect( - establishRestoredSandboxGatewayPairing("beta", { - restartRestoredSandboxGateway, - warmupScopeUpgrade, - approveRestoredClonePairing, - verifyGatewayPairing, - }), - ).rejects.toThrow( - "authenticated gateway verification run failed (gateway-connect-failure; approval=approve-failed)", - ); - expect(restartRestoredSandboxGateway).toHaveBeenCalledTimes(2); - expect(warmupScopeUpgrade).toHaveBeenCalledOnce(); - expect(approveRestoredClonePairing).toHaveBeenCalledOnce(); - expect(verifyGatewayPairing).toHaveBeenCalledOnce(); - }); - - it("does not retry a completed clone approval when verification remains pending (#7431)", async () => { - const restartRestoredSandboxGateway = vi.fn(); - const warmupScopeUpgrade = vi.fn(); - const approveRestoredClonePairing = vi.fn(() => "approved-one" as const); - const verifyGatewayPairing = vi.fn(() => ({ - ok: false as const, - failureLayer: "scope-upgrade-pending" as const, - })); - - await expect( - establishRestoredSandboxGatewayPairing("beta", { - restartRestoredSandboxGateway, - warmupScopeUpgrade, - approveRestoredClonePairing, - verifyGatewayPairing, - }), - ).rejects.toThrow( - "authenticated gateway verification run failed (scope-upgrade-pending; approval=approved-one)", - ); expect(restartRestoredSandboxGateway).toHaveBeenCalledTimes(2); expect(warmupScopeUpgrade).toHaveBeenCalledOnce(); expect(approveRestoredClonePairing).toHaveBeenCalledOnce(); diff --git a/src/lib/actions/sandbox/restore-gateway-pairing.ts b/src/lib/actions/sandbox/restore-gateway-pairing.ts index 7810fc42d16..50cff79caff 100644 --- a/src/lib/actions/sandbox/restore-gateway-pairing.ts +++ b/src/lib/actions/sandbox/restore-gateway-pairing.ts @@ -22,32 +22,13 @@ export type RestoreGatewayPairingDeps = { verifyGatewayPairing: (sandboxName: string) => RestoreGatewayPairingVerificationResult; }; -// Restored-clone approval performs one pairing-scoped credential-convergence -// list before the ordinary stored-auth list. Add that one bounded list to the -// connect-time outer cap while preserving its five seconds of startup slack. -export const RESTORED_CLONE_PAIRING_TIMEOUT_MS = - CONNECT_AUTO_PAIR_TIMEOUT_MS + CONNECT_AUTO_PAIR_LIST_TIMEOUT_S * 1000; - const RESTORED_CLONE_PAIRING_BUDGET = { maxApprovals: CONNECT_AUTO_PAIR_MAX_APPROVALS, listTimeoutS: CONNECT_AUTO_PAIR_LIST_TIMEOUT_S, approveTimeoutS: CONNECT_AUTO_PAIR_APPROVE_TIMEOUT_S, - timeoutMs: RESTORED_CLONE_PAIRING_TIMEOUT_MS, + timeoutMs: CONNECT_AUTO_PAIR_TIMEOUT_MS, } as const; -const RESTORED_CLONE_PAIRING_ATTEMPTS = 2; -const RETRYABLE_RESTORED_CLONE_APPROVAL_RECEIPTS = new Set([ - "credential-list-timeout", - "credential-list-failed", - "list-timeout", - "list-exec-failed", - "list-command-failed", - "list-empty-output", - "list-invalid-output", - "list-failed", - "approve-failed", -]); - class RestoreGatewayPairingClassifiedError extends Error {} type RestoredSandboxGatewayRestartDeps = { @@ -97,28 +78,17 @@ export async function establishRestoredSandboxGatewayPairing( targetSandbox: string, deps: RestoreGatewayPairingDeps = defaultRestoreGatewayPairingDeps(), ): Promise { + // Deliberately do not retry this authorization sequence internally. A fixed + // failure lets the caller retry the restore command as a new bounded attempt. try { - for (let attempt = 1; attempt <= RESTORED_CLONE_PAIRING_ATTEMPTS; attempt += 1) { - deps.restartRestoredSandboxGateway(targetSandbox); - deps.warmupScopeUpgrade(targetSandbox); - const approvalReceipt = deps.approveRestoredClonePairing(targetSandbox) ?? "exec-failed"; - // Publish the clone's approved pairing transition before the ordinary - // authenticated verifier. The verifier alone decides success. - deps.restartRestoredSandboxGateway(targetSandbox); - const verification = deps.verifyGatewayPairing(targetSandbox); - if (verification.ok) { - return; - } - // The bounded approval pass can fail while listing or approving the - // clone's local request. Retry once only when the authenticated verifier - // independently proves that exact scope-upgrade transition is pending. - if ( - attempt < RESTORED_CLONE_PAIRING_ATTEMPTS && - RETRYABLE_RESTORED_CLONE_APPROVAL_RECEIPTS.has(approvalReceipt) && - verification.failureLayer === "scope-upgrade-pending" - ) { - continue; - } + deps.restartRestoredSandboxGateway(targetSandbox); + deps.warmupScopeUpgrade(targetSandbox); + const approvalReceipt = deps.approveRestoredClonePairing(targetSandbox) ?? "exec-failed"; + // Publish the clone's approved pairing transition before the one ordinary + // authenticated verifier. The verifier alone decides success. + deps.restartRestoredSandboxGateway(targetSandbox); + const verification = deps.verifyGatewayPairing(targetSandbox); + if (!verification.ok) { throw new RestoreGatewayPairingClassifiedError( `the authenticated gateway verification run failed (${verification.failureLayer}; approval=${approvalReceipt})`, ); diff --git a/test/helpers/openclaw-device-self-approval-patch-harness.ts b/test/helpers/openclaw-device-self-approval-patch-harness.ts index 393c5c5049a..9b75450522f 100644 --- a/test/helpers/openclaw-device-self-approval-patch-harness.ts +++ b/test/helpers/openclaw-device-self-approval-patch-harness.ts @@ -49,36 +49,19 @@ const OPERATOR_ROLE = "operator"; const GATEWAY_CLIENT_NAMES = { CLI: "cli" }; const GATEWAY_CLIENT_MODES = { CLI: "cli" }; const KNOWN_NON_ADMIN_OPERATOR_SCOPES = new Set(["operator.pairing", "operator.read", "operator.write"]); -const process = { env: {} }; const gatewayCalls = []; -const runtimeJson = []; let pairingList = { pending: [], paired: [] }; let localPairingList = { pending: [], paired: [] }; -let localPairingFailure; -let listFailures = []; let approvalFailures = []; function setPairingLists(localList, liveList = localList) { localPairingList = localList; pairingList = liveList; } -function setStoredDeviceListMarker(value) { - if (value) process.env.NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH = "1"; - else delete process.env.NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH; -} -function setRequireStoredDeviceApprovalMarker(value) { - if (value) process.env.NEMOCLAW_OPENCLAW_REQUIRE_STORED_DEVICE_APPROVAL = "1"; - else delete process.env.NEMOCLAW_OPENCLAW_REQUIRE_STORED_DEVICE_APPROVAL; -} -function setLocalPairingFailure(value) { localPairingFailure = value; } -function setListFailures(errors) { listFailures = errors; } function withProgress(_options, callback) { return callback(); } function parseTimeoutMsWithFallback(value, fallback) { return value ?? fallback; } async function callGateway(options) { gatewayCalls.push(options); - if (options.method === "device.pair.list") { - if (listFailures.length > 0) throw listFailures.shift(); - return pairingList; - } + if (options.method === "device.pair.list") return pairingList; if (options.method === "device.pair.approve" && approvalFailures.length > 0) { throw approvalFailures.shift(); } @@ -149,7 +132,6 @@ function lookupPairedDevice(pairedByDeviceId, request) { return pairedByDeviceId.get(normalizeOptionalString(request.deviceId)); } async function listDevicePairing() { - if (localPairingFailure) throw localPairingFailure; return localPairingList; } async function listPairingWithFallback(opts) { @@ -159,19 +141,6 @@ async function listPairingWithFallback(opts) { throw error; } } -const defaultRuntime = { writeJson(value) { runtimeJson.push(value); } }; -async function runDevicesListCommand(opts) { - let list; - try { - list = await listPairingWithFallback(opts); - } catch (error) { - throw error; - } - if (opts.json) { - defaultRuntime.writeJson(list); - return; - } -} function resolveApprovePairingScopesForRequest(request, paired) { const operatorScopes = resolvePendingOperatorApprovalScopes(request, paired); if (operatorScopes.length === 0) return; diff --git a/test/helpers/openclaw-real-device-self-approval-proof.ts b/test/helpers/openclaw-real-device-self-approval-proof.ts index 77131a1bded..ed874deb97f 100644 --- a/test/helpers/openclaw-real-device-self-approval-proof.ts +++ b/test/helpers/openclaw-real-device-self-approval-proof.ts @@ -8,8 +8,6 @@ import net from "node:net"; import path from "node:path"; import { pathToFileURL } from "node:url"; -import { CONNECT_AUTO_PAIR_LIST_TIMEOUT_S } from "../../src/lib/actions/sandbox/connect-autopair-budget"; - interface ProofOptions { dist: string; nodeExecutable: string; @@ -212,18 +210,6 @@ function requireRealStoredDeviceAuthLinkage(sources: DistSource[], cliSource: Di ], "devices CLI stored-auth bridge", ); - requireOrderedMarkers( - cliSource.source, - [ - "async function resolveApprovePairingGatewayContext(opts, requestId)", - "NEMOCLAW_OPENCLAW_REQUIRE_STORED_DEVICE_APPROVAL", - "nemoclaw: require stored device auth for restored-clone approval", - "let nemoclawLocalStoredAuthCandidate = false;", - "if (nemoclawRequireStoredDeviceAuth && !nemoclawLocalStoredAuthCandidate)", - "nemoclawRefuseUnsafeApproval: true", - ], - "devices CLI restored-clone required stored auth", - ); requireOrderedMarkers( cliSource.source, [ @@ -238,20 +224,6 @@ function requireRealStoredDeviceAuthLinkage(sources: DistSource[], cliSource: Di ], "devices CLI bounded pairing-list preflight", ); - requireOrderedMarkers( - cliSource.source, - [ - "async function resolveNemoClawStoredDeviceListCallOpts(opts)", - "NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH", - "nemoclawCandidates.length !== 1", - "useStoredDeviceAuth: true", - "async function runDevicesListCommand(opts)", - "await resolveNemoClawStoredDeviceListCallOpts(opts)", - 'callGatewayCli("device.pair.list", opts, {}, nemoclawListCallOpts)', - "nemoclaw: select stored device auth for bounded standalone pairing list", - ], - "devices CLI bounded standalone stored-auth list", - ); requireOrderedMarkers( cliSource.source, [ @@ -990,9 +962,6 @@ async function runLiveStoredDeviceAuthSelfApprovalProof(options: ProofOptions): ); writeGatewayConfig({ mode: "none" }); const { - NEMOCLAW_OPENCLAW_FORCE_DEVICE_PAIRING: _forceDevicePairing, - NEMOCLAW_OPENCLAW_REQUIRE_STORED_DEVICE_APPROVAL: _requireStoredDeviceApproval, - NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH: _useStoredDeviceListAuth, OPENCLAW_GATEWAY_PASSWORD: _gatewayPassword, OPENCLAW_GATEWAY_PORT: _gatewayPort, OPENCLAW_GATEWAY_TOKEN: _gatewayToken, @@ -1009,19 +978,13 @@ async function runLiveStoredDeviceAuthSelfApprovalProof(options: ProofOptions): OPENCLAW_SKIP_CHANNELS: "1", OPENCLAW_SKIP_PROVIDERS: "1", OPENCLAW_STATE_DIR: stateDir, - // The child inherits VITEST=true, which otherwise suppresses real CLI JSON. - OPENCLAW_TEST_RUNTIME_LOG: "1", }; - const runCli = ( - args: string[], - cliEnv: NodeJS.ProcessEnv = env, - timeoutMs = Math.min(options.timeoutMs, 60_000), - ) => + const runCli = (args: string[]) => spawnSync(options.nodeExecutable, [openclawEntry, ...args], { cwd: packageDir, encoding: "utf8", - env: cliEnv, - timeout: Math.min(options.timeoutMs, timeoutMs), + env, + timeout: Math.min(options.timeoutMs, 60_000), }); const startGateway = (gatewayEnv: NodeJS.ProcessEnv, append: boolean) => { @@ -1080,7 +1043,7 @@ async function runLiveStoredDeviceAuthSelfApprovalProof(options: ProofOptions): ); await stopChild(gateway); - writeGatewayConfig({ mode: "token", token: gatewayToken }); + writeGatewayConfig({ mode: "token" }); gateway = startGateway({ ...env, OPENCLAW_GATEWAY_TOKEN: gatewayToken }, true); await waitForGatewayReady(gateway, port, options.timeoutMs); @@ -1134,108 +1097,15 @@ async function runLiveStoredDeviceAuthSelfApprovalProof(options: ProofOptions): typeof repair.requestId === "string" && repair.requestId.length > 0, "real same-device repair request id missing", ); - // Snapshot clone startup can retain the initial isRepair=false request - // after the pairing-only approval creates the matching paired baseline. - // Exercise that production ordering through the real CLI/gateway path, - // not only the standalone classifier proof below. - repair.isRepair = false; - fs.writeFileSync(pendingPath, JSON.stringify(pending)); const configuredBeforeApproval = readJsonObject(configPath, "real gateway config"); const configuredGateway = asRecord(configuredBeforeApproval.gateway); const configuredAuth = asRecord(configuredGateway?.auth); requireLiveProof( - configuredAuth?.mode === "token" && configuredAuth.token === gatewayToken, - "production-shaped gateway token auth configuration missing", - ); - - const markedListEnv = { - ...env, - NEMOCLAW_OPENCLAW_USE_STORED_DEVICE_LIST_AUTH: "1", - }; - const hiddenDeviceAuthPath = `${deviceAuthPath}.hidden`; - fs.renameSync(deviceAuthPath, hiddenDeviceAuthPath); - const listWithoutStoredAuth = runCli( - ["devices", "list", "--json"], - markedListEnv, - CONNECT_AUTO_PAIR_LIST_TIMEOUT_S * 1000, - ); - requireLiveProof( - listWithoutStoredAuth.status !== 0, - "marked device list fell back to the configured shared token without stored device auth", + configuredAuth?.mode === "token" && configuredAuth.token === undefined, + "gateway token auth was not isolated from the stored-device-auth client", ); - const credentialList = runCli( - ["devices", "list", "--json"], - { - ...env, - NEMOCLAW_OPENCLAW_FORCE_DEVICE_PAIRING: "1", - OPENCLAW_GATEWAY_TOKEN: gatewayToken, - }, - CONNECT_AUTO_PAIR_LIST_TIMEOUT_S * 1000, - ); - requireSuccess( - credentialList, - "converge the real pairing-scoped local device credential through shared auth", - ); - const convergedAuthStore = readJsonObject(deviceAuthPath, "converged real stored device auth"); - const convergedOperator = requireOperatorToken( - convergedAuthStore, - "converged real stored device auth", - ); - requireLiveProof( - convergedOperator.token === serverTokenBefore, - "credential convergence stored a token outside the paired device baseline", - ); - requireExactScopes( - convergedOperator.scopes, - ["operator.pairing"], - "converged real stored operator scopes", - ); - fs.rmSync(hiddenDeviceAuthPath); - - const strippedList = runCli( - ["devices", "list", "--json"], - markedListEnv, - CONNECT_AUTO_PAIR_LIST_TIMEOUT_S * 1000, - ); - requireSuccess( - strippedList, - "list the exact real same-device repair with the stripped post-bootstrap client", - ); - const strippedListValue: unknown = JSON.parse(String(strippedList.stdout)); - const strippedListObject = asRecord(strippedListValue); - requireLiveProof(strippedListObject, "stripped post-bootstrap device list was not an object"); - requireLiveProof( - Array.isArray(strippedListObject.pending) && Array.isArray(strippedListObject.paired), - "stripped post-bootstrap device list did not contain pending and paired arrays", - ); - const strippedPending = strippedListObject.pending - .map(asRecord) - .filter((request) => request !== null); - requireLiveProof( - strippedPending.length === 1, - `stripped post-bootstrap device list returned ${strippedPending.length} pending requests`, - ); - const listedRepair = strippedPending[0] as Record; - requireLiveProof( - listedRepair.requestId === repair.requestId && - listedRepair.deviceId === repair.deviceId && - listedRepair.publicKey === repair.publicKey && - listedRepair.clientId === repair.clientId && - listedRepair.clientMode === repair.clientMode && - listedRepair.isRepair === repair.isRepair, - "stripped post-bootstrap device list did not return the exact same-device repair", - ); - requireExactScopes( - listedRepair.scopes, - ["operator.write"], - "stripped post-bootstrap same-device repair scopes", - ); - - const approval = runCli(["devices", "approve", String(repair.requestId), "--json"], { - ...env, - NEMOCLAW_OPENCLAW_REQUIRE_STORED_DEVICE_APPROVAL: "1", - }); + const approval = runCli(["devices", "approve", String(repair.requestId), "--json"]); requireSuccess(approval, "approve real same-device repair with stored device auth"); const pendingAfter = readJsonObject(pendingPath, "real pending state after approval"); @@ -1282,7 +1152,7 @@ async function runLiveStoredDeviceAuthSelfApprovalProof(options: ProofOptions): const configuredGatewayAfter = asRecord(configuredAfterApproval.gateway); const configuredAuthAfter = asRecord(configuredGatewayAfter?.auth); requireLiveProof( - configuredAuthAfter?.mode === "token" && configuredAuthAfter.token === gatewayToken, + configuredAuthAfter?.mode === "token" && configuredAuthAfter.token === undefined, "gateway token auth configuration changed during stored-device-auth approval", ); } catch (error) { diff --git a/test/openclaw-device-stored-auth-patch.test.ts b/test/openclaw-device-stored-auth-patch.test.ts index 8ff7597af54..7ffbf7fd39d 100644 --- a/test/openclaw-device-stored-auth-patch.test.ts +++ b/test/openclaw-device-stored-auth-patch.test.ts @@ -16,160 +16,6 @@ import { } from "./helpers/openclaw-device-self-approval-patch-harness"; describe("OpenClaw bounded stored-device-auth selection (#4462)", () => { - it("uses pairing-scoped stored auth only for a marked exact same-device write transition", async () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-device-cli-marked-list-")); - const dist = path.join(tmp, "dist"); - fs.mkdirSync(dist); - writeFixtureDist(dist); - try { - expect(runPatch(dist).status).toBe(0); - const source = fs.readFileSync(path.join(dist, "devices-cli.runtime-fixture.js"), "utf8"); - const runtime = runFixture<{ - calls: Array>; - output: Array>; - list: (opts: Record) => Promise; - setListFailures: (errors: Error[]) => void; - setMarker: (value: boolean) => void; - setPairingLists: (local: Record, live?: Record) => void; - }>( - source, - `({ - calls: gatewayCalls, - output: runtimeJson, - list: runDevicesListCommand, - setListFailures, - setMarker: setStoredDeviceListMarker, - setPairingLists, - })`, - ); - const exactRepairList = { pending: [validPending()], paired: [validPaired()] }; - runtime.setPairingLists(exactRepairList); - - await runtime.list({ json: true }); - expect(runtime.calls).toHaveLength(1); - expect(runtime.calls[0]).not.toHaveProperty("useStoredDeviceAuth"); - expect(runtime.output).toEqual([exactRepairList]); - - runtime.setMarker(true); - for (const exactList of [ - exactRepairList, - { - pending: [validPending({ isRepair: false })], - paired: [validPaired()], - }, - ]) { - runtime.calls.length = 0; - runtime.output.length = 0; - runtime.setPairingLists(exactList); - await runtime.list({ json: true }); - expect(runtime.calls).toHaveLength(1); - expect(runtime.calls[0]).toMatchObject({ - method: "device.pair.list", - scopes: ["operator.pairing"], - useStoredDeviceAuth: true, - requiredStoredDeviceAuthScopes: ["operator.pairing"], - }); - expect(runtime.output).toEqual([exactList]); - } - - runtime.calls.length = 0; - runtime.output.length = 0; - runtime.setListFailures([new Error("stored device list denied")]); - await expect(runtime.list({ json: true })).rejects.toThrow("stored device list denied"); - expect(runtime.calls).toHaveLength(1); - expect(runtime.calls[0]).toMatchObject({ useStoredDeviceAuth: true }); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - - it("fails closed to the ordinary list path when marked local repair evidence is unsafe", async () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-device-cli-list-bounds-")); - const dist = path.join(tmp, "dist"); - fs.mkdirSync(dist); - writeFixtureDist(dist); - try { - expect(runPatch(dist).status).toBe(0); - const source = fs.readFileSync(path.join(dist, "devices-cli.runtime-fixture.js"), "utf8"); - const runtime = runFixture<{ - resolve: (opts: Record) => Promise | undefined>; - setFailure: (value: Error | undefined) => void; - setMarker: (value: boolean) => void; - setPairingLists: (local: Record, live?: Record) => void; - }>( - source, - `({ - resolve: resolveNemoClawStoredDeviceListCallOpts, - setFailure: setLocalPairingFailure, - setMarker: setStoredDeviceListMarker, - setPairingLists, - })`, - ); - const exactList = { pending: [validPending()], paired: [validPaired()] }; - - runtime.setPairingLists(exactList); - expect(await runtime.resolve({ json: true })).toBeUndefined(); - runtime.setMarker(true); - expect(await runtime.resolve({ json: false })).toBeUndefined(); - for (const explicit of [ - { url: "ws://127.0.0.1:18789" }, - { token: "explicit-token" }, - { password: "explicit-password" }, - ]) { - expect(await runtime.resolve({ json: true, ...explicit })).toBeUndefined(); - } - expect(await runtime.resolve({ json: true })).toEqual({ - scopes: ["operator.pairing"], - useStoredDeviceAuth: true, - requiredStoredDeviceAuthScopes: ["operator.pairing"], - }); - - const unsafeLists = [ - { pending: null, paired: null }, - { pending: [], paired: [validPaired()] }, - { pending: [validPending({ requestId: " " })], paired: [validPaired()] }, - { pending: [validPending()], paired: [] }, - { pending: [validPending({ publicKey: "other-key" })], paired: [validPaired()] }, - { pending: [validPending({ clientId: "openclaw-control-ui" })], paired: [validPaired()] }, - { pending: [validPending({ clientMode: "webchat" })], paired: [validPaired()] }, - { pending: [validPending({ role: "node", roles: ["node"] })], paired: [validPaired()] }, - { - pending: [validPending({ roles: ["operator", "node"] })], - paired: [validPaired()], - }, - { pending: [validPending({ scopes: [] })], paired: [validPaired()] }, - { pending: [validPending({ scopes: ["operator.pairing"] })], paired: [validPaired()] }, - { pending: [validPending({ scopes: ["operator.admin"] })], paired: [validPaired()] }, - { pending: [validPending({ scopes: ["operator.unknown"] })], paired: [validPaired()] }, - { - pending: [validPending({ scopes: ["operator.write", "operator.write"] })], - paired: [validPaired()], - }, - { pending: [validPending({ isRepair: false })], paired: [] }, - { - pending: [validPending(), validPending({ requestId: "request-2" })], - paired: [validPaired()], - }, - { - pending: [validPending(), validPending({ deviceId: "device-2" })], - paired: [validPaired()], - }, - { pending: [validPending()], paired: [validPaired(), validPaired()] }, - ]; - for (const unsafeList of unsafeLists) { - runtime.setPairingLists(unsafeList); - expect(await runtime.resolve({ json: true })).toBeUndefined(); - } - - runtime.setPairingLists(exactList); - runtime.setFailure(new Error("local pairing state unreadable")); - expect(await runtime.resolve({ json: true })).toBeUndefined(); - runtime.setFailure(undefined); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - it("forwards stored device auth only for exact same-device transitions", async () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-device-cli-stored-auth-")); const dist = path.join(tmp, "dist"); @@ -181,18 +27,15 @@ describe("OpenClaw bounded stored-device-auth selection (#4462)", () => { const runtime = runFixture<{ approve: (opts: Record, requestId: string) => Promise; calls: Array>; - setMarker: (value: boolean) => void; setList: (value: Record) => void; }>( source, `({ approve: approvePairingWithFallback, calls: gatewayCalls, - setMarker: setRequireStoredDeviceApprovalMarker, setList: setPairingLists, })`, ); - runtime.setMarker(true); for (const pending of [validPending(), validPending({ isRepair: false })]) { runtime.calls.length = 0; runtime.setList({ pending: [pending], paired: [validPaired()] }); @@ -217,62 +60,6 @@ describe("OpenClaw bounded stored-device-auth selection (#4462)", () => { } }); - it("does not downgrade a marked restored-clone approval when local evidence disappears", async () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-device-cli-required-auth-")); - const dist = path.join(tmp, "dist"); - fs.mkdirSync(dist); - writeFixtureDist(dist); - try { - expect(runPatch(dist).status).toBe(0); - const source = fs.readFileSync(path.join(dist, "devices-cli.runtime-fixture.js"), "utf8"); - const runtime = runFixture<{ - approve: (opts: Record, requestId: string) => Promise; - calls: Array>; - setMarker: (value: boolean) => void; - setLists: (local: Record, live?: Record) => void; - }>( - source, - `({ - approve: approvePairingWithFallback, - calls: gatewayCalls, - setMarker: setRequireStoredDeviceApprovalMarker, - setLists: setPairingLists, - })`, - ); - runtime.setLists( - { pending: [], paired: [] }, - { pending: [validPending()], paired: [validPaired()] }, - ); - runtime.setMarker(true); - - await expect( - runtime.approve({ json: true, token: "configured-shared-token" }, "request-1"), - ).rejects.toThrow("bounded same-device approval context changed before gateway approval"); - expect(runtime.calls).toEqual([]); - - runtime.setMarker(false); - await expect( - runtime.approve({ json: true, token: "configured-shared-token" }, "request-1"), - ).resolves.toEqual({ requestId: "request-1", approved: true }); - expect(runtime.calls).toHaveLength(2); - expect(runtime.calls).toEqual([ - expect.objectContaining({ - method: "device.pair.list", - token: "configured-shared-token", - }), - expect.objectContaining({ - method: "device.pair.approve", - token: "configured-shared-token", - }), - ]); - expect(runtime.calls).not.toContainEqual( - expect.objectContaining({ useStoredDeviceAuth: true }), - ); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - it.each([ ["missing paired view", validPending(), undefined, true], ["mismatched device", validPending(), validPaired({ deviceId: "device-2" }), true], diff --git a/test/sandbox-connect-inference/auto-pair-approval.test.ts b/test/sandbox-connect-inference/auto-pair-approval.test.ts index c4d5cda27ad..7ba528e9286 100644 --- a/test/sandbox-connect-inference/auto-pair-approval.test.ts +++ b/test/sandbox-connect-inference/auto-pair-approval.test.ts @@ -410,7 +410,7 @@ describe("sandbox connect scope-upgrade approval on recover/probe (#4504)", () = ); it( - "approve timeout matches the watcher, list exceeds OpenClaw's deadline, and both stay within the outer cap", + "approve timeout matches the watcher, cold list gets 5s, and both stay within the outer cap", testTimeoutOptions(20_000), () => { const { tmpDir, stateFile, sandboxName } = setupFixture( @@ -439,9 +439,9 @@ describe("sandbox connect scope-upgrade approval on recover/probe (#4504)", () = expect(script).toContain(`MAX_APPROVALS = ${CONNECT_AUTO_PAIR_MAX_APPROVALS}`); // Approve budget matches the in-sandbox watcher RUN_TIMEOUT_SECS = 10; - // list budget exceeds OpenClaw 2026.7.1's own 10s gateway-call deadline. + // list budget covers a cold OpenClaw 2026.6.10 CLI load. expect(CONNECT_AUTO_PAIR_APPROVE_TIMEOUT_S).toBe(10); - expect(CONNECT_AUTO_PAIR_LIST_TIMEOUT_S).toBe(15); + expect(CONNECT_AUTO_PAIR_LIST_TIMEOUT_S).toBe(5); // Budget invariant: the inner worst case (list + approve × MAX_APPROVALS) // must stay STRICTLY below the outer spawnSync cap. The outer timer starts