diff --git a/docs/deployment/deploy-to-headless-server.mdx b/docs/deployment/deploy-to-headless-server.mdx index 2d31351d367..8c47522c4d3 100644 --- a/docs/deployment/deploy-to-headless-server.mdx +++ b/docs/deployment/deploy-to-headless-server.mdx @@ -303,8 +303,9 @@ Model traffic uses the OpenShell-managed `inference.local` route. -OpenClaw generates a new gateway token each time the sandbox container starts. -Retrieve the dashboard URL or token again after the container restarts or a replacement sandbox is created. +OpenClaw generates a new gateway token when the sandbox container starts with mutable configuration. +If Shields are up, a non-root start preserves the sealed token because the sandbox user cannot replace the protected configuration. +Retrieve the dashboard URL or token again after the container starts or a replacement sandbox is created. @@ -389,7 +390,7 @@ Use `$$nemoclaw headless-agent rebuild` when you need the current agent image wh | Manually installed system or global packages | Usually remain in the same writable layer | Not preserved | Not preserved | | Direct edits to generated profile, config, or environment files | May remain until regeneration | Agent-specific and usually excluded or filtered | Regenerated or filtered by the current manifest | -| OpenClaw gateway token | Rotated when the container starts | Not captured; a replacement sandbox generates a new token | Rotated for the replacement sandbox | +| OpenClaw gateway token | Rotated when the container starts with mutable configuration; preserved for a non-root start while Shields are up | Not captured; a replacement sandbox generates a new token | Rotated for the replacement sandbox | | Hermes `API_SERVER_KEY` | Preserved | Not captured; a replacement sandbox generates a new token | Rotated for the replacement sandbox | diff --git a/docs/manage-sandboxes/run-sandboxes.mdx b/docs/manage-sandboxes/run-sandboxes.mdx index 1c4b4724335..d19c5eb2057 100644 --- a/docs/manage-sandboxes/run-sandboxes.mdx +++ b/docs/manage-sandboxes/run-sandboxes.mdx @@ -132,10 +132,11 @@ $$nemoclaw start ``` -NemoClaw restarts the container and repairs the in-sandbox gateway and host forwards. +After Docker reports the existing container as running, NemoClaw checks the selected agent's managed startup state and recovers missing processes when needed. +NemoClaw completes this recovery before its final OpenShell readiness and host-forward checks. -NemoClaw restarts the container so the managed terminal runtime can run again. +After Docker reports the existing container as running, NemoClaw verifies the managed terminal runtime before the command reports success. Refer to [`$$nemoclaw stop`](../../reference/commands#$$nemoclaw-name-stop) and [`$$nemoclaw start`](../../reference/commands#$$nemoclaw-name-start) for details. Use [`$$nemoclaw destroy`](../../reference/commands#$$nemoclaw-name-destroy) when you want to delete the sandbox instead. diff --git a/docs/security/best-practices.mdx b/docs/security/best-practices.mdx index 2143c9a3b02..ef699ee868c 100644 --- a/docs/security/best-practices.mdx +++ b/docs/security/best-practices.mdx @@ -297,8 +297,10 @@ After the top-level config binding is frozen, lockdown makes containment monoton It removes unsafe symlinks, special entries, and protected-root names that are not directories through descriptor-relative operations without following their targets. For protected regular files, lockdown publishes a fresh inode, severing hardlinks while preserving file content, read/execute mode, timestamps, and supported extended attributes; this also revokes write authority held through a descriptor opened before `shields up`. The OpenClaw gateway (a member of the `sandbox` group) keeps read access to plugin and agent code; the sandbox user can no longer write them. -The same workflow also locks the secret-bearing directories (`credentials`, `identity`, `pairing`) to `root:root 700` with `chmod -R go-rwX`. -Neither the sandbox user nor the gateway can read those secrets while the lock is active. +The same workflow locks `identity`, `pairing`, and a non-empty `credentials` directory to `root:root 0700`. +An empty `credentials` directory uses `root:sandbox 0710` so OpenClaw can confirm that optional credential files are absent during startup. +The sandbox group cannot list, create, or remove entries, and it cannot read credential files. +Neither the sandbox user nor the gateway can read stored secrets while the lock is active. Restoring the mutable-default posture returns those directories to `sandbox:sandbox 2770`. The list is the union of state directories declared by every shipped agent manifest. The lock helper silently skips dirs that are not present in a given agent's config tree. diff --git a/docs/security/credential-storage.mdx b/docs/security/credential-storage.mdx index deb3d8a93d1..7c77e983444 100644 --- a/docs/security/credential-storage.mdx +++ b/docs/security/credential-storage.mdx @@ -16,7 +16,15 @@ When you provide a provider credential, either interactively during `$$nemoclaw The gateway stores the credential and the OpenShell L7 proxy substitutes it into outbound requests at egress, so sandboxed agents see placeholders instead of the raw secret. -The sandbox-side OpenClaw gateway token is generated at container startup and is not rotated through provider credential commands. +The sandbox-side OpenClaw gateway token is generated when the container starts with mutable configuration. +Provider credential commands do not rotate this token. +A non-root start while Shields are up preserves the sealed token because the sandbox user cannot replace the protected configuration. +If the sealed gateway token or required auth profile is unavailable, the start fails instead of weakening the sealed configuration. +The error tells you to lower Shields before you restart. +When the sealed `credentials` directory is empty, NemoClaw grants the sandbox group search-only access with mode `0710`. +This access lets OpenClaw confirm that optional credential files are absent during start. +The sandbox group cannot list, create, or remove entries, and it cannot read credential files. +A non-empty `credentials` directory remains root-only. NemoClaw manages Hermes API credentials and provider credentials through the same OpenShell provider boundary. diff --git a/scripts/nemoclaw-start.sh b/scripts/nemoclaw-start.sh index a60be1250ca..ebd1f9d5202 100755 --- a/scripts/nemoclaw-start.sh +++ b/scripts/nemoclaw-start.sh @@ -2409,6 +2409,21 @@ needs_gateway_token_for_current_command() { prepare_gateway_token_for_current_command() { if [ ${#NEMOCLAW_CMD[@]} -eq 0 ]; then + # OpenShell launches the persisted workload as the sandbox user. When + # Shields are up, the root-owned config seal deliberately prevents that + # identity from replacing openclaw.json. Preserve the sealed startup token + # rather than weakening the lock; mutable and root-owned startup paths keep + # rotating it. A sealed config without a token cannot be repaired safely by + # this identity, so fail before attempting a write. + if [ "$(id -u)" -ne 0 ] \ + && [ "$(openclaw_config_dir_owner /sandbox/.openclaw)" = "root" ]; then + if [ -n "$(_read_gateway_token)" ]; then + printf '[token] Shields are up; preserving the sealed gateway token for startup\n' >&2 + return 0 + fi + printf '[SECURITY] Shields are up but the sealed OpenClaw config has no gateway token; lower Shields before restarting\n' >&2 + return 1 + fi ensure_gateway_token return $? fi @@ -2436,6 +2451,17 @@ write_auth_profile() { # fallback in v0.0.90. # See: https://github.com/NVIDIA/NemoClaw/issues/1332 local provider_key="${NEMOCLAW_INFERENCE_PROVIDER_ID:-${NEMOCLAW_PROVIDER_KEY:-inference}}" + local auth_profile_path="${HOME}/.openclaw/agents/main/agent/auth-profiles.json" + + if [ "$(id -u)" -ne 0 ] \ + && [ "$(openclaw_config_dir_owner "${HOME}/.openclaw")" = "root" ]; then + if [ -L "$auth_profile_path" ] || [ ! -f "$auth_profile_path" ]; then + printf '[SECURITY] Shields are up but the sealed OpenClaw auth profile is unavailable; lower Shields before restarting\n' >&2 + return 1 + fi + printf '[auth] Shields are up; preserving the sealed OpenClaw auth profile\n' >&2 + return 0 + fi python3 - "$provider_key" <<'PYAUTH' import json @@ -4625,6 +4651,7 @@ seed_default_workspace_templates_as_sandbox() { setup_auth_profile_as_sandbox() { run_step_down_as_sandbox \ "export HOME=/sandbox; write_auth_profile; harden_auth_profiles" \ + openclaw_config_dir_owner \ write_auth_profile \ harden_auth_profiles } diff --git a/scripts/state-dir-guard.py b/scripts/state-dir-guard.py index a4cc994cf78..87205a1cf04 100755 --- a/scripts/state-dir-guard.py +++ b/scripts/state-dir-guard.py @@ -89,7 +89,7 @@ FS_IOC_GETFLAGS = 0x80086601 FS_IOC_SETFLAGS = 0x40086602 -Action = Literal["preflight", "lock", "unlock"] +Action = Literal["preflight", "lock", "unlock", "startup"] Policy = Literal["high-risk", "confidentiality"] @@ -921,6 +921,32 @@ def _set_dir_metadata( os.fchmod(dir_fd, _expected_dir_mode(policy, action)) +def _set_empty_credentials_startup_metadata( + dir_fd: int, + identity: Identity, +) -> None: + """Allow the sandbox group to probe names in an empty sealed credentials dir.""" + + os.fchown(dir_fd, identity.root_uid, identity.sandbox_gid) + os.fchmod(dir_fd, 0o710) + + +def _is_empty_credentials_root( + relative_dir: str, + policy: Policy, + action: Action, + is_root: bool, + names: list[str], +) -> bool: + return ( + action == "lock" + and policy == "confidentiality" + and is_root + and relative_dir == "credentials" + and not names + ) + + def _copy_extent( source_fd: int, temp_fd: int, @@ -1337,6 +1363,14 @@ def _mutate_dir( names = _bounded_directory_names( dir_fd, context.display(relative_dir), context.budget ) + if _is_empty_credentials_root( + relative_dir, policy, action, is_root, names + ): + # OpenClaw probes optional credential paths while starting. An + # empty directory contains no secret metadata to expose, so allow + # the sandbox group to traverse it without granting list, read, or + # write access. Non-empty confidentiality roots remain root-only. + _set_empty_credentials_startup_metadata(dir_fd, identity) except OSError as exc: raise GuardOperationError( _os_issue( @@ -1510,8 +1544,12 @@ def _verify_metadata( policy: Policy, action: Action, identity: Identity, + empty_credentials_root: bool = False, ) -> Issue | None: - expected_uid, expected_gid = _expected_ids(policy, action, identity) + if empty_credentials_root: + expected_uid, expected_gid = identity.root_uid, identity.sandbox_gid + else: + expected_uid, expected_gid = _expected_ids(policy, action, identity) if st.st_uid != expected_uid or st.st_gid != expected_gid: return Issue( "verification-owner-mismatch", @@ -1522,7 +1560,7 @@ def _verify_metadata( return None mode = stat.S_IMODE(st.st_mode) if entry_type == "directory": - expected_mode = _expected_dir_mode(policy, action) + expected_mode = 0o710 if empty_credentials_root else _expected_dir_mode(policy, action) if mode != expected_mode: return Issue( "verification-mode-mismatch", @@ -1579,16 +1617,6 @@ def _verify_dir( ) ) return - dir_issue = _verify_metadata( - context.display(relative_dir), - os.fstat(dir_fd), - "directory", - policy, - action, - identity, - ) - if dir_issue is not None: - issues.append(dir_issue) try: names = _bounded_directory_names( dir_fd, context.display(relative_dir), context.budget @@ -1600,6 +1628,19 @@ def _verify_dir( ) ) return + dir_issue = _verify_metadata( + context.display(relative_dir), + os.fstat(dir_fd), + "directory", + policy, + action, + identity, + _is_empty_credentials_root( + relative_dir, policy, action, relative_dir == "credentials", names + ), + ) + if dir_issue is not None: + issues.append(dir_issue) for name in names: relative_path = posixpath.join(relative_dir, name) path = context.display(relative_path) @@ -1697,6 +1738,118 @@ def _verify_dir( ) +def _restore_empty_credentials_startup_access( + config_dir: str, + identity: Identity, + deadline: float, +) -> GuardResult: + """Restore only the empty credentials traversal needed during startup.""" + + result = GuardResult(action="startup") + config_fd = -1 + credentials_fd = -1 + path = _display_path(config_dir, "credentials") + try: + config_fd = _open_absolute_dir_nofollow(config_dir) + config_st = os.fstat(config_fd) + config_mode = stat.S_IMODE(config_st.st_mode) + if config_st.st_uid != identity.root_uid or config_mode & 0o022: + result.issues.append( + Issue( + "startup-posture-mismatch", + config_dir, + "sealed config directory must be root-owned and not group/world writable", + ) + ) + return result + try: + credentials_st = os.stat( + "credentials", dir_fd=config_fd, follow_symlinks=False + ) + except FileNotFoundError: + return result + if ( + not stat.S_ISDIR(credentials_st.st_mode) + or credentials_st.st_dev != config_st.st_dev + ): + result.issues.append( + Issue( + "unsafe-startup-credentials-root", + path, + "credentials root must be a directory on the config filesystem", + ) + ) + return result + credentials_fd = _open_child_dir(config_fd, "credentials", credentials_st) + names = _bounded_directory_names( + credentials_fd, path, WorkBudget(deadline) + ) + current = os.fstat(credentials_fd) + mode = stat.S_IMODE(current.st_mode) + root_only = ( + current.st_uid == identity.root_uid + and current.st_gid == identity.root_gid + and mode == 0o700 + ) + startup_traversable = ( + current.st_uid == identity.root_uid + and current.st_gid == identity.sandbox_gid + and mode == 0o710 + ) + if not root_only and not startup_traversable: + result.issues.append( + Issue( + "startup-posture-mismatch", + path, + f"credentials root has unexpected owner or mode {mode:04o}", + ) + ) + return result + if names: + if startup_traversable: + _set_dir_metadata( + credentials_fd, "confidentiality", "lock", identity + ) + os.fsync(credentials_fd) + return result + _set_empty_credentials_startup_metadata(credentials_fd, identity) + after = os.fstat(credentials_fd) + if ( + after.st_uid != identity.root_uid + or after.st_gid != identity.sandbox_gid + or stat.S_IMODE(after.st_mode) != 0o710 + ): + result.issues.append( + Issue( + "startup-verification-failed", + path, + "empty credentials root did not reach root:sandbox 0710", + ) + ) + return result + os.fsync(credentials_fd) + result.roots = 1 + result.directories = 1 + return result + except (OSError, GuardOperationError) as exc: + result.issues.append( + exc.issue + if isinstance(exc, GuardOperationError) + else _os_issue( + "startup-restore-failed", + path, + "restore empty credentials startup access", + exc, + ) + ) + return result + finally: + if credentials_fd >= 0: + os.close(credentials_fd) + if config_fd >= 0: + os.close(config_fd) + + def _run_guard_unserialized( action: Action, config_dir: str, @@ -1707,6 +1860,10 @@ def _run_guard_unserialized( result = GuardResult(action=action) deadline = time.monotonic() + MAX_GUARD_SECONDS normalized_config = posixpath.normpath(config_dir) + if action == "startup": + return _restore_empty_credentials_startup_access( + normalized_config, identity, deadline + ) fail_closed_config_root = action == "lock" and ( normalized_config in PRODUCTION_FAIL_CLOSED_CONFIG_DIRS or os.environ.get("NEMOCLAW_TEST_OPENCLAW_FAIL_CLOSED") == "1" @@ -2003,9 +2160,9 @@ def _production_identity() -> Identity: def _parse_args(argv: list[str]) -> argparse.Namespace: parser = argparse.ArgumentParser( - description="Safely preflight, lock, or unlock recursive agent state directories" + description="Safely manage recursive agent state directories" ) - parser.add_argument("action", choices=("preflight", "lock", "unlock")) + parser.add_argument("action", choices=("preflight", "lock", "unlock", "startup")) parser.add_argument("--config-dir", required=True) return parser.parse_args(argv) diff --git a/src/lib/actions/sandbox/connect.ts b/src/lib/actions/sandbox/connect.ts index 930caffe474..082fc7b84b7 100644 --- a/src/lib/actions/sandbox/connect.ts +++ b/src/lib/actions/sandbox/connect.ts @@ -891,6 +891,10 @@ function maybeEnsureHermesToolGatewayBroker(sb: SandboxEntry | null): void { } } +export function restoreSandboxStartupState(sandboxName: string): void { + checkAndRecoverSandboxProcesses(sandboxName, { quiet: true }); +} + function restoreInteractiveTerminal(): void { if (!process.stdin.isTTY) return; diff --git a/src/lib/actions/sandbox/start.test.ts b/src/lib/actions/sandbox/start.test.ts index 70e89bac072..e6ebea0d12b 100644 --- a/src/lib/actions/sandbox/start.test.ts +++ b/src/lib/actions/sandbox/start.test.ts @@ -10,7 +10,7 @@ import { } from "../../onboard/runtime-provider/docker"; import { createRuntimeProviderBundleRegistry } from "../../onboard/runtime-provider/registry"; import type { SandboxEntry } from "../../state/registry"; -import { type SandboxStartDeps, startSandbox } from "./start"; +import { restoreStoppedSandboxStartupState, type SandboxStartDeps, startSandbox } from "./start"; function sandbox(values: Partial = {}): SandboxEntry { return { name: "my-sandbox", ...values }; @@ -45,6 +45,7 @@ function harness(overrides: Partial = {}) { const verifyGateway = vi.fn>(() => Promise.resolve(), ); + const restoreStartupState = vi.fn>(); const log = vi.fn<(message: string) => void>(); const runtimeProviders = createRuntimeProviderBundleRegistry([ [ @@ -62,6 +63,7 @@ function harness(overrides: Partial = {}) { const deps: SandboxStartDeps = { getSandbox, runtimeProviders, + restoreStartupState, verifyGateway, log, ...overrides, @@ -74,21 +76,78 @@ function harness(overrides: Partial = {}) { isDockerRuntimeDown, log, printDockerRuntimeDownGuidance, - verifyGateway, recoverDockerDriverSandbox, + restoreStartupState, + verifyGateway, }; } describe("startSandbox", () => { - it("starts the stopped container and then probes gateway health (#6026)", async () => { + it("restores sealed access before recovering sandbox processes (#8112)", () => { + const restoreAccess = vi.fn(); + const restoreProcesses = vi.fn(); + + restoreStoppedSandboxStartupState("my-sandbox", { + agent: "openclaw", + restoreLockedStartupAccess: restoreAccess, + restoreProcessState: restoreProcesses, + }); + + expect(restoreAccess).toHaveBeenCalledWith("my-sandbox"); + expect(restoreProcesses).toHaveBeenCalledWith("my-sandbox"); + expect(restoreAccess.mock.invocationCallOrder[0]).toBeLessThan( + restoreProcesses.mock.invocationCallOrder[0], + ); + }); + + it("keeps Hermes sealed state untouched while recovering sandbox processes (#8112)", () => { + const restoreAccess = vi.fn(); + const restoreProcesses = vi.fn(); + + restoreStoppedSandboxStartupState("my-sandbox", { + agent: "hermes", + restoreLockedStartupAccess: restoreAccess, + restoreProcessState: restoreProcesses, + }); + + expect(restoreAccess).not.toHaveBeenCalled(); + expect(restoreProcesses).toHaveBeenCalledWith("my-sandbox"); + }); + + it("restores startup state before probing readiness after a stopped container starts (#8112)", async () => { const h = harness(); const result = await startSandbox("my-sandbox", h.deps); expect(result.exitCode).toBe(0); - expect(h.recoverDockerDriverSandbox).toHaveBeenCalledWith("my-sandbox"); + expect(h.recoverDockerDriverSandbox).toHaveBeenCalledWith("my-sandbox", { + readiness: "runtime-running", + }); + expect(h.restoreStartupState).toHaveBeenCalledWith("my-sandbox"); expect(h.verifyGateway).toHaveBeenCalledWith("my-sandbox"); expect(h.recoverDockerDriverSandbox.mock.invocationCallOrder[0]).toBeLessThan( + h.restoreStartupState.mock.invocationCallOrder[0], + ); + expect(h.restoreStartupState.mock.invocationCallOrder[0]).toBeLessThan( + h.verifyGateway.mock.invocationCallOrder[0], + ); + }); + + it("attempts startup restoration again when start is rerun after a failure (#8112)", async () => { + const h = harness(); + h.restoreStartupState.mockImplementationOnce(() => { + throw new Error("restore failed"); + }); + + await expect(startSandbox("my-sandbox", h.deps)).rejects.toThrow("restore failed"); + expect(h.verifyGateway).not.toHaveBeenCalled(); + + const result = await startSandbox("my-sandbox", h.deps); + + expect(result.exitCode).toBe(0); + expect(h.restoreStartupState).toHaveBeenCalledTimes(2); + expect(h.verifyGateway).toHaveBeenCalledOnce(); + expect(h.restoreStartupState.mock.invocationCallOrder[1]).toBeLessThan( h.verifyGateway.mock.invocationCallOrder[0], ); }); @@ -116,7 +175,11 @@ describe("startSandbox", () => { const result = await startSandbox("my-sandbox", h.deps); expect(result.exitCode).toBe(0); + expect(h.restoreStartupState).toHaveBeenCalledWith("my-sandbox"); expect(h.verifyGateway).toHaveBeenCalledWith("my-sandbox"); + expect(h.restoreStartupState.mock.invocationCallOrder[0]).toBeLessThan( + h.verifyGateway.mock.invocationCallOrder[0], + ); const output = h.log.mock.calls.map(([line]) => line).join("\n"); expect(output).toContain("already running"); }); @@ -139,7 +202,11 @@ describe("startSandbox", () => { timeout: 30_000, }); expect(h.recoverDockerDriverSandbox).not.toHaveBeenCalled(); + expect(h.restoreStartupState).toHaveBeenCalledWith("my-sandbox"); expect(h.verifyGateway).toHaveBeenCalledWith("my-sandbox"); + expect(h.restoreStartupState.mock.invocationCallOrder[0]).toBeLessThan( + h.verifyGateway.mock.invocationCallOrder[0], + ); const output = h.log.mock.calls.map(([line]) => line).join("\n"); expect(output).toContain("unpaused"); }); @@ -160,6 +227,7 @@ describe("startSandbox", () => { expect(result.exitCode).toBe(1); expect(result.message).toContain("openshell-my-sandbox"); expect(result.message).toContain("125"); + expect(h.restoreStartupState).not.toHaveBeenCalled(); expect(h.verifyGateway).not.toHaveBeenCalled(); }); @@ -189,6 +257,7 @@ describe("startSandbox", () => { retryCommand: "start", }); expect(h.recoverDockerDriverSandbox).not.toHaveBeenCalled(); + expect(h.restoreStartupState).not.toHaveBeenCalled(); expect(h.verifyGateway).not.toHaveBeenCalled(); }); @@ -206,6 +275,7 @@ describe("startSandbox", () => { expect(result.exitCode).toBe(1); expect(result.message).toContain("no Docker container labeled"); expect(result.message).toContain("rebuild"); + expect(h.restoreStartupState).not.toHaveBeenCalled(); expect(h.verifyGateway).not.toHaveBeenCalled(); }); @@ -218,6 +288,7 @@ describe("startSandbox", () => { expect(result.exitCode).toBe(1); expect(result.message).toContain("not registered"); expect(h.recoverDockerDriverSandbox).not.toHaveBeenCalled(); + expect(h.restoreStartupState).not.toHaveBeenCalled(); }); it("refuses non-direct drivers instead of guessing at container control (#6026)", async () => { @@ -231,6 +302,7 @@ describe("startSandbox", () => { expect(result.message).toContain("does not authorize 'start' mutation"); expect(h.findLabeledSandboxContainers).not.toHaveBeenCalled(); expect(h.recoverDockerDriverSandbox).not.toHaveBeenCalled(); + expect(h.restoreStartupState).not.toHaveBeenCalled(); expect(h.verifyGateway).not.toHaveBeenCalled(); }); @@ -249,6 +321,7 @@ describe("startSandbox", () => { expect(h.findLabeledSandboxContainers).not.toHaveBeenCalled(); expect(h.dockerUnpause).not.toHaveBeenCalled(); expect(h.recoverDockerDriverSandbox).not.toHaveBeenCalled(); + expect(h.restoreStartupState).not.toHaveBeenCalled(); expect(h.verifyGateway).not.toHaveBeenCalled(); }); diff --git a/src/lib/actions/sandbox/start.ts b/src/lib/actions/sandbox/start.ts index 6e63ca3f6cf..7e904befd0a 100644 --- a/src/lib/actions/sandbox/start.ts +++ b/src/lib/actions/sandbox/start.ts @@ -5,6 +5,7 @@ import { CURRENT_RUNTIME_PROVIDER_BUNDLES, type RuntimeProviderBundleRegistry, } from "../../onboard/runtime-provider/access"; +import type { SandboxEntry } from "../../state/registry"; import * as registry from "../../state/registry"; import { resolveSandboxLifecycleProvider, @@ -18,17 +19,46 @@ function verifyGateway(sandboxName: string): Promise { return connectSandbox(sandboxName, { probeOnly: true }); } +function restoreProcessState(sandboxName: string): void { + const { restoreSandboxStartupState } = require("./connect") as typeof import("./connect"); + restoreSandboxStartupState(sandboxName); +} + +function restoreLockedStartupAccess(sandboxName: string): void { + const { restoreLockedStateDirStartupAccess } = + require("../../shields") as typeof import("../../shields"); + restoreLockedStateDirStartupAccess(sandboxName); +} + +export interface SandboxStartupStateDeps { + agent?: SandboxEntry["agent"]; + restoreLockedStartupAccess?: (sandboxName: string) => void; + restoreProcessState?: (sandboxName: string) => void; +} + +export function restoreStoppedSandboxStartupState( + sandboxName: string, + deps: SandboxStartupStateDeps = {}, +): void { + if ((deps.agent ?? "openclaw") === "openclaw") { + (deps.restoreLockedStartupAccess ?? restoreLockedStartupAccess)(sandboxName); + } + (deps.restoreProcessState ?? restoreProcessState)(sandboxName); +} + export interface SandboxStartDeps { environment?: NodeJS.ProcessEnv; getSandbox?: typeof registry.getSandbox; runtimeProviders?: RuntimeProviderBundleRegistry; + restoreStartupState?: (sandboxName: string) => void; verifyGateway?: (sandboxName: string) => Promise; log?: (message: string) => void; } /** * Restart a stopped sandbox through the lifecycle facet bound to its durable - * provider identity, then restore gateway health and host forwards. + * provider identity, then restore startup state before verifying readiness and + * host forwards. */ export async function startSandbox( sandboxName: string, @@ -55,7 +85,17 @@ export async function startSandbox( const result = resolved.lifecycle.start(input); if (result.exitCode !== 0) return result; - log(" Checking gateway health and host forwards…"); - await resolved.lifecycle.verifyStarted(input, deps.verifyGateway ?? verifyGateway); + await resolved.lifecycle.verifyStarted(input, async (name) => { + log(" Restoring sandbox startup state…"); + const restoreStartupState = + deps.restoreStartupState ?? + ((sandboxNameToRestore: string) => + restoreStoppedSandboxStartupState(sandboxNameToRestore, { + agent: resolved.sandbox.agent, + })); + restoreStartupState(name); + log(" Checking gateway health and host forwards…"); + await (deps.verifyGateway ?? verifyGateway)(name); + }); return { exitCode: 0 }; } diff --git a/src/lib/onboard/docker-driver-sandbox-recovery.test.ts b/src/lib/onboard/docker-driver-sandbox-recovery.test.ts index e7c98d4ace4..53c7eca7fe7 100644 --- a/src/lib/onboard/docker-driver-sandbox-recovery.test.ts +++ b/src/lib/onboard/docker-driver-sandbox-recovery.test.ts @@ -195,6 +195,25 @@ describe("recoverDockerDriverSandbox — stopped original (start)", () => { expect(sleep).toHaveBeenCalledWith(1_000); }); + it("hands a running container to lifecycle verification while Docker health is starting (#8112)", () => { + const sleep = vi.fn(); + const result = recoverDockerDriverSandbox("e2e-x", { + dockerCapture: fakeCapture("openshell-e2e-x\tExited (137) 30 seconds ago\n", [ + "running\tstarting", + ]), + dockerStart: fakeStart(0), + readiness: "runtime-running", + sleep, + }); + + expect(result).toEqual({ + recovered: true, + via: "started-stopped-original", + containerName: "openshell-e2e-x", + }); + expect(sleep).not.toHaveBeenCalled(); + }); + it("accepts a running container whose image has no Docker health check", () => { const sleep = vi.fn(); const result = recoverDockerDriverSandbox("e2e-x", { @@ -319,6 +338,27 @@ describe("recoverDockerDriverSandbox — backup-only (rename + start)", () => { ); }); + it("hands a renamed backup to lifecycle verification while Docker health is starting (#8112)", () => { + const sleep = vi.fn(); + const result = recoverDockerDriverSandbox("e2e-x", { + dockerCapture: fakeCapture( + "openshell-e2e-x-nemoclaw-gpu-backup-1717280000000\tExited (0) 5 minutes ago\n", + ["running\tstarting"], + ), + dockerRename: fakeRename(0), + dockerStart: fakeStart(0), + readiness: "runtime-running", + sleep, + }); + + expect(result).toEqual({ + recovered: true, + via: "renamed-and-started-backup", + containerName: "openshell-e2e-x", + }); + expect(sleep).not.toHaveBeenCalled(); + }); + it("picks the most recent backup when several siblings exist", () => { const rename = vi.fn(fakeRename(0)); const start = vi.fn(fakeStart(0)); diff --git a/src/lib/onboard/docker-driver-sandbox-recovery.ts b/src/lib/onboard/docker-driver-sandbox-recovery.ts index 50d47d3ec07..71c6c49be1a 100644 --- a/src/lib/onboard/docker-driver-sandbox-recovery.ts +++ b/src/lib/onboard/docker-driver-sandbox-recovery.ts @@ -119,6 +119,13 @@ export interface DockerDriverRecoveryDeps { sleep?: (ms: number) => void; /** Injectable clock for deterministic readiness deadlines in tests. */ now?: () => number; + /** + * Readiness boundary for the caller. Recovery keeps Docker health as its + * default authority. The explicit sandbox lifecycle start path stops at a + * running container because its provider-owned verification then proves + * OpenShell registration, the managed agent gateway, and host forwards. + */ + readiness?: "docker-health" | "runtime-running"; } interface LabeledContainer { @@ -242,6 +249,7 @@ interface ContainerReadinessResult { function waitForRecoveredContainerReady( containerName: string, deps: ReturnType, + readiness: NonNullable, ): ContainerReadinessResult { let lastState = "unknown"; const deadlineMs = deps.now() + DOCKER_RECOVERY_READY_TIMEOUT_MS; @@ -264,7 +272,10 @@ function waitForRecoveredContainerReady( ? `runtime=${runtimeState || "unknown"}, health=${healthState}` : `runtime=${runtimeState || "unknown"}`; - if (runtimeState === "running" && (healthState === "healthy" || healthState === "none")) { + if ( + runtimeState === "running" && + (readiness === "runtime-running" || healthState === "healthy" || healthState === "none") + ) { return { ready: true, detail: lastState }; } // These states cannot become ready without another lifecycle action. @@ -284,16 +295,17 @@ function recoveredAfterReadiness( containerName: string, via: DockerDriverRecoveryVia, deps: ReturnType, + readiness: NonNullable, ): DockerDriverRecoveryResult { - const readiness = waitForRecoveredContainerReady(containerName, deps); - if (!readiness.ready) { + const result = waitForRecoveredContainerReady(containerName, deps, readiness); + if (!result.ready) { return { recovered: false, via: null, containerName, detail: `docker container ${containerName} did not become ready after recovery ` + - `(${readiness.detail})`, + `(${result.detail})`, }; } return { @@ -322,6 +334,7 @@ export function recoverDockerDriverSandbox( deps: DockerDriverRecoveryDeps = {}, ): DockerDriverRecoveryResult { const d = depsWithDefaults(deps); + const readiness = deps.readiness ?? "docker-health"; const containers = findLabeledSandboxContainers(sandboxName, deps); if (containers.length === 0) { return { @@ -347,9 +360,9 @@ export function recoverDockerDriverSandbox( detail: `docker unpause ${runningOriginal.name} failed (exit ${unpause.status ?? "unknown"}).`, }; } - return recoveredAfterReadiness(runningOriginal.name, "unpaused-original", d); + return recoveredAfterReadiness(runningOriginal.name, "unpaused-original", d, readiness); } - return recoveredAfterReadiness(runningOriginal.name, "started-running-original", d); + return recoveredAfterReadiness(runningOriginal.name, "started-running-original", d, readiness); } if (stoppedOriginal) { @@ -358,7 +371,12 @@ export function recoverDockerDriverSandbox( timeout: DOCKER_OPERATION_TIMEOUT_MS, }); if (result.status === 0) { - return recoveredAfterReadiness(stoppedOriginal.name, "started-stopped-original", d); + return recoveredAfterReadiness( + stoppedOriginal.name, + "started-stopped-original", + d, + readiness, + ); } return { recovered: false, @@ -385,7 +403,7 @@ export function recoverDockerDriverSandbox( timeout: DOCKER_OPERATION_TIMEOUT_MS, }); if (startResult.status === 0) { - return recoveredAfterReadiness(restoreName, "renamed-and-started-backup", d); + return recoveredAfterReadiness(restoreName, "renamed-and-started-backup", d, readiness); } return { recovered: false, diff --git a/src/lib/onboard/runtime-provider/docker.ts b/src/lib/onboard/runtime-provider/docker.ts index 63945928895..6fd2b59dde3 100644 --- a/src/lib/onboard/runtime-provider/docker.ts +++ b/src/lib/onboard/runtime-provider/docker.ts @@ -158,7 +158,13 @@ function startDockerSandbox( return { exitCode: 0 }; } - const recovery = deps.recoverSandbox(input.sandboxName); + // Docker health is an image-level signal, not the lifecycle authority for + // `start`. Once the container is running, verifyStarted performs the + // provider-owned OpenShell, managed gateway, and host-forward recovery. + // Waiting for Docker health here can prevent that repair from running. + const recovery = deps.recoverSandbox(input.sandboxName, { + readiness: "runtime-running", + }); if (!recovery.recovered) { return { exitCode: 1, diff --git a/src/lib/shields/index.ts b/src/lib/shields/index.ts index 0410802afaa..8dd81baa2e2 100644 --- a/src/lib/shields/index.ts +++ b/src/lib/shields/index.ts @@ -84,6 +84,7 @@ const { applyStateDirLockMode, preflightStateDirLock, restoreStateDirLockPosture, + restoreStateDirStartupAccess, }: typeof import("./state-dir-lock") = require("./state-dir-lock"); const { OPENCLAW_CONFIG_DIR, @@ -2113,6 +2114,20 @@ function repairMutableConfigPerms(sandboxName: string): MutableConfigRepairResul }); } +function restoreLockedStateDirStartupAccess(sandboxName: string): void { + validateName(sandboxName, "sandbox name"); + prepareExpiredAutoRestoreHostLockTakeover(sandboxName); + withTimerBoundShieldsMutationLock(sandboxName, "restore locked startup access", () => { + const posture = getShieldsPostureWithoutHostLock(sandboxName, true); + if (!posture.locked) return; + const target = ensureConfigHashSensitiveFile(resolveAgentConfig(sandboxName)); + const issues = restoreStateDirStartupAccess(stateDirLockExec(sandboxName), target.configDir); + if (issues.length > 0) { + throw new Error(`Locked startup access could not be restored: ${issues.join(", ")}`); + } + }); +} + // --------------------------------------------------------------------------- // Config lock — used by shields-up (opt-in lockdown), auto-restore timer, // and rollback @@ -3652,6 +3667,7 @@ export { parseDuration, prepareAutoRestoreTransitionTakeover, repairMutableConfigPerms, + restoreLockedStateDirStartupAccess, resolvePersistedAutoRestoreTarget, shieldsDown, shieldsStatus, diff --git a/src/lib/shields/state-dir-lock.test.ts b/src/lib/shields/state-dir-lock.test.ts index 91ddd2dd303..022333c6204 100644 --- a/src/lib/shields/state-dir-lock.test.ts +++ b/src/lib/shields/state-dir-lock.test.ts @@ -7,6 +7,7 @@ import { applyStateDirLockMode, preflightStateDirLock, restoreStateDirLockPosture, + restoreStateDirStartupAccess, } from "./state-dir-lock"; type RunCall = { cmd: string[]; input?: string }; @@ -99,6 +100,26 @@ describe("recursive state-dir lock host wiring", () => { expect(invocation?.input).toContain("Descriptor-safe recursive state-directory"); }); + it("uses the current host guard for the narrow startup repair (#8112)", () => { + const { calls, privileged } = createExec(); + + expect(restoreStateDirStartupAccess(privileged, "/sandbox/.openclaw")).toEqual([]); + expect(calls).toHaveLength(1); + expect(calls[0]?.cmd).toEqual([ + "timeout", + "--signal=TERM", + "--kill-after=5s", + "12m", + "python3", + "-I", + "-", + "startup", + "--config-dir", + "/sandbox/.openclaw", + ]); + expect(calls[0]?.input).toContain('choices=("preflight", "lock", "unlock", "startup")'); + }); + it("surfaces structured helper findings and rejects contradictory exit contracts", () => { const privileged: PrivilegedExec = { run: (cmd) => { diff --git a/src/lib/shields/state-dir-lock.ts b/src/lib/shields/state-dir-lock.ts index 7f11a4a329f..fdf2de18da2 100644 --- a/src/lib/shields/state-dir-lock.ts +++ b/src/lib/shields/state-dir-lock.ts @@ -52,7 +52,7 @@ const CONTAINER_HELPER = "/usr/local/lib/nemoclaw/state-dir-guard.py"; const HOST_HELPER = path.resolve(__dirname, "../../../scripts/state-dir-guard.py"); const CONTAINER_TIMEOUT = ["timeout", "--signal=TERM", "--kill-after=5s", "12m"]; -type GuardAction = "preflight" | "lock" | "unlock"; +type GuardAction = "preflight" | "lock" | "unlock" | "startup"; type GuardIssue = { type: "issue"; @@ -108,7 +108,10 @@ function parseGuardOutput(action: GuardAction, result: PrivilegedExecResult): st } if ( record.type === "result" && - (record.action === "preflight" || record.action === "lock" || record.action === "unlock") && + (record.action === "preflight" || + record.action === "lock" || + record.action === "unlock" || + record.action === "startup") && (record.status === "ok" || record.status === "failed") && typeof record.issueCount === "number" && Number.isInteger(record.issueCount) @@ -200,6 +203,22 @@ function runStateDirGuard( return parseGuardOutput(action, privileged.run(command, input)); } +function runHostStateDirGuard( + privileged: PrivilegedExec, + action: GuardAction, + configDir: string, +): string[] { + let input: string; + try { + input = readHostHelper(); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return [`host state-dir helper cannot be read: ${message}`]; + } + const command = [...CONTAINER_TIMEOUT, "python3", "-I", "-", action, "--config-dir", configDir]; + return parseGuardOutput(action, privileged.run(command, input)); +} + // Read-only recursive validation. Call this before top-level config mutation so // a hostile nested link, hardlink, special entry, or cross-device mount fails // without partially changing the protected tree. @@ -237,3 +256,13 @@ export function restoreStateDirLockPosture( if (preflightIssues.length > 0) return preflightIssues; return applyStateDirLockMode(privileged, configDir, "root:sandbox", true); } + +// Existing sandboxes may contain an older in-image guard, so startup always +// injects the trusted helper shipped with the current host CLI. The startup +// action changes only an empty, already sealed credentials root. +export function restoreStateDirStartupAccess( + privileged: PrivilegedExec, + configDir: string, +): string[] { + return runHostStateDirGuard(privileged, "startup", configDir); +} diff --git a/test/e2e/live/hermes-shields-config.test.ts b/test/e2e/live/hermes-shields-config.test.ts index 3d2b192fdb5..1287288b1be 100644 --- a/test/e2e/live/hermes-shields-config.test.ts +++ b/test/e2e/live/hermes-shields-config.test.ts @@ -102,6 +102,38 @@ async function expectShieldsStatus( expect(resultText(status)).toContain(`Shields: ${expected}`); } +async function expectStopStartRecovery( + host: HostCliClient, + posture: "DOWN" | "UP", + artifactPrefix: string, +): Promise { + const stop = await host.nemoclaw([SANDBOX_NAME, "stop"], { + artifactName: `${artifactPrefix}-stop`, + env: commandEnv(), + redactionValues: [COMPATIBLE_API_KEY], + timeoutMs: 5 * 60_000, + }); + assertExitZero(stop, `stop Hermes with shields ${posture.toLowerCase()}`); + + const start = await host.nemoclaw([SANDBOX_NAME, "start"], { + artifactName: `${artifactPrefix}-start`, + env: commandEnv(), + redactionValues: [COMPATIBLE_API_KEY], + timeoutMs: 5 * 60_000, + }); + assertExitZero(start, `start Hermes with shields ${posture.toLowerCase()}`); + + const status = await host.nemoclaw([SANDBOX_NAME, "status"], { + artifactName: `${artifactPrefix}-status`, + env: commandEnv(), + redactionValues: [COMPATIBLE_API_KEY], + timeoutMs: 5 * 60_000, + }); + assertExitZero(status, `read Hermes status with shields ${posture.toLowerCase()}`); + expect(stripAnsi(resultText(status))).toMatch(/Phase:\s*Ready/i); + await expectShieldsStatus(host, posture, `${artifactPrefix}-shields-status`); +} + async function expectMutablePosture(sandbox: SandboxClient, cycle: number): Promise { const result = await sandboxShell( sandbox, @@ -152,7 +184,7 @@ async function completeShieldsCycle( await expectLockedPosture(sandbox, cycle); } -test("hermes-shields-config: fresh non-root Hermes sandbox completes two shields cycles (#6381)", { +test("hermes-shields-config: stopped Hermes restores under both Shields postures (#6381, #8112)", { timeout: HERMES_SHIELDS_CONFIG_TEST_TIMEOUT_MS, meta: { e2ePhases: [ @@ -160,6 +192,8 @@ test("hermes-shields-config: fresh non-root Hermes sandbox completes two shields "onboard non-root Hermes sandbox", "verify fresh Hermes runtime state", "complete first shields cycle", + "restart Hermes with shields up", + "unlock shields and restart Hermes", "complete second shields cycle", "verify preserved config and ready state", ], @@ -167,11 +201,14 @@ test("hermes-shields-config: fresh non-root Hermes sandbox completes two shields }, async ({ artifacts, cleanup: cleanupRegistry, host, progress, sandbox }) => { await artifacts.target.declare({ id: "hermes-shields-config", - boundary: "fresh CPU-only Hermes onboard plus two real shields down/up transitions", + boundary: + "fresh CPU-only Hermes onboard plus two real shields down/up transitions and stopped-sandbox recovery", contracts: [ "fresh OpenShell-managed non-root Hermes startup mints its API key", "the first shields-down reconciles the startup hash anchor", "shields-up establishes the root-owned locked posture", + "start restores a stopped Hermes sandbox while shields are up", + "start restores a stopped Hermes sandbox while shields are down", "a second down/up cycle completes without corrupting config state", ], issue: "#6381", @@ -289,8 +326,28 @@ test("hermes-shields-config: fresh non-root Hermes sandbox completes two shields progress.phase("complete first shields cycle"); await completeShieldsCycle(host, sandbox, 1); + + progress.phase("restart Hermes with shields up"); + await expectStopStartRecovery(host, "UP", "cycle-1-shields-up-start-recovery"); + await expectLockedPosture(sandbox, 1); + + progress.phase("unlock shields and restart Hermes"); + const down = await runShields( + host, + ["down", "--timeout", "15m", "--reason", "Hermes live E2E cycle 2"], + "cycle-2-shields-down", + ); + assertExitZero(down, "unlock fresh Hermes config in cycle 2"); + await expectShieldsStatus(host, "DOWN", "cycle-2-status-down"); + await expectMutablePosture(sandbox, 2); + await expectStopStartRecovery(host, "DOWN", "cycle-2-shields-down-start-recovery"); + await expectMutablePosture(sandbox, 2); + progress.phase("complete second shields cycle"); - await completeShieldsCycle(host, sandbox, 2); + const up = await runShields(host, ["up"], "cycle-2-shields-up"); + assertExitZero(up, "lock fresh Hermes config in cycle 2"); + await expectShieldsStatus(host, "UP", "cycle-2-status-up"); + await expectLockedPosture(sandbox, 2); progress.phase("verify preserved config and ready state"); const configHashAfter = await sandboxShell( @@ -317,6 +374,8 @@ test("hermes-shields-config: fresh non-root Hermes sandbox completes two shields configPreserved: true, freshNonrootTrigger: true, firstCycle: true, + shieldsDownStartRecovery: true, + shieldsUpStartRecovery: true, secondCycle: true, }, }); diff --git a/test/e2e/live/shields-config.test.ts b/test/e2e/live/shields-config.test.ts index 8e58e981ef1..5667b3cee7a 100644 --- a/test/e2e/live/shields-config.test.ts +++ b/test/e2e/live/shields-config.test.ts @@ -29,6 +29,7 @@ import { expect, test } from "../fixtures/e2e-test.ts"; import { requireHostedInferenceConfig } from "../fixtures/hosted-inference.ts"; import { REPO_ROOT } from "../fixtures/paths.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; +import { stripAnsi } from "./json-envelope.ts"; const CONFIG_PATH = "/sandbox/.openclaw/openclaw.json"; const CONFIG_DIR = path.dirname(CONFIG_PATH); @@ -143,6 +144,77 @@ async function statPath( return { ...parsed, raw: result.stdout.trim() }; } +async function expectStopStartRecovery( + host: HostCliClient, + posture: "DOWN" | "UP", + artifactPrefix: string, +): Promise { + const stop = await runNemoclaw(host, [SANDBOX_NAME, "stop"], { + artifactName: `${artifactPrefix}-stop`, + timeoutMs: 5 * 60_000, + }); + expect(stop.exitCode, resultText(stop)).toBe(0); + + const start = await runNemoclaw(host, [SANDBOX_NAME, "start"], { + artifactName: `${artifactPrefix}-start`, + timeoutMs: 5 * 60_000, + }); + expect(start.exitCode, resultText(start)).toBe(0); + + const status = await runNemoclaw(host, [SANDBOX_NAME, "status"], { + artifactName: `${artifactPrefix}-status`, + timeoutMs: 5 * 60_000, + }); + expect(status.exitCode, resultText(status)).toBe(0); + expect(stripAnsi(resultText(status))).toMatch(/Phase:\s*Ready/i); + + const shields = await runNemoclaw(host, [SANDBOX_NAME, "shields", "status"], { + artifactName: `${artifactPrefix}-shields-status`, + }); + expect(shields.exitCode, resultText(shields)).toBe(0); + expect(resultText(shields)).toContain(`Shields: ${posture}`); +} + +async function expectCredentialsTraversalBoundary( + host: HostCliClient, + sandbox: SandboxClient, + containerId: string, +): Promise { + const credentialsDir = "/sandbox/.openclaw/credentials"; + const seededPath = `${credentialsDir}/.nemoclaw-permission-probe`; + const seeded = await docker( + host, + ["exec", "--user", "0", containerId, "sh", "-c", `umask 077; : > ${seededPath}`], + { artifactName: "phase-5a-seed-credential-permission-probe" }, + ); + expect(seeded.exitCode, resultText(seeded)).toBe(0); + + try { + const metadata = await statPath( + sandbox, + credentialsDir, + "phase-5a-credential-directory-metadata", + ); + expect(metadata.mode).toBe("710"); + expect(metadata.owner).toBe("root:sandbox"); + + const boundary = await sandboxShell( + sandbox, + `python3 - <<'PY'\nimport os\n\ndirectory = ${JSON.stringify(credentialsDir)}\nseeded = ${JSON.stringify(seededPath)}\noptional = os.path.join(directory, "optional.json")\n\ntry:\n os.stat(optional)\nexcept FileNotFoundError:\n print("traversal=allowed")\nelse:\n raise RuntimeError("optional credential path unexpectedly exists")\n\noperations = (\n ("listing", lambda: os.listdir(directory)),\n ("reading", lambda: open(seeded, "rb").read()),\n ("creation", lambda: open(os.path.join(directory, "created"), "wb").close()),\n ("removal", lambda: os.unlink(seeded)),\n)\nfor label, operation in operations:\n try:\n operation()\n except PermissionError:\n print(f"{label}=denied")\n else:\n raise RuntimeError(f"{label} unexpectedly allowed")\nPY`, + { artifactName: "phase-5a-credential-traversal-boundary" }, + ); + expect(boundary.exitCode, resultText(boundary)).toBe(0); + expect(boundary.stdout).toContain("traversal=allowed"); + for (const operation of ["listing", "reading", "creation", "removal"]) { + expect(boundary.stdout).toContain(`${operation}=denied`); + } + } finally { + await docker(host, ["exec", "--user", "0", containerId, "rm", "-f", seededPath], { + artifactName: "phase-5a-remove-credential-permission-probe", + }); + } +} + async function preCleanSandbox( host: HostCliClient, sandbox: SandboxClient, @@ -222,16 +294,18 @@ function readTimerMarker(sandboxName: string): { return JSON.parse(fs.readFileSync(TIMER_FILE(sandboxName), "utf8")); } -test("shields-config: live shields up/down locks config and detects drift", { +test("shields-config: live Shields lifecycle restores stopped OpenClaw under both postures (#8112)", { timeout: TEST_TIMEOUT_MS, meta: { e2ePhases: [ "confirm Docker and onboard the shields sandbox", "establish the mutable unified OpenClaw config", "lock config and workspace and inspect redaction", + "restart OpenClaw with shields up", "detect host-root config drift and refuse resealing", "re-seal a perms-only .config-hash drift instead of failing closed", "unlock shields and inspect the audit trail", + "restart OpenClaw with shields down", "recover shields after a dead restore timer", "reject duplicate shields transitions", "record shields contract evidence", @@ -246,9 +320,12 @@ test("shields-config: live shields up/down locks config and detects drift", { "default config starts mutable with unified .openclaw layout", "documented nemoclaw exec doctor path preserves 2770/660 and gateway writes", "shields up locks config/workspace and config get redacts secrets", + "start restores a stopped OpenClaw sandbox while shields are up", + "empty sealed credentials allow traversal but deny sandbox identity access", "host-root chmod-write-chmod tamper is detected as content drift", "a perms-only .config-hash drift is re-sealed by shields up, not failed closed", "shields down restores mutable modes and records audit JSONL", + "start restores a stopped OpenClaw sandbox while shields are down", "dead auto-restore timer inline recovery re-locks config and .config-hash", "double shields-up/down operations are rejected", ], @@ -512,6 +589,17 @@ test("shields-config: live shields up/down locks config and detects drift", { expect(statusUp.exitCode, resultText(statusUp)).toBe(0); expect(statusUp.stdout).toContain("Shields: UP"); + progress.phase("restart OpenClaw with shields up"); + await expectStopStartRecovery(host, "UP", "phase-5a-shields-up-start-recovery"); + const configAfterLockedRestart = await statPath( + sandbox, + CONFIG_PATH, + "phase-5a-config-after-shields-up-start-recovery", + ); + expect(configAfterLockedRestart.mode).toMatch(/^4[0-4][0-4]$/); + expect(configAfterLockedRestart.owner).toBe("root:root"); + await expectCredentialsTraversalBoundary(host, sandbox, containerId); + progress.phase("detect host-root config drift and refuse resealing"); const originalConfig = path.join(os.tmpdir(), `nemoclaw-shields-orig-${process.pid}.json`); await readOriginalConfig(host, containerId, originalConfig); @@ -632,7 +720,7 @@ test("shields-config: live shields up/down locks config and detects drift", { progress.phase("unlock shields and inspect the audit trail"); const shieldsDown = await runNemoclaw( host, - [SANDBOX_NAME, "shields", "down", "--timeout", "5m", "--reason", "E2E shields lifecycle test"], + [SANDBOX_NAME, "shields", "down", "--timeout", "15m", "--reason", "E2E shields lifecycle test"], { artifactName: "phase-6-shields-down" }, ); expect(shieldsDown.exitCode, resultText(shieldsDown)).toBe(0); @@ -659,6 +747,18 @@ test("shields-config: live shields up/down locks config and detects drift", { expect(statusDown.stdout).toContain("E2E shields lifecycle test"); expect(statusDown.stdout).toMatch(/Auto-lockdown in:|remaining/i); + progress.phase("restart OpenClaw with shields down"); + await expectStopStartRecovery(host, "DOWN", "phase-7a-shields-down-start-recovery"); + const configAfterMutableRestart = await statPath( + sandbox, + CONFIG_PATH, + "phase-7a-config-after-shields-down-start-recovery", + ); + expect(configAfterMutableRestart).toMatchObject({ + mode: "660", + owner: "sandbox:sandbox", + }); + const restoreUp = await runNemoclaw(host, [SANDBOX_NAME, "shields", "up"], { artifactName: "phase-7-restore-shields-up", }); diff --git a/test/headless-server-docs.test.ts b/test/headless-server-docs.test.ts index 7af6f45dcd6..16203b89012 100644 --- a/test/headless-server-docs.test.ts +++ b/test/headless-server-docs.test.ts @@ -95,9 +95,14 @@ describe("headless server deployment guide contracts", () => { }); it("keeps dashboard access and token lifecycles specific to each agent (#7180)", () => { - expect(openclawGuide).toContain("OpenClaw generates a new gateway token each time"); expect(openclawGuide).toContain( - "| OpenClaw gateway token | Rotated when the container starts |", + "OpenClaw generates a new gateway token when the sandbox container starts with mutable configuration", + ); + expect(openclawGuide).toContain( + "preserves the sealed token because the sandbox user cannot replace the protected configuration", + ); + expect(openclawGuide).toContain( + "| OpenClaw gateway token | Rotated when the container starts with mutable configuration; preserved for a non-root start while Shields are up |", ); expect(openclawGuide).not.toContain("Hermes preserves its `API_SERVER_KEY`"); diff --git a/test/nemoclaw-start-sealed-restart.test.ts b/test/nemoclaw-start-sealed-restart.test.ts new file mode 100644 index 00000000000..03caaeeb8b7 --- /dev/null +++ b/test/nemoclaw-start-sealed-restart.test.ts @@ -0,0 +1,141 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; + +const START_SCRIPT = path.join(import.meta.dirname, "..", "scripts", "nemoclaw-start.sh"); + +describe("nemoclaw-start sealed restart", () => { + it("preserves the sealed gateway token during non-root startup with Shields up (#8112)", () => { + const script = [ + "set -euo pipefail", + `eval "$(sed -n '/^needs_gateway_token_for_current_command() {$/,/^}$/p' "$1")"`, + `eval "$(sed -n '/^prepare_gateway_token_for_current_command() {$/,/^}$/p' "$1")"`, + "id() { echo 998; }", + "openclaw_config_dir_owner() { echo root; }", + "_read_gateway_token() { echo sealed-token; }", + 'ensure_gateway_token() { echo "SHOULD_NOT_ROTATE"; exit 75; }', + 'ensure_gateway_token_if_missing() { echo "SHOULD_NOT_ENSURE"; exit 76; }', + "NEMOCLAW_CMD=()", + "prepare_gateway_token_for_current_command", + ].join("\n"); + + const result = spawnSync("bash", ["-s", "--", START_SCRIPT], { + input: script, + encoding: "utf-8", + timeout: 5000, + }); + + expect(result.status).toBe(0); + expect(result.stdout).not.toContain("SHOULD_NOT"); + expect(result.stderr).toContain( + "Shields are up; preserving the sealed gateway token for startup", + ); + expect(result.stderr).not.toContain("sealed-token"); + }); + + it("refuses non-root startup when the sealed config has no gateway token (#8112)", () => { + const script = [ + "set -euo pipefail", + `eval "$(sed -n '/^needs_gateway_token_for_current_command() {$/,/^}$/p' "$1")"`, + `eval "$(sed -n '/^prepare_gateway_token_for_current_command() {$/,/^}$/p' "$1")"`, + "id() { echo 998; }", + "openclaw_config_dir_owner() { echo root; }", + "_read_gateway_token() { :; }", + 'ensure_gateway_token() { echo "SHOULD_NOT_ROTATE"; exit 75; }', + 'ensure_gateway_token_if_missing() { echo "SHOULD_NOT_ENSURE"; exit 76; }', + "NEMOCLAW_CMD=()", + "prepare_gateway_token_for_current_command", + ].join("\n"); + + const result = spawnSync("bash", ["-s", "--", START_SCRIPT], { + input: script, + encoding: "utf-8", + timeout: 5000, + }); + + expect(result.status).toBe(1); + expect(result.stdout).not.toContain("SHOULD_NOT"); + expect(result.stderr).toContain( + "Shields are up but the sealed OpenClaw config has no gateway token", + ); + }); + + it("preserves an unreadable sealed auth profile during non-root startup (#8112)", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-sealed-auth-test-")); + const authPath = path.join(home, ".openclaw", "agents", "main", "agent", "auth-profiles.json"); + const before = JSON.stringify({ + "openai:manual": { + type: "api_key", + provider: "openai", + keyRef: { source: "env", id: "NVIDIA_INFERENCE_API_KEY" }, + profileId: "openai:manual", + }, + }); + fs.mkdirSync(path.dirname(authPath), { recursive: true }); + fs.writeFileSync(authPath, before, { mode: 0o600 }); + const authFd = fs.openSync(authPath, "r"); + const authInode = fs.fstatSync(authFd).ino; + fs.fchmodSync(authFd, 0o000); + const script = [ + "set -euo pipefail", + `eval "$(sed -n '/^write_auth_profile() {$/,/^}$/p' "$1")"`, + "id() { echo 998; }", + "openclaw_config_dir_owner() { echo root; }", + "write_auth_profile", + ].join("\n"); + + try { + const result = spawnSync("bash", ["-s", "--", START_SCRIPT], { + input: script, + env: { + PATH: process.env.PATH, + HOME: home, + NVIDIA_INFERENCE_API_KEY: "secret", + NEMOCLAW_INFERENCE_PROVIDER_ID: "openai", + }, + encoding: "utf-8", + }); + expect(result.status, result.stderr).toBe(0); + expect(result.stderr).toContain("preserving the sealed OpenClaw auth profile"); + expect(fs.fstatSync(authFd).mode & 0o777).toBe(0o000); + expect(fs.lstatSync(authPath).ino).toBe(authInode); + expect(fs.readFileSync(authFd, "utf-8")).toBe(before); + } finally { + fs.closeSync(authFd); + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it("refuses a missing sealed auth profile during non-root startup (#8112)", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-missing-sealed-auth-test-")); + const script = [ + "set -euo pipefail", + `eval "$(sed -n '/^write_auth_profile() {$/,/^}$/p' "$1")"`, + "id() { echo 998; }", + "openclaw_config_dir_owner() { echo root; }", + "write_auth_profile", + ].join("\n"); + + try { + const result = spawnSync("bash", ["-s", "--", START_SCRIPT], { + input: script, + env: { + PATH: process.env.PATH, + HOME: home, + NVIDIA_INFERENCE_API_KEY: "secret", + NEMOCLAW_INFERENCE_PROVIDER_ID: "openai", + }, + encoding: "utf-8", + }); + expect(result.status).toBe(1); + expect(result.stderr).toContain("sealed OpenClaw auth profile is unavailable"); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); +}); diff --git a/test/nemoclaw-start.test.ts b/test/nemoclaw-start.test.ts index b248c388c09..6a1c41407a0 100644 --- a/test/nemoclaw-start.test.ts +++ b/test/nemoclaw-start.test.ts @@ -4482,7 +4482,6 @@ describe("setup_auth_profile_as_sandbox", () => { extractShellFunctionFromSource(src, "run_step_down_as_sandbox"), ].join("\n"); const setup = extractShellFunctionFromSource(src, "setup_auth_profile_as_sandbox"); - it("runs the auth-profile setup under HOME=/sandbox even when the parent env has HOME=/root", () => { // setpriv preserves the parent shell's environment, so the root // entrypoint's HOME=/root would otherwise leak into the step-down @@ -4500,6 +4499,7 @@ describe("setup_auth_profile_as_sandbox", () => { "set -euo pipefail", "export HOME=/root", "STEP_DOWN_PREFIX_SANDBOX=(env)", + "openclaw_config_dir_owner() { echo sandbox; }", `write_auth_profile() { printf '%s\\n' "$HOME" >${JSON.stringify(observedHome)}; }`, "harden_auth_profiles() { :; }", helper, diff --git a/test/state-dir-guard.test.ts b/test/state-dir-guard.test.ts index 201a619f4e4..77faeb01f2b 100644 --- a/test/state-dir-guard.test.ts +++ b/test/state-dir-guard.test.ts @@ -231,7 +231,7 @@ function fixture(configDirName = ".agent"): { root: string; configDir: string } } function runGuard( - action: "preflight" | "lock" | "unlock", + action: "preflight" | "lock" | "unlock" | "startup", configDir: string, env: Record = {}, ) { @@ -937,6 +937,26 @@ describe("state-dir-guard", () => { expect(mode(agentDir)).toBe(0o2770); }); + it("restores startup traversal only for an empty sealed credentials root (#8112)", () => { + const { configDir } = fixture(); + const credentialsDir = path.join(configDir, "credentials"); + fs.mkdirSync(credentialsDir); + fs.chmodSync(credentialsDir, 0o700); + + const restored = runGuard("startup", configDir); + + expect(restored.status, JSON.stringify(restored.lines)).toBe(0); + expect(mode(credentialsDir)).toBe(0o710); + + fs.writeFileSync(path.join(credentialsDir, "token.json"), "secret\n", { mode: 0o600 }); + + const nonemptyStartup = runGuard("startup", configDir); + + expect(nonemptyStartup.status, nonemptyStartup.stderr).toBe(0); + expect(mode(credentialsDir)).toBe(0o700); + expect(mode(path.join(credentialsDir, "token.json"))).toBe(0o600); + }); + it("creates a missing sessions carveout during lock so a first-boot agent can write sessions (#7545)", () => { const { configDir } = fixture(".openclaw"); const agentDir = path.join(configDir, "agents", "main");