diff --git a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx index 1b6be097676..a455c4abafd 100644 --- a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx +++ b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx @@ -25,6 +25,18 @@ This path preserves the sandbox workspace and repairs the agent runtime and host If the container is paused, follow the printed `docker unpause` guidance instead. If the container is missing or OpenShell reports another terminal phase such as `Failed`, follow the printed `rebuild --yes` guidance so NemoClaw can recreate the sandbox from its recorded metadata. + +When the sandbox has shields up and the OpenClaw gateway does not come back after the container restarts, lower shields before reaching for a rebuild: + +```bash +$$nemoclaw shields down +``` + +NemoClaw accepts `shields down` once it confirms, while holding the config mutation lock, that no startup process is running and no readiness lease was published. +Other locked-config operations still require the lease. +Start the sandbox again after shields are down, then re-run `shields up` when it is healthy. + + ## Recover the Agent Runtime diff --git a/scripts/openclaw-config-guard.py b/scripts/openclaw-config-guard.py index 5bcaaddfbbd..cafef14aab8 100755 --- a/scripts/openclaw-config-guard.py +++ b/scripts/openclaw-config-guard.py @@ -50,8 +50,13 @@ "publish-startup-ready", "write-config", "recover", + "unlock-failed-startup", ] StartupIdentity = tuple[int, str, int] +# One action only. It unseals both layers in a single mutex window, so the +# multi-step host sequence can never mutate state on stale evidence (#8304). +STARTUP_FAILURE_RECOVERY_ACTIONS = frozenset({"unlock-failed-startup"}) +INSTALLED_STATE_DIR_GUARD = "/usr/local/lib/nemoclaw/state-dir-guard.py" CONFIG_FILES = ("openclaw.json", ".config-hash") PRODUCTION_CONFIG_DIR = "/sandbox/.openclaw" MAX_FILE_BYTES = { @@ -73,6 +78,11 @@ NODE_BINARY_PATH = "/usr/local/bin/node" JSON5_MODULE_PATH = "/opt/nemoclaw/node_modules/json5" JSON5_VALIDATION_TIMEOUT_SECONDS = 5 +# Whole-action budget for `unlock-failed-startup`: the recursive unseal and its +# relock rollback share it. Keep it below RECOVERY_CONTAINER_TIMEOUT in +# src/lib/shields/openclaw-config-lock.ts, or the host kills the guard before +# the rollback can finish. +STATE_DIR_GUARD_TIMEOUT_SECONDS = 12 * 60 INSTALLED_HELPER_PATH = "/usr/local/lib/nemoclaw/openclaw-config-guard.py" COPY_BUFFER_BYTES = 1024 * 1024 STABLE_READ_ATTEMPTS = 3 @@ -986,14 +996,15 @@ def _pinned_process_matches_supervised_nonroot_start( os.close(proc_pid_fd) -def _openshell_supervised_nonroot_start_is_live( +def _openshell_supervised_nonroot_start_census( expected_root_uid: int, expected_sandbox_uid: int, - required_pid: int | None = None, -) -> bool: +) -> tuple[int, int | None] | None: + """Return a stable OpenShell start-child census, or ``None`` on uncertainty.""" + supervisor_identity = _openshell_supervisor_identity(expected_root_uid) if supervisor_identity is None: - return False + return None proc_root_fd = -1 try: proc_root_fd = _open_proc_root() @@ -1006,7 +1017,7 @@ def _openshell_supervised_nonroot_start_is_live( continue observed += 1 if observed > MAX_PROC_ENTRIES: - return False + return None if _pinned_process_matches_supervised_nonroot_start( proc_root_fd, entry.name, @@ -1016,20 +1027,44 @@ def _openshell_supervised_nonroot_start_is_live( matches += 1 matched_pid = int(entry.name, 10) if matches > 1: - return False - return bool( - matches == 1 - and (required_pid is None or matched_pid == required_pid) - and _openshell_supervisor_identity(expected_root_uid) - == supervisor_identity - ) + return matches, None + if _openshell_supervisor_identity(expected_root_uid) != supervisor_identity: + return None + return matches, matched_pid except OSError: - return False + return None finally: if proc_root_fd >= 0: os.close(proc_root_fd) +def _openshell_supervised_nonroot_start_is_live( + expected_root_uid: int, + expected_sandbox_uid: int, + required_pid: int | None = None, +) -> bool: + census = _openshell_supervised_nonroot_start_census( + expected_root_uid, expected_sandbox_uid + ) + return bool( + census is not None + and census[0] == 1 + and (required_pid is None or census[1] == required_pid) + ) + + +def _openshell_supervised_nonroot_start_is_absent( + expected_root_uid: int, + expected_sandbox_uid: int, +) -> bool: + """Return whether a stable OpenShell supervisor has no start child.""" + + census = _openshell_supervised_nonroot_start_census( + expected_root_uid, expected_sandbox_uid + ) + return census is not None and census[0] == 0 + + def _pid1_effective_uid() -> int | None: """Read PID 1's effective UID from a pinned procfs descriptor.""" @@ -1325,7 +1360,14 @@ def _revoke_startup_ready(identity: Identity) -> None: def _validate_action_readiness( action: Action, startup_owner: bool, identity: Identity -) -> None: +) -> bool: + """Authorize ``action`` and report whether only the failed-startup path allowed it. + + A ``True`` result is provisional: it rests on a live procfs census taken + before the mutation mutex, so the caller must reconfirm it under the mutex + with ``_reconfirm_startup_failure_recovery`` before any effect. + """ + startup_action = action in {"revoke-startup-ready", "publish-startup-ready"} if startup_action: if not _pid1_is_nemoclaw_start() or not startup_owner or os.getppid() != 1: @@ -1334,7 +1376,7 @@ def _validate_action_readiness( STARTUP_READY_PATH, f"{action} is restricted to the PID 1 startup transaction", ) - return + return False installed_current = os.path.realpath(__file__) == os.path.realpath( INSTALLED_HELPER_PATH ) @@ -1343,11 +1385,14 @@ def _validate_action_readiness( # A source helper injected into an older image, and the local unit # harness, retain their explicit compatibility path. Current images # use the installed helper and authenticate a namespace remap below. - return + return False protocol_active, startup_ready = _startup_lease_state(identity) if not pid1_is_nemoclaw_start and not protocol_active: if ( installed_current + # The recovery action is authorized by the escape below and by + # nothing else. + and action not in STARTUP_FAILURE_RECOVERY_ACTIONS and _startup_markers_absent(identity) and _openshell_supervised_nonroot_start_is_live( identity.root_uid, identity.sandbox_uid @@ -1360,19 +1405,29 @@ def _validate_action_readiness( # and NSpid evidence selects the same two topologies. They cannot # publish root-owned readiness markers, so authenticate the stable # supervisor/child pair while refusing stale or malformed markers. - return - if installed_current: - raise GuardError( - "startup-not-ready", - STARTUP_READY_PATH, - "installed config guard requires NemoClaw PID 1", + return False + if ( + installed_current + and action in STARTUP_FAILURE_RECOVERY_ACTIONS + and _startup_markers_absent(identity) + and _openshell_supervised_nonroot_start_is_absent( + identity.root_uid, identity.sandbox_uid ) - return + ): + # Provisional: a child can still appear after this scan, so main() + # reconfirms under the mutation mutex before any effect (#8304). + return True + # The early return above leaves `installed_current` true here. + raise GuardError( + "startup-not-ready", + STARTUP_READY_PATH, + "installed config guard requires NemoClaw PID 1", + ) # Source injected into an older image retains compatibility until that # image explicitly opts in. The trusted installed helper requires the # protocol from its very first exec, closing the pre-revoke boot race. if not installed_current and not protocol_active: - return + return False if installed_current and not protocol_active: # The supported --user sandbox entrypoint cannot create a root-owned # readiness capability. It also explicitly disables gateway privilege @@ -1382,7 +1437,7 @@ def _validate_action_readiness( # capability opts even a non-root PID 1 into the strict lease below. pid1_euid = _pid1_effective_uid() if pid1_euid is not None and pid1_euid != identity.root_uid: - return + return False early_recover = action == "recover" and not startup_ready if early_recover: if not startup_owner or os.getppid() != 1: @@ -1391,7 +1446,7 @@ def _validate_action_readiness( STARTUP_READY_PATH, f"{action} is restricted to the PID 1 startup transaction", ) - return + return False if action in { "lock", "unlock", @@ -1404,6 +1459,136 @@ def _validate_action_readiness( STARTUP_READY_PATH, "OpenClaw startup is not ready for host config mutations", ) + return False + + +def _reconfirm_startup_failure_recovery(action: Action, identity: Identity) -> None: + """Re-prove a failed startup while the mutation mutex is held. + + The first scan runs before the mutex. Repeating it here binds the + authorization to the effect. + """ + + if _startup_markers_absent(identity) and _openshell_supervised_nonroot_start_is_absent( + identity.root_uid, identity.sandbox_uid + ): + return + raise GuardError( + "startup-not-ready", + STARTUP_READY_PATH, + f"{action} lost its failed-startup authorization before taking effect", + ) + + +def _run_state_dir_guard( + action: str, + config_dir: str, + plan_json: str, + mutation_lock_fd: int, + deadline: float, +) -> None: + """Run the recursive state-dir guard under this process's mutation mutex. + + The child takes the same mutex, so pass the held descriptor: it inherits + the lock instead of deadlocking on it. + """ + + if not os.path.isfile(INSTALLED_STATE_DIR_GUARD): + raise GuardError( + "state-dir-guard-missing", + INSTALLED_STATE_DIR_GUARD, + "recursive state guard is required for failed-startup recovery", + ) + remaining = deadline - time.monotonic() + if remaining <= 0: + raise GuardError( + "state-dir-transition-timeout", + config_dir, + f"no recovery budget left for state-dir {action}", + ) + try: + completed = subprocess.run( # noqa: S603 + [ + sys.executable, + "-I", + INSTALLED_STATE_DIR_GUARD, + action, + "--config-dir", + config_dir, + "--plan-json", + plan_json, + "--transition-lock-fd", + str(mutation_lock_fd), + ], + capture_output=True, + text=True, + timeout=remaining, + check=False, + pass_fds=(mutation_lock_fd,), + ) + except subprocess.TimeoutExpired as exc: + raise GuardError( + "state-dir-transition-timeout", + config_dir, + f"state-dir {action} exceeded the remaining recovery budget", + ) from exc + except subprocess.SubprocessError as exc: + raise GuardError( + "state-dir-transition-failed", + config_dir, + f"state-dir {action} could not complete: {exc}", + ) from exc + if completed.returncode != 0: + detail = (completed.stderr.strip() or completed.stdout.strip())[:400] + raise GuardError( + "state-dir-transition-failed", + config_dir, + f"state-dir {action} failed: {detail}", + ) + + +def _run_failed_startup_unlock( + opened: OpenConfig, + identity: Identity, + config_dir: str, + plan_json: str, + mutation_lock_fd: int, + *, + quarantine_untrusted: bool, +) -> None: + """Unseal both OpenClaw state layers or restore their locked posture. + + One deadline covers the unseal and its rollback, so a slow unseal cannot + leave the relock without budget, and neither can outlast the host. + """ + + deadline = time.monotonic() + STATE_DIR_GUARD_TIMEOUT_SECONDS + + try: + _run_state_dir_guard("unlock", config_dir, plan_json, mutation_lock_fd, deadline) + _transition( + "unlock", + opened, + identity, + quarantine_untrusted=quarantine_untrusted, + ) + except (GuardError, OSError) as exc: + rollback_errors: list[str] = [] + try: + _transition("lock", opened, identity) + except (GuardError, OSError) as rollback_exc: + rollback_errors.append(f"config lock: {rollback_exc}") + try: + _run_state_dir_guard("lock", config_dir, plan_json, mutation_lock_fd, deadline) + except (GuardError, OSError) as rollback_exc: + rollback_errors.append(f"state-dir lock: {rollback_exc}") + + detail = str(exc) + if rollback_errors: + detail += "; rollback issues: " + "; ".join(rollback_errors) + if isinstance(exc, GuardError): + raise GuardError(exc.code, exc.path, detail) from exc + raise GuardError("operation-failed", config_dir, detail) from exc def _write_secondary_journal(record: dict[str, object], identity: Identity) -> None: @@ -4130,11 +4315,13 @@ def _parser() -> argparse.ArgumentParser: "publish-startup-ready", "write-config", "recover", + "unlock-failed-startup", ), ) parser.add_argument("--config-dir", default=PRODUCTION_CONFIG_DIR) parser.add_argument("--expected-config-sha256", default="") parser.add_argument("--startup-owner", action="store_true") + parser.add_argument("--plan-json", default=None) return parser @@ -4163,9 +4350,26 @@ def main(argv: list[str] | None = None) -> int: f"helper is restricted to {PRODUCTION_CONFIG_DIR}", ) identity = _production_identity() - _validate_action_readiness(action, args.startup_owner, identity) + startup_failure_recovery = _validate_action_readiness( + action, args.startup_owner, identity + ) + if action == "unlock-failed-startup" and not startup_failure_recovery: + # Never let this action inherit the ordinary lease path. + raise GuardError( + "startup-not-ready", + STARTUP_READY_PATH, + "unlock-failed-startup requires a proven terminal startup failure", + ) read_only = action in {"preflight", "preflight-restart"} mutex = _acquire_mutation_mutex(action, identity, exclusive=not read_only) + if startup_failure_recovery: + _reconfirm_startup_failure_recovery(action, identity) + if action == "unlock-failed-startup" and args.plan_json is None: + raise GuardError( + "invalid-state-lock-plan", + "--plan-json", + "unlock-failed-startup requires the agent state lock plan", + ) if action in {"revoke-startup-ready", "publish-startup-ready"}: if action == "revoke-startup-ready": @@ -4312,6 +4516,19 @@ def main(argv: list[str] | None = None) -> int: recovery, new_digest, original_locked = _recover_any_transaction( opened, identity, pending_journal ) + elif action == "unlock-failed-startup": + # Every check that can refuse this action has already run, so no + # refusal can leave state unsealed under a locked config. + _run_failed_startup_unlock( + opened, + identity, + args.config_dir, + args.plan_json, + mutex.fd, + quarantine_untrusted=untrusted_reserved_entry, + ) + new_digest = None + recovery = None else: _transition( action, diff --git a/scripts/state-dir-guard.py b/scripts/state-dir-guard.py index 73704ef8e1b..a470870d682 100755 --- a/scripts/state-dir-guard.py +++ b/scripts/state-dir-guard.py @@ -47,6 +47,7 @@ {"/sandbox/.openclaw", "/sandbox/.hermes", "/sandbox/.deepagents"} ) OPENCLAW_MUTATION_MUTEX_PATH = "/run/nemoclaw/openclaw-config-mutation.lock" +MAX_TRANSITION_LOCK_BYTES = 16 * 1024 # Keep this exact source/target contract aligned with # src/lib/state/openclaw-managed-extensions.ts. OPENCLAW_IMAGE_PACKAGE_PATHS = frozenset( @@ -2378,16 +2379,108 @@ def _acquire_transition_lock(path: str, identity: Identity) -> int: os.close(parent_fd) +def _run_guard_with_transition_lock_fd( + action: Action, + config_dir: str, + identity: Identity, + plan: AgentStateLockPlan, + lock_path: str, + lock_fd: int, +) -> GuardResult: + """Run under the caller's inherited OpenClaw mutation-mutex description. + + The caller must hold the mutex exclusively. Re-locking an inherited + descriptor succeeds while a separately opened one blocks, which is what + proves inheritance. A shared lock would pass without giving exclusion, so + do not reuse this path for read-only actions. + """ + + try: + opened = os.fstat(lock_fd) + current = os.stat(lock_path, follow_symlinks=False) + if ( + not stat.S_ISREG(opened.st_mode) + or opened.st_nlink != 1 + or not _same_entry(opened, current) + or opened.st_uid != identity.root_uid + or opened.st_gid != identity.root_gid + or stat.S_IMODE(opened.st_mode) != 0o600 + or opened.st_size > MAX_TRANSITION_LOCK_BYTES + ): + raise GuardOperationError( + Issue( + "unsafe-transition-lock", + lock_path, + "inherited mutation mutex must be the private root-owned lock file", + ) + ) + try: + # An inherited descriptor shares the parent's flock ownership; a + # separately opened one blocks and cannot bypass serialization. + fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError as exc: + raise GuardOperationError( + Issue( + "transition-lock-not-inherited", + lock_path, + "mutation mutex descriptor does not share the caller's lock", + ) + ) from exc + after = os.stat(lock_path, follow_symlinks=False) + if not _same_entry(opened, after): + raise GuardOperationError( + Issue( + "transition-lock-raced", + lock_path, + "transition mutex changed during inherited-lock verification", + ) + ) + return _run_guard_unserialized(action, config_dir, identity, plan) + except GuardOperationError as exc: + result = GuardResult(action=action) + result.issues.append(exc.issue) + return result + except OSError as exc: + result = GuardResult(action=action) + result.issues.append( + _os_issue( + "transition-lock-failed", lock_path, "verify inherited mutation mutex", exc + ) + ) + return result + + def run_guard( action: Action, config_dir: str, identity: Identity, plan: AgentStateLockPlan, + *, + transition_lock_fd: int | None = None, ) -> GuardResult: """Serialize production OpenClaw recursive transitions with its top guard.""" normalized_config = posixpath.normpath(config_dir) lock_path = _transition_lock_path(normalized_config) + if transition_lock_fd is not None: + if lock_path is None: + result = GuardResult(action=action) + result.issues.append( + Issue( + "unexpected-transition-lock-fd", + normalized_config, + "an inherited transition mutex is valid only for serialized OpenClaw state", + ) + ) + return result + return _run_guard_with_transition_lock_fd( + action, + normalized_config, + identity, + plan, + lock_path, + transition_lock_fd, + ) if lock_path is None: return _run_guard_unserialized(action, normalized_config, identity, plan) @@ -2445,6 +2538,7 @@ def _parse_args(argv: list[str]) -> argparse.Namespace: plan_source = parser.add_mutually_exclusive_group() plan_source.add_argument("--plan-json") plan_source.add_argument("--plan-file") + parser.add_argument("--transition-lock-fd", type=int, help=argparse.SUPPRESS) return parser.parse_args(argv) @@ -2510,7 +2604,13 @@ def main(argv: list[str] | None = None) -> int: ) ) else: - result = run_guard(args.action, args.config_dir, identity, plan) + result = run_guard( + args.action, + args.config_dir, + identity, + plan, + transition_lock_fd=args.transition_lock_fd, + ) for issue in result.issues: print(json.dumps(issue.as_json(), sort_keys=True, separators=(",", ":"))) diff --git a/src/lib/shields/index.ts b/src/lib/shields/index.ts index fc239035411..c7e5e0e2879 100644 --- a/src/lib/shields/index.ts +++ b/src/lib/shields/index.ts @@ -2122,6 +2122,39 @@ function assertNoLegacyStateLayout(sandboxName: string, configDir: string): void // read_only) + chown/chmod below. // --------------------------------------------------------------------------- +/** Whether a guard error reports the OpenClaw startup readiness lease. */ +function isOpenClawStartupNotReady(error: unknown): boolean { + return (error instanceof Error ? error.message : String(error)).includes("[startup-not-ready]"); +} + +/** The guard's refusal when the sandbox is simply not in a failed startup. */ +const NOT_A_FAILED_STARTUP = "requires a proven terminal startup failure"; + +/** + * Lower shields on an OpenClaw sandbox whose startup terminally failed. + * + * Returns false when the sandbox is not in that state. The guard proves a + * stable supervisor with no startup process and no readiness marker, then + * unseals both layers in one mutex window. + */ +function recoverOpenClawFailedStartupShields( + sandboxName: string, + target: AgentConfigTarget, +): boolean { + assertCanonicalOpenClawConfigTarget(target); + const result = runOpenClawConfigGuard( + openClawConfigGuardExec(sandboxName), + "unlock-failed-startup", + { planJson: JSON.stringify(requireStateLockPlan(target)) }, + ); + if (result.issues.length === 0) return true; + // Only "not a failed startup" falls back. A transition, rollback, contract, + // parse, or timeout failure must surface instead of being masked. + const notApplicable = result.issues.every((issue) => issue.includes(NOT_A_FAILED_STARTUP)); + if (notApplicable) return false; + throw new Error(`Failed-startup shields recovery failed: ${result.issues.join(", ")}`); +} + function unlockAgentConfigUnderMutationLock( sandboxName: string, rawTarget: AgentConfigTarget, @@ -2161,7 +2194,16 @@ function unlockAgentConfigUnderMutationLock( let openClawMutationStarted = false; try { if (openClawProtocol) { - transitionOpenClawTopConfig(sandboxName, target, "preflight"); + try { + transitionOpenClawTopConfig(sandboxName, target, "preflight"); + } catch (preflightError) { + // Preflight is read-only, so nothing is mutated yet. Hand the whole + // unseal to the guard, which does it atomically (#8304). + if (!isOpenClawStartupNotReady(preflightError)) throw preflightError; + if (!recoverOpenClawFailedStartupShields(sandboxName, target)) throw preflightError; + console.log(" Lowered shields on a sandbox whose startup never completed."); + return; + } } if (target.agentName === "hermes" && !legacyHermesProtocol) { transaction = beginHermesConfigShields( diff --git a/src/lib/shields/openclaw-config-lock.test.ts b/src/lib/shields/openclaw-config-lock.test.ts index 36cc6416c95..d6902dccb59 100644 --- a/src/lib/shields/openclaw-config-lock.test.ts +++ b/src/lib/shields/openclaw-config-lock.test.ts @@ -400,3 +400,43 @@ describe("OpenClaw top-config guard host wiring", () => { expect(parseOpenClawConfigGuardOutput("lock", plain).resealedDrift).toBeUndefined(); }); }); + +describe("OpenClaw config guard failed-startup recovery wiring (#8304)", () => { + it("accepts the recovery action's result record instead of discarding it", () => { + const { privileged } = createExec(true); + + const result = runOpenClawConfigGuard(privileged, "unlock-failed-startup", { + planJson: '{"version":1}', + }); + + // A missing entry in the parser's action set turns a successful guard run + // into an "unknown record" issue, which silently disables the whole path. + expect(result.issues).toEqual([]); + }); + + it("outlasts the guard's own recursive fan-out budget and forwards the plan", () => { + const { calls, privileged } = createExec(true); + + runOpenClawConfigGuard(privileged, "unlock-failed-startup", { planJson: '{"version":1}' }); + const recovery = calls + .map(({ cmd }) => cmd) + .find((cmd) => cmd.includes("unlock-failed-startup")); + + // The guard allows the state-dir fan-out 12m, so a 5m host timeout would + // kill it mid-unseal, past its rollback and its JSON error contract. + expect(recovery?.slice(0, 4)).toEqual(["timeout", "--signal=TERM", "--kill-after=5s", "15m"]); + expect(recovery).toContain("--plan-json"); + }); + + it("refuses the recovery action when the sandbox has no installed guard", () => { + const { privileged } = createExec(false); + + const result = runOpenClawConfigGuard(privileged, "unlock-failed-startup", { + planJson: '{"version":1}', + }); + + expect(result.issues).toEqual([ + "OpenClaw config guard is absent in the sandbox; rebuild before recovering a failed startup", + ]); + }); +}); diff --git a/src/lib/shields/openclaw-config-lock.ts b/src/lib/shields/openclaw-config-lock.ts index 91214bc2807..afe53480294 100644 --- a/src/lib/shields/openclaw-config-lock.ts +++ b/src/lib/shields/openclaw-config-lock.ts @@ -19,6 +19,11 @@ export const OPENCLAW_CONFIG_HASH_PATH = `${OPENCLAW_CONFIG_DIR}/.config-hash`; const CONTAINER_HELPER = "/usr/local/lib/nemoclaw/openclaw-config-guard.py"; const HOST_HELPER = path.resolve(__dirname, "../../../scripts/openclaw-config-guard.py"); const CONTAINER_TIMEOUT = ["timeout", "--signal=TERM", "--kill-after=5s", "5m"]; +// Must exceed STATE_DIR_GUARD_TIMEOUT_SECONDS (12m) in +// scripts/openclaw-config-guard.py, which is the guard's whole-action budget +// for the unseal and its rollback together. A shorter host timeout lands the +// kill mid-unseal, past the rollback and the JSON contract. +const RECOVERY_CONTAINER_TIMEOUT = ["timeout", "--signal=TERM", "--kill-after=5s", "15m"]; const SCHEMA_VALIDATION_TIMEOUT = ["timeout", "--signal=TERM", "--kill-after=5s", "30s"]; const MAX_SCHEMA_CANDIDATE_BYTES = 16 * 1024 * 1024; // OpenClaw resolves relative includes from the config file's directory. @@ -41,12 +46,15 @@ export type OpenClawConfigGuardAction = | "write-config" | "recover" | "revoke-startup-ready" - | "publish-startup-ready"; + | "publish-startup-ready" + | "unlock-failed-startup"; export type OpenClawConfigGuardOptions = { expectedConfigSha256?: string; input?: string; startupOwner?: boolean; + /** Agent state lock plan, required by `unlock-failed-startup`. */ + planJson?: string; }; type GuardIssue = { @@ -91,6 +99,7 @@ const GUARD_ACTIONS = new Set([ "recover", "revoke-startup-ready", "publish-startup-ready", + "unlock-failed-startup", ]); function executionFailure(label: string, result: PrivilegedExecResult): string { @@ -353,11 +362,13 @@ export function runOpenClawConfigGuard( } const capability = privileged.run(["test", "-r", CONTAINER_HELPER]); + const timeoutPrefix = + action === "unlock-failed-startup" ? RECOVERY_CONTAINER_TIMEOUT : CONTAINER_TIMEOUT; let command: string[]; let input: string | undefined; if (capability.status === 0 && capability.signal === null && !capability.error) { command = [ - ...CONTAINER_TIMEOUT, + ...timeoutPrefix, "python3", "-I", CONTAINER_HELPER, @@ -378,6 +389,16 @@ export function runOpenClawConfigGuard( chattrApplied: false, }; } + if (action === "unlock-failed-startup") { + // Needs the in-image state guard and the installed helper. An injected + // copy satisfies neither, so refuse instead of half-running it. + return { + issues: [ + "OpenClaw config guard is absent in the sandbox; rebuild before recovering a failed startup", + ], + chattrApplied: false, + }; + } try { input = readHostHelper(); } catch (error) { @@ -389,15 +410,7 @@ export function runOpenClawConfigGuard( chattrApplied: false, }; } - command = [ - ...CONTAINER_TIMEOUT, - "python3", - "-I", - "-", - action, - "--config-dir", - OPENCLAW_CONFIG_DIR, - ]; + command = [...timeoutPrefix, "python3", "-I", "-", action, "--config-dir", OPENCLAW_CONFIG_DIR]; } else { return { issues: [executionFailure("OpenClaw config guard capability probe failed", capability)], @@ -409,6 +422,7 @@ export function runOpenClawConfigGuard( command.push("--expected-config-sha256", options.expectedConfigSha256); } if (options.startupOwner) command.push("--startup-owner"); + if (options.planJson) command.push("--plan-json", options.planJson); return parseOpenClawConfigGuardOutput(action, privileged.run(command, input)); } diff --git a/test/e2e/live/shields-config.test.ts b/test/e2e/live/shields-config.test.ts index 4c4c495ac6b..682c0c93757 100644 --- a/test/e2e/live/shields-config.test.ts +++ b/test/e2e/live/shields-config.test.ts @@ -34,6 +34,12 @@ import { stripAnsi } from "./json-envelope.ts"; const CONFIG_PATH = "/sandbox/.openclaw/openclaw.json"; const CONFIG_DIR = path.dirname(CONFIG_PATH); const CONFIG_HASH_PATH = `${CONFIG_DIR}/.config-hash`; +const CONFIG_GUARD_PATH = "/usr/local/lib/nemoclaw/openclaw-config-guard.py"; +const STATE_LOCK_PLAN_PATH = "/usr/local/share/nemoclaw/state-lock-plan.json"; +const STARTUP_MARKER_PATHS = [ + "/run/nemoclaw/openclaw-config-ready-v1.capability.json", + "/run/nemoclaw/openclaw-config-ready.json", +] as const; const AUDIT_FILE = path.join(os.homedir(), ".nemoclaw", "state", "shields-audit.jsonl"); const STATE_FILE = (sandboxName: string) => path.join(os.homedir(), ".nemoclaw", "state", `shields-${sandboxName}.json`); @@ -257,6 +263,72 @@ async function findSandboxContainer(host: HostCliClient): Promise { return containerId; } +type StartupCensus = { count: number; pid: number | null }; + +async function installedStartupCensus( + host: HostCliClient, + containerId: string, + artifactName: string, +): Promise { + const script = [ + "import json, runpy", + `guard = runpy.run_path(${JSON.stringify(CONFIG_GUARD_PATH)})`, + "identity = guard['_production_identity']()", + "census = guard['_openshell_supervised_nonroot_start_census'](identity.root_uid, identity.sandbox_uid)", + "assert census is not None", + "print(json.dumps({'count': census[0], 'pid': census[1]}))", + ].join("\n"); + const result = await docker( + host, + ["exec", "--user", "0", containerId, "python3", "-I", "-c", script], + { artifactName, timeoutMs: 30_000 }, + ); + expect(result.exitCode, resultText(result)).toBe(0); + return JSON.parse(result.stdout.trim()) as StartupCensus; +} + +async function runInstalledFailedStartupUnlock( + host: HostCliClient, + containerId: string, + artifactName: string, +): Promise { + const script = [ + "set -eu", + `plan_json=$(cat ${STATE_LOCK_PLAN_PATH})`, + `exec timeout --signal=TERM --kill-after=5s 15m python3 -I ${CONFIG_GUARD_PATH} unlock-failed-startup --config-dir ${CONFIG_DIR} --plan-json "$plan_json"`, + ].join("\n"); + return docker(host, ["exec", "--user", "0", containerId, "sh", "-c", script], { + artifactName, + timeoutMs: 16 * 60_000, + }); +} + +async function waitForChildlessStartup( + host: HostCliClient, + containerId: string, + startupPid: number, +): Promise { + expect(Number.isSafeInteger(startupPid) && startupPid > 1).toBe(true); + const terminate = await docker( + host, + ["exec", "--user", "0", containerId, "kill", "-TERM", String(startupPid)], + { artifactName: "phase-12-terminate-startup-child", timeoutMs: 30_000 }, + ); + expect(terminate.exitCode, resultText(terminate)).toBe(0); + + let lastCensus: StartupCensus | undefined; + for (let attempt = 1; attempt <= 20; attempt += 1) { + lastCensus = await installedStartupCensus( + host, + containerId, + `phase-12-childless-census-${attempt}`, + ); + if (lastCensus.count === 0) return; + await delay(500); + } + throw new Error(`startup child remained live after termination: ${JSON.stringify(lastCensus)}`); +} + async function readOriginalConfig( host: HostCliClient, containerId: string, @@ -308,6 +380,7 @@ test("shields-config: live Shields lifecycle restores stopped OpenClaw under bot "restart OpenClaw with shields down", "recover shields after a dead restore timer", "reject duplicate shields transitions", + "prove installed failed-startup recovery refuses a live child and unlocks childless state", "record shields contract evidence", ], }, @@ -329,6 +402,7 @@ test("shields-config: live Shields lifecycle restores stopped OpenClaw under bot "start restores a stopped OpenClaw sandbox while shields are down", "dead auto-restore timer inline recovery re-locks config and .config-hash", "double shields-up/down operations are rejected", + "installed failed-startup recovery refuses a live supervised child and atomically unlocks childless state", ], }); @@ -889,6 +963,76 @@ test("shields-config: live Shields lifecycle restores stopped OpenClaw under bot expect(finalUp.exitCode, resultText(finalUp)).toBe(0); expect(resultText(finalUp)).toContain("Lockdown active"); + progress.phase( + "prove installed failed-startup recovery refuses a live child and unlocks childless state", + ); + const recoveryContainerId = await findSandboxContainer(host); + const removeMarkers = await docker( + host, + ["exec", "--user", "0", recoveryContainerId, "rm", "-f", ...STARTUP_MARKER_PATHS], + { artifactName: "phase-12-remove-startup-markers", timeoutMs: 30_000 }, + ); + expect(removeMarkers.exitCode, resultText(removeMarkers)).toBe(0); + + const liveCensus = await installedStartupCensus( + host, + recoveryContainerId, + "phase-12-live-startup-census", + ); + expect(liveCensus).toMatchObject({ count: 1, pid: expect.any(Number) }); + if (liveCensus.pid === null) throw new Error("live startup census did not return its process ID"); + const liveChildRefusal = await runInstalledFailedStartupUnlock( + host, + recoveryContainerId, + "phase-12-live-child-refusal", + ); + expect(liveChildRefusal.exitCode, resultText(liveChildRefusal)).not.toBe(0); + expect(resultText(liveChildRefusal)).toContain('"code": "startup-not-ready"'); + expect(await statPath(sandbox, CONFIG_PATH, "phase-12-config-still-locked")).toMatchObject({ + mode: "444", + owner: "root:root", + }); + + await waitForChildlessStartup(host, recoveryContainerId, liveCensus.pid); + const childlessUnlock = await runInstalledFailedStartupUnlock( + host, + recoveryContainerId, + "phase-12-childless-unlock", + ); + expect(childlessUnlock.exitCode, resultText(childlessUnlock)).toBe(0); + expect(resultText(childlessUnlock)).toContain('"action": "unlock-failed-startup"'); + expect(resultText(childlessUnlock)).toContain('"status": "ok"'); + expect(await statPath(sandbox, CONFIG_PATH, "phase-12-config-unlocked")).toMatchObject({ + mode: "660", + owner: "sandbox:sandbox", + }); + expect( + await statPath(sandbox, `${CONFIG_DIR}/workspace`, "phase-12-state-tree-unlocked"), + ).toMatchObject({ mode: "2770", owner: "sandbox:sandbox" }); + + // Reconcile the host-side Shields receipt after the direct installed-guard + // proof, then restart the failed sandbox and return cleanup to lockdown. + const reconcileDown = await runNemoclaw( + host, + [ + SANDBOX_NAME, + "shields", + "down", + "--timeout", + "5m", + "--reason", + "Installed failed-startup recovery E2E", + ], + { artifactName: "phase-12-reconcile-shields-down", timeoutMs: 16 * 60_000 }, + ); + expect(reconcileDown.exitCode, resultText(reconcileDown)).toBe(0); + await expectStopStartRecovery(host, "DOWN", "phase-12-restart-after-recovery"); + const relockAfterRecovery = await runNemoclaw(host, [SANDBOX_NAME, "shields", "up"], { + artifactName: "phase-12-relock-after-recovery", + }); + expect(relockAfterRecovery.exitCode, resultText(relockAfterRecovery)).toBe(0); + expect(resultText(relockAfterRecovery)).toContain("Lockdown active"); + progress.phase("record shields contract evidence"); await artifacts.target.complete({ id: "shields-config", @@ -905,6 +1049,9 @@ test("shields-config: live Shields lifecycle restores stopped OpenClaw under bot auditTrail: true, deadTimerInlineAutoRestore: true, doubleOperationRejection: true, + installedFailedStartupLiveChildRefusal: true, + installedFailedStartupChildlessUnlock: true, + inheritedMutationLockAcceptedByStateGuard: true, }, }); }); diff --git a/test/openclaw-config-guard-startup-failure-gate.test.ts b/test/openclaw-config-guard-startup-failure-gate.test.ts new file mode 100644 index 00000000000..3b50ed035fe --- /dev/null +++ b/test/openclaw-config-guard-startup-failure-gate.test.ts @@ -0,0 +1,362 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; + +const GUARD_PATH = path.resolve("scripts/openclaw-config-guard.py"); +const STATE_GUARD_PATH = path.resolve("scripts/state-dir-guard.py"); +const PYTHON = process.platform === "win32" ? "python" : "python3"; + +const HARNESS = String.raw` +import importlib.util +import json +import os +import sys +import tempfile +import typing + +spec = importlib.util.spec_from_file_location("guard", sys.argv[1]) +guard = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = guard +spec.loader.exec_module(guard) + +identity = guard.Identity(root_uid=0, root_gid=0, sandbox_uid=1000, sandbox_gid=1000) +guard.INSTALLED_HELPER_PATH = guard.__file__ +guard._pid1_is_nemoclaw_start = lambda: False +guard._startup_lease_state = lambda _identity: (False, False) +nul = bytes([0]) +start_cmdline = b"bash" + nul + b"/usr/local/bin/nemoclaw-start" + nul +supervisor_cmdline = b"/opt/openshell/bin/openshell-sandbox" + nul + +def write_process(proc_root, pid, cmdline, namespace_path, uid, parent_pid): + process_dir = os.path.join(proc_root, str(pid)) + os.makedirs(os.path.join(process_dir, "ns")) + fields = ["S", str(parent_pid)] + (["0"] * 17) + ["424242"] + with open(os.path.join(process_dir, "stat"), "w", encoding="ascii") as stream: + stream.write(f"{pid} (nemoclaw) {' '.join(fields)}\n") + with open(os.path.join(process_dir, "cmdline"), "wb") as stream: + stream.write(cmdline) + with open(os.path.join(process_dir, "status"), "w", encoding="ascii") as stream: + stream.write( + f"Uid:\t{uid}\t{uid}\t{uid}\t{uid}\n" + f"NSpid:\t{pid}\t{pid}\n" + ) + os.link(namespace_path, os.path.join(process_dir, "ns", "pid")) + +def with_proc(children, markers_absent, supervisor, limit): + root = tempfile.mkdtemp() + proc_root = os.path.join(root, "proc") + os.mkdir(proc_root) + namespace_path = os.path.join(root, "shared") + with open(namespace_path, "wb") as stream: + stream.write(b"shared") + write_process(proc_root, 1, supervisor, namespace_path, 0, 0) + for pid in children: + write_process(proc_root, pid, start_cmdline, namespace_path, 1000, 1) + guard.PROC_ROOT = proc_root + guard.MAX_PROC_ENTRIES = limit + guard._startup_markers_absent = lambda _identity: markers_absent + +def gate(action, children, markers_absent=True, supervisor=supervisor_cmdline, limit=32768): + with_proc(children, markers_absent, supervisor, limit) + try: + guard._validate_action_readiness(action, False, identity) + return "allowed" + except guard.GuardError as error: + return error.code + +def provisional(action, children): + with_proc(children, True, supervisor_cmdline, 32768) + try: + return bool(guard._validate_action_readiness(action, False, identity)) + except guard.GuardError: + return "refused" + +def reconfirm(children, markers_absent=True): + with_proc(children, markers_absent, supervisor_cmdline, 32768) + try: + guard._reconfirm_startup_failure_recovery("unlock", identity) + return "ok" + except guard.GuardError as error: + return error.code + +def cli_accepts(action): + parser = guard._parser() + choices = next(a.choices for a in parser._actions if a.dest == "action") + return action in set(choices) and set(choices) == set(typing.get_args(guard.Action)) + +print(json.dumps({ + # The CLI choices tuple is separate from the Action type, so an action can + # exist in code and still be unreachable through the entry point. + "cli_exposes_recovery": cli_accepts("unlock-failed-startup"), + "failed_recovery": gate("unlock-failed-startup", []), + "failed_preflight": gate("preflight", []), + "failed_unlock": gate("unlock", []), + "failed_lock": gate("lock", []), + "failed_write": gate("write-config", []), + "failed_seal": gate("seal-restart", []), + "failed_recover": gate("recover", []), + "live_lock": gate("lock", [412]), + "duplicate_unlock": gate("unlock-failed-startup", [412, 413]), + "foreign_unlock": gate("unlock-failed-startup", [], supervisor=b"/usr/bin/foreign" + nul), + "stale_marker_unlock": gate("unlock-failed-startup", [], markers_absent=False), + "bounded_scan_unlock": gate("unlock-failed-startup", [], limit=0), + "provisional_failed_recovery": provisional("unlock-failed-startup", []), + "provisional_live_lock": provisional("lock", [412]), + "live_recovery": gate("unlock-failed-startup", [412]), + "reconfirm_still_childless": reconfirm([]), + "reconfirm_child_appeared": reconfirm([412]), + "reconfirm_marker_appeared": reconfirm([], markers_absent=False), +})) +`; + +const TRANSACTION_HARNESS = String.raw` +import importlib.util +import json +import subprocess +import time +import sys +import types + +if sys.platform == "win32": + for name in ("fcntl", "grp", "pwd"): + sys.modules[name] = types.ModuleType(name) + +spec = importlib.util.spec_from_file_location("guard", sys.argv[1]) +guard = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = guard +spec.loader.exec_module(guard) + +guard.os.path.isfile = lambda _path: True +run_call = {} +def timeout_run(command, **kwargs): + run_call["command"] = command + run_call["pass_fds"] = kwargs.get("pass_fds") + raise subprocess.TimeoutExpired("state-dir-guard", guard.STATE_DIR_GUARD_TIMEOUT_SECONDS) + +guard.subprocess.run = timeout_run +try: + guard._run_state_dir_guard( + "unlock", + guard.PRODUCTION_CONFIG_DIR, + "{}", + 91, + time.monotonic() + guard.STATE_DIR_GUARD_TIMEOUT_SECONDS, + ) +except guard.GuardError as error: + timeout_code = error.code + +events = [] +def transition(action, _opened, _identity, **_kwargs): + events.append(f"config-{action}") + if action == "unlock": + raise guard.MutableHandoffError( + "mutable-handoff-incomplete", guard.PRODUCTION_CONFIG_DIR, "handoff failed" + ) + +def state_dir(action, _config_dir, _plan_json, lock_fd, _deadline): + assert lock_fd == 91 + events.append(f"state-{action}") + if action == "lock": + raise guard.GuardError("state-lock-failed", guard.PRODUCTION_CONFIG_DIR, "lock failed") + +guard._transition = transition +guard._run_state_dir_guard = state_dir +try: + guard._run_failed_startup_unlock( + object(), object(), guard.PRODUCTION_CONFIG_DIR, "{}", 91, quarantine_untrusted=False + ) +except guard.GuardError as error: + transaction_error = { + "code": error.code, + "detail": error.detail, + } + +timeout_events = [] +def timeout_transition(action, _opened, _identity, **_kwargs): + timeout_events.append(f"config-{action}") + +def timeout_state_dir(action, _config_dir, _plan_json, lock_fd, _deadline): + assert lock_fd == 91 + timeout_events.append(f"state-{action}") + if action == "unlock": + raise guard.GuardError( + "state-dir-transition-timeout", guard.PRODUCTION_CONFIG_DIR, "unlock timed out" + ) + +guard._transition = timeout_transition +guard._run_state_dir_guard = timeout_state_dir +try: + guard._run_failed_startup_unlock( + object(), object(), guard.PRODUCTION_CONFIG_DIR, "{}", 91, quarantine_untrusted=False + ) +except guard.GuardError as error: + timeout_transaction_code = error.code + +print(json.dumps({ + "timeout_code": timeout_code, + "lock_fd_flag": run_call["command"][-2:], + "pass_fds": run_call["pass_fds"], + "events": events, + "transaction_error": transaction_error, + "timeout_events": timeout_events, + "timeout_transaction_code": timeout_transaction_code, +})) +`; + +const LOCK_HANDOFF_HARNESS = String.raw` +import fcntl +import json +import os +import subprocess +import sys +import tempfile + +guard_path = sys.argv[1] +root = tempfile.mkdtemp() +config_dir = os.path.join(root, ".openclaw") +lock_path = os.path.join(root, ".openclaw-config-mutation.lock") +os.mkdir(config_dir) +owner_fd = os.open(lock_path, os.O_RDWR | os.O_CREAT, 0o600) +foreign_fd = os.open(lock_path, os.O_RDWR) +fcntl.flock(owner_fd, fcntl.LOCK_EX) +plan_json = json.dumps({ + "version": 1, + "readOnlyRoots": [], + "confidentialRoots": [], + "readOnlyPrefixes": [], + "confidentialPrefixes": [], + "writableSubpaths": [], +}) +child = r''' +import importlib.util +import json +import os +import sys + +guard_path, config_dir, plan_json, lock_fd = sys.argv[1:5] +spec = importlib.util.spec_from_file_location("state_guard", guard_path) +guard = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = guard +spec.loader.exec_module(guard) +identity = guard.Identity( + root_uid=os.getuid(), root_gid=os.getgid(), + sandbox_uid=os.getuid(), sandbox_gid=os.getgid(), +) +guard.os.geteuid = lambda: 0 +guard._production_identity = lambda: identity +raise SystemExit(guard.main([ + "unlock", "--config-dir", config_dir, "--plan-json", plan_json, + "--transition-lock-fd", lock_fd, +])) +''' +env = {**os.environ, "NEMOCLAW_TEST_OPENCLAW_TRANSACTION_LOCK": "1"} + +def invoke(lock_fd): + return subprocess.run( + [sys.executable, "-c", child, guard_path, config_dir, plan_json, str(lock_fd)], + capture_output=True, + text=True, + timeout=5, + env=env, + pass_fds=(lock_fd,), + check=False, + ) + +inherited = invoke(owner_fd) +foreign = invoke(foreign_fd) +print(json.dumps({ + "inherited_status": inherited.returncode, + "inherited_records": [json.loads(line) for line in inherited.stdout.splitlines()], + "foreign_status": foreign.returncode, + "foreign_records": [json.loads(line) for line in foreign.stdout.splitlines()], +})) +`; + +describe("OpenClaw failed-startup unlock transaction (#8304)", () => { + it("relocks both state layers after a config handoff failure", () => { + const result = spawnSync(PYTHON, ["-c", TRANSACTION_HARNESS, GUARD_PATH], { + encoding: "utf-8", + timeout: 10000, + }); + + expect(result.status, result.stderr).toBe(0); + expect(JSON.parse(result.stdout)).toEqual({ + timeout_code: "state-dir-transition-timeout", + lock_fd_flag: ["--transition-lock-fd", "91"], + pass_fds: [91], + events: ["state-unlock", "config-unlock", "config-lock", "state-lock"], + transaction_error: { + code: "mutable-handoff-incomplete", + detail: "handoff failed; rollback issues: state-dir lock: lock failed", + }, + timeout_events: ["state-unlock", "config-lock", "state-lock"], + timeout_transaction_code: "state-dir-transition-timeout", + }); + }); +}); + +describe.skipIf(process.platform === "win32")( + "OpenClaw config guard startup-failure gate (#8304)", + () => { + it("shares the held mutation lock with the recursive state guard", () => { + const result = spawnSync(PYTHON, ["-c", LOCK_HANDOFF_HARNESS, STATE_GUARD_PATH], { + encoding: "utf-8", + timeout: 10000, + }); + + expect(result.status, result.stderr).toBe(0); + const outcome = JSON.parse(result.stdout); + expect(outcome.inherited_status).toBe(0); + expect(outcome.inherited_records).toContainEqual( + expect.objectContaining({ type: "result", action: "unlock", status: "ok" }), + ); + expect(outcome.foreign_status).toBe(1); + expect(outcome.foreign_records).toContainEqual( + expect.objectContaining({ type: "issue", code: "transition-lock-not-inherited" }), + ); + }); + + it("combines the real process census with the action gate", () => { + const result = spawnSync(PYTHON, ["-c", HARNESS, GUARD_PATH], { + encoding: "utf-8", + timeout: 10000, + }); + + expect(result.status, result.stderr).toBe(0); + expect(JSON.parse(result.stdout)).toEqual({ + cli_exposes_recovery: true, + // Only the dedicated atomic action is reachable through the escape. + failed_recovery: "allowed", + // The multi-step host sequence stays refused, so its first step fails + // closed and no recursive state is mutated on stale evidence. + failed_preflight: "startup-not-ready", + failed_unlock: "startup-not-ready", + failed_lock: "startup-not-ready", + failed_write: "startup-not-ready", + failed_seal: "startup-not-ready", + failed_recover: "startup-not-ready", + live_lock: "allowed", + duplicate_unlock: "startup-not-ready", + foreign_unlock: "startup-not-ready", + stale_marker_unlock: "startup-not-ready", + // A bounded-scan overflow makes the census undeterminable. Both + // predicates must map that onto False so neither path authenticates. + bounded_scan_unlock: "startup-not-ready", + // Only the failed-startup path reports a provisional authorization, so + // the mutex-held reconfirm runs for that path and nothing else. + provisional_failed_recovery: true, + provisional_live_lock: false, + // A healthy sandbox must never reach the recovery action. + live_recovery: "startup-not-ready", + // The reconfirm is what binds the census to the effect: a start child + // or a marker appearing after the pre-mutex scan revokes the escape. + reconfirm_still_childless: "ok", + reconfirm_child_appeared: "startup-not-ready", + reconfirm_marker_appeared: "startup-not-ready", + }); + }); + }, +);